diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 18ae4782b..bc3f49726 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -14,6 +14,10 @@ jobs: runs-on: macos-14 env: CARGO_TERM_COLOR: always + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository @@ -52,13 +56,11 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Get Verus commit + - name: Record toolchain revisions id: verus shell: bash run: | - VERUS_COMMIT=$(git ls-remote https://github.com/asterinas/verus HEAD | cut -f1) - echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus commit: $VERUS_COMMIT" + echo "Using pinned Asterinas Verus base: $VERUS_BASE_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -74,7 +76,7 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verus-irc11-weak-memory-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/bootstrap-verus-irc11.sh', 'tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Bootstrap Verus (if needed) shell: bash @@ -82,9 +84,27 @@ jobs: if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus..." + echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - cargo dv bootstrap + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + bash tools/bootstrap-verus-irc11.sh + fi + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + test -f tools/verus/source/vstd/atomic_weak.rs + test -f tools/verus/source/vstd/thread_view.rs + + - name: Enable IRC11 alongside existing SC atomics + shell: bash + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" fi - name: Run verification diff --git a/.github/workflows/ci-upstream-verus.yml b/.github/workflows/ci-upstream-verus.yml index 817f1207e..3a821ce35 100644 --- a/.github/workflows/ci-upstream-verus.yml +++ b/.github/workflows/ci-upstream-verus.yml @@ -1,4 +1,4 @@ -name: Verify VOSTD (Main) with verus-lang/verus +name: Verify VOSTD with rebased Verus IRC11 on: push: @@ -23,6 +23,10 @@ jobs: runs-on: ubuntu-24.04 env: CARGO_TERM_COLOR: always + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Get PR head commit @@ -53,7 +57,7 @@ jobs: sha: '${{ steps.pr.outputs.head_sha }}', state: 'pending', context: 'ci/upstream-verus', - description: 'Upstream Verus verification is running', + description: 'Upstream Verus IRC11 verification is running', target_url: runUrl, }); @@ -68,10 +72,27 @@ jobs: sudo apt update -qq sudo apt install -y build-essential unzip pkg-config libssl-dev llvm - - name: Run dv bootstrap with upstream verus - run: cargo dv bootstrap --upstream-verus + - name: Bootstrap rebased Verus IRC11 + run: | + rm -rf tools/verus + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + bash tools/bootstrap-verus-irc11.sh + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + + - name: Enable IRC11 alongside existing SC atomics + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" + fi - - name: Verify ostd with upstream verus + - name: Verify ostd with upstream Verus IRC11 run: make - name: Report upstream Verus verification status @@ -85,8 +106,8 @@ jobs: const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; const state = '${{ job.status }}' === 'success' ? 'success' : 'failure'; const description = state === 'success' - ? 'Upstream Verus verification passed' - : 'Upstream Verus verification failed'; + ? 'Upstream Verus IRC11 verification passed' + : 'Upstream Verus IRC11 verification failed'; await github.rest.repos.createCommitStatus({ owner, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fa79da4c..e6c3abe89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,10 @@ jobs: runs-on: ubuntu-24.04 env: CARGO_TERM_COLOR: always + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository @@ -61,12 +65,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Get Verus commit + - name: Record toolchain revisions id: verus run: | - VERUS_COMMIT=$(git ls-remote https://github.com/asterinas/verus HEAD | cut -f1) - echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus commit: $VERUS_COMMIT" + echo "Using pinned Asterinas Verus base: $VERUS_BASE_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -82,31 +84,48 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verus-irc11-weak-memory-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/bootstrap-verus-irc11.sh', 'tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt uses: actions/cache@v6 with: path: ~/.cargo/bin/verusfmt - key: ${{ runner.os }}-verusfmt-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BASE_COMMIT }} - name: Bootstrap Verus (if needed) run: | if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus..." + echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - cargo dv bootstrap + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + bash tools/bootstrap-verus-irc11.sh fi if ! command -v verusfmt >/dev/null 2>&1; then - echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap + echo "verusfmt not found, installing..." + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/verus-lang/verusfmt/releases/latest/download/verusfmt-installer.sh | sh fi + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + test -f tools/verus/source/vstd/atomic_weak.rs + test -f tools/verus/source/vstd/thread_view.rs verusfmt --version + - name: Enable IRC11 alongside existing SC atomics + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" + fi + - name: Run verification run: | set -o pipefail @@ -179,4 +198,4 @@ jobs: else echo "- Verification warnings: ✅ none" fi - } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index b698a592b..93a497ae9 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -21,6 +21,11 @@ concurrency: jobs: build: runs-on: ubuntu-latest + env: + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository uses: actions/checkout@v7 @@ -62,12 +67,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Get Verus commit + - name: Record toolchain revisions id: verus run: | - VERUS_COMMIT=$(git ls-remote https://github.com/asterinas/verus HEAD | cut -f1) - echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus commit: $VERUS_COMMIT" + echo "Using pinned Asterinas Verus base: $VERUS_BASE_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -83,31 +86,48 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verus-irc11-weak-memory-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/bootstrap-verus-irc11.sh', 'tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt uses: actions/cache@v6 with: path: ~/.cargo/bin/verusfmt - key: ${{ runner.os }}-verusfmt-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BASE_COMMIT }} - name: Bootstrap Verus (if needed) run: | if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus..." + echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - cargo dv bootstrap + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + bash tools/bootstrap-verus-irc11.sh fi if ! command -v verusfmt >/dev/null 2>&1; then - echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap + echo "verusfmt not found, installing..." + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/verus-lang/verusfmt/releases/latest/download/verusfmt-installer.sh | sh fi + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + test -f tools/verus/source/vstd/atomic_weak.rs + test -f tools/verus/source/vstd/thread_view.rs verusfmt --version + - name: Enable IRC11 alongside existing SC atomics + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" + fi + - name: Build docs run: make doc diff --git a/Cargo.lock b/Cargo.lock index 92c2f90e6..f1945994d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -595,11 +595,11 @@ dependencies = [ [[package]] name = "verus_builtin" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" [[package]] name = "verus_builtin_macros" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" dependencies = [ "convert_case", "proc-macro2", @@ -612,7 +612,7 @@ dependencies = [ [[package]] name = "verus_prettyplease" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" dependencies = [ "proc-macro2", "verus_syn", @@ -651,7 +651,7 @@ checksum = "af8ca9a5d4debca0633e697c88269395493cebf2e10db21ca2dbde37c1356452" [[package]] name = "vstd" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" dependencies = [ "verus_builtin", "verus_builtin_macros", diff --git a/Cargo.toml b/Cargo.toml index 0963ca334..2dc375bf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,4 +50,4 @@ codegen-units = 1 [workspace.dependencies] # Verus -vstd = { path = "tools/verus/source/vstd", default-features = false, features = ["alloc"] } +vstd = { path = "tools/verus/source/vstd", default-features = false, features = ["alloc", "weak-memory"] } diff --git a/ostd/specs/mm/cpu.rs b/ostd/specs/mm/cpu.rs index 0072227e5..ac5c0634c 100644 --- a/ostd/specs/mm/cpu.rs +++ b/ostd/specs/mm/cpu.rs @@ -1,16 +1,179 @@ +use core::sync::atomic::Ordering; + use vstd::prelude::*; +use vstd::resource::Loc; +use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; + +use crate::task::RunningTaskContext; verus! { pub struct CpuId(u32); -pub struct AtomicCpuSet; +/// The fixed set of CPUs participating in system-wide protocols. +/// +/// OSTD does not support CPU hotplug, so the executable `num_cpus()` value is +/// stable after boot. Keeping that set abstract here avoids exposing the +/// executable bitset representation to clients of this specification. +pub uninterp spec fn online_cpus() -> Set; + +impl CpuId { + #[verifier::external_body] + pub fn current(Tracked(context): Tracked<&RunningTaskContext>) -> (res: Self) + requires + context.wf(), + ensures + res == context.cpu(), + online_cpus().contains(res), + { + unimplemented!() + } +} + +/// Linear shadow state for an [`AtomicCpuSet`]. +/// +/// The token is deliberately separate from the executable bitset. Clients +/// that serialize all operations (the RCU monitor does so with its state +/// lock) move this token through `store`, `add`, and `load`, obtaining an exact +/// logical view despite the executable type using atomic words internally. +pub tracked struct AtomicCpuSetToken { + identity: GhostVarAuth<()>, + ghost cpus: Set, +} + +impl AtomicCpuSetToken { + pub closed spec fn id(self) -> Loc { + self.identity.id() + } + + pub closed spec fn cpus(self) -> Set { + self.cpus + } + + pub closed spec fn wf(self) -> bool { + self.cpus().subset_of(online_cpus()) + } +} + +pub struct AtomicCpuSet { + tracked_identity: Tracked>>, + tracked_peer: Tracked>, + ghost_initial: Ghost>, +} impl AtomicCpuSet { + pub closed spec fn id(self) -> Loc { + self.tracked_peer@.id() + } + + pub closed spec fn token_available(self) -> bool { + self.tracked_identity@ is Some + } + + pub closed spec fn initial_cpus(self) -> Set { + self.ghost_initial@ + } + + pub closed spec fn wf(self) -> bool { + &&& self.initial_cpus().subset_of(online_cpus()) + &&& self.token_available() ==> self.tracked_identity@->Some_0.id() == self.id() + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } + + pub fn new(_initial: CpuSet) -> (res: Self) + requires + _initial.cpus.subset_of(online_cpus()), + ensures + res.token_available(), + res.initial_cpus() == _initial.cpus, + res.wf(), + { + let tracked (identity, peer) = GhostVarAuth::new(()); + Self { + tracked_identity: Tracked(Some(identity)), + tracked_peer: Tracked(peer), + ghost_initial: Ghost(_initial.cpus), + } + } + + /// Extracts the unique logical state used to verify serialized operations + /// on this atomic set. + pub proof fn tracked_take_token(tracked &mut self) -> (tracked res: AtomicCpuSetToken) + requires + old(self).token_available(), + ensures + !final(self).token_available(), + final(self).id() == old(self).id(), + final(self).initial_cpus() == old(self).initial_cpus(), + res.id() == final(self).id(), + res.cpus() == final(self).initial_cpus(), + res.wf(), + { + use_type_invariant(&*self); + let tracked identity = self.tracked_identity.borrow_mut().tracked_take(); + let tracked res = AtomicCpuSetToken { identity, cpus: self.ghost_initial@ }; + assert(res.id() == self.id()); + res + } + + #[verus_spec(res => + with + Tracked(token): Tracked<&AtomicCpuSetToken>, + requires + token.id() == self.id(), + token.wf(), + ensures + res.cpus == token.cpus(), + )] #[verifier::external_body] - pub fn new(_initial: CpuSet) -> Self { + pub fn load(&self, _ordering: Ordering) -> CpuSet + no_unwind + { unimplemented!() } + + #[verus_spec( + with + Tracked(token): Tracked<&mut AtomicCpuSetToken>, + requires + old(token).id() == self.id(), + _value.cpus.subset_of(online_cpus()), + ensures + final(token).id() == old(token).id(), + final(token).cpus() == _value.cpus, + final(token).wf(), + )] + pub fn store(&self, _value: &CpuSet, _ordering: Ordering) + no_unwind + { + proof { + token.cpus = _value.cpus; + } + } + + #[verus_spec( + with + Tracked(token): Tracked<&mut AtomicCpuSetToken>, + requires + old(token).id() == self.id(), + old(token).wf(), + online_cpus().contains(_cpu), + ensures + final(token).id() == old(token).id(), + final(token).cpus() == old(token).cpus().insert(_cpu), + final(token).wf(), + )] + pub fn add(&self, _cpu: CpuId, _ordering: Ordering) + no_unwind + { + proof { + token.cpus = token.cpus.insert(_cpu); + } + } } pub struct CpuSet { @@ -27,6 +190,16 @@ impl CpuSet { pub fn new_empty() -> Self returns Self::new_empty_spec(), + no_unwind + { + unimplemented!() + } + + #[verifier::external_body] + pub fn is_full(&self) -> (res: bool) + ensures + res == (self.cpus == online_cpus()), + no_unwind { unimplemented!() } diff --git a/ostd/specs/mod.rs b/ostd/specs/mod.rs index bc8234611..ef9e3ee9e 100644 --- a/ostd/specs/mod.rs +++ b/ostd/specs/mod.rs @@ -12,7 +12,7 @@ pub mod arch; pub mod mm; #[allow(unused_parens)] #[allow(unused_braces)] -mod sync; +pub mod sync; #[allow(unused_parens)] #[allow(unused_braces)] pub mod task; diff --git a/ostd/specs/sync/mod.rs b/ostd/specs/sync/mod.rs index e5784b635..3a616868b 100644 --- a/ostd/specs/sync/mod.rs +++ b/ostd/specs/sync/mod.rs @@ -2,3 +2,7 @@ pub mod abstract_lock; pub mod mutex; //pub mod mutex_verussync; +pub mod rcu; +pub mod rcu_cpu; +pub mod sc_model; +pub mod weak_memory; diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs new file mode 100644 index 000000000..738bccea2 --- /dev/null +++ b/ostd/specs/sync/rcu.rs @@ -0,0 +1,6160 @@ +//! Weak-memory RCU base and traversal specification. +//! +//! This module models the shape of the traversal specification from the RCU +//! relaxed-memory paper: +//! +//! - the base layer provides registration-time allocation IDs, persistent +//! block information, unique retire permissions, and the +//! `Inactive(tid) <-> Guard(tid, X, G)` reader protocol; +//! - the traversal layer reasons about link histories (`RcuPointsTo`) and +//! incoming-link histories (`RcuPointedBy`); +//! - concrete data structures instantiate the traversal trait. +//! +//! Allocation IDs, removed sets, link views, and incoming edges are all keyed +//! by AId, not by physical address. Physical addresses only appear in +//! `BlockInfo` and the guard's `address -> AId` protection map. This distinction +//! is required to handle stale weak-memory messages after address reuse. +//! +//! The module remains proof-only. The executable `Rcu

` adapter is a +//! direct-root specialization: replacing its atomic publication detaches the +//! owned `P` handle from that root. This must not be reused as the detach rule +//! for an internal node with arbitrary incoming links; such nodes require the +//! tracked `RcuPointedBy` transition from the traversal layer. +//! +//! The paper-level reclamation chain is connected through persistent reader +//! start snapshots, per-CPU closed-generation resources, physical read leases, +//! and type-erased callback reclaim permits. The linked-list acceptance path +//! additionally connects native internal-link loads and unlink CAS events to +//! AId-keyed traversal authority. Remaining limitations are integration +//! boundaries rather than missing logical steps: the linked-list adapter is +//! still private and acceptance-specific, `read_with()` retains a trusted +//! shared-reference bridge, and implicit Rust `Drop` cannot yet carry the +//! verified consuming transition. +use core::marker::PhantomData; + +use crate::specs::mm::cpu::CpuId; + +use vstd::invariant::InvariantPredicate; +use vstd::prelude::*; +use vstd::resource::Loc; +use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; +use vstd::thread_view::Objective; +use vstd_extra::atomic_irc11::{ + AtomicHistory as Irc11History, AtomicId as Irc11AtomicId, AtomicPointsTo, + ThreadView as Irc11ThreadView, +}; + +verus! { + +broadcast use {vstd::atomic_weak::group_view_history, vstd::thread_view::group_thread_view_axioms}; + +pub type LinkIndex = nat; + +pub type LinkEdge = (nat, LinkIndex); + +/// Scheduler identity of the execution context that owns one RCU reader. +/// +/// `session` is the fresh preemption-session resource identity created when +/// the scheduler checks a task in on `cpu`. Recording the full tuple prevents +/// an RCU guard from being detached, in the proof, from the preemption guard +/// that keeps its task on that CPU. +pub ghost struct RcuReaderContext { + pub scheduler: Loc, + pub task: Loc, + pub session: Loc, + pub cpu: CpuId, + /// Implementation generation in which this reader started. + /// + /// For a [`super::rcu_cpu::CpuRcuReadGuardToken`], this is required to + /// equal the persistent CPU participant generation. The legacy + /// task-session generation must not be substituted here when the two + /// authorities have not been connected. + pub generation: nat, +} + +/// Proof summary for a type-erased RCU callback. +/// +/// The executable callback may close over any sized Rust value, but the RCU +/// proof only needs to know which logical object it will reclaim and which +/// grace-period generation retired that object. `domain` identifies the RCU +/// protection domain, and `obj` identifies the reclaimed allocation/object +/// inside that domain. `retire_view` is the retiring task's weak-memory view +/// after unlink and before the callback is enqueued; completion must eventually +/// prove that every CPU report has advanced beyond this view. +pub ghost struct RcuCallbackSummary { + /// Scheduler whose CPU participants must complete the grace period. + pub scheduler: Loc, + /// The RCU protection domain whose grace period governs this callback. + pub domain: Loc, + /// Logical identity of the retired object inside `domain`. + pub obj: nat, + /// Root-atomic removal observation retained from `Retired(a, Q)`. + pub removal: RcuRemovalObservation, + /// Authoritative observation map that recorded `removal`. + pub retire_observation_registry: Loc, + /// The domain-local epoch in which `obj` was retired. + pub retire_epoch: nat, + /// Weak-memory observations that must precede safe reclamation. + pub retire_view: Irc11ThreadView, +} + +/// Persistent identity of one completed base-retirement transition. +/// +/// The observation-registry identity is part of the record. A domain-local +/// allocation ID and a numerically equal removal observation are not enough to +/// compare resources unless they also belong to the same authoritative +/// observation map. +pub ghost struct RcuRetiredRecord { + pub domain: Loc, + pub obj: nat, + pub removal: RcuRemovalObservation, + pub retire_observation_registry: Loc, +} + +impl RcuCallbackSummary { + pub open spec fn retired_record(self) -> RcuRetiredRecord { + RcuRetiredRecord { + domain: self.domain, + obj: self.obj, + removal: self.removal, + retire_observation_registry: self.retire_observation_registry, + } + } +} + +/// The paper's detachment observation `Q` for a root publication. +/// +/// A view observes this fact once it has advanced to at least `timestamp` in +/// the root atomic's modification history. The message at `timestamp` is the +/// first publication after the retired object ceased to be the root. +pub ghost struct RcuRemovalObservation { + pub root: Loc, + pub timestamp: nat, + pub message_view: Irc11ThreadView, +} + +impl RcuRemovalObservation { + pub open spec fn observed_by(self, view: Irc11ThreadView) -> bool { + view.contains(self.message_view) + } +} + +/// Logical identity attached to one non-null publication in an RCU root. +/// +/// The paper distinguishes a physical address from an allocation ID because +/// an address may be reused after reclamation. `domain` identifies this RCU +/// registry, while `obj` identifies one registration within that domain. The +/// same address may therefore occur in multiple `RcuPublishedObject` values +/// without introducing an ABA-style identity collision. +pub ghost struct RcuPublishedObject { + pub domain: Loc, + pub obj: nat, + pub addr: usize, +} + +/// Resources created by one application of the paper's registration rule. +/// +/// `BlockInfo` is persistent and may justify any number of publications. The +/// base retire permission is unique and must survive until traversal proves +/// that the registered allocation has been removed. +pub type RcuRegistration = (RcuBlockInfo, RcuBaseRetirePerm); + +/// Complete linear ownership associated with one registered allocation. +/// +/// The RCU base protocol treats `ownership` abstractly. The executable OSTD +/// instance uses `P::Permission`, while proof examples may use `()` or another +/// client resource. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuOwnedObject { + registration: RcuRegistration, + ownership: O, +} + +/// Complete ownership of a detached root after the base retire transition. +/// +/// The persistent object identity justifies the erased callback summary, +/// `retired` proves that traversal removal happened, and `ownership` is the +/// physical resource consumed by the callback body. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetiredOwnedObject { + object: RcuObjectId, + retired: RcuRetired, + ownership: O, +} + +impl RcuRetiredOwnedObject { + pub closed spec fn object(self) -> RcuObjectId { + self.object + } + + pub closed spec fn retired(self) -> RcuRetired { + self.retired + } + + pub closed spec fn ownership(self) -> O { + self.ownership + } + + pub closed spec fn domain(self) -> Loc { + self.object().domain() + } + + pub closed spec fn obj(self) -> nat { + self.object().obj() + } + + pub closed spec fn ptr(self) -> *mut T { + self.object().ptr() + } + + pub proof fn tracked_into_parts(tracked self) -> (tracked res: ( + RcuObjectId, + RcuRetired, + O, + )) + ensures + res.0 == self.object(), + res.1 == self.retired(), + res.2 == self.ownership(), + res.0.domain() == res.1.domain(), + res.0.obj() == res.1.obj(), + res.0.ptr() == res.1.ptr(), + { + use_type_invariant(&self); + (self.object, self.retired, self.ownership) + } + + pub open spec fn wf(self) -> bool { + &&& self.object().domain() == self.retired().domain() + &&& self.object().obj() == self.retired().obj() + &&& self.object().ptr() == self.retired().ptr() + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +impl RcuOwnedObject { + pub closed spec fn registration(self) -> RcuRegistration { + self.registration + } + + pub closed spec fn block_info(self) -> RcuBlockInfo { + self.registration().0 + } + + pub closed spec fn retire_perm(self) -> RcuBaseRetirePerm { + self.registration().1 + } + + pub closed spec fn ownership(self) -> O { + self.ownership + } + + pub proof fn tracked_into_parts(tracked self) -> (tracked res: (RcuRegistration, O)) + ensures + res.0 == self.registration(), + res.1 == self.ownership(), + { + (self.registration, self.ownership) + } +} + +/// Agreement between one registration resource and publication metadata. +pub open spec fn registration_matches_publication( + registration: RcuRegistration, + object: RcuPublishedObject, +) -> bool { + &&& registration.0.wf() + &&& registration.1.wf() + &&& registration.0.domain() == object.domain + &&& registration.0.obj() == object.obj + &&& registration.0.addr() == object.addr + &&& registration.0.obj() == registration.1.obj() + &&& registration.0.domain() == registration.1.domain() + &&& registration.0.ptr() == registration.1.ptr() +} + +pub open spec fn current_registration_matches( + root: RcuRootGhost, + registration: Option>, +) -> bool { + match (root.current(), registration) { + (None, None) => true, + (Some(object), Some(registration)) => { + &&& registration_matches_publication(registration, object) + &&& registration.1.belongs_to(root.domain_auth()) + }, + _ => false, + } +} + +/// Publication metadata paired with an RCU root's atomic message history. +/// +/// Entry `publications[i]` describes atomic message `i`. A null message has no +/// allocation identity; a non-null message refers to an allocation ID obtained +/// from `RcuDomainAuth::tracked_register`. In particular, the allocation ID is +/// not the history index `i`. +/// +/// This state intentionally does not contain the traversal removed set or a +/// grace-period epoch. In the paper, removal of an internal node belongs to +/// `SeenRemoved` and incoming-link histories, while expiration/reclamation +/// belongs to the base RCU protocol. A root store only detaches the directly +/// owned root publication; it cannot prove that an arbitrary internal node is +/// unreachable from every incoming link. +pub tracked struct RcuRootGhost { + domain: RcuDomainAuth, + ghost publications: Map>, + ghost current_timestamp: nat, +} + +impl RcuRootGhost { + pub closed spec fn domain_auth(self) -> RcuDomainAuth { + self.domain + } + + pub closed spec fn domain(self) -> Loc { + self.domain.id() + } + + pub closed spec fn objects(self) -> Map { + self.domain.objects() + } + + pub closed spec fn domain_wf(self) -> bool { + self.domain.wf() + } + + pub closed spec fn publications(self) -> Map> { + self.publications + } + + pub closed spec fn current_timestamp(self) -> nat { + self.current_timestamp + } + + pub open spec fn published_at(self, ts: nat) -> Option + recommends + self.publications().contains_key(ts), + { + match self.publications()[ts] { + Some(obj) => Some( + RcuPublishedObject { domain: self.domain(), obj, addr: self.objects()[obj] }, + ), + None => None, + } + } + + /// Allocation identity carried by the latest atomic message. + pub open spec fn current(self) -> Option + recommends + self.publications().contains_key(self.current_timestamp()), + { + self.published_at(self.current_timestamp()) + } + + /// Allocate a fresh publication registry containing the initial message. + /// + /// A non-null initial value is registered exactly once and the registration + /// resources are returned to the caller. The root history retains only the + /// allocation ID; it does not consume the unique retire permission. + pub proof fn tracked_initial( + ptr: *mut T, + history: Irc11History<*mut T>, + timestamp: nat, + message_view: Irc11ThreadView, + ) -> (tracked res: (Self, Option>)) + requires + history.is_singleton(timestamp, (ptr, message_view)), + ensures + rcu_root_history_inv(history, res.0), + (res.1 is Some) == (ptr.addr() != 0), + res.1 is Some ==> res.1->Some_0.0.ptr() == ptr, + res.1 is Some ==> res.1->Some_0.0.obj() == res.1->Some_0.1.obj(), + res.1 is Some ==> res.1->Some_0.0.domain() == res.0.domain(), + res.1 is Some ==> res.0.publications()[timestamp] == Some(res.1->Some_0.0.obj()), + res.1 is Some ==> res.1->Some_0.0.wf(), + match res.1 { + Some(registration) => res.0.objects() == Map::empty().insert( + registration.0.obj(), + ptr.addr(), + ), + None => res.0.objects() == Map::empty(), + }, + current_registration_matches(res.0, res.1), + res.0.domain_auth().retired() == Set::::empty(), + res.0.domain_auth().retire_observations() == Map::::empty(), + { + let tracked mut domain = RcuDomainAuth::tracked_new(); + assert(history.is_max_timestamp(timestamp)); + assert(history.dom() == Set::empty().insert(timestamp)) by { + assert forall|ts: nat| + history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { + if history.dom().contains(ts) { + assert(history.contains_timestamp(ts)); + assert(ts == timestamp); + } + }; + }; + if ptr.addr() == 0 { + ( + RcuRootGhost { + domain, + publications: Map::empty().insert(timestamp, None), + current_timestamp: timestamp, + }, + None, + ) + } else { + let tracked (block_info, retire_perm) = domain.tracked_register(ptr); + let ghost obj = block_info.obj(); + assert(domain.objects().contains_pair(obj, ptr.addr())); + ( + RcuRootGhost { + domain, + publications: Map::empty().insert(timestamp, Some(obj)), + current_timestamp: timestamp, + }, + Some((block_info, retire_perm)), + ) + } + } + + /// Publish a freshly introduced allocation. + /// + /// This combines the paper's registration rule with the first publication + /// of that registration. The returned resources must remain associated + /// with the allocation; in particular, the retire permission is not part + /// of the append-only atomic history. + pub proof fn tracked_push_fresh( + tracked &mut self, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, + ) -> (tracked res: Option>) + requires + rcu_root_history_inv(prev, *old(self)), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), + ensures + rcu_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + final(self).domain_auth().retire_registry() == old( + self, + ).domain_auth().retire_registry(), + final(self).domain_auth().reader_registry() == old( + self, + ).domain_auth().reader_registry(), + final(self).domain_auth().retire_observation_registry() == old( + self, + ).domain_auth().retire_observation_registry(), + final(self).domain_auth().retired() == old(self).domain_auth().retired(), + final(self).domain_auth().retire_observations() == old( + self, + ).domain_auth().retire_observations(), + (res is Some) == (value.addr() != 0), + res is Some ==> res->Some_0.0.ptr() == value, + res is Some ==> res->Some_0.0.obj() == res->Some_0.1.obj(), + res is Some ==> !old(self).objects().contains_key(res->Some_0.0.obj()), + final(self).publications() == old(self).publications().insert( + new_timestamp, + match res { + Some(registration) => Some(registration.0.obj()), + None => None, + }, + ), + match res { + Some(registration) => final(self).objects() == old(self).objects().insert( + registration.0.obj(), + value.addr(), + ), + None => final(self).objects() == old(self).objects(), + }, + current_registration_matches(*final(self), res), + { + let tracked res = if value.addr() == 0 { + self.publications = self.publications.insert(new_timestamp, None); + None + } else { + let tracked (block_info, retire_perm) = self.domain.tracked_register(value); + let ghost obj = block_info.obj(); + self.publications = self.publications.insert(new_timestamp, Some(obj)); + Some((block_info, retire_perm)) + }; + self.current_timestamp = new_timestamp; + + assert forall|ts: nat| next.contains_timestamp(ts) implies { + match #[trigger] self.publications()[ts] { + None => next.value(ts).addr() == 0, + Some(obj) => { + &&& next.value(ts).addr() != 0 + &&& self.objects().contains_pair(obj, next.value(ts).addr()) + }, + } + } by { + if ts == new_timestamp { + } else { + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + assert(self.publications()[ts] == old(self).publications()[ts]); + match self.publications()[ts] { + Some(obj) => { + assert(old(self).objects().contains_pair(obj, prev.value(ts).addr())); + assert(self.objects().contains_pair(obj, next.value(ts).addr())); + }, + None => {}, + } + } + }; + res + } + + /// Re-publish an allocation that was registered earlier. + /// + /// Unlike [`tracked_push_fresh`](Self::tracked_push_fresh), this rule does + /// not allocate a new AId. Every message published with the same persistent + /// `BlockInfo` therefore carries the same allocation identity. + pub proof fn tracked_push_registered( + tracked &mut self, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, + tracked info: &RcuBlockInfo, + ) + requires + rcu_root_history_inv(prev, *old(self)), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), + info.domain() == old(self).domain(), + info.ptr() == value, + info.wf(), + ensures + rcu_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + final(self).domain_auth().retire_registry() == old( + self, + ).domain_auth().retire_registry(), + final(self).domain_auth().reader_registry() == old( + self, + ).domain_auth().reader_registry(), + final(self).domain_auth().retire_observation_registry() == old( + self, + ).domain_auth().retire_observation_registry(), + final(self).domain_auth().retired() == old(self).domain_auth().retired(), + final(self).domain_auth().retire_observations() == old( + self, + ).domain_auth().retire_observations(), + final(self).objects() == old(self).objects(), + final(self).publications() == old(self).publications().insert( + new_timestamp, + Some(info.obj()), + ), + { + self.domain.lemma_block_info_agree(info); + self.publications = self.publications.insert(new_timestamp, Some(info.obj())); + self.current_timestamp = new_timestamp; + assert(self.objects() == old(self).objects()); + + assert forall|ts: nat| next.contains_timestamp(ts) implies { + match #[trigger] self.publications()[ts] { + None => next.value(ts).addr() == 0, + Some(obj) => { + &&& next.value(ts).addr() != 0 + &&& self.objects().contains_pair(obj, next.value(ts).addr()) + }, + } + } by { + if ts == new_timestamp { + assert(info.addr() == value.addr()); + assert(value.addr() != 0); + assert(self.publications()[ts] == Some(info.obj())); + assert(self.objects().contains_pair(info.obj(), next.value(ts).addr())); + } else { + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + assert(self.publications()[ts] == old(self).publications()[ts]); + match self.publications()[ts] { + Some(obj) => { + assert(old(self).objects().contains_pair(obj, prev.value(ts).addr())); + assert(self.objects().contains_pair(obj, next.value(ts).addr())); + }, + None => {}, + } + } + }; + } +} + +/// Agreement between the weak-memory message history and RCU allocation IDs. +pub open spec fn rcu_root_history_inv( + history: Irc11History<*mut T>, + ghost: RcuRootGhost, +) -> bool { + &&& ghost.domain_wf() + &&& ghost.publications().dom() == history.dom() + &&& history.is_max_timestamp(ghost.current_timestamp()) + &&& forall|ts: nat| + history.contains_timestamp(ts) ==> { + match #[trigger] ghost.publications()[ts] { + None => history.value(ts).addr() == 0, + Some(obj) => { + &&& history.value(ts).addr() != 0 + &&& ghost.objects().contains_pair(obj, history.value(ts).addr()) + }, + } + } +} + +pub open spec fn rcu_history_inv(nullable: bool, history: Irc11History<*mut T>) -> bool { + &&& !history.dom().is_empty() + &&& !nullable ==> forall|ts: nat| + history.contains_timestamp(ts) ==> #[trigger] history.value(ts).addr() != 0 +} + +/// Scheduler registry used by the kernel's singleton RCU domain. +/// +/// Scheduler integration must establish that every `RunningTaskContext` +/// passed to the global RCU API is checked out from this registry. +pub uninterp spec fn rcu_scheduler() -> Loc; + +/// Immutable identity carried by an executable RCU root atomic. +/// +/// Besides nullability, the key records the two resource locations needed to +/// associate read-side guard tokens with the same root invariant after the +/// invariant has been closed. +pub ghost struct RcuRootKey { + pub nullable: bool, + /// Scheduler registry whose canonical CPU participants protect this root. + pub scheduler: Loc, + pub domain: Loc, + pub reader_registry: Loc, + pub retire_observation_registry: Loc, + pub reclaim_registry: Loc, + pub active_lease_registry: Loc, +} + +/// Typed ownership state paired with one executable RCU root atomic. +/// +/// `root` owns the append-only publication registry. `current` owns the unique +/// registration resources for the latest non-null root value. Historical +/// messages retain only persistent allocation metadata, so replacing the root +/// can move the old unique retire permission out exactly once. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRootOwnedGhost { + root: RcuRootGhost, + current: Option>, + infos: Map>, + ghost removals: Map, +} + +// The root ghost owns only global resource-algebra state. Its payload remains +// objective exactly when the client ownership stored in it is objective. +unsafe impl Objective for RcuRootOwnedGhost { + +} + +impl RcuRootOwnedGhost { + pub closed spec fn root(self) -> RcuRootGhost { + self.root + } + + pub closed spec fn domain(self) -> Loc { + self.root().domain() + } + + pub closed spec fn publications(self) -> Map> { + self.root().publications() + } + + pub closed spec fn reader_registry(self) -> Loc { + self.root().domain_auth().reader_registry() + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.root().domain_auth().retire_observation_registry() + } + + /// Agrees a persistent callback retirement fact with this root's + /// authoritative removal map. + pub proof fn lemma_retired_fact_agrees(tracked &self, tracked fact: &RcuRetiredFact) + requires + self.root().domain_wf(), + self.removals() == self.root().domain_auth().retire_observations(), + fact.wf(), + fact.domain() == self.domain(), + fact.retire_observation_registry() == self.retire_observation_registry(), + ensures + self.removals().contains_pair(fact.obj(), fact.removal()), + { + fact.lemma_observation_agrees(&self.root.domain); + } + + pub open spec fn published_at(self, ts: nat) -> Option + recommends + self.publications().contains_key(ts), + { + self.root().published_at(ts) + } + + pub closed spec fn current_registration(self) -> Option> { + match self.current { + Some(owned) => Some(owned.registration()), + None => None, + } + } + + pub closed spec fn current_owned(self) -> Option> { + self.current + } + + pub closed spec fn current_ownership(self) -> Option { + match self.current { + Some(owned) => Some(owned.ownership()), + None => None, + } + } + + pub closed spec fn infos(self) -> Map> { + self.infos + } + + pub closed spec fn removals(self) -> Map { + self.removals + } + + pub open spec fn ownership_wf(self) -> bool { + current_registration_matches(self.root(), self.current_registration()) + } + + /// Every registered allocation retains a persistent typed identity token. + /// + /// Entries are append-only. Retiring an object moves its unique ownership + /// and retire permission out of `current`, but leaves this persistent + /// `BlockInfo` available to justify stale weak-memory history reads. + pub open spec fn infos_wf(self) -> bool { + &&& self.infos().dom() == self.root().objects().dom() + &&& match self.current_owned() { + Some(owned) => { + &&& self.infos().contains_key(owned.block_info().obj()) + &&& equal(self.infos()[owned.block_info().obj()].ptr(), owned.block_info().ptr()) + }, + None => true, + } + &&& forall|obj: nat| + self.infos().contains_key(obj) ==> { + let info = #[trigger] self.infos()[obj]; + &&& info.wf() + &&& info.domain() == self.domain() + &&& info.obj() == obj + &&& self.root().objects().contains_pair(obj, info.addr()) + } + } + + /// Root-history interpretation of the paper's detachment observations. + /// + /// Once `removals[obj] = ts`, no message at or after `ts` may publish that + /// allocation ID again. The currently owned registration is therefore + /// never in the removed domain. + pub open spec fn removals_wf(self, history: Irc11History<*mut T>) -> bool { + &&& self.removals().dom().subset_of(self.infos().dom()) + &&& match self.current_registration() { + Some(registration) => !self.removals().contains_key(registration.0.obj()), + None => true, + } + &&& forall|obj: nat| + self.removals().contains_key(obj) ==> { + let ts = (#[trigger] self.removals()[obj]).timestamp; + &&& history.contains_timestamp(ts) + &&& forall|later: nat| + history.contains_timestamp(later) && ts <= later + ==> #[trigger] self.publications()[later] != Some(obj) + } + } + + /// Copies the persistent identity corresponding to one published message. + pub proof fn tracked_info_for(tracked &self, object: RcuPublishedObject) -> (tracked res: + RcuBlockInfo) + requires + self.infos_wf(), + object.domain == self.domain(), + self.root().objects().contains_pair(object.obj, object.addr), + ensures + res.wf(), + res.domain() == object.domain, + res.obj() == object.obj, + res.addr() == object.addr, + equal(res.ptr(), self.infos()[object.obj].ptr()), + { + let tracked info = self.infos.tracked_borrow(object.obj); + info.tracked_duplicate() + } + + /// Resolves one atomic-history timestamp to its persistent typed identity. + /// + /// This is the proof interface used by weak atomic loads. It keeps the + /// root's internal publication and identity maps opaque to the atomic + /// wrapper while exporting exact pointer provenance, not just an address. + pub proof fn tracked_info_at( + tracked &self, + history: Irc11History<*mut T>, + ts: nat, + ) -> (tracked res: Option>) + requires + rcu_owned_root_history_inv(history, *self), + history.contains_timestamp(ts), + ensures + self.publications().contains_key(ts), + match (self.published_at(ts), res) { + (None, None) => history.value(ts).addr() == 0, + (Some(object), Some(info)) => { + &&& object.domain == self.domain() + &&& object.addr == history.value(ts).addr() + &&& self.infos().contains_key(info.obj()) + &&& info.wf() + &&& info.domain() == object.domain + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), history.value(ts)) + &&& equal(info.ptr(), self.infos()[info.obj()].ptr()) + }, + _ => false, + }, + { + assert(self.publications().contains_key(ts)); + match self.publications()[ts] { + Some(obj) => { + let ghost object = RcuPublishedObject { + domain: self.domain(), + obj, + addr: self.root().objects()[obj], + }; + assert(self.published_at(ts) == Some(object)); + let tracked info = self.tracked_info_for(object); + assert(equal(info.ptr(), history.value(ts))); + Some(info) + }, + None => { + assert(self.published_at(ts) is None); + None + }, + } + } + + /// Extracts the allocation ID stored in a non-null root publication. + pub proof fn lemma_published_object_id( + tracked &self, + history: Irc11History<*mut T>, + ts: nat, + object: RcuPublishedObject, + ) + requires + rcu_owned_root_history_inv(history, *self), + history.contains_timestamp(ts), + self.published_at(ts) == Some(object), + ensures + self.publications()[ts] == Some(object.obj), + { + match self.publications()[ts] { + Some(obj) => { + assert(self.root().objects().contains_pair(obj, history.value(ts).addr())); + assert(self.published_at(ts) == Some( + RcuPublishedObject { + domain: self.domain(), + obj, + addr: self.root().objects()[obj], + }, + )); + }, + None => { + assert(self.published_at(ts) is None); + }, + } + } + + /// Opens the paper's entry-time expired-set membership into the recorded + /// root-removal observation for that allocation. + pub proof fn lemma_observed_retired( + tracked &self, + history: Irc11History<*mut T>, + root: Loc, + view: Irc11ThreadView, + obj: nat, + ) + requires + rcu_owned_root_history_inv(history, *self), + self.root().domain_auth().observed_retired(root, view).contains(obj), + ensures + self.removals().contains_key(obj), + self.removals()[obj].root == root, + self.removals()[obj].observed_by(view), + { + assert(self.root().domain_auth().wf()); + assert(self.root().domain_auth().retired().contains(obj)); + assert(self.root().domain_auth().retire_observations().dom() + == self.root().domain_auth().retired()); + assert(self.root().domain_auth().retire_observations().dom().contains(obj)); + assert(self.root().domain_auth().retire_observations().contains_key(obj)); + } + + /// Relates an observed persistent retirement-fact collection to this + /// root's entry-time expired set. + pub proof fn lemma_retired_facts_observed( + tracked &self, + history: Irc11History<*mut T>, + tracked facts: &RcuRetiredFacts, + root: Loc, + view: Irc11ThreadView, + ) + requires + rcu_owned_root_history_inv(history, *self), + facts.observed_by(view), + ensures + forall|record: RcuRetiredRecord| #[trigger] + facts.records().contains(record) && record.domain == self.domain() + && record.retire_observation_registry == self.retire_observation_registry() + && record.removal.root == root ==> self.root().domain_auth().observed_retired( + root, + view, + ).contains(record.obj), + { + facts.lemma_matching_records_observed_retired(&self.root.domain, root, view); + } + + /// Registers and starts one fresh logical reader instance. + /// + /// The paper leaves `TId` abstract. This implementation allocates one + /// proof-only identity per critical section so nested kernel readers remain + /// distinguishable. It implements the paper's base specification, but is + /// not the fixed `LOCALS[tid]` identity used by its concrete epoch + /// algorithm. + pub proof fn tracked_start_reader( + tracked &mut self, + history: Irc11History<*mut T>, + root: Loc, + start_view: Irc11ThreadView, + reader: RcuReaderContext, + ) -> (tracked res: RcuBaseGuard) + requires + rcu_owned_root_history_inv(history, *old(self)), + ensures + rcu_owned_root_history_inv(history, *final(self)), + final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).current_owned() == old(self).current_owned(), + final(self).current_registration() == old(self).current_registration(), + final(self).publications() == old(self).publications(), + final(self).infos() == old(self).infos(), + final(self).removals() == old(self).removals(), + final(self).root().domain_auth().retired() == old(self).root().domain_auth().retired(), + final(self).root().domain_auth().retire_observations() == old( + self, + ).root().domain_auth().retire_observations(), + res.wf(), + res.domain() == final(self).domain(), + res.reader_registry() == final(self).reader_registry(), + res.reader() == reader, + res.root() == root, + res.start_view() == start_view, + res.retire_observation_registry() + == final(self).root().domain_auth().retire_observation_registry(), + res.retire_observation_registry() == old(self).retire_observation_registry(), + res.expired() == final(self).root().domain_auth().observed_retired(root, start_view), + { + let tracked inactive = self.root.domain.tracked_register_reader(reader); + let tracked guard = self.root.domain.tracked_guard_start(inactive, root, start_view); + assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.infos_wf()); + guard + } + + /// Initializes root history and retains the initial registration as the + /// current unique ownership resource. + pub proof fn tracked_initial( + ptr: *mut T, + tracked ownership: Option, + history: Irc11History<*mut T>, + timestamp: nat, + message_view: Irc11ThreadView, + ) -> (tracked res: Self) + requires + (ownership is Some) == (ptr.addr() != 0), + history.is_singleton(timestamp, (ptr, message_view)), + ensures + rcu_owned_root_history_inv(history, res), + (res.current_registration() is Some) == (ptr.addr() != 0), + res.current_registration() is Some ==> res.current_registration()->Some_0.0.ptr() + == ptr, + res.current_ownership() == ownership, + res.removals() == Map::::empty(), + res.infos().dom() == match res.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }, + match res.current_owned() { + Some(owned) => { + &&& ptr.addr() != 0 + &&& equal(owned.block_info().ptr(), ptr) + &&& ownership == Some(owned.ownership()) + }, + None => { + &&& ptr.addr() == 0 + &&& ownership is None + }, + }, + { + let tracked (root, registration) = RcuRootGhost::tracked_initial( + ptr, + history, + timestamp, + message_view, + ); + let tracked mut infos = Map::>::tracked_empty(); + let tracked current = match registration { + Some(registration) => { + let ghost obj = registration.0.obj(); + let tracked info = registration.0.tracked_duplicate(); + infos.tracked_insert(obj, info); + assert(infos.dom() == root.objects().dom()); + assert forall|registered: nat| infos.contains_key(registered) implies { + let saved = #[trigger] infos[registered]; + &&& saved.wf() + &&& saved.domain() == root.domain() + &&& saved.obj() == registered + &&& root.objects().contains_pair(registered, saved.addr()) + } by { + assert(registered == obj); + }; + Some(RcuOwnedObject { registration, ownership: ownership.tracked_unwrap() }) + }, + None => { + assert(infos.dom() == root.objects().dom()); + None + }, + }; + let tracked res = RcuRootOwnedGhost { root, current, infos, removals: Map::empty() }; + assert(res.infos().dom() == match res.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert(res.infos_wf()); + assert(res.removals_wf(history)); + assert(res.removals() == res.root().domain_auth().retire_observations()); + res + } + + /// Publishes a fresh allocation and retires the previously current root. + /// + /// In this direct-root specialization, replacement is the complete removal + /// event for the old owned publication: this cell is the only managed edge + /// for that `P` handle. This rule is not the paper's general + /// `RcuPointedBy-detach` rule for internal nodes. + pub proof fn tracked_push_fresh( + tracked &mut self, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, + root: Loc, + tracked ownership: Option, + ) -> (tracked detached: Option>) where + OwnPred: RcuRootOwnershipPredicate, + + requires + rcu_owned_root_history_inv(prev, *old(self)), + rcu_current_ownership_inv::(*old(self)), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), + (ownership is Some) == (value.addr() != 0), + ensures + rcu_owned_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + match detached { + Some(detached) => { + &&& old(self).current_registration() is Some + &&& detached.object() == old(self).current_registration()->Some_0.0 + &&& detached.object().domain() == old(self).domain() + &&& detached.obj() == old(self).current_registration()->Some_0.0.obj() + &&& equal(detached.ptr(), old(self).infos()[detached.obj()].ptr()) + &&& detached.retired().domain() == detached.domain() + &&& detached.retired().obj() == detached.obj() + &&& detached.retired().ptr() == detached.ptr() + &&& detached.retired().removal() == (RcuRemovalObservation { + root, + timestamp: new_timestamp, + message_view, + }) + &&& detached.retired().retire_observation_registry() == old( + self, + ).retire_observation_registry() + &&& equal(detached.ptr(), prev.value(old_timestamp)) + &&& old(self).current_ownership() == Some(detached.ownership()) + &&& OwnPred::owns(detached.ptr(), detached.ownership()) + }, + None => old(self).current_registration() is None, + }, + (final(self).current_registration() is Some) == (value.addr() != 0), + final(self).current_registration() is Some + ==> final(self).current_registration()->Some_0.0.ptr() == value, + final(self).current_ownership() == ownership, + match final(self).current_registration() { + Some(registration) => { + &&& !old(self).infos().contains_key(registration.0.obj()) + &&& final(self).infos().dom() == old(self).infos().dom().insert( + registration.0.obj(), + ) + &&& forall|obj: nat| #[trigger] + old(self).infos().contains_key(obj) ==> final(self).infos()[obj] == old( + self, + ).infos()[obj] + }, + None => final(self).infos() == old(self).infos(), + }, + final(self).removals() == match detached { + Some(detached) => old(self).removals().insert( + detached.obj(), + detached.retired().removal(), + ), + None => old(self).removals(), + }, + match final(self).current_owned() { + Some(owned) => { + &&& value.addr() != 0 + &&& equal(owned.block_info().ptr(), value) + &&& ownership == Some(owned.ownership()) + }, + None => { + &&& value.addr() == 0 + &&& ownership is None + }, + }, + { + assert(current_registration_matches(self.root(), self.current_registration())); + let ghost removed_obj = match self.current_registration() { + Some(registration) => Some(registration.0.obj()), + None => None, + }; + let tracked old_current = if self.current is Some { + Some(self.current.tracked_take()) + } else { + None + }; + let tracked new_registration = self.root.tracked_push_fresh( + prev, + next, + old_timestamp, + new_timestamp, + value, + message_view, + ); + let tracked new_current = match new_registration { + Some(registration) => { + let ghost obj = registration.0.obj(); + let tracked info = registration.0.tracked_duplicate(); + self.infos.tracked_insert(obj, info); + assert(self.infos.dom() == self.root.objects().dom()); + assert forall|registered: nat| self.infos.contains_key(registered) implies { + let saved = #[trigger] self.infos[registered]; + &&& saved.wf() + &&& saved.domain() == self.root.domain() + &&& saved.obj() == registered + &&& self.root.objects().contains_pair(registered, saved.addr()) + } by { + if registered != obj { + assert(old(self).infos().contains_key(registered)); + assert(self.infos[registered] == old(self).infos()[registered]); + assert(old(self).root().objects().contains_pair( + registered, + self.infos[registered].addr(), + )); + } + }; + Some(RcuOwnedObject { registration, ownership: ownership.tracked_unwrap() }) + }, + None => { + assert(self.infos_wf()); + None + }, + }; + let ghost removal = RcuRemovalObservation { root, timestamp: new_timestamp, message_view }; + let tracked detached = match old_current { + Some(owned) => { + let tracked (registration, old_ownership) = owned.tracked_into_parts(); + let tracked (object, base) = registration; + assert(base.belongs_to(self.root.domain)); + let ghost seen_removed = RcuSeenRemoved { + removed: Set::empty().insert(object.obj()), + link_view: RcuLinkView::empty(), + }; + let tracked retire = lift_direct_root_retire_perm(base, seen_removed); + let tracked retired = self.root.domain.tracked_retire(retire, removal); + Some(RcuRetiredOwnedObject { object, retired, ownership: old_ownership }) + }, + None => None, + }; + self.current = new_current; + assert(match self.current_registration() { + Some(registration) => { + &&& !old(self).infos().contains_key(registration.0.obj()) + &&& self.infos().dom() == old(self).infos().dom().insert(registration.0.obj()) + }, + None => self.infos() == old(self).infos(), + }); + self.removals = match removed_obj { + Some(obj) => self.removals.insert(obj, removal), + None => self.removals, + }; + assert(self.removals() == self.root().domain_auth().retire_observations()) by { + match removed_obj { + Some(obj) => { + assert(old(self).removals() == old( + self, + ).root().domain_auth().retire_observations()); + assert(self.removals() == old(self).removals().insert(obj, removal)); + assert(self.root().domain_auth().retire_observations() == old( + self, + ).root().domain_auth().retire_observations().insert(obj, removal)); + }, + None => { + assert(old(self).removals() == old( + self, + ).root().domain_auth().retire_observations()); + assert(self.removals() == old(self).removals()); + assert(self.root().domain_auth().retire_observations() == old( + self, + ).root().domain_auth().retire_observations()); + }, + } + }; + assert(self.removals() == match detached { + Some(detached) => old(self).removals().insert( + detached.obj(), + detached.retired().removal(), + ), + None => old(self).removals(), + }); + assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.infos_wf()); + assert(self.removals_wf(next)) by { + assert forall|obj: nat| self.removals().contains_key(obj) implies { + let ts = (#[trigger] self.removals()[obj]).timestamp; + &&& next.contains_timestamp(ts) + &&& forall|later: nat| + next.contains_timestamp(later) && ts <= later + ==> #[trigger] self.publications()[later] != Some(obj) + } by { + if removed_obj == Some(obj) { + assert(self.removals()[obj] == removal); + assert(self.removals()[obj].timestamp == new_timestamp); + assert(self.publications()[new_timestamp] == match new_registration { + Some(registration) => Some(registration.0.obj()), + None => None, + }); + if new_registration is Some { + assert(!old(self).root().objects().contains_key( + new_registration->Some_0.0.obj(), + )); + assert(old(self).infos().contains_key(obj)); + } + } else { + assert(old(self).removals().contains_key(obj)); + assert(self.removals()[obj] == old(self).removals()[obj]); + assert forall|later: nat| + next.contains_timestamp(later) && self.removals()[obj].timestamp + <= later implies #[trigger] self.publications()[later] != Some(obj) by { + if later != new_timestamp { + assert(prev.contains_timestamp(later)); + assert(self.publications()[later] == old(self).publications()[later]); + } else { + if new_registration is Some { + assert(!old(self).root().objects().contains_key( + new_registration->Some_0.0.obj(), + )); + assert(old(self).infos().contains_key(obj)); + } + } + }; + } + }; + } + detached + } + + /// Re-publishes the currently owned registration without changing its AId + /// or releasing its unique retire permission. + pub proof fn tracked_republish_current( + tracked &mut self, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, + ) + requires + rcu_owned_root_history_inv(prev, *old(self)), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), + old(self).current_registration() is Some, + old(self).current_registration()->Some_0.0.ptr() == value, + ensures + rcu_owned_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).current_registration() == old(self).current_registration(), + { + let tracked owned = self.current.tracked_take(); + self.root.tracked_push_registered( + prev, + next, + old_timestamp, + new_timestamp, + value, + message_view, + &owned.registration.0, + ); + self.current = Some(owned); + assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.removals() == self.root().domain_auth().retire_observations()); + assert(self.removals_wf(next)) by { + assert forall|obj: nat| self.removals().contains_key(obj) implies { + let ts = (#[trigger] self.removals()[obj]).timestamp; + &&& next.contains_timestamp(ts) + &&& forall|later: nat| + next.contains_timestamp(later) && ts <= later + ==> #[trigger] self.publications()[later] != Some(obj) + } by { + assert(old(self).removals().contains_key(obj)); + assert(!old(self).removals().contains_key(owned.registration.0.obj())); + assert(obj != owned.registration.0.obj()); + assert forall|later: nat| + next.contains_timestamp(later) && self.removals()[obj].timestamp + <= later implies #[trigger] self.publications()[later] != Some(obj) by { + if later != new_timestamp { + assert(prev.contains_timestamp(later)); + assert(self.publications()[later] == old(self).publications()[later]); + } else { + assert(self.publications()[later] == Some(owned.registration.0.obj())); + } + }; + }; + } + } +} + +/// The current ownership resource agrees with the latest publication, while +/// older history entries need only agree with persistent registration metadata. +pub open spec fn rcu_owned_root_history_inv( + history: Irc11History<*mut T>, + ghost: RcuRootOwnedGhost, +) -> bool { + &&& rcu_root_history_inv(history, ghost.root()) + &&& ghost.ownership_wf() + &&& ghost.infos_wf() + &&& ghost.removals_wf(history) + &&& ghost.removals() == ghost.root().domain_auth().retire_observations() + &&& forall|ts: nat| + history.contains_timestamp(ts) ==> { + match #[trigger] ghost.publications()[ts] { + Some(obj) => equal(ghost.infos()[obj].ptr(), history.value(ts)), + None => true, + } + } + &&& match ghost.current_registration() { + Some(registration) => equal( + registration.0.ptr(), + history.value(ghost.root().current_timestamp()), + ), + None => history.value(ghost.root().current_timestamp()).addr() == 0, + } +} + +/// Client relation between a pointer and its physical ownership resource. +pub trait RcuRootOwnershipPredicate { + spec fn owns(ptr: *mut T, ownership: O) -> bool; +} + +/// Trivial ownership relation used by proof-only examples carrying `()`. +pub struct UnitRcuRootOwnership; + +impl RcuRootOwnershipPredicate for UnitRcuRootOwnership { + open spec fn owns(_ptr: *mut T, _ownership: ()) -> bool { + true + } +} + +pub open spec fn rcu_current_ownership_inv( + ghost: RcuRootOwnedGhost, +) -> bool where OwnPred: RcuRootOwnershipPredicate { + match ghost.current_owned() { + Some(owned) => OwnPred::owns(owned.block_info().ptr(), owned.ownership()), + None => true, + } +} + +/// Opens the structural current-ownership relation for atomic clients. +pub proof fn lemma_current_owned_resources( + history: Irc11History<*mut T>, + tracked ghost: &RcuRootOwnedGhost, +) where OwnPred: RcuRootOwnershipPredicate + requires + rcu_owned_root_history_inv(history, *ghost), + rcu_current_ownership_inv::(*ghost), + ensures + match ghost.current_owned() { + Some(owned) => { + &&& owned.block_info().wf() + &&& equal(owned.block_info().ptr(), history.value(ghost.root().current_timestamp())) + &&& OwnPred::owns(owned.block_info().ptr(), owned.ownership()) + }, + None => history.value(ghost.root().current_timestamp()).addr() == 0, + }, +{ + match ghost.current_owned() { + Some(owned) => { + assert(ghost.current_registration() == Some(owned.registration())); + }, + None => {}, + } +} + +/// RCU weak-atomic invariant with typed ownership for the current root value. +pub struct RcuOwnedWeakAtomicInv { + _marker: PhantomData, +} + +impl InvariantPredicate< + (RcuRootKey, Irc11AtomicId), + (AtomicPointsTo<*mut T>, RcuRootOwnedGhost), +> for RcuOwnedWeakAtomicInv where OwnPred: RcuRootOwnershipPredicate { + open spec fn inv( + key_loc: (RcuRootKey, Irc11AtomicId), + pair: (AtomicPointsTo<*mut T>, RcuRootOwnedGhost), + ) -> bool { + let (key, loc) = key_loc; + let (points_to, g) = pair; + &&& points_to.loc() == loc + &&& key.domain == g.domain() + &&& key.reader_registry == g.reader_registry() + &&& key.retire_observation_registry == g.retire_observation_registry() + &&& rcu_history_inv(key.nullable, points_to.hist()) + &&& rcu_owned_root_history_inv(points_to.hist(), g) + &&& rcu_current_ownership_inv::(g) + &&& forall|obj: nat| + g.removals().contains_key(obj) ==> { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } + } +} + +/// Proof-facing summary of one grace period. +/// +/// `epoch` is assigned by the monitor, not by callback producers. Every +/// callback in a batch carries exactly this epoch, so completion of an older +/// grace period cannot authorize a callback queued for a later one. +pub ghost struct GracePeriodView { + pub epoch: nat, + pub callbacks: Seq, + pub is_complete: bool, +} + +impl GracePeriodView { + /// The state of the grace period when the monitor is created: complete, + /// with no callbacks attached. + pub open spec fn initial() -> Self { + GracePeriodView { epoch: 0, callbacks: Seq::empty(), is_complete: true } + } + + pub open spec fn has_pending_work(self) -> bool { + !self.is_complete || self.callbacks.len() > 0 + } + + /// Lock-protected well-formedness: a completed grace period has already + /// had its callbacks taken. The monitor may break this transiently inside + /// a critical section (between completing a grace period and taking its + /// callbacks), but it must hold whenever the monitor lock is released. + pub open spec fn wf(self) -> bool { + &&& self.is_complete ==> self.callbacks.len() == 0 + &&& forall|i: int| + 0 <= i < self.callbacks.len() ==> (#[trigger] self.callbacks[i]).retire_epoch + == self.epoch + } +} + +/// Proof-facing summary of the monitor state protected by the RCU monitor's +/// lock. +pub ghost struct MonitorStateView { + pub current_gp: GracePeriodView, + pub next_callbacks: Seq, +} + +impl MonitorStateView { + /// The monitor state at creation: a complete grace period and no queued + /// callbacks. + pub open spec fn initial() -> Self { + MonitorStateView { current_gp: GracePeriodView::initial(), next_callbacks: Seq::empty() } + } + + /// All callback summaries the monitor is still responsible for. + pub open spec fn pending_summaries(self) -> Seq { + self.current_gp.callbacks.add(self.next_callbacks) + } + + pub open spec fn has_pending_work(self) -> bool { + self.current_gp.has_pending_work() || self.next_callbacks.len() > 0 + } + + pub open spec fn no_pending_work(self) -> bool { + !self.has_pending_work() + } + + /// Lock-protected well-formedness: when the current grace period is + /// complete, the monitor has either restarted it with the queued callbacks + /// or stopped monitoring, so both callback lists are empty. + pub open spec fn wf(self) -> bool { + &&& self.current_gp.wf() + &&& self.current_gp.is_complete ==> self.next_callbacks.len() == 0 + &&& forall|i: int| + 0 <= i < self.next_callbacks.len() ==> (#[trigger] self.next_callbacks[i]).retire_epoch + == self.current_gp.epoch + 1 + } +} + +/// Under the lock-protected invariant, "has pending work" collapses to "the +/// current grace period is incomplete": a complete grace period implies both +/// callback lists are empty. +pub proof fn monitor_state_pending_iff_incomplete(state: MonitorStateView) + requires + state.wf(), + ensures + state.has_pending_work() <==> !state.current_gp.is_complete, + state.no_pending_work() <==> state.current_gp.is_complete, +{ +} + +/// `no_pending_work` certifies that the pending-summary sequence is empty. +pub proof fn monitor_state_no_pending_no_summaries(state: MonitorStateView) + requires + state.no_pending_work(), + ensures + state.pending_summaries() == Seq::::empty(), +{ +} + +/// Ghost summary paired with the RCU monitor's `is_monitoring` flag. +/// +/// `states[ts]` summarizes the lock-protected monitor state stored with flag +/// message timestamp `ts`. This is intentionally a summary: the concrete +/// callback vectors live in the monitor state protected by its lock, and the +/// agreement between `states[ts]` and that state is established by the writer, +/// which performs every flag store while holding the monitor lock. +pub tracked struct RcuMonitorFlagGhost { + pub ghost states: Map, +} + +unsafe impl Objective for RcuMonitorFlagGhost { + +} + +impl RcuMonitorFlagGhost { + pub open spec fn initial(timestamp: nat) -> Self { + RcuMonitorFlagGhost { states: Map::empty().insert(timestamp, MonitorStateView::initial()) } + } + + /// Proof-mode constructor for the tracked ghost state stored inside the + /// monitor flag's weak atomic invariant. + pub proof fn tracked_initial(timestamp: nat) -> (tracked res: Self) + ensures + res == Self::initial(timestamp), + { + RcuMonitorFlagGhost { states: Map::empty().insert(timestamp, MonitorStateView::initial()) } + } + + pub open spec fn insert(self, timestamp: nat, state: MonitorStateView) -> Self { + RcuMonitorFlagGhost { states: self.states.insert(timestamp, state) } + } + + pub proof fn tracked_insert( + tracked self, + timestamp: nat, + state: MonitorStateView, + ) -> (tracked res: Self) + ensures + res == self.insert(timestamp, state), + { + RcuMonitorFlagGhost { states: self.states.insert(timestamp, state) } + } + + /// Whether the state recorded at flag message `timestamp` had work pending. + pub open spec fn pending_at(self, timestamp: nat) -> bool + recommends + self.states.contains_key(timestamp), + { + self.states[timestamp].has_pending_work() + } +} + +/// Weak-memory invariant for the monitor's fast-path flag. +/// +/// Every flag message carries a well-formed snapshot of the monitor state, and +/// the invariant is deliberately one-way: a `false` flag message certifies that +/// the state recorded at that message had no pending monitor work. A `true` +/// flag is conservative and may over-approximate pending work. +/// +/// Note the weak-memory reading: a relaxed load may observe a stale message, +/// so a `false` read only certifies "no pending work as of that message", not +/// "no pending work now". That is exactly what the monitor fast path needs: +/// callbacks enqueued after that message were published together with a `true` +/// flag message, so skipping the slow path can only delay their grace period, +/// never lose them. +pub open spec fn rcu_monitor_flag_history_inv( + history: Irc11History, + ghost: RcuMonitorFlagGhost, +) -> bool { + &&& !history.dom().is_empty() + &&& ghost.states.dom() == history.dom() + &&& forall|timestamp: nat| + history.contains_timestamp(timestamp) ==> (#[trigger] ghost.states[timestamp]).wf() + &&& forall|timestamp: nat| + history.contains_timestamp(timestamp) ==> { + !(#[trigger] history.value(timestamp)) ==> ghost.states[timestamp].no_pending_work() + } +} + +pub struct RcuMonitorFlagInv; + +impl InvariantPredicate< + Irc11AtomicId, + (AtomicPointsTo, RcuMonitorFlagGhost), +> for RcuMonitorFlagInv { + open spec fn inv( + loc: Irc11AtomicId, + pair: (AtomicPointsTo, RcuMonitorFlagGhost), + ) -> bool { + &&& pair.0.loc() == loc + &&& rcu_monitor_flag_history_inv(pair.0.hist(), pair.1) + } +} + +pub proof fn rcu_monitor_flag_initial_inv( + history: Irc11History, + timestamp: nat, + message_view: Irc11ThreadView, +) + requires + history.is_singleton(timestamp, (false, message_view)), + ensures + rcu_monitor_flag_history_inv(history, RcuMonitorFlagGhost::initial(timestamp)), +{ + assert(history.dom() == Set::empty().insert(timestamp)) by { + assert forall|ts: nat| + #![auto] + history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { + if history.dom().contains(ts) { + assert(history.contains_timestamp(ts)); + assert(ts == timestamp); + } + }; + }; +} + +/// Inserting one flag message preserves the history invariant, provided the +/// writer records a well-formed state snapshot and only writes `false` when +/// that snapshot has no pending work. +/// +/// This is the proof obligation discharged by `set_monitoring`: it +/// stores the flag while holding the monitor lock, so it can supply the +/// lock-protected state view as the snapshot. +pub proof fn preserve_rcu_monitor_flag_inv_on_insert( + prev: Irc11History, + next: Irc11History, + timestamp: nat, + value: bool, + message_view: Irc11ThreadView, + prev_ghost: RcuMonitorFlagGhost, + next_ghost: RcuMonitorFlagGhost, + state: MonitorStateView, +) + requires + rcu_monitor_flag_history_inv(prev, prev_ghost), + !prev.contains_timestamp(timestamp), + next == prev.insert(timestamp, value, message_view), + next_ghost == prev_ghost.insert(timestamp, state), + state.wf(), + !value ==> state.no_pending_work(), + ensures + rcu_monitor_flag_history_inv(next, next_ghost), +{ + assert(next_ghost.states.dom() == next.dom()); + assert forall|ts: nat| next.contains_timestamp(ts) implies ( + #[trigger] next_ghost.states[ts]).wf() by { + if ts == timestamp { + assert(next_ghost.states[ts] == state); + } else { + assert(prev.contains_timestamp(ts)); + assert(next_ghost.states[ts] == prev_ghost.states[ts]); + } + }; + assert forall|ts: nat| next.contains_timestamp(ts) implies { + !(#[trigger] next.value(ts)) ==> next_ghost.states[ts].no_pending_work() + } by { + if ts == timestamp { + assert(next.value(ts) == value); + assert(next_ghost.states[ts] == state); + } else { + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + assert(next_ghost.states[ts] == prev_ghost.states[ts]); + } + }; +} + +/// The key safety fact behind the monitor fast path: observing a `false` flag +/// message certifies that the monitor state recorded at that message had no +/// queued callbacks and no incomplete grace period. +pub proof fn rcu_monitor_flag_false_has_no_pending( + history: Irc11History, + ghost: RcuMonitorFlagGhost, + ts: nat, +) + requires + rcu_monitor_flag_history_inv(history, ghost), + history.contains_timestamp(ts), + !history.value(ts), + ensures + ghost.states[ts].no_pending_work(), + ghost.states[ts].pending_summaries() == Seq::::empty(), + ghost.states[ts].current_gp.is_complete, +{ + monitor_state_pending_iff_incomplete(ghost.states[ts]); + monitor_state_no_pending_no_summaries(ghost.states[ts]); +} + +pub proof fn preserve_rcu_history_inv_on_push( + nullable: bool, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, +) + requires + rcu_history_inv(nullable, prev), + !prev.contains_timestamp(timestamp), + next == prev.insert(timestamp, value, message_view), + nullable || value.addr() != 0, + ensures + rcu_history_inv(nullable, next), +{ + assert(!next.dom().is_empty()); + if !nullable { + assert forall|ts: nat| next.contains_timestamp(ts) implies #[trigger] next.value(ts).addr() + != 0 by { + if ts == timestamp { + assert(next.value(ts) == value); + } else { + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + } + }; + } +} + +pub proof fn rcu_history_inv_read_nonnull(history: Irc11History<*mut T>, ts: nat) + requires + rcu_history_inv(false, history), + history.contains_timestamp(ts), + ensures + history.value(ts).addr() != 0, + !history.value(ts).is_null(), +{ + assert(history.value(ts).addr() != 0); +} + +/// Link view carried by an RCU read-side guard. +/// +/// `seen_at(a) = n` means the guard has observed link-history events for source +/// AId `a` up to at least `n`. Following a loaded link at index `k` is allowed +/// only when `seen_at(a) <= k`; otherwise the pointer may be too stale. +#[verifier::reject_recursive_types(T)] +pub ghost struct RcuLinkView { + pub seen: Map, + pub marker: Option<*mut T>, +} + +impl RcuLinkView { + pub open spec fn empty() -> Self { + RcuLinkView { seen: Map::empty(), marker: None } + } + + pub open spec fn seen_at(self, obj: nat) -> LinkIndex { + if self.seen.contains_key(obj) { + self.seen[obj] + } else { + 0nat + } + } + + pub open spec fn observe(self, obj: nat, n: LinkIndex) -> Self { + RcuLinkView { + seen: self.seen.insert( + obj, + if self.seen_at(obj) <= n { + n + } else { + self.seen_at(obj) + }, + ), + marker: self.marker, + } + } + + /// Observing one source never moves any source's traversal position + /// backwards. + pub proof fn lemma_observe_monotonic(self, obj: nat, n: LinkIndex, other: nat) + ensures + self.seen_at(other) <= self.observe(obj, n).seen_at(other), + self.seen_at(obj) <= n ==> self.observe(obj, n).seen_at(obj) == n, + { + } +} + +/// Paper-style `SeenRemoved(D, LV)`. +/// +/// `removed` is the set `D` observed by the guard; `link_view` is `LV`. +/// A dead incoming edge is either from a removed predecessor AId or overwritten +/// by a later observed link event. +#[verifier::reject_recursive_types(T)] +pub ghost struct RcuSeenRemoved { + pub removed: Set, + pub link_view: RcuLinkView, +} + +impl RcuSeenRemoved { + pub open spec fn empty() -> Self { + RcuSeenRemoved { removed: Set::empty(), link_view: RcuLinkView::empty() } + } + + pub open spec fn seen_at(self, obj: nat) -> LinkIndex { + self.link_view.seen_at(obj) + } + + pub open spec fn dead_edge(self, edge: LinkEdge) -> bool { + self.removed.contains(edge.0) || self.seen_at(edge.0) > edge.1 + } +} + +/// Authoritative ghost handle for one RCU protection domain. +/// +/// The concrete implementation owns this token in its invariant. We keep the +/// fields private so clients cannot manufacture domain authority. `readers` +/// only registers fresh logical reader-instance identities. Active/inactive +/// phase is represented linearly by [`RcuInactive`] and [`RcuBaseGuard`], so +/// ending a critical section does not need this authority. +pub tracked struct RcuDomainAuth { + objects: GhostMapAuth, + retire_perms: GhostMapAuth, + retire_observation_cells: GhostMapAuth>, + readers: GhostMapAuth, + ghost next_obj: nat, + ghost next_reader: nat, + ghost retired: Set, + ghost retire_observations: Map, +} + +impl RcuDomainAuth { + /// The paper's RCU location `l` is the identity of the allocation registry. + /// Every object registered through this authority belongs to this domain. + pub closed spec fn id(self) -> Loc { + self.objects.id() + } + + pub closed spec fn objects(self) -> Map { + self.objects@ + } + + pub closed spec fn next_obj(self) -> nat { + self.next_obj + } + + pub closed spec fn retired(self) -> Set { + self.retired + } + + /// Physical detachment observation recorded for each retired allocation. + /// + /// A guard's implementation-specific expired set `X` is derived from this + /// map at critical-section entry: it contains exactly the retired + /// allocations whose detachment observation is already covered by the + /// entering thread's weak-memory view. + pub closed spec fn retire_observations(self) -> Map { + self.retire_observations + } + + pub open spec fn observed_retired(self, root: Loc, view: Irc11ThreadView) -> Set { + self.retired().filter( + |obj: nat| + self.retire_observations()[obj].root == root + && self.retire_observations()[obj].observed_by(view), + ) + } + + pub closed spec fn reader_registry(self) -> Loc { + self.readers.id() + } + + pub closed spec fn retire_registry(self) -> Loc { + self.retire_perms.id() + } + + /// Resource registry that agrees every retired object with its unique + /// detachment observation. + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retire_observation_cells.id() + } + + pub closed spec fn next_reader(self) -> nat { + self.next_reader + } + + /// Internal consistency of the resource algebras used by the base RCU + /// model. + pub closed spec fn wf(self) -> bool { + &&& self.objects@ == self.retire_perms@ + &&& self.retire_observation_cells@.dom() == self.objects@.dom() + &&& forall|obj: nat| #[trigger] + self.objects@.contains_key(obj) ==> { + match self.retire_observation_cells@[obj] { + Some(removal) => self.retire_observations().contains_pair(obj, removal), + None => !self.retire_observations().contains_key(obj), + } + } + &&& forall|obj: nat| #[trigger] self.objects@.contains_key(obj) ==> obj < self.next_obj() + &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> tid < self.next_reader() + &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> !self.readers@[tid] + &&& self.retired().subset_of(self.objects().dom()) + &&& self.retire_observations().dom() == self.retired() + } + + /// Allocates a fresh RCU protection domain. + pub proof fn tracked_new() -> (tracked res: Self) + ensures + res.wf(), + res.objects() == Map::::empty(), + res.next_obj() == 0, + res.retired() == Set::::empty(), + res.retire_observations() == Map::::empty(), + { + let tracked (objects, _objects_entries) = GhostMapAuth::new(Map::empty()); + let tracked (retire_perms, _retire_entries) = GhostMapAuth::new(Map::empty()); + let tracked (retire_observation_cells, _retire_observation_entries) = GhostMapAuth::new( + Map::empty(), + ); + let tracked (readers, _reader_entries) = GhostMapAuth::new(Map::empty()); + RcuDomainAuth { + objects, + retire_perms, + retire_observation_cells, + readers, + next_obj: 0, + next_reader: 0, + retired: Set::empty(), + retire_observations: Map::empty(), + } + } + + /// Implements the paper's `rcu-register` rule. + /// + /// The allocation ID is chosen here, once per registration. It is not an + /// atomic-history timestamp. Registration returns both persistent block + /// information and the unique base retire permission for the allocation. + pub proof fn tracked_register(tracked &mut self, ptr: *mut T) -> (tracked res: ( + RcuBlockInfo, + RcuBaseRetirePerm, + )) + requires + old(self).wf(), + ptr.addr() != 0, + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).next_obj() == old(self).next_obj() + 1, + final(self).retired() == old(self).retired(), + final(self).retire_observations() == old(self).retire_observations(), + final(self).objects() == old(self).objects().insert(old(self).next_obj(), ptr.addr()), + res.0.domain() == final(self).id(), + res.0.obj() == old(self).next_obj(), + res.0.ptr() == ptr, + res.0.addr() == ptr.addr(), + res.0.wf(), + res.1.domain() == final(self).id(), + res.1.obj() == res.0.obj(), + res.1.ptr() == ptr, + res.1.wf(), + res.1.belongs_to(*final(self)), + { + let ghost obj = self.next_obj; + assert(!self.objects@.contains_key(obj)); + assert(!self.retire_perms@.contains_key(obj)); + + let tracked object = self.objects.insert(obj, ptr.addr()); + let tracked block_info = object.persist(); + let tracked retire_perm = self.retire_perms.insert(obj, ptr.addr()); + let tracked retire_observation = self.retire_observation_cells.insert(obj, None); + self.next_obj = self.next_obj + 1; + + assert forall|registered: nat| #[trigger] + self.objects@.contains_key(registered) implies registered < self.next_obj by { + if registered != obj { + assert(old(self).objects().contains_key(registered)); + } + }; + + ( + RcuBlockInfo { info: block_info, ptr }, + RcuBaseRetirePerm { + domain: self.id(), + perm: retire_perm, + observation: retire_observation, + ptr, + }, + ) + } + + /// Establishes agreement between domain authority and persistent block + /// information supplied by a client or an atomic-history invariant. + pub proof fn lemma_block_info_agree(tracked &self, tracked info: &RcuBlockInfo) + requires + self.wf(), + info.domain() == self.id(), + ensures + self.objects().contains_pair(info.obj(), info.addr()), + { + info.info.agree(&self.objects); + } + + /// Registers one reader slot and returns the paper's `Inactive(tid)` + /// resource for it. + pub proof fn tracked_register_reader( + tracked &mut self, + reader: RcuReaderContext, + ) -> (tracked res: RcuInactive) + requires + old(self).wf(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).objects() == old(self).objects(), + final(self).retired() == old(self).retired(), + final(self).retire_observations() == old(self).retire_observations(), + res.domain() == final(self).id(), + res.tid() == old(self).next_reader(), + res.reader() == reader, + res.belongs_to(*final(self)), + res.wf(), + { + let ghost tid = self.next_reader; + assert(!self.readers@.contains_key(tid)); + let tracked state = self.readers.insert(tid, false); + self.next_reader = self.next_reader + 1; + assert forall|registered: nat| #[trigger] + self.readers@.contains_key(registered) implies registered < self.next_reader by { + if registered != tid { + assert(old(self).readers@.contains_key(registered)); + } + }; + assert forall|registered: nat| #[trigger] + self.readers@.contains_key(registered) implies !self.readers@[registered] by { + if registered != tid { + assert(old(self).readers@.contains_key(registered)); + } + }; + RcuInactive { domain: self.id(), state, reader } + } + + /// Starts a read-side critical section, snapshotting the set `X` of AIds + /// whose root-removal observation is covered by the entering thread's + /// weak-memory view. + /// + /// The paper only requires `X` to be a subset of all retired allocations. + /// Using `retired` itself here would be too strong: a reader may safely + /// observe a newly retired stale pointer while that retirement's grace + /// period is still in progress. + pub proof fn tracked_guard_start( + tracked &self, + tracked inactive: RcuInactive, + root: Loc, + start_view: Irc11ThreadView, + ) -> (tracked res: RcuBaseGuard) + requires + self.wf(), + inactive.belongs_to(*self), + inactive.wf(), + ensures + res.belongs_to(*self), + res.tid() == inactive.tid(), + res.reader() == inactive.reader(), + res.root() == root, + res.start_view() == start_view, + res.retire_observation_registry() == self.retire_observation_registry(), + res.expired() == self.observed_retired(root, start_view), + res.protected() == Map::::empty(), + res.wf(), + { + let ghost tid = inactive.tid(); + inactive.state.agree(&self.readers); + assert(self.readers@.contains_key(tid)); + RcuBaseGuard { + domain: self.id(), + state: inactive.state, + reader: inactive.reader, + root, + start_view, + retire_observation_registry: self.retire_observation_registry(), + expired: self.observed_retired(root, start_view), + protected: Map::empty(), + } + } + + /// Implements the base `rcu-retire` transition by adding the detached AId + /// to `RcuState.R` and consuming its unique traversal retire permission. + pub proof fn tracked_retire( + tracked &mut self, + tracked retire: RcuRetirePerm, + removal: RcuRemovalObservation, + ) -> (tracked res: RcuRetired) + requires + old(self).wf(), + retire.belongs_to(*old(self)), + retire.wf(), + retire.ready_to_retire(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).objects() == old(self).objects(), + final(self).retired() == old(self).retired().insert(retire.obj()), + final(self).retire_observations() == old(self).retire_observations().insert( + retire.obj(), + removal, + ), + res.domain() == final(self).id(), + res.obj() == retire.obj(), + res.ptr() == retire.ptr(), + res.removal() == removal, + res.retire_observation_registry() == final(self).retire_observation_registry(), + res.wf(), + { + let ghost domain = retire.domain(); + let ghost obj = retire.obj(); + let ghost ptr = retire.ptr(); + retire.base.perm.agree(&self.retire_perms); + retire.base.observation.agree(&self.retire_observation_cells); + assert(self.objects().contains_key(obj)); + let tracked mut observation = retire.base.observation; + observation.update(&mut self.retire_observation_cells, Some(removal)); + let tracked observation = observation.persist(); + self.retired = self.retired.insert(obj); + self.retire_observations = self.retire_observations.insert(obj, removal); + assert(self.objects@ == old(self).objects@); + assert(self.retire_perms@ == old(self).retire_perms@); + assert(self.readers@ == old(self).readers@); + assert(self.next_obj() == old(self).next_obj()); + assert(self.next_reader() == old(self).next_reader()); + assert(self.retire_observation_cells@ == old(self).retire_observation_cells@.insert( + obj, + Some(removal), + )); + assert(self.retire_observation_cells@.dom() == old(self).retire_observation_cells@.dom()); + assert(self.retire_observation_cells@.dom() == self.objects@.dom()); + assert forall|registered: nat| #[trigger] self.objects@.contains_key(registered) implies { + match self.retire_observation_cells@[registered] { + Some(recorded) => self.retire_observations().contains_pair(registered, recorded), + None => !self.retire_observations().contains_key(registered), + } + } by { + if registered == obj { + assert(self.retire_observation_cells@[registered] == Some(removal)); + assert(self.retire_observations().contains_pair(registered, removal)); + } else { + assert(self.retire_observation_cells@[registered] == old( + self, + ).retire_observation_cells@[registered]); + assert(self.retire_observations().contains_key(registered) == old( + self, + ).retire_observations().contains_key(registered)); + } + }; + assert(self.retired().subset_of(self.objects().dom())); + assert(self.retire_observations().dom() == self.retired()); + let tracked fact = retire.base.perm.persist(); + RcuRetired { fact: RcuRetiredFact { domain, fact, observation }, ptr } + } +} + +/// The paper's unique `Inactive(tid)` reader resource. +pub tracked struct RcuInactive { + ghost domain: Loc, + state: GhostPointsTo, + ghost reader: RcuReaderContext, +} + +impl RcuInactive { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn tid(self) -> nat { + self.state.key() + } + + pub closed spec fn reader(self) -> RcuReaderContext { + self.reader + } + + pub closed spec fn reader_registry(self) -> Loc { + self.state.id() + } + + pub closed spec fn wf(self) -> bool { + true + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + &&& self.domain() == domain.id() + &&& self.state.id() == domain.reader_registry() + } +} + +/// The base paper guard `Guard(tid, X, G)`. +/// +/// `expired` is the start-time snapshot `X`. `protected[addr] = a` is the +/// mutable protection map `G` populated by successful protect operations. +/// `root` and `start_view` are implementation-refinement metadata: they are +/// not additional assumptions in the paper's abstract `Guard(tid, X, G)`, but +/// retain the witness from which `X` was computed. +pub tracked struct RcuBaseGuard { + ghost domain: Loc, + state: GhostPointsTo, + ghost reader: RcuReaderContext, + ghost root: Loc, + ghost start_view: Irc11ThreadView, + ghost retire_observation_registry: Loc, + ghost expired: Set, + ghost protected: Map, +} + +impl RcuBaseGuard { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn tid(self) -> nat { + self.state.key() + } + + pub closed spec fn reader_registry(self) -> Loc { + self.state.id() + } + + pub closed spec fn reader(self) -> RcuReaderContext { + self.reader + } + + /// Root atomic whose removal history determined this guard's expired set. + pub closed spec fn root(self) -> Loc { + self.root + } + + /// Weak-memory view captured when this read-side critical section began. + pub closed spec fn start_view(self) -> Irc11ThreadView { + self.start_view + } + + /// Registry that issued persistent retirement facts for this domain. + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retire_observation_registry + } + + pub closed spec fn expired(self) -> Set { + self.expired + } + + pub closed spec fn protected(self) -> Map { + self.protected + } + + pub closed spec fn wf(self) -> bool { + &&& !self.state.value() + &&& forall|addr: usize| #[trigger] + self.protected().dom().contains(addr) ==> !self.expired().contains( + self.protected()[addr], + ) + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + &&& self.domain() == domain.id() + &&& self.state.id() == domain.reader_registry() + &&& self.retire_observation_registry() == domain.retire_observation_registry() + } + + pub closed spec fn protects(self, addr: usize, obj: nat) -> bool { + self.protected().contains_pair(addr, obj) + } + + /// Implements the paper's local `Guard -> Inactive` unlock rule. + /// + /// The domain registry only certifies the reader-instance identity; it + /// does not track critical-section phase. Consequently this transition + /// consumes the linear guard without opening `RcuState` or the root atomic + /// invariant. + pub proof fn tracked_stop(tracked self) -> (tracked res: RcuInactive) + requires + self.wf(), + ensures + res.domain() == self.domain(), + res.tid() == self.tid(), + res.reader() == self.reader(), + res.wf(), + res.reader_registry() == self.reader_registry(), + { + RcuInactive { domain: self.domain, state: self.state, reader: self.reader } + } + + /// Implements the base `Guard-protect` update. An object already in the + /// guard's start snapshot `X` cannot be newly protected by this guard. + pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) + requires + old(self).wf(), + info.wf(), + info.domain() == old(self).domain(), + !old(self).expired().contains(info.obj()), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), + final(self).protects(info.addr(), info.obj()), + { + self.protected = self.protected.insert(info.addr(), info.obj()); + assert forall|addr: usize| #[trigger] + self.protected().dom().contains(addr) implies !self.expired().contains( + self.protected()[addr], + ) by { + if addr == info.addr() { + assert(self.protected()[addr] == info.obj()); + } else { + assert(old(self).protected().dom().contains(addr)); + assert(self.protected()[addr] == old(self).protected()[addr]); + } + }; + } +} + +/// Persistent `BlockInfo(l, a, P)` for one RCU-managed allocation. +/// +/// The current Verus cut records the allocation's typed pointer and physical +/// address; the client-owned block predicate `P` remains represented by the +/// corresponding `P::Permission` at the executable boundary. The resource +/// token is persistent and can therefore be copied into every weak-memory +/// history entry that publishes this allocation. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuBlockInfo { + info: GhostPersistentPointsTo, + ghost ptr: *mut T, +} + +impl RcuBlockInfo { + pub closed spec fn domain(self) -> Loc { + self.info.id() + } + + pub closed spec fn obj(self) -> nat { + self.info.key() + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } + + pub closed spec fn addr(self) -> usize { + self.info.value() + } + + pub closed spec fn wf(self) -> bool { + &&& self.addr() == self.ptr().addr() + &&& self.ptr().addr() != 0 + } + + /// Opens the address facts hidden by the block-info abstraction. + pub proof fn lemma_wf_facts(tracked &self) + requires + self.wf(), + ensures + self.addr() == self.ptr().addr(), + self.ptr().addr() != 0, + { + } + + /// Persistent block information can be retained by both the client and + /// every historical atomic message that mentions the allocation. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.domain() == self.domain(), + res.obj() == self.obj(), + res.ptr() == self.ptr(), + res.addr() == self.addr(), + res.wf() == self.wf(), + { + let tracked info = self.info.duplicate(); + RcuBlockInfo { info, ptr: self.ptr } + } +} + +/// Compatibility name used by the callback/traversal boundary. +pub type RcuObjectId = RcuBlockInfo; + +/// Regression proof for the allocation-ID discipline required by relaxed +/// memory RCU. +/// +/// `res.0` and `res.1` are two persistent copies of one registration, so they +/// carry the same AId. `res.2` is a second registration at the same physical +/// address, so it carries a fresh AId. This is the distinction that rules out +/// identifying stale weak-memory messages by address alone. +pub proof fn registration_distinguishes_reused_address(ptr: *mut T) -> (tracked res: ( + RcuBlockInfo, + RcuBlockInfo, + RcuBlockInfo, +)) + requires + ptr.addr() != 0, + ensures + res.0.domain() == res.1.domain(), + res.0.domain() == res.2.domain(), + res.0.obj() == res.1.obj(), + res.0.obj() != res.2.obj(), + res.0.addr() == ptr.addr(), + res.1.addr() == ptr.addr(), + res.2.addr() == ptr.addr(), +{ + let tracked mut domain = RcuDomainAuth::tracked_new(); + let tracked (first, _first_retire) = domain.tracked_register(ptr); + let tracked first_history_copy = first.tracked_duplicate(); + let tracked (second, _second_retire) = domain.tracked_register(ptr); + assert(first.obj() < second.obj()); + (first, first_history_copy, second) +} + +/// Regression proof for registration-time identity across re-publication. +/// +/// Both history entries are justified by the same persistent `BlockInfo`, so +/// they contain the same AId even though they have different atomic +/// timestamps. This is the paper's required separation between allocation +/// identity and modification-order position. +pub proof fn registered_republication_preserves_allocation_id(ptr: *mut T) -> (tracked res: ( + RcuRootGhost, + RcuRegistration, +)) + requires + ptr.addr() != 0, + ensures + res.0.publications().dom() == Set::empty().insert(0nat).insert(1nat), + res.0.publications()[0] == Some(res.1.0.obj()), + res.0.publications()[1] == Some(res.1.0.obj()), + res.1.0.domain() == res.0.domain(), + res.1.0.obj() == res.1.1.obj(), +{ + let ghost view = Irc11ThreadView::empty(); + let ghost initial = Irc11History(Map::empty().insert(0nat, (ptr, view))); + let tracked (mut root, registration_opt) = RcuRootGhost::tracked_initial(ptr, initial, 0, view); + let tracked registration = registration_opt.tracked_unwrap(); + let ghost next = initial.insert(1, ptr, view); + root.tracked_push_registered(initial, next, 0, 1, ptr, view, ®istration.0); + (root, registration) +} + +/// Regression proof for ownership transfer on root replacement. +/// +/// The old registration leaves the atomic ownership state exactly once, while +/// the fresh registration for `next_ptr` becomes current. This is the resource +/// handoff that later feeds traversal retirement and callback construction. +pub proof fn owned_root_replacement_retires_previous_registration( + first_ptr: *mut T, + next_ptr: *mut T, +) -> (tracked res: (RcuRootOwnedGhost, RcuRetiredOwnedObject)) + requires + first_ptr.addr() != 0, + next_ptr.addr() != 0, + ensures + res.1.ptr() == first_ptr, + res.1.object().obj() == res.1.retired().obj(), + res.0.current_registration() is Some, + res.0.current_registration()->Some_0.0.ptr() == next_ptr, + res.0.current_registration()->Some_0.0.obj() + == res.0.current_registration()->Some_0.1.obj(), + res.1.domain() == res.0.domain(), +{ + let ghost view = Irc11ThreadView::empty(); + let ghost initial = Irc11History(Map::empty().insert(0nat, (first_ptr, view))); + let tracked mut root = RcuRootOwnedGhost::tracked_initial( + first_ptr, + Some(()), + initial, + 0, + view, + ); + let ghost next_history = initial.insert(1, next_ptr, view); + let tracked detached = root.tracked_push_fresh::( + initial, + next_history, + 0, + 1, + next_ptr, + view, + root.domain(), + Some(()), + ); + let tracked detached = detached.tracked_unwrap(); + (root, detached) +} + +/// Low-level base retire permission. +/// +/// This is the paper's unique `BaseRetirePerm(l, a)`. The embedded owning +/// points-to resource makes duplication impossible. By itself it is not enough +/// to retire or reclaim an object; traversal must first establish that every +/// relevant incoming edge has been removed. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuBaseRetirePerm { + ghost domain: Loc, + perm: GhostPointsTo, + observation: GhostPointsTo>, + ghost ptr: *mut T, +} + +impl RcuBaseRetirePerm { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } + + pub closed spec fn obj(self) -> nat { + self.perm.key() + } + + pub closed spec fn addr(self) -> usize { + self.perm.value() + } + + pub closed spec fn wf(self) -> bool { + &&& self.addr() == self.ptr().addr() + &&& self.observation.key() == self.obj() + &&& self.observation.value() is None + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + &&& self.domain() == domain.id() + &&& self.perm.id() == domain.retire_registry() + &&& self.observation.id() == domain.retire_observation_registry() + } +} + +/// High-level retire permission. +/// +/// This corresponds to `RetirePerm(l, a) = BaseRetirePerm(l, a) * +/// exists D LV. SeenRemoved(D, LV) * a in D`. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetirePerm { + base: RcuBaseRetirePerm, + ghost seen_removed: RcuSeenRemoved, +} + +impl RcuRetirePerm { + pub closed spec fn base(self) -> RcuBaseRetirePerm { + self.base + } + + pub closed spec fn domain(self) -> Loc { + self.base.domain() + } + + pub closed spec fn ptr(self) -> *mut T { + self.base.ptr() + } + + pub closed spec fn obj(self) -> nat { + self.base.obj() + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.seen_removed + } + + pub open spec fn wf(self) -> bool { + self.base().wf() + } + + /// The traversal layer has established that this object may be retired. + /// Reclamation still requires a completed base-RCU grace period. + pub open spec fn ready_to_retire(self) -> bool { + self.seen_removed().removed.contains(self.obj()) + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + self.base.belongs_to(domain) + } +} + +/// Internal bridge for the directly owned root-pointer specialization. +/// +/// This is deliberately not a public traversal rule. `RcuSeenRemoved` is only +/// a logical view and can be constructed freely, so exposing this function +/// would let a client claim detachment without owning the paper's +/// `RcuPointedBy`/incoming-link authority. The executable `Rcu

` root uses +/// this bridge only while replacing its directly owned root publication. A +/// general linked structure must instead obtain a retire permission from a +/// future tracked traversal-state transition. +proof fn lift_direct_root_retire_perm( + tracked base: RcuBaseRetirePerm, + seen_removed: RcuSeenRemoved, +) -> (tracked perm: RcuRetirePerm) + requires + base.wf(), + seen_removed.removed.contains(base.obj()), + ensures + perm.base() == base, + perm.domain() == base.domain(), + perm.ptr() == base.ptr(), + perm.obj() == base.obj(), + perm.seen_removed() == seen_removed, + perm.wf(), + perm.ready_to_retire(), +{ + RcuRetirePerm { base, seen_removed } +} + +/// Persistent, type-erased evidence that one allocation passed the base +/// `rcu-retire` transition. +/// +/// `fact` comes from consuming the allocation's unique `BaseRetirePerm`. +/// `observation` comes from updating that permission's domain-owned +/// observation cell from `None` to `Some(removal)`. Their keys agree, so +/// clients cannot attach an unrelated detachment observation to a registered +/// object. Both facts are persistent and remain duplicable after callback type +/// erasure. +pub tracked struct RcuRetiredFact { + ghost domain: Loc, + fact: GhostPersistentPointsTo, + observation: GhostPersistentPointsTo>, +} + +impl RcuRetiredFact { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn obj(self) -> nat { + self.fact.key() + } + + pub closed spec fn addr(self) -> usize { + self.fact.value() + } + + pub closed spec fn removal(self) -> RcuRemovalObservation { + self.observation.value()->Some_0 + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.observation.id() + } + + pub closed spec fn wf(self) -> bool { + &&& self.fact.key() == self.observation.key() + &&& self.observation.value() is Some + } + + pub open spec fn record(self) -> RcuRetiredRecord { + RcuRetiredRecord { + domain: self.domain(), + obj: self.obj(), + retire_observation_registry: self.retire_observation_registry(), + removal: self.removal(), + } + } + + pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { + &&& summary.domain == self.domain() + &&& summary.obj == self.obj() + &&& summary.removal == self.removal() + &&& summary.retire_observation_registry == self.retire_observation_registry() + } + + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + requires + self.wf(), + ensures + res.wf(), + res.record() == self.record(), + res.domain() == self.domain(), + res.obj() == self.obj(), + res.addr() == self.addr(), + res.removal() == self.removal(), + res.retire_observation_registry() == self.retire_observation_registry(), + { + let tracked fact = self.fact.duplicate(); + let tracked observation = self.observation.duplicate(); + RcuRetiredFact { domain: self.domain, fact, observation } + } + + /// Establishes that this callback fact contains the observation recorded by + /// the corresponding domain authority. + pub proof fn lemma_observation_agrees(tracked &self, tracked domain: &RcuDomainAuth) + requires + self.wf(), + domain.wf(), + self.domain() == domain.id(), + self.retire_observation_registry() == domain.retire_observation_registry(), + ensures + domain.retired().contains(self.obj()), + domain.retire_observations().contains_pair(self.obj(), self.removal()), + { + self.observation.agree(&domain.retire_observation_cells); + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +/// A finite collection of persistent retirement facts. +/// +/// The map key is the complete retirement record, rather than just an AId. +/// This lets CPU-generation state accumulate facts from independent RCU +/// domains without assuming that domain IDs determine observation-registry +/// identities by pure equality alone. +pub tracked struct RcuRetiredFacts { + facts: Map, +} + +impl RcuRetiredFacts { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + forall|record: RcuRetiredRecord| #[trigger] + self.facts.contains_key(record) ==> { + &&& self.facts[record].wf() + &&& self.facts[record].record() == record + } + } + + /// Complete set of certified retirement records in this collection. + pub closed spec fn records(self) -> Set { + self.facts.dom() + } + + pub closed spec fn contains(self, record: RcuRetiredRecord) -> bool { + self.facts.contains_key(record) + } + + /// Every retained detachment observation is covered by `view`. + /// + /// The retirement facts themselves are persistent, but this predicate is + /// the separate weak-memory premise needed before a CPU report may publish + /// them to readers in a later quiescent generation. + pub open spec fn observed_by(self, view: Irc11ThreadView) -> bool { + forall|record: RcuRetiredRecord| #[trigger] + self.records().contains(record) ==> record.removal.observed_by(view) + } + + proof fn lemma_matching_subset_observed_retired( + tracked &self, + tracked domain: &RcuDomainAuth, + root: Loc, + view: Irc11ThreadView, + records: Set, + ) + requires + domain.wf(), + self.observed_by(view), + records.subset_of(self.records()), + ensures + forall|record: RcuRetiredRecord| #[trigger] + records.contains(record) && record.domain == domain.id() + && record.retire_observation_registry == domain.retire_observation_registry() + && record.removal.root == root ==> domain.observed_retired(root, view).contains( + record.obj, + ), + decreases records.len(), + { + if !records.is_empty() { + let ghost record = records.choose(); + let ghost rest = records.remove(record); + let tracked fact = self.tracked_borrow(record); + assert(self.records().contains(record)); + assert(record.removal.observed_by(view)); + if record.domain == domain.id() && record.retire_observation_registry + == domain.retire_observation_registry() && record.removal.root == root { + fact.lemma_observation_agrees(domain); + assert(domain.retired().contains(record.obj)); + assert(domain.retire_observations().contains_pair(record.obj, record.removal)); + } + Self::lemma_matching_subset_observed_retired(self, domain, root, view, rest); + assert forall|candidate: RcuRetiredRecord| #[trigger] + records.contains(candidate) && candidate.domain == domain.id() + && candidate.retire_observation_registry == domain.retire_observation_registry() + && candidate.removal.root == root implies domain.observed_retired( + root, + view, + ).contains(candidate.obj) by { + if candidate == record { + } else { + assert(rest.contains(candidate)); + } + }; + } + } + + /// Converts persistent retirement facts whose detachments have been + /// observed into membership in the domain's entry-time expired set. + /// + /// This is the paper's `Retired(a, Q)` plus observation-of-`Q` step. The + /// retirement fact alone is deliberately insufficient. + pub proof fn lemma_matching_records_observed_retired( + tracked &self, + tracked domain: &RcuDomainAuth, + root: Loc, + view: Irc11ThreadView, + ) + requires + domain.wf(), + self.observed_by(view), + ensures + forall|record: RcuRetiredRecord| #[trigger] + self.records().contains(record) && record.domain == domain.id() + && record.retire_observation_registry == domain.retire_observation_registry() + && record.removal.root == root ==> domain.observed_retired(root, view).contains( + record.obj, + ), + { + Self::lemma_matching_subset_observed_retired(self, domain, root, view, self.records()); + } + + /// Creates an empty retirement-fact collection. + pub proof fn empty() -> (tracked res: Self) + ensures + res.records() == Set::::empty(), + { + RcuRetiredFacts { facts: Map::tracked_empty() } + } + + /// Borrows the persistent fact for one record. + pub proof fn tracked_borrow(tracked &self, record: RcuRetiredRecord) -> (tracked res: + &RcuRetiredFact) + requires + self.records().contains(record), + ensures + res.wf(), + res.record() == record, + { + use_type_invariant(self); + let tracked res = self.facts.tracked_borrow(record); + res + } + + /// Inserts a persistent copy of `fact`. + pub proof fn tracked_insert(tracked &mut self, tracked fact: &RcuRetiredFact) + requires + fact.wf(), + ensures + final(self).records() == old(self).records().insert(fact.record()), + final(self).contains(fact.record()), + { + use_type_invariant(&*self); + use_type_invariant(fact); + let ghost record = fact.record(); + let ghost old_records = self.facts.dom(); + if !self.contains(record) { + let tracked duplicate = fact.tracked_duplicate(); + self.facts.tracked_insert(record, duplicate); + assert forall|saved: RcuRetiredRecord| #[trigger] + self.facts.contains_key(saved) implies { + &&& self.facts[saved].wf() + &&& self.facts[saved].record() == saved + } by { + if saved == record { + } else { + assert(old_records.contains(saved)); + } + }; + } + } + + proof fn tracked_duplicate_keys( + tracked source: &Map, + keys: Set, + ) -> (tracked duplicates: Map) + requires + keys.subset_of(source.dom()), + forall|record: RcuRetiredRecord| #[trigger] + source.dom().contains(record) ==> { + &&& source[record].wf() + &&& source[record].record() == record + }, + ensures + duplicates.dom() == keys, + forall|record: RcuRetiredRecord| #[trigger] + keys.contains(record) ==> { + &&& duplicates[record].wf() + &&& duplicates[record].record() == record + }, + decreases keys.len(), + { + if keys.is_empty() { + Map::tracked_empty() + } else { + let ghost record = keys.choose(); + let ghost rest = keys.remove(record); + let tracked mut duplicates = Self::tracked_duplicate_keys(source, rest); + let tracked fact = source.tracked_borrow(record); + let tracked duplicate = fact.tracked_duplicate(); + duplicates.tracked_insert(record, duplicate); + assert(keys == rest.insert(record)); + assert forall|saved: RcuRetiredRecord| #[trigger] keys.contains(saved) implies { + &&& duplicates[saved].wf() + &&& duplicates[saved].record() == saved + } by { + if saved == record { + } else { + assert(rest.contains(saved)); + } + }; + duplicates + } + } + + /// Duplicates this persistent fact collection. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.records() == self.records(), + { + use_type_invariant(self); + let tracked facts = Self::tracked_duplicate_keys(&self.facts, self.facts.dom()); + RcuRetiredFacts { facts } + } + + proof fn tracked_merge_keys( + tracked target: &mut RcuRetiredFacts, + tracked source: &RcuRetiredFacts, + keys: Set, + ) + requires + keys.subset_of(source.records()), + ensures + final(target).records() == old(target).records().union(keys), + decreases keys.len(), + { + if !keys.is_empty() { + let ghost record = keys.choose(); + let ghost rest = keys.remove(record); + let tracked fact = source.tracked_borrow(record); + target.tracked_insert(fact); + Self::tracked_merge_keys(target, source, rest); + assert(keys == rest.insert(record)); + } + } + + /// Adds persistent copies of all facts in `other`. + pub proof fn tracked_merge(tracked &mut self, tracked other: &RcuRetiredFacts) + ensures + final(self).records() == old(self).records().union(other.records()), + { + Self::tracked_merge_keys(self, other, other.records()); + } +} + +/// Typed objective record that an allocation has passed the base +/// `rcu-retire` transition. It is safe to enqueue its callback, but not yet +/// safe to execute it; execution additionally needs monitor grace-period +/// completion. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetired { + fact: RcuRetiredFact, + ghost ptr: *mut T, +} + +impl RcuRetired { + pub closed spec fn domain(self) -> Loc { + self.fact.domain() + } + + pub closed spec fn obj(self) -> nat { + self.fact.obj() + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } + + pub closed spec fn addr(self) -> usize { + self.fact.addr() + } + + pub closed spec fn removal(self) -> RcuRemovalObservation { + self.fact.removal() + } + + /// Authoritative observation registry that issued this retirement fact. + pub closed spec fn retire_observation_registry(self) -> Loc { + self.fact.retire_observation_registry() + } + + pub closed spec fn wf(self) -> bool { + &&& self.fact.wf() + &&& self.addr() == self.ptr().addr() + } + + proof fn tracked_into_fact(tracked self) -> (tracked res: RcuRetiredFact) + requires + self.wf(), + ensures + res.domain() == self.domain(), + res.obj() == self.obj(), + res.addr() == self.ptr().addr(), + res.removal() == self.removal(), + res.retire_observation_registry() == self.retire_observation_registry(), + { + self.fact + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +/// Regression proof for the distinction between retirement and expiration. +/// +/// Even a reader registered after `retire` may protect the object while no +/// grace period has expired it. This is the stale-read case required by the +/// paper's relaxed-memory base specification. +pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> (tracked res: ( + RcuBaseGuard, + RcuBlockInfo, +)) + requires + ptr.addr() != 0, + ensures + res.1.ptr() == ptr, + res.0.domain() == res.1.domain(), + res.0.expired() == Set::::empty(), + res.0.protects(res.1.addr(), res.1.obj()), +{ + let tracked mut domain = RcuDomainAuth::tracked_new(); + let tracked (info, base) = domain.tracked_register(ptr); + let ghost reader = arbitrary(); + let tracked inactive = domain.tracked_register_reader(reader); + let tracked mut guard = domain.tracked_guard_start( + inactive, + domain.id(), + Irc11ThreadView::empty(), + ); + let ghost seen_removed = RcuSeenRemoved { + removed: Set::empty().insert(info.obj()), + link_view: RcuLinkView::empty(), + }; + let tracked retire = lift_direct_root_retire_perm(base, seen_removed); + let ghost removal = RcuRemovalObservation { + root: domain.id(), + timestamp: 1, + message_view: Irc11ThreadView::empty(), + }; + let tracked _retired = domain.tracked_retire(retire, removal); + assert(guard.expired() == Set::::empty()); + assert(!guard.expired().contains(info.obj())); + guard.tracked_protect(&info); + (guard, info) +} + +/// Regression proof that observing a retirement makes it expired for a new +/// guard. +/// +/// Timestamp zero is covered by an empty weak-memory view. Consequently the +/// retired allocation enters the new guard's `X` snapshot and cannot be added +/// to its protection map. +pub proof fn observed_retired_object_enters_guard_expired(ptr: *mut T) -> (tracked res: ( + RcuBaseGuard, + RcuBlockInfo, +)) + requires + ptr.addr() != 0, + ensures + res.1.ptr() == ptr, + res.0.domain() == res.1.domain(), + res.0.expired().contains(res.1.obj()), +{ + let tracked mut domain = RcuDomainAuth::tracked_new(); + let tracked (info, base) = domain.tracked_register(ptr); + let ghost seen_removed = RcuSeenRemoved { + removed: Set::empty().insert(info.obj()), + link_view: RcuLinkView::empty(), + }; + let tracked retire = lift_direct_root_retire_perm(base, seen_removed); + let ghost removal = RcuRemovalObservation { + root: domain.id(), + timestamp: 0, + message_view: Irc11ThreadView::empty(), + }; + let tracked _retired = domain.tracked_retire(retire, removal); + + let ghost reader = arbitrary(); + let tracked inactive = domain.tracked_register_reader(reader); + let tracked guard = domain.tracked_guard_start(inactive, domain.id(), Irc11ThreadView::empty()); + assert(removal.observed_by(Irc11ThreadView::empty())); + assert(guard.expired().contains(info.obj())); + (guard, info) +} + +/// Non-generic proof certificate carried across the type-erasure boundary. +/// +/// A certificate can only be produced from a typed traversal retire permission, +/// but after that point the monitor only needs the erased callback summary. +pub tracked struct RcuCallbackSafety { + retired: RcuRetiredFact, +} + +impl RcuCallbackSafety { + pub closed spec fn domain(self) -> Loc { + self.retired.domain() + } + + pub closed spec fn obj(self) -> nat { + self.retired.obj() + } + + pub closed spec fn removal(self) -> RcuRemovalObservation { + self.retired.removal() + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retired.retire_observation_registry() + } + + /// The monitor may assign any future batch generation, but it cannot + /// change the retired object's domain or allocation identity. + pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { + self.retired.matches(summary) + } + + /// Establishes the abstract matching predicate without exposing the + /// persistent retirement resource to callback-monitor clients. + pub proof fn lemma_matches(tracked &self, summary: RcuCallbackSummary) + requires + summary.domain == self.domain(), + summary.obj == self.obj(), + summary.removal == self.removal(), + summary.retire_observation_registry == self.retire_observation_registry(), + ensures + self.matches(summary), + { + } + + /// Duplicates the persistent base-retirement fact for an object-level + /// reclaim permit. + pub proof fn tracked_retired_fact(tracked &self, summary: RcuCallbackSummary) -> (tracked res: + RcuRetiredFact) + requires + self.matches(summary), + ensures + res.wf(), + res.domain() == self.domain(), + res.obj() == self.obj(), + res.removal() == self.removal(), + res.retire_observation_registry() == self.retire_observation_registry(), + res.matches(summary), + res.record() == summary.retired_record(), + { + use_type_invariant(&self.retired); + self.retired.tracked_duplicate() + } +} + +pub open spec fn callback_safety_from_traversal( + cert: RcuCallbackSafety, + object: RcuObjectId, +) -> bool { + &&& cert.domain() == object.domain() + &&& cert.obj() == object.obj() +} + +/// Consume a typed traversal retire permission and compress it into the +/// non-generic summary needed by the type-erased callback monitor. +pub proof fn certify_callback_from_retired( + tracked object: &RcuObjectId, + tracked retired: RcuRetired, +) -> (tracked cert: RcuCallbackSafety) + requires + object.domain() == retired.domain(), + object.obj() == retired.obj(), + object.ptr() == retired.ptr(), + ensures + cert.domain() == retired.domain(), + cert.obj() == object.obj(), + cert.removal() == retired.removal(), + cert.retire_observation_registry() == retired.retire_observation_registry(), + callback_safety_from_traversal(cert, *object), +{ + use_type_invariant(&retired); + let tracked fact = retired.tracked_into_fact(); + RcuCallbackSafety { retired: fact } +} + +/// Read-side guard token for one critical section. +/// +/// This is the traversal-level guard: it includes the base guard protection and +/// the `SeenRemoved(D, LV)` observation used to rule out stale links. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuReadGuardToken { + base: RcuBaseGuard, + ghost seen_removed: RcuSeenRemoved, +} + +impl RcuReadGuardToken { + pub closed spec fn domain(self) -> Loc { + self.base.domain() + } + + pub closed spec fn tid(self) -> nat { + self.base.tid() + } + + pub closed spec fn reader_registry(self) -> Loc { + self.base.reader_registry() + } + + pub closed spec fn reader(self) -> RcuReaderContext { + self.base.reader() + } + + pub closed spec fn root(self) -> Loc { + self.base.root() + } + + pub closed spec fn start_view(self) -> Irc11ThreadView { + self.base.start_view() + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.base.retire_observation_registry() + } + + pub closed spec fn expired(self) -> Set { + self.base.expired() + } + + pub closed spec fn protected(self) -> Map { + self.base.protected() + } + + pub closed spec fn protects(self, addr: usize, obj: nat) -> bool { + self.base.protects(addr, obj) + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.seen_removed + } + + pub closed spec fn link_view(self) -> RcuLinkView { + self.seen_removed().link_view + } + + pub open spec fn seen_at(self, obj: nat) -> LinkIndex { + self.seen_removed().seen_at(obj) + } + + pub closed spec fn wf(self) -> bool { + &&& self.base.wf() + &&& self.expired().subset_of(self.seen_removed().removed) + } + + pub closed spec fn is_for(self, domain: RcuDomainAuth) -> bool { + self.base.belongs_to(domain) + } + + /// Exposes the traversal-side consequence of a well-formed guard without + /// opening the guard representation in client modules. + pub proof fn lemma_expired_is_removed(tracked &self) + requires + self.wf(), + ensures + self.expired().subset_of(self.seen_removed().removed), + { + } + + pub proof fn lemma_protected_not_expired(tracked &self, addr: usize, obj: nat) + requires + self.wf(), + self.protects(addr, obj), + ensures + !self.expired().contains(obj), + { + } + + /// Preconditions of the paper's base `Guard-protect` rule. + /// + /// This deliberately says only that the allocation was not already + /// expired when the critical section started. Whether a stale traversal + /// may still reach the allocation is proved separately from + /// `SeenRemoved` and the link history. + pub open spec fn can_base_protect(self, info: RcuBlockInfo) -> bool { + &&& self.wf() + &&& info.wf() + &&& info.domain() == self.domain() + &&& !self.expired().contains(info.obj()) + } + + /// A base-protectable allocation that traversal has also proved is not in + /// the guard's observed removed set. + pub open spec fn can_protect(self, info: RcuBlockInfo) -> bool { + &&& self.can_base_protect(info) + &&& !self.seen_removed().removed.contains(info.obj()) + } + + /// Combines the paper's base guard with traversal `SeenRemoved(D, LV)`. + pub proof fn tracked_new( + tracked base: RcuBaseGuard, + seen_removed: RcuSeenRemoved, + ) -> (tracked res: Self) + requires + base.wf(), + base.expired().subset_of(seen_removed.removed), + ensures + res.wf(), + res.domain() == base.domain(), + res.tid() == base.tid(), + res.reader_registry() == base.reader_registry(), + res.reader() == base.reader(), + res.root() == base.root(), + res.start_view() == base.start_view(), + res.retire_observation_registry() == base.retire_observation_registry(), + res.expired() == base.expired(), + res.protected() == base.protected(), + res.seen_removed() == seen_removed, + { + RcuReadGuardToken { base, seen_removed } + } + + /// Lift a base guard using its start-time expired set as the initial + /// traversal observation. + pub proof fn tracked_from_base(tracked base: RcuBaseGuard) -> (tracked res: Self) + requires + base.wf(), + ensures + res.wf(), + res.domain() == base.domain(), + res.tid() == base.tid(), + res.reader_registry() == base.reader_registry(), + res.reader() == base.reader(), + res.root() == base.root(), + res.start_view() == base.start_view(), + res.retire_observation_registry() == base.retire_observation_registry(), + res.expired() == base.expired(), + res.seen_removed().removed == base.expired(), + res.link_view() == RcuLinkView::::empty(), + { + let ghost seen_removed = RcuSeenRemoved { + removed: base.expired(), + link_view: RcuLinkView::empty(), + }; + RcuReadGuardToken::tracked_new(base, seen_removed) + } + + /// Consume the traversal wrapper when ending the read-side critical + /// section. + pub proof fn tracked_into_base(tracked self) -> (tracked res: RcuBaseGuard) + requires + self.wf(), + ensures + res.wf(), + res.domain() == self.domain(), + res.tid() == self.tid(), + res.reader_registry() == self.reader_registry(), + res.reader() == self.reader(), + res.root() == self.root(), + res.start_view() == self.start_view(), + res.retire_observation_registry() == self.retire_observation_registry(), + res.expired() == self.expired(), + res.protected() == self.protected(), + { + self.base + } + + /// Records one successful base `Guard-protect` operation in `G`. + pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) + requires + old(self).can_protect(*info), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).seen_removed() == old(self).seen_removed(), + final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), + final(self).protects(info.addr(), info.obj()), + { + self.base.tracked_protect(info); + } + + /// Records a coherent observation of the next link event from an already + /// protected source. The returned source witness is refreshed to carry the + /// guard's advanced link-view snapshot. + pub proof fn tracked_observe_link( + tracked &mut self, + tracked from: RcuProtectedPtr, + n: LinkIndex, + ) -> (tracked res: RcuProtectedPtr) + requires + old(self).wf(), + from.protected_by(*old(self)), + old(self).seen_at(from.obj()) <= n, + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected(), + final(self).seen_removed().removed == old(self).seen_removed().removed, + final(self).link_view() == old(self).link_view().observe(from.obj(), n), + final(self).seen_at(from.obj()) == n, + res.domain() == from.domain(), + res.obj() == from.obj(), + res.ptr() == from.ptr(), + res.protected_by(*final(self)), + { + let ghost old_expired = self.expired(); + let ghost old_removed = self.seen_removed.removed; + let ghost seen_removed = RcuSeenRemoved { + removed: old_removed, + link_view: self.seen_removed.link_view.observe(from.obj(), n), + }; + self.seen_removed = seen_removed; + assert(self.expired() == old_expired); + assert(self.seen_removed.removed == old_removed); + assert(self.expired().subset_of(self.seen_removed.removed)); + assert(self.wf()); + RcuProtectedPtr { domain: from.domain(), obj: from.obj(), ptr: from.ptr(), seen_removed } + } + + /// In-place form of [`tracked_observe_link`](Self::tracked_observe_link). + /// This is convenient for atomic loads, which must advance the guard and + /// its source protection witness to the same traversal snapshot. + pub proof fn tracked_observe_link_in_place( + tracked &mut self, + tracked from: &mut RcuProtectedPtr, + n: LinkIndex, + ) + requires + old(self).wf(), + old(from).protected_by(*old(self)), + old(self).seen_at(old(from).obj()) <= n, + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected(), + final(self).seen_removed().removed == old(self).seen_removed().removed, + final(self).link_view() == old(self).link_view().observe(old(from).obj(), n), + final(self).seen_at(old(from).obj()) == n, + final(from).domain() == old(from).domain(), + final(from).obj() == old(from).obj(), + final(from).ptr() == old(from).ptr(), + final(from).protected_by(*final(self)), + { + let ghost seen_removed = RcuSeenRemoved { + removed: self.seen_removed.removed, + link_view: self.seen_removed.link_view.observe(from.obj(), n), + }; + self.seen_removed = seen_removed; + from.seen_removed = seen_removed; + assert(self.wf()); + assert(from.protected_by(*self)); + } +} + +/// A pointer protected by a live read-side guard. +/// +/// It records the same `SeenRemoved` snapshot as the guard. This lets traversal +/// proofs preserve the fact that the protected pointer is not in the guard's +/// removed set. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuProtectedPtr { + ghost domain: Loc, + ghost obj: nat, + ghost ptr: *mut T, + ghost seen_removed: RcuSeenRemoved, +} + +impl RcuProtectedPtr { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } + + pub closed spec fn obj(self) -> nat { + self.obj + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.seen_removed + } + + /// Duplicates the persistent fact that this pointer is protected by its + /// recorded guard snapshot. + /// + /// The witness contains only ghost identities and observations; it owns + /// no linear physical resource. Traversal can therefore retain one copy + /// while another is recorded in a physical read lease. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res == *self, + { + RcuProtectedPtr { + domain: self.domain, + obj: self.obj, + ptr: self.ptr, + seen_removed: self.seen_removed, + } + } + + pub open spec fn protected_by(self, guard: RcuReadGuardToken) -> bool { + &&& self.domain() == guard.domain() + &&& self.seen_removed() == guard.seen_removed() + &&& !self.seen_removed().removed.contains(self.obj()) + &&& guard.protects(self.ptr().addr(), self.obj()) + } + + /// Materializes the linear protection witness for a pointer already + /// installed in a direct-root guard's protection map. + /// + /// Generic traversal clients should normally use [`protect_root`] or + /// [`protect_next`]. This constructor is for the executable `Rcu

` + /// adapter, whose guarded atomic load performs the root protection + /// transition before returning to the caller. + pub proof fn tracked_from_guard( + tracked guard: &RcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: Self) + requires + guard.wf(), + info.wf(), + info.domain() == guard.domain(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + ensures + res.domain() == info.domain(), + res.obj() == info.obj(), + res.ptr() == info.ptr(), + res.protected_by(*guard), + { + RcuProtectedPtr { + domain: info.domain(), + obj: info.obj(), + ptr: info.ptr(), + seen_removed: guard.seen_removed(), + } + } +} + +/// Traversal specification for an RCU-protected data structure. +/// +/// `link_inv(from, n, to, g)` is the client-facing analogue of a +/// `RcuPointsTo(from, ...)` snapshot containing the `n`th link event from +/// `from` to `to`. `seen_removed_sound` is the client-facing analogue of the +/// `RcuPointedBy`/`SeenRemoved` invariant for partially ordered link histories. +pub trait RcuTraversalSafety: Sized { + type Node; + + type Ghost; + + spec fn root_inv(p: *mut Self::Node, obj: nat, g: Self::Ghost) -> bool; + + spec fn node_inv(p: *mut Self::Node, obj: nat, g: Self::Ghost) -> bool; + + spec fn link_inv( + from: *mut Self::Node, + from_obj: nat, + n: LinkIndex, + to: *mut Self::Node, + to_obj: nat, + g: Self::Ghost, + ) -> bool; + + spec fn seen_removed_sound(seen_removed: RcuSeenRemoved, g: Self::Ghost) -> bool; + + proof fn root_is_node_inv(p: *mut Self::Node, obj: nat, g: Self::Ghost) + requires + Self::root_inv(p, obj, g), + ensures + Self::node_inv(p, obj, g), + ; + + proof fn link_preserves_protection( + from: *mut Self::Node, + from_obj: nat, + n: LinkIndex, + to: *mut Self::Node, + to_obj: nat, + seen_removed: RcuSeenRemoved, + g: Self::Ghost, + ) + requires + Self::node_inv(from, from_obj, g), + Self::link_inv(from, from_obj, n, to, to_obj, g), + Self::seen_removed_sound(seen_removed, g), + !seen_removed.removed.contains(from_obj), + seen_removed.seen_at(from_obj) <= n, + ensures + Self::node_inv(to, to_obj, g), + !seen_removed.removed.contains(to_obj), + ; +} + +/// Protect a freshly acquired root pointer. +pub proof fn protect_root( + tracked guard: &mut RcuReadGuardToken, + tracked info: &RcuBlockInfo, + p: *mut S::Node, + g: S::Ghost, +) -> (tracked root: RcuProtectedPtr) + requires + old(guard).can_protect(*info), + info.ptr() == p, + S::root_inv(p, info.obj(), g), + ensures + root.ptr() == p, + root.obj() == info.obj(), + root.domain() == old(guard).domain(), + root.protected_by(*final(guard)), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).expired() == old(guard).expired(), + final(guard).seen_removed() == old(guard).seen_removed(), + S::node_inv(p, info.obj(), g), +{ + S::root_is_node_inv(p, info.obj(), g); + guard.tracked_protect(info); + RcuProtectedPtr { + domain: guard.domain(), + obj: info.obj(), + ptr: p, + seen_removed: guard.seen_removed(), + } +} + +/// Protect a child reached by following a non-stale link-history event. +pub proof fn protect_link( + tracked guard: &mut RcuReadGuardToken, + tracked from: &RcuProtectedPtr, + tracked to_info: &RcuBlockInfo, + n: LinkIndex, + to: *mut S::Node, + g: S::Ghost, +) -> (tracked to_protected: RcuProtectedPtr) + requires + from.protected_by(*old(guard)), + old(guard).can_base_protect(*to_info), + to_info.ptr() == to, + S::node_inv(from.ptr(), from.obj(), g), + S::link_inv(from.ptr(), from.obj(), n, to, to_info.obj(), g), + S::seen_removed_sound(guard.seen_removed(), g), + guard.seen_at(from.obj()) <= n, + ensures + to_protected.ptr() == to, + to_protected.obj() == to_info.obj(), + to_protected.domain() == from.domain(), + to_protected.protected_by(*final(guard)), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).tid() == old(guard).tid(), + final(guard).reader_registry() == old(guard).reader_registry(), + final(guard).reader() == old(guard).reader(), + final(guard).root() == old(guard).root(), + final(guard).start_view() == old(guard).start_view(), + final(guard).retire_observation_registry() == old(guard).retire_observation_registry(), + final(guard).expired() == old(guard).expired(), + final(guard).seen_removed() == old(guard).seen_removed(), + S::node_inv(to, to_info.obj(), g), +{ + S::link_preserves_protection( + from.ptr(), + from.obj(), + n, + to, + to_info.obj(), + guard.seen_removed(), + g, + ); + assert(old(guard).can_protect(*to_info)); + guard.tracked_protect(to_info); + RcuProtectedPtr { + domain: from.domain(), + obj: to_info.obj(), + ptr: to, + seen_removed: guard.seen_removed(), + } +} + +/// Minimal ghost-only node used to demonstrate the traversal contract. +pub struct LinkedListNode; + +/// Paper-style ghost state for a linked list. +/// +/// `successors[p]` is the successor history for `p`, corresponding to +/// `RcuPointsTo(p, s)`. A non-null event records both the pointer and its AId; +/// retaining the AId prevents an old history event from being reinterpreted as +/// a different allocation after address reuse. +/// +/// `incoming_all[p]` is the set of all incoming edges that have ever pointed to +/// `p`, corresponding to the authoritative incoming set in `RcuPointedBy(p, B)`. +/// +pub ghost struct LinkedListGhost { + pub root: *mut LinkedListNode, + pub root_obj: nat, + /// Historical allocation registry, keyed by AId rather than address. + /// Distinct reclaimed and newly registered objects may therefore retain + /// the same pointer without collapsing their identities. + pub objects: Map, + /// Per-allocation link histories. An address is not a valid history key: + /// after reuse, the old and new allocations must have disjoint histories. + pub successors: Map>>, + pub incoming_all: Map>, +} + +impl LinkedListGhost { + /// Current `RcuPointedBy` set, derived from the latest event of every + /// source. Keeping it derived avoids a second mutable representation of + /// the same link relation. + pub open spec fn current_incoming(self, to_obj: nat) -> Set { + if self.incoming_all.contains_key(to_obj) { + self.incoming_all[to_obj].filter( + |edge: LinkEdge| + self.objects.contains_key(edge.0) && self.successors.contains_key(edge.0) + && self.successors[edge.0].len() > 0 && edge.1 + == self.successors[edge.0].len() - 1 + && self.successors[edge.0].last() is Some + && self.successors[edge.0].last()->Some_0.1 == to_obj, + ) + } else { + Set::empty() + } + } + + /// Every recorded history event names a registered allocation and appears + /// in that allocation's authoritative incoming-edge history. + pub open spec fn wf(self) -> bool { + &&& self.objects.contains_pair(self.root_obj, self.root) + &&& self.successors.dom() == self.objects.dom() + &&& self.incoming_all.dom() == self.objects.dom() + &&& forall|from_obj: nat, n: LinkIndex| + #![trigger self.objects.contains_key(from_obj), self.successors[from_obj][n as int]] + self.objects.contains_key(from_obj) && n < self.successors[from_obj].len() + && self.successors[from_obj][n as int] is Some ==> { + let event = self.successors[from_obj][n as int]->Some_0; + &&& self.objects.contains_pair(event.1, event.0) + &&& self.incoming_all[event.1].contains((from_obj, n)) + } + } + + /// A link view cannot claim an event newer than the source's history. + pub open spec fn bounds(self, view: RcuLinkView) -> bool { + forall|from_obj: nat| #[trigger] + self.objects.contains_key(from_obj) && view.seen.contains_key(from_obj) ==> { + &&& self.successors[from_obj].len() > 0 + &&& view.seen_at(from_obj) < self.successors[from_obj].len() + } + } +} + +/// Linear writer authority for the linked-list traversal history. +/// +/// Clients can inspect [`LinkedListGhost`] snapshots, but only this tracked +/// authority can append link events and turn a node's unique base permission +/// into [`RcuRetirePerm`]. This is the proof-only analogue of owning all +/// `RcuPointsTo`/`RcuPointedBy` resources for one list. +pub tracked struct LinkedListTraversalAuth { + ghost domain: Loc, + ghost state: LinkedListGhost, + ghost removed: Set, + infos: Map>, + retire_perms: Map>, +} + +impl LinkedListTraversalAuth { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn state(self) -> LinkedListGhost { + self.state + } + + /// Objects whose removal has already been certified by this authority. + pub closed spec fn removed(self) -> Set { + self.removed + } + + pub closed spec fn has_retire_perm(self, obj: nat) -> bool { + self.retire_perms.contains_key(obj) + } + + pub closed spec fn has_info(self, obj: nat) -> bool { + self.infos.contains_key(obj) + } + + pub closed spec fn info(self, obj: nat) -> RcuBlockInfo + recommends + self.has_info(obj), + { + self.infos[obj] + } + + pub closed spec fn wf(self) -> bool { + &&& self.state().wf() + &&& self.infos.dom() == self.state().incoming_all.dom() + &&& forall|obj: nat| #[trigger] + self.infos.contains_key(obj) ==> { + let info = self.infos[obj]; + &&& info.wf() + &&& info.domain() == self.domain() + &&& info.obj() == obj + &&& self.state().objects.contains_pair(obj, info.ptr()) + } + &&& forall|obj: nat| #[trigger] + self.retire_perms.contains_key(obj) ==> { + let perm = self.retire_perms[obj]; + &&& perm.wf() + &&& perm.domain() == self.domain() + &&& self.state().objects.contains_pair(obj, perm.ptr()) + &&& perm.obj() == obj + } + } + + /// Opens the registry-domain equalities hidden by [`Self::wf`]. + /// Native-link adapters use these facts to move between an AId's object, + /// successor, incoming-edge, and persistent-info entries. + pub proof fn lemma_registry_domains(tracked &self) + requires + self.wf(), + ensures + self.state().wf(), + self.state().successors.dom() == self.state().objects.dom(), + self.state().incoming_all.dom() == self.state().objects.dom(), + forall|obj: nat| #[trigger] + self.has_info(obj) <==> self.state().objects.contains_key(obj), + { + } + + /// A fresh AId cannot already own any list-local registration resource. + pub proof fn lemma_unregistered_has_no_resources(tracked &self, obj: nat) + requires + self.wf(), + !self.state().objects.contains_key(obj), + ensures + !self.state().successors.contains_key(obj), + !self.state().incoming_all.contains_key(obj), + !self.has_info(obj), + !self.has_retire_perm(obj), + { + self.lemma_registry_domains(); + if self.has_retire_perm(obj) { + assert(self.retire_perms.contains_key(obj)); + assert(self.state().objects.contains_pair(obj, self.retire_perms[obj].ptr())); + } + } + + /// Starts an authoritative list with one registered root allocation. + pub proof fn tracked_new( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + ) -> (tracked res: Self) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + ensures + res.wf(), + res.domain() == root_info.domain(), + res.state().root == root_info.ptr(), + res.state().root_obj == root_info.obj(), + res.state().objects == Map::empty().insert(root_info.obj(), root_info.ptr()), + res.state().successors == Map::empty().insert(root_info.obj(), Seq::empty()), + res.state().incoming_all == Map::empty().insert(root_info.obj(), Set::empty()), + res.removed() == Set::empty(), + res.has_info(root_info.obj()), + res.info(root_info.obj()).ptr() == root_info.ptr(), + res.has_retire_perm(root_info.obj()), + forall|obj: nat| #[trigger] res.has_retire_perm(obj) <==> obj == root_info.obj(), + { + let ghost state = LinkedListGhost { + root: root_info.ptr(), + root_obj: root_info.obj(), + objects: Map::empty().insert(root_info.obj(), root_info.ptr()), + successors: Map::empty().insert(root_info.obj(), Seq::empty()), + incoming_all: Map::empty().insert(root_info.obj(), Set::empty()), + }; + let tracked mut infos = Map::tracked_empty(); + let tracked saved_root_info = root_info.tracked_duplicate(); + infos.tracked_insert(root_info.obj(), saved_root_info); + let tracked mut retire_perms = Map::tracked_empty(); + retire_perms.tracked_insert(root_info.obj(), root_retire); + LinkedListTraversalAuth { + domain: root_info.domain(), + state, + removed: Set::empty(), + infos, + retire_perms, + } + } + + /// Adds a separately registered allocation to this list's traversal + /// authority. Registration alone does not publish an incoming link. + pub proof fn tracked_register_node( + tracked &mut self, + tracked info: &RcuBlockInfo, + tracked retire: RcuBaseRetirePerm, + ) + requires + old(self).wf(), + info.wf(), + retire.wf(), + info.domain() == old(self).domain(), + retire.domain() == old(self).domain(), + retire.obj() == info.obj(), + retire.ptr() == info.ptr(), + !old(self).state().objects.contains_key(info.obj()), + !old(self).state().incoming_all.contains_key(info.obj()), + !old(self).has_retire_perm(info.obj()), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects.insert(info.obj(), info.ptr()), + final(self).state().successors == old(self).state().successors.insert( + info.obj(), + Seq::empty(), + ), + final(self).state().incoming_all == old(self).state().incoming_all.insert( + info.obj(), + Set::empty(), + ), + final(self).removed() == old(self).removed(), + final(self).has_info(info.obj()), + final(self).info(info.obj()).ptr() == info.ptr(), + final(self).has_retire_perm(info.obj()), + forall|obj: nat| + obj != info.obj() ==> #[trigger] final(self).has_retire_perm(obj) == old( + self, + ).has_retire_perm(obj), + { + let ghost old_state = self.state; + self.state = LinkedListGhost { + root: self.state.root, + root_obj: self.state.root_obj, + objects: self.state.objects.insert(info.obj(), info.ptr()), + successors: self.state.successors.insert(info.obj(), Seq::empty()), + incoming_all: self.state.incoming_all.insert(info.obj(), Set::empty()), + }; + let tracked saved_info = info.tracked_duplicate(); + self.infos.tracked_insert(info.obj(), saved_info); + self.retire_perms.tracked_insert(info.obj(), retire); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert(self.state.incoming_all.dom() == self.state.objects.dom()); + assert forall|from_obj: nat, n: LinkIndex| + #![trigger self.state.objects.contains_key(from_obj), self.state.successors[from_obj][n as int]] + self.state.objects.contains_key(from_obj) && n < self.state.successors[from_obj].len() + && self.state.successors[from_obj][n as int] is Some implies { + let event = self.state.successors[from_obj][n as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((from_obj, n)) + } by { + assert(from_obj != info.obj()); + assert(old_state.objects.contains_key(from_obj)); + assert(old_state.successors[from_obj] == self.state.successors[from_obj]); + }; + assert(self.state.wf()); + assert(self.infos.dom() == self.state.incoming_all.dom()); + assert forall|obj: nat| obj != info.obj() implies #[trigger] self.has_retire_perm(obj) + == old(self).has_retire_perm(obj) by {}; + assert forall|obj: nat| #[trigger] self.infos.contains_key(obj) implies { + let saved = self.infos[obj]; + &&& saved.wf() + &&& saved.domain() == self.domain() + &&& saved.obj() == obj + &&& self.state.objects.contains_pair(obj, saved.ptr()) + } by { + if obj != info.obj() { + assert(old(self).infos.contains_key(obj)); + } + }; + assert forall|obj: nat| #[trigger] self.retire_perms.contains_key(obj) implies { + let perm = self.retire_perms[obj]; + &&& perm.wf() + &&& perm.domain() == self.domain() + &&& self.state.objects.contains_pair(obj, perm.ptr()) + &&& perm.obj() == obj + } by { + if obj != info.obj() { + assert(old(self).retire_perms.contains_key(obj)); + } + }; + } + + /// Copies the persistent allocation identity used by an atomic link + /// message without exposing the authority's internal registry. + pub proof fn tracked_info_for(tracked &self, obj: nat) -> (tracked res: RcuBlockInfo< + LinkedListNode, + >) + requires + self.wf(), + self.has_info(obj), + ensures + res.wf(), + res.domain() == self.domain(), + res.obj() == obj, + res.ptr() == self.info(obj).ptr(), + self.state().objects.contains_pair(obj, res.ptr()), + { + let tracked info = self.infos.tracked_borrow(obj); + info.tracked_duplicate() + } + + /// Opens the registry-domain consequence needed by traversal adapters. + pub proof fn lemma_has_info_for_object(tracked &self, obj: nat) + requires + self.wf(), + self.state().incoming_all.contains_key(obj), + ensures + self.has_info(obj), + { + assert(self.infos.dom() == self.state().incoming_all.dom()); + } + + /// Installs the initial null event for a newly created atomic link. + /// Subsequent changes to this source must use the publish/unlink rules. + pub proof fn tracked_initialize_null( + tracked &mut self, + from: *mut LinkedListNode, + from_obj: nat, + ) -> (n: LinkIndex) + requires + old(self).wf(), + old(self).state().objects.contains_pair(from_obj, from), + old(self).state().successors[from_obj].len() == 0, + !old(self).removed().contains(from_obj), + ensures + n == 0, + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects, + final(self).state().successors == old(self).state().successors.insert( + from_obj, + old(self).state().successors[from_obj].push(None), + ), + final(self).state().incoming_all == old(self).state().incoming_all, + final(self).removed() == old(self).removed(), + forall|obj: nat| #[trigger] + final(self).has_retire_perm(obj) == old(self).has_retire_perm(obj), + { + let ghost old_state = self.state; + self.state = LinkedListGhost { + root: old_state.root, + root_obj: old_state.root_obj, + objects: old_state.objects, + successors: old_state.successors.insert( + from_obj, + old_state.successors[from_obj].push(None), + ), + incoming_all: old_state.incoming_all, + }; + assert(old_state.successors.contains_key(from_obj)); + assert(self.state.objects.contains_pair(self.state.root_obj, self.state.root)); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert forall|source: *mut LinkedListNode, source_obj: nat, i: LinkIndex| + #![trigger self.state.objects.contains_pair(source_obj, source), self.state.successors[source_obj][i as int]] + self.state.objects.contains_pair(source_obj, source) + && self.state.successors.contains_key(source_obj) && i + < self.state.successors[source_obj].len() + && self.state.successors[source_obj][i as int] is Some implies { + let event = self.state.successors[source_obj][i as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((source_obj, i)) + } by { + if source_obj == from_obj { + assert(i == 0); + assert(self.state.successors[source_obj][i as int] is None); + assert(false); + } + assert(old_state.objects.contains_pair(source_obj, source)); + assert(old_state.successors[source_obj] == self.state.successors[source_obj]); + }; + assert(self.state.wf()); + assert(self.infos.dom() == self.state.incoming_all.dom()); + 0 + } + + /// Publishes (or replaces with) a non-null successor and returns the new + /// source-history index. + pub proof fn tracked_publish_link( + tracked &mut self, + from: *mut LinkedListNode, + from_obj: nat, + to: *mut LinkedListNode, + to_obj: nat, + ) -> (n: LinkIndex) + requires + old(self).wf(), + old(self).state().objects.contains_pair(from_obj, from), + old(self).state().objects.contains_pair(to_obj, to), + !old(self).removed().contains(from_obj), + !old(self).removed().contains(to_obj), + ensures + n == old(self).state().successors[from_obj].len(), + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects, + final(self).state().successors == old(self).state().successors.insert( + from_obj, + old(self).state().successors[from_obj].push(Some((to, to_obj))), + ), + final(self).state().incoming_all == old(self).state().incoming_all.insert( + to_obj, + old(self).state().incoming_all[to_obj].insert((from_obj, n)), + ), + final(self).removed() == old(self).removed(), + forall|obj: nat| #[trigger] + final(self).has_retire_perm(obj) == old(self).has_retire_perm(obj), + LinkedListTraversalSpec::link_inv(from, from_obj, n, to, to_obj, final(self).state()), + { + let ghost old_state = self.state; + let n = old_state.successors[from_obj].len(); + self.state = LinkedListGhost { + root: old_state.root, + root_obj: old_state.root_obj, + objects: old_state.objects, + successors: old_state.successors.insert( + from_obj, + old_state.successors[from_obj].push(Some((to, to_obj))), + ), + incoming_all: old_state.incoming_all.insert( + to_obj, + old_state.incoming_all[to_obj].insert((from_obj, n)), + ), + }; + assert(old_state.successors.contains_key(from_obj)); + assert(old_state.incoming_all.contains_key(to_obj)); + assert(self.state.objects.contains_pair(self.state.root_obj, self.state.root)); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert(self.state.incoming_all.dom() == self.state.objects.dom()); + assert forall|source: *mut LinkedListNode, source_obj: nat, i: LinkIndex| + #![trigger self.state.objects.contains_pair(source_obj, source), self.state.successors[source_obj][i as int]] + self.state.objects.contains_pair(source_obj, source) + && self.state.successors.contains_key(source_obj) && i + < self.state.successors[source_obj].len() + && self.state.successors[source_obj][i as int] is Some implies { + let event = self.state.successors[source_obj][i as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((source_obj, i)) + } by { + if source_obj == from_obj && i == n { + assert(self.state.successors[source_obj][i as int] == Some((to, to_obj))); + } else { + assert(i < old_state.successors[source_obj].len()); + assert(self.state.successors[source_obj][i as int] + == old_state.successors[source_obj][i as int]); + let event = old_state.successors[source_obj][i as int]->Some_0; + assert(old_state.incoming_all[event.1].contains((source_obj, i))); + assert(self.state.incoming_all[event.1].contains((source_obj, i))); + } + }; + assert(self.state.wf()); + assert(self.infos.dom() == self.state.incoming_all.dom()); + assert forall|obj: nat| #[trigger] self.infos.contains_key(obj) implies { + let info = self.infos[obj]; + &&& info.wf() + &&& info.domain() == self.domain() + &&& info.obj() == obj + &&& self.state.objects.contains_pair(obj, info.ptr()) + } by { + let ghost saved = self.infos[obj]; + assert(old_state.objects.contains_pair(obj, saved.ptr())); + }; + n + } + + /// Appends a null event after the expected current successor. The old + /// incoming edge remains in `incoming_all`, but is no longer current. + pub proof fn tracked_unlink( + tracked &mut self, + from: *mut LinkedListNode, + from_obj: nat, + to: *mut LinkedListNode, + to_obj: nat, + ) -> (n: LinkIndex) + requires + old(self).wf(), + old(self).state().objects.contains_pair(from_obj, from), + old(self).state().objects.contains_pair(to_obj, to), + !old(self).removed().contains(from_obj), + old(self).state().successors[from_obj].len() > 0, + old(self).state().successors[from_obj].last() == Some((to, to_obj)), + ensures + n == old(self).state().successors[from_obj].len(), + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects, + final(self).state().successors == old(self).state().successors.insert( + from_obj, + old(self).state().successors[from_obj].push(None), + ), + final(self).state().incoming_all == old(self).state().incoming_all, + final(self).removed() == old(self).removed(), + forall|obj: nat| #[trigger] + final(self).has_retire_perm(obj) == old(self).has_retire_perm(obj), + !final(self).state().current_incoming(to_obj).contains((from_obj, (n - 1) as nat)), + { + let ghost old_state = self.state; + let n = old_state.successors[from_obj].len(); + self.state = LinkedListGhost { + root: old_state.root, + root_obj: old_state.root_obj, + objects: old_state.objects, + successors: old_state.successors.insert( + from_obj, + old_state.successors[from_obj].push(None), + ), + incoming_all: old_state.incoming_all, + }; + assert(old_state.successors.contains_key(from_obj)); + assert(self.state.objects.contains_pair(self.state.root_obj, self.state.root)); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert forall|source: *mut LinkedListNode, source_obj: nat, i: LinkIndex| + #![trigger self.state.objects.contains_pair(source_obj, source), self.state.successors[source_obj][i as int]] + self.state.objects.contains_pair(source_obj, source) + && self.state.successors.contains_key(source_obj) && i + < self.state.successors[source_obj].len() + && self.state.successors[source_obj][i as int] is Some implies { + let event = self.state.successors[source_obj][i as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((source_obj, i)) + } by { + if source_obj == from_obj && i == n { + assert(self.state.successors[source_obj][i as int] is None); + assert(false); + } + assert(i < old_state.successors[source_obj].len()); + assert(self.state.successors[source_obj][i as int] + == old_state.successors[source_obj][i as int]); + }; + assert(self.state.wf()); + n + } + + /// Applies the paper's traversal retire rule. The prior observation must + /// already cover every historical incoming edge; this authority is the + /// only public producer of the resulting high-level retire permission. + pub proof fn tracked_retire_node( + tracked &mut self, + obj: nat, + prior: RcuSeenRemoved, + ) -> (tracked res: RcuRetirePerm) + requires + old(self).wf(), + old(self).has_retire_perm(obj), + old(self).has_info(obj), + obj != old(self).state().root_obj, + old(self).state().incoming_all[obj].len() > 0, + prior.removed == old(self).removed(), + old(self).state().bounds(prior.link_view), + LinkedListTraversalSpec::seen_removed_sound(prior, old(self).state()), + forall|edge: LinkEdge| #[trigger] + old(self).state().incoming_all[obj].contains(edge) ==> prior.dead_edge(edge), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state() == old(self).state(), + final(self).removed() == old(self).removed().insert(obj), + !final(self).has_retire_perm(obj), + forall|other: nat| + other != obj ==> #[trigger] final(self).has_retire_perm(other) == old( + self, + ).has_retire_perm(other), + final(self).has_info(obj), + final(self).info(obj) == old(self).info(obj), + forall|other: nat| #[trigger] final(self).has_info(other) == old(self).has_info(other), + forall|other: nat| #[trigger] + old(self).has_info(other) ==> final(self).info(other) == old(self).info(other), + res.wf(), + res.ready_to_retire(), + res.domain() == old(self).domain(), + res.obj() == obj, + old(self).state().objects.contains_pair(obj, res.ptr()), + res.seen_removed().removed == prior.removed.insert(obj), + res.seen_removed().link_view == prior.link_view, + LinkedListTraversalSpec::seen_removed_sound(res.seen_removed(), final(self).state()), + { + assert(self.state().objects.contains_pair(obj, self.retire_perms[obj].ptr())); + let tracked base = self.retire_perms.tracked_remove(obj); + assert forall|other: nat| other != obj implies #[trigger] self.has_retire_perm(other) + == old(self).has_retire_perm(other) by {}; + assert forall|other: nat| #[trigger] + self.has_info(other) == old(self).has_info(other) by {}; + assert forall|other: nat| #[trigger] old(self).has_info(other) implies self.info(other) + == old(self).info(other) by {}; + let ghost seen_removed = RcuSeenRemoved { + removed: prior.removed.insert(obj), + link_view: prior.link_view, + }; + self.removed = self.removed.insert(obj); + assert forall|to_obj: nat| #[trigger] seen_removed.removed.contains(to_obj) implies { + &&& self.state.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + self.state.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge) + } by { + if to_obj == obj { + assert(self.state.incoming_all.contains_key(obj)); + } else { + assert(prior.removed.contains(to_obj)); + } + }; + RcuRetirePerm { base, seen_removed } + } +} + +/// Native IRC11 timestamp metadata for one linked-list atomic link. +/// +/// Native histories use abstract timestamps, whereas the traversal proof uses +/// dense per-source event indices. This linear ghost state records their +/// explicit correspondence; no equality between the two namespaces is +/// assumed. +pub tracked struct LinkedListLinkObservation { + fact: GhostPersistentPointsTo, + native_fact: GhostPersistentPointsTo, +} + +impl LinkedListLinkObservation { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + self.native_fact.value().2 == self.timestamp() + } + + /// Persistent registry that certifies this timestamp/index pair. + pub closed spec fn registry(self) -> Loc { + self.fact.id() + } + + /// Persistent registry that certifies the native view observation. + pub closed spec fn native_registry(self) -> Loc { + self.native_fact.id() + } + + /// Fresh identifier allocated for this native load observation. + pub closed spec fn native_observation_id(self) -> nat { + self.native_fact.key() + } + + /// Native IRC11 timestamp observed by the load. + pub closed spec fn timestamp(self) -> nat { + self.fact.key() + } + + /// Dense traversal-history index corresponding to [`Self::timestamp`]. + pub closed spec fn index(self) -> LinkIndex { + self.fact.value() + } + + /// Native atomic location whose timestamp was observed. + pub closed spec fn loc(self) -> Irc11AtomicId { + self.native_fact.value().0 + } + + /// Subjective view immediately after the load that minted this token. + pub closed spec fn view(self) -> Irc11ThreadView { + self.native_fact.value().1 + } + + /// Duplicates the persistent timestamp/index observation. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.registry() == self.registry(), + res.native_registry() == self.native_registry(), + res.timestamp() == self.timestamp(), + res.index() == self.index(), + res.loc() == self.loc(), + res.view() == self.view(), + { + use_type_invariant(self); + LinkedListLinkObservation { + fact: self.fact.duplicate(), + native_fact: self.native_fact.duplicate(), + } + } +} + +pub tracked struct LinkedListAtomicLinkGhost { + ghost source: *mut LinkedListNode, + ghost source_obj: nat, + ghost timestamp_to_index: Map, + timestamp_registry: GhostMapAuth, + timestamp_facts: Map>, + native_observation_registry: GhostMapAuth, + ghost next_observation: nat, + ghost current_timestamp: nat, +} + +impl LinkedListAtomicLinkGhost { + pub closed spec fn source(self) -> *mut LinkedListNode { + self.source + } + + pub closed spec fn source_obj(self) -> nat { + self.source_obj + } + + pub closed spec fn timestamps(self) -> Map { + self.timestamp_to_index + } + + /// Append-only registry used to issue persistent load observations. + pub closed spec fn timestamp_registry(self) -> Loc { + self.timestamp_registry.id() + } + + pub closed spec fn certified_timestamps(self) -> Map { + self.timestamp_registry@ + } + + pub closed spec fn observation_facts(self) -> Map< + nat, + GhostPersistentPointsTo, + > { + self.timestamp_facts + } + + /// Append-only registry of native `(location, view, timestamp)` load facts. + pub closed spec fn native_observation_registry(self) -> Loc { + self.native_observation_registry.id() + } + + pub closed spec fn native_observations(self) -> Map< + nat, + (Irc11AtomicId, Irc11ThreadView, nat), + > { + self.native_observation_registry@ + } + + pub closed spec fn next_observation(self) -> nat { + self.next_observation + } + + pub closed spec fn current_timestamp(self) -> nat { + self.current_timestamp + } + + pub open spec fn index_at(self, timestamp: nat) -> LinkIndex + recommends + self.timestamps().contains_key(timestamp), + { + self.timestamps()[timestamp] + } + + /// Agreement among the native atomic history, dense traversal history, + /// and persistent allocation identities retained by the list authority. + pub open spec fn wf( + self, + history: Irc11History<*mut LinkedListNode>, + auth: LinkedListTraversalAuth, + ) -> bool { + &&& auth.wf() + &&& auth.state().objects.contains_pair(self.source_obj(), self.source()) + &&& !auth.removed().contains(self.source_obj()) + &&& history.is_max_timestamp(self.current_timestamp()) + &&& self.timestamps().dom() == history.dom() + &&& self.certified_timestamps() == self.timestamps() + &&& self.observation_facts().dom() == self.timestamps().dom() + &&& forall|timestamp: nat| #[trigger] + self.observation_facts().contains_key(timestamp) ==> { + let fact = self.observation_facts()[timestamp]; + &&& fact.id() == self.timestamp_registry() + &&& fact.key() == timestamp + &&& fact.value() == self.timestamps()[timestamp] + } + &&& forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) ==> observation_id + < self.next_observation() + &&& auth.state().successors[self.source_obj()].len() > 0 + &&& self.index_at(self.current_timestamp()) + 1 + == auth.state().successors[self.source_obj()].len() + &&& forall|timestamp: nat| + history.contains_timestamp(timestamp) ==> { + let n = #[trigger] self.timestamps()[timestamp]; + &&& n < auth.state().successors[self.source_obj()].len() + &&& match auth.state().successors[self.source_obj()][n as int] { + None => history.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& history.value(timestamp).addr() != 0 + &&& equal(ptr, history.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } + &&& forall|earlier: nat, later: nat| + #![trigger self.timestamps()[earlier], self.timestamps()[later]] + history.contains_timestamp(earlier) && history.contains_timestamp(later) ==> (earlier + < later <==> self.index_at(earlier) < self.index_at(later)) + } + + /// Every issued native observation remains valid for the current + /// append-only atomic points-to resource. + pub open spec fn native_observations_wf( + self, + points_to: AtomicPointsTo<*mut LinkedListNode>, + ) -> bool { + forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) ==> { + let observation = self.native_observations()[observation_id]; + &&& observation.0 == points_to.loc() + &&& points_to.get_timestamp(observation.1) == Some(observation.2) + } + } + + /// Creates the timestamp mapping for an atomic link initialized to null. + pub proof fn tracked_initial_null( + history: Irc11History<*mut LinkedListNode>, + timestamp: nat, + message_view: Irc11ThreadView, + tracked auth: &LinkedListTraversalAuth, + source: *mut LinkedListNode, + source_obj: nat, + ) -> (tracked res: Self) + requires + auth.wf(), + auth.state().objects.contains_pair(source_obj, source), + !auth.removed().contains(source_obj), + auth.state().successors[source_obj] == Seq::empty().push(None), + history.is_singleton(timestamp, (core::ptr::null_mut(), message_view)), + ensures + res.wf(history, *auth), + res.source() == source, + res.source_obj() == source_obj, + res.timestamps() == Map::empty().insert(timestamp, 0), + res.native_observations() == Map::empty(), + res.next_observation() == 0, + res.current_timestamp() == timestamp, + { + assert(history.is_max_timestamp(timestamp)); + assert(history.dom() == Set::empty().insert(timestamp)) by { + assert forall|ts: nat| + #![auto] + history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { + if history.dom().contains(ts) { + assert(history.contains_timestamp(ts)); + assert(ts == timestamp); + } + }; + }; + let tracked (mut timestamp_registry, _timestamp_entries) = GhostMapAuth::new(Map::empty()); + let tracked initial_fact = timestamp_registry.insert(timestamp, 0).persist(); + let tracked mut timestamp_facts = Map::tracked_empty(); + timestamp_facts.tracked_insert(timestamp, initial_fact); + let tracked (native_observation_registry, _native_observations) = GhostMapAuth::new( + Map::empty(), + ); + let tracked res = LinkedListAtomicLinkGhost { + source, + source_obj, + timestamp_to_index: Map::empty().insert(timestamp, 0), + timestamp_registry, + timestamp_facts, + native_observation_registry, + next_observation: 0, + current_timestamp: timestamp, + }; + assert forall|ts: nat| history.contains_timestamp(ts) implies { + let n = #[trigger] res.timestamps()[ts]; + &&& n < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][n as int] { + None => history.value(ts).addr() == 0, + Some((ptr, obj)) => { + &&& history.value(ts).addr() != 0 + &&& equal(ptr, history.value(ts)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + assert(ts == timestamp); + }; + assert forall|earlier: nat, later: nat| + #![trigger res.timestamps()[earlier], res.timestamps()[later]] + history.contains_timestamp(earlier) && history.contains_timestamp(later) implies ( + earlier < later <==> res.index_at(earlier) < res.index_at(later)) by { + assert(earlier == timestamp); + assert(later == timestamp); + }; + res + } + + /// Registers a fresh target while preserving the native link/history + /// correspondence. Registration adds an empty successor history and an + /// empty incoming-edge set for the new AId, so every pre-existing native + /// timestamp continues to resolve to the same traversal event. + pub proof fn tracked_register_target( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &mut LinkedListTraversalAuth, + tracked info: &RcuBlockInfo, + tracked retire: RcuBaseRetirePerm, + ) + requires + self.wf(history, *old(auth)), + info.wf(), + retire.wf(), + info.domain() == old(auth).domain(), + retire.domain() == old(auth).domain(), + retire.obj() == info.obj(), + retire.ptr() == info.ptr(), + !old(auth).state().objects.contains_key(info.obj()), + !old(auth).state().incoming_all.contains_key(info.obj()), + !old(auth).has_retire_perm(info.obj()), + ensures + self.wf(history, *final(auth)), + final(auth).wf(), + final(auth).domain() == old(auth).domain(), + final(auth).state().root == old(auth).state().root, + final(auth).state().root_obj == old(auth).state().root_obj, + final(auth).state().objects == old(auth).state().objects.insert(info.obj(), info.ptr()), + final(auth).state().successors == old(auth).state().successors.insert( + info.obj(), + Seq::empty(), + ), + final(auth).state().incoming_all == old(auth).state().incoming_all.insert( + info.obj(), + Set::empty(), + ), + final(auth).removed() == old(auth).removed(), + final(auth).has_info(info.obj()), + final(auth).info(info.obj()).ptr() == info.ptr(), + final(auth).has_retire_perm(info.obj()), + forall|obj: nat| + obj != info.obj() ==> #[trigger] final(auth).has_retire_perm(obj) == old( + auth, + ).has_retire_perm(obj), + { + let ghost old_state = auth.state(); + let ghost source_obj = self.source_obj(); + assert(old_state.objects.contains_key(source_obj)); + assert(info.obj() != source_obj); + auth.tracked_register_node(info, retire); + auth.lemma_registry_domains(); + assert(auth.state().successors[source_obj] == old_state.successors[source_obj]); + assert forall|timestamp: nat| history.contains_timestamp(timestamp) implies { + let n = #[trigger] self.timestamps()[timestamp]; + &&& n < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][n as int] { + None => history.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& history.value(timestamp).addr() != 0 + &&& equal(ptr, history.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + assert(auth.state().successors[source_obj][self.timestamps()[timestamp] as int] + == old_state.successors[source_obj][self.timestamps()[timestamp] as int]); + match old_state.successors[source_obj][self.timestamps()[timestamp] as int] { + None => {}, + Some((ptr, obj)) => { + assert(old_state.objects.contains_pair(obj, ptr)); + assert(auth.state().objects.contains_pair(obj, ptr)); + assert(auth.has_info(obj)); + assert(auth.info(obj).ptr() == ptr); + }, + } + }; + assert(self.wf(history, *auth)); + } + + /// Issues persistent evidence for one native timestamp's dense index. + pub proof fn tracked_observation_at( + tracked &mut self, + tracked points_to: &AtomicPointsTo<*mut LinkedListNode>, + tracked auth: &LinkedListTraversalAuth, + timestamp: nat, + view: Irc11ThreadView, + ) -> (tracked res: LinkedListLinkObservation) + requires + old(self).wf(points_to.hist(), *auth), + old(self).native_observations_wf(*points_to), + points_to.hist().contains_timestamp(timestamp), + points_to.get_timestamp(view) == Some(timestamp), + ensures + final(self).wf(points_to.hist(), *auth), + final(self).native_observations_wf(*points_to), + final(self).source() == old(self).source(), + final(self).source_obj() == old(self).source_obj(), + final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).native_observation_registry() == old(self).native_observation_registry(), + final(self).timestamps() == old(self).timestamps(), + final(self).current_timestamp() == old(self).current_timestamp(), + res.registry() == final(self).timestamp_registry(), + res.native_registry() == final(self).native_observation_registry(), + res.timestamp() == timestamp, + res.index() == final(self).index_at(timestamp), + res.loc() == points_to.loc(), + res.view() == view, + { + let ghost old_native_observations = self.native_observations(); + let ghost observation_id = self.next_observation; + assert(!old_native_observations.contains_key(observation_id)) by { + if old_native_observations.contains_key(observation_id) { + assert(observation_id < self.next_observation); + assert(false); + } + }; + let tracked native_fact = self.native_observation_registry.insert( + observation_id, + (points_to.loc(), view, timestamp), + ).persist(); + self.next_observation = observation_id + 1; + let tracked fact = self.timestamp_facts.tracked_borrow(timestamp).duplicate(); + let tracked res = LinkedListLinkObservation { fact, native_fact }; + assert forall|id: nat| #[trigger] self.native_observations().contains_key(id) implies id + < self.next_observation by { + if id == observation_id { + } else { + assert(old_native_observations.contains_key(id)); + assert(old_native_observations[id] == self.native_observations()[id]); + } + }; + assert forall|id: nat| #[trigger] self.native_observations().contains_key(id) implies { + let observation = self.native_observations()[id]; + &&& observation.0 == points_to.loc() + &&& points_to.get_timestamp(observation.1) == Some(observation.2) + } by { + if id == observation_id { + } else { + assert(old_native_observations.contains_key(id)); + assert(old_native_observations[id] == self.native_observations()[id]); + } + }; + res + } + + /// Agrees a prior persistent observation with the current append-only + /// timestamp map. This remains valid after later CAS updates. + pub proof fn lemma_observation_agrees( + tracked &self, + tracked observation: &LinkedListLinkObservation, + ) + requires + observation.registry() == self.timestamp_registry(), + self.certified_timestamps() == self.timestamps(), + ensures + self.timestamps().contains_pair(observation.timestamp(), observation.index()), + { + observation.fact.agree(&self.timestamp_registry); + } + + /// Agrees a prior persistent native observation with the current + /// append-only observation registry. + pub proof fn lemma_native_observation_agrees( + tracked &self, + tracked observation: &LinkedListLinkObservation, + ) + requires + observation.native_registry() == self.native_observation_registry(), + ensures + self.native_observations().contains_pair( + observation.native_observation_id(), + (observation.loc(), observation.view(), observation.timestamp()), + ), + { + use_type_invariant(observation); + observation.native_fact.agree(&self.native_observation_registry); + } + + /// Records a successful native CAS that publishes a non-null successor. + /// + /// `load_timestamp` and `store_timestamp` are supplied by the native + /// [`UpdateData`](vstd::atomic_weak::UpdateData). The traversal index is + /// allocated independently by [`LinkedListTraversalAuth`]. + pub proof fn tracked_cas_publish( + tracked &mut self, + tracked auth: &mut LinkedListTraversalAuth, + prev: Irc11History<*mut LinkedListNode>, + next: Irc11History<*mut LinkedListNode>, + load_timestamp: nat, + store_timestamp: nat, + to: *mut LinkedListNode, + to_obj: nat, + message_view: Irc11ThreadView, + ) -> (n: LinkIndex) + requires + old(self).wf(prev, *old(auth)), + prev.is_max_timestamp(load_timestamp), + store_timestamp == load_timestamp + 1, + next == prev.insert(store_timestamp, to, message_view), + to.addr() != 0, + old(auth).state().objects.contains_pair(to_obj, to), + !old(auth).removed().contains(to_obj), + ensures + n == old(auth).state().successors[old(self).source_obj()].len(), + final(self).wf(next, *final(auth)), + final(self).source() == old(self).source(), + final(self).source_obj() == old(self).source_obj(), + final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).native_observation_registry() == old(self).native_observation_registry(), + final(self).native_observations() == old(self).native_observations(), + final(self).next_observation() == old(self).next_observation(), + final(self).timestamps() == old(self).timestamps().insert(store_timestamp, n), + final(self).current_timestamp() == store_timestamp, + final(auth).domain() == old(auth).domain(), + final(auth).state().root == old(auth).state().root, + final(auth).state().root_obj == old(auth).state().root_obj, + final(auth).state().objects == old(auth).state().objects, + final(auth).state().successors == old(auth).state().successors.insert( + old(self).source_obj(), + old(auth).state().successors[old(self).source_obj()].push(Some((to, to_obj))), + ), + final(auth).state().incoming_all == old(auth).state().incoming_all.insert( + to_obj, + old(auth).state().incoming_all[to_obj].insert((old(self).source_obj(), n)), + ), + final(auth).removed() == old(auth).removed(), + forall|obj: nat| #[trigger] + final(auth).has_retire_perm(obj) == old(auth).has_retire_perm(obj), + LinkedListTraversalSpec::link_inv( + final(self).source(), + final(self).source_obj(), + n, + to, + to_obj, + final(auth).state(), + ), + { + let ghost source = self.source; + let ghost source_obj = self.source_obj; + let ghost old_timestamps = self.timestamp_to_index; + let ghost old_native_observations = self.native_observations(); + let ghost old_next_observation = self.next_observation(); + let ghost old_state = auth.state(); + let ghost old_current = self.current_timestamp; + + assert(prev.contains_timestamp(old_current)); + assert(prev.contains_timestamp(load_timestamp)); + assert(load_timestamp <= old_current); + assert(old_current <= load_timestamp); + assert(load_timestamp == old_current); + assert(!prev.contains_timestamp(store_timestamp)); + + let n = auth.tracked_publish_link(source, source_obj, to, to_obj); + self.timestamp_to_index = old_timestamps.insert(store_timestamp, n); + let tracked fact = self.timestamp_registry.insert(store_timestamp, n).persist(); + self.timestamp_facts.tracked_insert(store_timestamp, fact); + self.current_timestamp = store_timestamp; + + assert(next.is_max_timestamp(store_timestamp)) by { + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies timestamp + <= store_timestamp by { + if timestamp != store_timestamp { + assert(prev.contains_timestamp(timestamp)); + assert(timestamp <= load_timestamp); + } + }; + }; + assert(self.timestamps().dom() == next.dom()); + assert(self.native_observations() == old_native_observations); + assert(self.next_observation() == old_next_observation); + assert forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) implies observation_id + < self.next_observation() by {}; + assert(self.index_at(store_timestamp) == n); + assert(n + 1 == auth.state().successors[source_obj].len()); + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies { + let index = #[trigger] self.timestamps()[timestamp]; + &&& index < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][index as int] { + None => next.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& next.value(timestamp).addr() != 0 + &&& equal(ptr, next.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + if timestamp == store_timestamp { + assert(auth.state().successors[source_obj][n as int] == Some((to, to_obj))); + assert(auth.has_info(to_obj)); + assert(auth.info(to_obj).ptr() == to); + } else { + assert(prev.contains_timestamp(timestamp)); + let ghost index = old_timestamps[timestamp]; + assert(index < old_state.successors[source_obj].len()); + assert(auth.state().successors[source_obj][index as int] + == old_state.successors[source_obj][index as int]); + assert(next.value(timestamp) == prev.value(timestamp)); + match old_state.successors[source_obj][index as int] { + None => {}, + Some((ptr, obj)) => { + assert(old(auth).has_info(obj)); + assert(auth.has_info(obj)); + assert(auth.info(obj).ptr() == ptr); + }, + } + } + }; + assert forall|earlier: nat, later: nat| + #![trigger self.timestamps()[earlier], self.timestamps()[later]] + next.contains_timestamp(earlier) && next.contains_timestamp(later) implies (earlier + < later <==> self.index_at(earlier) < self.index_at(later)) by { + if earlier == store_timestamp { + if later != store_timestamp { + assert(prev.contains_timestamp(later)); + assert(later <= load_timestamp); + assert(old_timestamps[later] < old_state.successors[source_obj].len()); + assert(self.index_at(later) < n); + } + } else if later == store_timestamp { + assert(prev.contains_timestamp(earlier)); + assert(earlier <= load_timestamp); + assert(old_timestamps[earlier] < old_state.successors[source_obj].len()); + assert(self.index_at(earlier) < n); + } else { + assert(prev.contains_timestamp(earlier)); + assert(prev.contains_timestamp(later)); + } + }; + n + } + + /// Records a successful native CAS that replaces the current successor + /// with null. The detached edge remains in the append-only traversal + /// history and can subsequently be discharged by a reader observation. + pub proof fn tracked_cas_unlink( + tracked &mut self, + tracked auth: &mut LinkedListTraversalAuth, + prev: Irc11History<*mut LinkedListNode>, + next: Irc11History<*mut LinkedListNode>, + load_timestamp: nat, + store_timestamp: nat, + to: *mut LinkedListNode, + to_obj: nat, + message_view: Irc11ThreadView, + ) -> (n: LinkIndex) + requires + old(self).wf(prev, *old(auth)), + prev.is_max_timestamp(load_timestamp), + store_timestamp == load_timestamp + 1, + next == prev.insert(store_timestamp, core::ptr::null_mut(), message_view), + old(auth).state().successors[old(self).source_obj()].last() == Some((to, to_obj)), + ensures + n == old(auth).state().successors[old(self).source_obj()].len(), + final(self).wf(next, *final(auth)), + final(self).source() == old(self).source(), + final(self).source_obj() == old(self).source_obj(), + final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).native_observation_registry() == old(self).native_observation_registry(), + final(self).native_observations() == old(self).native_observations(), + final(self).next_observation() == old(self).next_observation(), + final(self).timestamps() == old(self).timestamps().insert(store_timestamp, n), + final(self).current_timestamp() == store_timestamp, + final(auth).domain() == old(auth).domain(), + final(auth).state().root == old(auth).state().root, + final(auth).state().root_obj == old(auth).state().root_obj, + final(auth).state().objects == old(auth).state().objects, + final(auth).state().successors == old(auth).state().successors.insert( + old(self).source_obj(), + old(auth).state().successors[old(self).source_obj()].push(None), + ), + final(auth).state().incoming_all == old(auth).state().incoming_all, + final(auth).removed() == old(auth).removed(), + forall|obj: nat| #[trigger] + final(auth).has_retire_perm(obj) == old(auth).has_retire_perm(obj), + final(auth).state().successors[final(self).source_obj()][n as int] is None, + !final(auth).state().current_incoming(to_obj).contains( + (final(self).source_obj(), (n - 1) as nat), + ), + { + let ghost source = self.source; + let ghost source_obj = self.source_obj; + let ghost old_timestamps = self.timestamp_to_index; + let ghost old_native_observations = self.native_observations(); + let ghost old_next_observation = self.next_observation(); + let ghost old_state = auth.state(); + let ghost old_current = self.current_timestamp; + + assert(prev.contains_timestamp(old_current)); + assert(prev.contains_timestamp(load_timestamp)); + assert(load_timestamp <= old_current); + assert(old_current <= load_timestamp); + assert(load_timestamp == old_current); + assert(!prev.contains_timestamp(store_timestamp)); + + let n = auth.tracked_unlink(source, source_obj, to, to_obj); + self.timestamp_to_index = old_timestamps.insert(store_timestamp, n); + let tracked fact = self.timestamp_registry.insert(store_timestamp, n).persist(); + self.timestamp_facts.tracked_insert(store_timestamp, fact); + self.current_timestamp = store_timestamp; + + assert(next.is_max_timestamp(store_timestamp)) by { + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies timestamp + <= store_timestamp by { + if timestamp != store_timestamp { + assert(prev.contains_timestamp(timestamp)); + assert(timestamp <= load_timestamp); + } + }; + }; + assert(self.timestamps().dom() == next.dom()); + assert(self.native_observations() == old_native_observations); + assert(self.next_observation() == old_next_observation); + assert forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) implies observation_id + < self.next_observation() by {}; + assert(self.index_at(store_timestamp) == n); + assert(n + 1 == auth.state().successors[source_obj].len()); + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies { + let index = #[trigger] self.timestamps()[timestamp]; + &&& index < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][index as int] { + None => next.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& next.value(timestamp).addr() != 0 + &&& equal(ptr, next.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + if timestamp == store_timestamp { + assert(auth.state().successors[source_obj][n as int] is None); + } else { + assert(prev.contains_timestamp(timestamp)); + let ghost index = old_timestamps[timestamp]; + assert(index < old_state.successors[source_obj].len()); + assert(auth.state().successors[source_obj][index as int] + == old_state.successors[source_obj][index as int]); + assert(next.value(timestamp) == prev.value(timestamp)); + match old_state.successors[source_obj][index as int] { + None => {}, + Some((ptr, obj)) => { + assert(old(auth).has_info(obj)); + assert(auth.has_info(obj)); + assert(auth.info(obj).ptr() == ptr); + }, + } + } + }; + assert forall|earlier: nat, later: nat| + #![trigger self.timestamps()[earlier], self.timestamps()[later]] + next.contains_timestamp(earlier) && next.contains_timestamp(later) implies (earlier + < later <==> self.index_at(earlier) < self.index_at(later)) by { + if earlier == store_timestamp { + if later != store_timestamp { + assert(prev.contains_timestamp(later)); + assert(later <= load_timestamp); + assert(old_timestamps[later] < old_state.successors[source_obj].len()); + assert(self.index_at(later) < n); + } + } else if later == store_timestamp { + assert(prev.contains_timestamp(earlier)); + assert(earlier <= load_timestamp); + assert(old_timestamps[earlier] < old_state.successors[source_obj].len()); + assert(self.index_at(earlier) < n); + } else { + assert(prev.contains_timestamp(earlier)); + assert(prev.contains_timestamp(later)); + } + }; + n + } + + /// Resolves an observed native atomic message to its traversal event and + /// persistent child identity. + pub proof fn tracked_info_at( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &LinkedListTraversalAuth, + timestamp: nat, + ) -> (tracked res: Option>) + requires + self.wf(history, *auth), + history.contains_timestamp(timestamp), + ensures + self.index_at(timestamp) < auth.state().successors[self.source_obj()].len(), + match res { + None => { + &&& history.value(timestamp).addr() == 0 + &&& auth.state().successors[self.source_obj()][self.index_at( + timestamp, + ) as int] is None + }, + Some(info) => { + &&& history.value(timestamp).addr() != 0 + &&& info.wf() + &&& info.domain() == auth.domain() + &&& equal(info.ptr(), history.value(timestamp)) + &&& LinkedListTraversalSpec::link_inv( + self.source(), + self.source_obj(), + self.index_at(timestamp), + info.ptr(), + info.obj(), + auth.state(), + ) + }, + }, + { + let ghost n = self.index_at(timestamp); + match auth.state().successors[self.source_obj()][n as int] { + None => None, + Some((ptr, obj)) => { + let tracked info = auth.tracked_info_for(obj); + assert(equal(info.ptr(), history.value(timestamp))); + Some(info) + }, + } + } + + /// Connects a native atomic load to the paper's guarded traversal rule. + /// + /// The source witness is refreshed in place with the observed link index. + /// A non-null message additionally installs the loaded allocation in the + /// guard's protection map and returns its protected witness. + pub proof fn tracked_load_and_protect( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &LinkedListTraversalAuth, + tracked guard: &mut RcuReadGuardToken, + tracked from: &mut RcuProtectedPtr, + timestamp: nat, + ) -> (tracked res: Option>) + requires + self.wf(history, *auth), + history.contains_timestamp(timestamp), + old(guard).wf(), + old(guard).domain() == auth.domain(), + old(from).protected_by(*old(guard)), + old(from).ptr() == self.source(), + old(from).obj() == self.source_obj(), + old(guard).seen_at(old(from).obj()) <= self.index_at(timestamp), + LinkedListTraversalSpec::seen_removed_sound(old(guard).seen_removed(), auth.state()), + ensures + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).tid() == old(guard).tid(), + final(guard).reader_registry() == old(guard).reader_registry(), + final(guard).reader() == old(guard).reader(), + final(guard).root() == old(guard).root(), + final(guard).start_view() == old(guard).start_view(), + final(guard).retire_observation_registry() == old(guard).retire_observation_registry(), + final(guard).expired() == old(guard).expired(), + final(guard).seen_removed().removed == old(guard).seen_removed().removed, + final(guard).seen_at(self.source_obj()) == self.index_at(timestamp), + LinkedListTraversalSpec::seen_removed_sound(final(guard).seen_removed(), auth.state()), + final(from).ptr() == self.source(), + final(from).obj() == self.source_obj(), + final(from).domain() == final(guard).domain(), + final(from).seen_removed() == final(guard).seen_removed(), + !final(from).seen_removed().removed.contains(final(from).obj()), + res is None ==> final(guard).protected() == old(guard).protected(), + (res is Some) == (history.value(timestamp).addr() != 0), + match res { + None => history.value(timestamp).addr() == 0, + Some(child) => { + &&& equal(child.ptr(), history.value(timestamp)) + &&& child.domain() == auth.domain() + &&& child.protected_by(*final(guard)) + &&& LinkedListTraversalSpec::node_inv(child.ptr(), child.obj(), auth.state()) + &&& LinkedListTraversalSpec::link_inv( + self.source(), + self.source_obj(), + self.index_at(timestamp), + child.ptr(), + child.obj(), + auth.state(), + ) + }, + }, + { + let ghost n = self.index_at(timestamp); + let tracked info = self.tracked_info_at(history, auth, timestamp); + let ghost old_seen_removed = guard.seen_removed(); + let ghost old_domain = guard.domain(); + let ghost old_tid = guard.tid(); + let ghost old_reader_registry = guard.reader_registry(); + let ghost old_reader = guard.reader(); + let ghost old_root = guard.root(); + let ghost old_start_view = guard.start_view(); + let ghost old_retire_observation_registry = guard.retire_observation_registry(); + let ghost old_expired = guard.expired(); + let ghost old_protected = guard.protected(); + assert(old_domain == old(guard).domain()); + assert(old_tid == old(guard).tid()); + assert(old_reader_registry == old(guard).reader_registry()); + assert(old_reader == old(guard).reader()); + assert(old_root == old(guard).root()); + assert(old_start_view == old(guard).start_view()); + assert(old_retire_observation_registry == old(guard).retire_observation_registry()); + assert(old_expired == old(guard).expired()); + assert(old_protected == old(guard).protected()); + guard.tracked_observe_link_in_place(from, n); + linked_list_observe_preserves_seen_removed_sound( + old_seen_removed, + auth.state(), + self.source_obj(), + n, + ); + assert(from.protected_by(*guard)); + assert(guard.domain() == old_domain); + assert(guard.tid() == old_tid); + assert(guard.reader_registry() == old_reader_registry); + assert(guard.reader() == old_reader); + assert(guard.root() == old_root); + assert(guard.start_view() == old_start_view); + assert(guard.retire_observation_registry() == old_retire_observation_registry); + assert(guard.expired() == old_expired); + assert(guard.protected() == old_protected); + let tracked res; + match info { + None => { + assert(guard.domain() == old(guard).domain()); + assert(guard.tid() == old(guard).tid()); + assert(guard.reader_registry() == old(guard).reader_registry()); + assert(guard.reader() == old(guard).reader()); + assert(guard.root() == old(guard).root()); + assert(guard.start_view() == old(guard).start_view()); + assert(guard.retire_observation_registry() == old( + guard, + ).retire_observation_registry()); + assert(guard.expired() == old(guard).expired()); + assert(guard.protected() == old(guard).protected()); + res = None; + }, + Some(info) => { + assert(LinkedListTraversalSpec::node_inv(from.ptr(), from.obj(), auth.state())); + let tracked child = protect_link::( + guard, + from, + &info, + n, + info.ptr(), + auth.state(), + ); + res = Some(child); + }, + } + assert(guard.domain() == old(guard).domain()); + assert(guard.tid() == old(guard).tid()); + assert(guard.reader_registry() == old(guard).reader_registry()); + assert(guard.reader() == old(guard).reader()); + assert(guard.root() == old(guard).root()); + assert(guard.start_view() == old(guard).start_view()); + assert(guard.retire_observation_registry() == old(guard).retire_observation_registry()); + assert(guard.expired() == old(guard).expired()); + res + } +} + +pub struct LinkedListTraversalSpec; + +impl RcuTraversalSafety for LinkedListTraversalSpec { + type Node = LinkedListNode; + + type Ghost = LinkedListGhost; + + open spec fn root_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { + &&& p == g.root + &&& obj == g.root_obj + &&& g.objects.contains_pair(obj, p) + &&& g.successors.contains_key(obj) + &&& g.incoming_all.contains_key(obj) + } + + open spec fn node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { + &&& g.objects.contains_pair(obj, p) + &&& g.successors.contains_key(obj) + &&& g.incoming_all.contains_key(obj) + } + + open spec fn link_inv( + from: *mut LinkedListNode, + from_obj: nat, + n: LinkIndex, + to: *mut LinkedListNode, + to_obj: nat, + g: LinkedListGhost, + ) -> bool { + &&& g.objects.contains_pair(from_obj, from) + &&& g.objects.contains_pair(to_obj, to) + &&& g.successors.contains_key(from_obj) + &&& n < g.successors[from_obj].len() + &&& g.successors[from_obj][n as int] == Some((to, to_obj)) + &&& g.successors.contains_key(to_obj) + &&& g.incoming_all.contains_key(to_obj) + &&& g.incoming_all[to_obj].contains((from_obj, n)) + } + + open spec fn seen_removed_sound( + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + ) -> bool { + forall|to_obj: nat| #[trigger] + seen_removed.removed.contains(to_obj) ==> { + &&& g.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge) + } + } + + proof fn root_is_node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) { + } + + proof fn link_preserves_protection( + from: *mut LinkedListNode, + from_obj: nat, + n: LinkIndex, + to: *mut LinkedListNode, + to_obj: nat, + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + ) { + if seen_removed.removed.contains(to_obj) { + assert(g.incoming_all[to_obj].contains((from_obj, n))); + assert(seen_removed.dead_edge((from_obj, n))); + assert(false); + } + } +} + +/// Advancing one source observation preserves every previously established +/// dead-edge fact in a linked-list `SeenRemoved` snapshot. +pub proof fn linked_list_observe_preserves_seen_removed_sound( + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + source_obj: nat, + n: LinkIndex, +) + requires + LinkedListTraversalSpec::seen_removed_sound(seen_removed, g), + ensures + LinkedListTraversalSpec::seen_removed_sound( + RcuSeenRemoved { + removed: seen_removed.removed, + link_view: seen_removed.link_view.observe(source_obj, n), + }, + g, + ), +{ + let ghost observed = RcuSeenRemoved { + removed: seen_removed.removed, + link_view: seen_removed.link_view.observe(source_obj, n), + }; + assert forall|to_obj: nat| #[trigger] observed.removed.contains(to_obj) implies { + &&& g.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> observed.dead_edge(edge) + } by { + assert(seen_removed.removed.contains(to_obj)); + assert(g.incoming_all.contains_key(to_obj)); + assert forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) implies observed.dead_edge(edge) by { + assert(seen_removed.dead_edge(edge)); + if !seen_removed.removed.contains(edge.0) { + seen_removed.link_view.lemma_observe_monotonic(source_obj, n, edge.0); + assert(observed.seen_at(edge.0) >= seen_removed.seen_at(edge.0)); + } + }; + }; +} + +/// A latest link from a predecessor not already in `D` cannot be dead in a +/// bounded link view. Consequently the traversal retire rule cannot be +/// applied while that incoming edge remains live. +pub proof fn linked_list_live_edge_blocks_retire( + g: LinkedListGhost, + seen_removed: RcuSeenRemoved, + from: *mut LinkedListNode, + from_obj: nat, + n: LinkIndex, + to: *mut LinkedListNode, + to_obj: nat, +) + requires + g.wf(), + g.bounds(seen_removed.link_view), + LinkedListTraversalSpec::link_inv(from, from_obj, n, to, to_obj, g), + g.successors[from_obj].len() == n + 1, + !seen_removed.removed.contains(from_obj), + ensures + g.current_incoming(to_obj).contains((from_obj, n)), + !seen_removed.dead_edge((from_obj, n)), + !(forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge)), +{ + if seen_removed.link_view.seen.contains_key(from_obj) { + assert(seen_removed.seen_at(from_obj) < g.successors[from_obj].len()); + } else { + assert(seen_removed.seen_at(from_obj) == 0); + } + assert(seen_removed.seen_at(from_obj) <= n); + assert(g.successors[from_obj].last() == Some((to, to_obj))); + assert(g.current_incoming(to_obj).contains((from_obj, n))) by { + assert(g.objects.contains_key(from_obj)); + assert(g.successors.contains_key(from_obj)); + } +} + +/// End-to-end ghost example for an internal list node. Publishing creates the +/// historical incoming edge at index 0; unlinking appends a newer null event; +/// observing index 1 then lets the authority consume the child's unique base +/// permission and produce the traversal-level retire permission. +pub proof fn linked_list_unlink_enables_retire( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked child_info: &RcuBlockInfo, + tracked child_retire: RcuBaseRetirePerm, +) -> (tracked retired: RcuRetirePerm) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + child_info.wf(), + child_retire.wf(), + child_info.domain() == root_info.domain(), + child_retire.domain() == root_info.domain(), + child_retire.obj() == child_info.obj(), + child_retire.ptr() == child_info.ptr(), + child_info.ptr() != root_info.ptr(), + child_info.obj() != root_info.obj(), + ensures + retired.wf(), + retired.ready_to_retire(), + retired.domain() == root_info.domain(), + retired.obj() == child_info.obj(), + LinkedListTraversalSpec::seen_removed_sound( + retired.seen_removed(), + LinkedListGhost { + root: root_info.ptr(), + root_obj: root_info.obj(), + objects: Map::empty().insert(root_info.obj(), root_info.ptr()).insert( + child_info.obj(), + child_info.ptr(), + ), + successors: Map::empty().insert(root_info.obj(), Seq::empty()).insert( + child_info.obj(), + Seq::empty(), + ).insert( + root_info.obj(), + Seq::empty().push(Some((child_info.ptr(), child_info.obj()))).push(None), + ), + incoming_all: Map::empty().insert(root_info.obj(), Set::empty()).insert( + child_info.obj(), + Set::empty().insert((root_info.obj(), 0)), + ), + }, + ), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + auth.tracked_register_node(child_info, child_retire); + let n = auth.tracked_publish_link( + root_info.ptr(), + root_info.obj(), + child_info.ptr(), + child_info.obj(), + ); + assert(n == 0); + let observed = auth.tracked_unlink( + root_info.ptr(), + root_info.obj(), + child_info.ptr(), + child_info.obj(), + ); + assert(observed == 1); + let ghost prior = RcuSeenRemoved { + removed: Set::empty(), + link_view: RcuLinkView::empty().observe(root_info.obj(), observed), + }; + assert(auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut LinkedListNode, from_obj: nat| #[trigger] + auth.state().objects.contains_pair(from_obj, from) && prior.link_view.seen.contains_key( + from_obj, + ) implies { + &&& auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) < auth.state().successors[from_obj].len() + } by { + assert(from == root_info.ptr()); + assert(from_obj == root_info.obj()); + }; + } + assert(LinkedListTraversalSpec::seen_removed_sound(prior, auth.state())); + assert forall|edge: LinkEdge| #[trigger] + auth.state().incoming_all[child_info.obj()].contains(edge) implies prior.dead_edge( + edge, + ) by { + assert(edge == (root_info.obj(), 0)); + assert(prior.seen_at(root_info.obj()) == observed); + }; + auth.tracked_retire_node(child_info.obj(), prior) +} + +/// Regression proof for the fully dynamic traversal authority. +/// +/// The old allocation is reachable from two distinct predecessors. Both +/// incoming edges must be unlinked and observed before retirement. A fresh +/// AId is then registered at exactly the same address, published, unlinked, +/// and published again. The old history entries continue naming `old_obj`, +/// while both publications of the reused allocation name `reused_obj`. +pub proof fn linked_list_multiple_predecessors_republish_reused_address( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked left_info: &RcuBlockInfo, + tracked left_retire: RcuBaseRetirePerm, + tracked right_info: &RcuBlockInfo, + tracked right_retire: RcuBaseRetirePerm, + tracked old_info: &RcuBlockInfo, + tracked old_retire: RcuBaseRetirePerm, + tracked reused_info: &RcuBlockInfo, + tracked reused_retire: RcuBaseRetirePerm, +) -> (tracked res: (LinkedListTraversalAuth, RcuRetirePerm)) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + left_info.wf(), + left_retire.wf(), + left_info.domain() == root_info.domain(), + left_retire.domain() == root_info.domain(), + left_retire.obj() == left_info.obj(), + left_retire.ptr() == left_info.ptr(), + right_info.wf(), + right_retire.wf(), + right_info.domain() == root_info.domain(), + right_retire.domain() == root_info.domain(), + right_retire.obj() == right_info.obj(), + right_retire.ptr() == right_info.ptr(), + old_info.wf(), + old_retire.wf(), + old_info.domain() == root_info.domain(), + old_retire.domain() == root_info.domain(), + old_retire.obj() == old_info.obj(), + old_retire.ptr() == old_info.ptr(), + reused_info.wf(), + reused_retire.wf(), + reused_info.domain() == root_info.domain(), + reused_retire.domain() == root_info.domain(), + reused_retire.obj() == reused_info.obj(), + reused_retire.ptr() == reused_info.ptr(), + old_info.ptr() == reused_info.ptr(), + root_info.obj() != left_info.obj(), + root_info.obj() != right_info.obj(), + root_info.obj() != old_info.obj(), + root_info.obj() != reused_info.obj(), + left_info.obj() != right_info.obj(), + left_info.obj() != old_info.obj(), + left_info.obj() != reused_info.obj(), + right_info.obj() != old_info.obj(), + right_info.obj() != reused_info.obj(), + old_info.obj() != reused_info.obj(), + ensures + res.0.wf(), + res.0.removed().contains(old_info.obj()), + res.0.has_retire_perm(reused_info.obj()), + res.0.state().objects.contains_pair(old_info.obj(), old_info.ptr()), + res.0.state().objects.contains_pair(reused_info.obj(), old_info.ptr()), + res.0.state().successors[left_info.obj()][0] == Some((old_info.ptr(), old_info.obj())), + res.0.state().successors[left_info.obj()][2] == Some( + (reused_info.ptr(), reused_info.obj()), + ), + res.0.state().successors[left_info.obj()][4] == Some( + (reused_info.ptr(), reused_info.obj()), + ), + res.0.state().incoming_all[old_info.obj()].contains((left_info.obj(), 0)), + res.0.state().incoming_all[old_info.obj()].contains((right_info.obj(), 0)), + res.0.state().incoming_all[reused_info.obj()].contains((left_info.obj(), 2)), + res.0.state().incoming_all[reused_info.obj()].contains((left_info.obj(), 4)), + res.0.state().current_incoming(reused_info.obj()).contains((left_info.obj(), 4)), + res.1.wf(), + res.1.ready_to_retire(), + res.1.obj() == old_info.obj(), + res.1.ptr() == old_info.ptr(), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + auth.tracked_register_node(left_info, left_retire); + auth.tracked_register_node(right_info, right_retire); + auth.tracked_register_node(old_info, old_retire); + + let root_left = auth.tracked_publish_link( + root_info.ptr(), + root_info.obj(), + left_info.ptr(), + left_info.obj(), + ); + let left_old = auth.tracked_publish_link( + left_info.ptr(), + left_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + let root_right = auth.tracked_publish_link( + root_info.ptr(), + root_info.obj(), + right_info.ptr(), + right_info.obj(), + ); + let right_old = auth.tracked_publish_link( + right_info.ptr(), + right_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + assert(root_left == 0); + assert(left_old == 0); + assert(root_right == 1); + assert(right_old == 0); + assert(auth.state().current_incoming(old_info.obj()).contains((left_info.obj(), left_old))); + assert(auth.state().current_incoming(old_info.obj()).contains((right_info.obj(), right_old))); + + let left_unlink = auth.tracked_unlink( + left_info.ptr(), + left_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + let right_unlink = auth.tracked_unlink( + right_info.ptr(), + right_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + assert(left_unlink == 1); + assert(right_unlink == 1); + let ghost prior = RcuSeenRemoved { + removed: Set::empty(), + link_view: RcuLinkView::empty().observe(left_info.obj(), left_unlink).observe( + right_info.obj(), + right_unlink, + ), + }; + assert(auth.state().bounds(prior.link_view)) by { + assert forall|from_obj: nat| #[trigger] + auth.state().objects.contains_key(from_obj) && prior.link_view.seen.contains_key( + from_obj, + ) implies { + &&& auth.state().successors[from_obj].len() > 0 + &&& prior.seen_at(from_obj) < auth.state().successors[from_obj].len() + } by { + if from_obj == left_info.obj() { + assert(prior.seen_at(from_obj) == left_unlink); + } else { + assert(from_obj == right_info.obj()); + assert(prior.seen_at(from_obj) == right_unlink); + } + }; + } + assert(LinkedListTraversalSpec::seen_removed_sound(prior, auth.state())); + assert forall|edge: LinkEdge| #[trigger] + auth.state().incoming_all[old_info.obj()].contains(edge) implies prior.dead_edge(edge) by { + if edge.0 == left_info.obj() { + assert(edge == (left_info.obj(), left_old)); + assert(prior.seen_at(left_info.obj()) == left_unlink); + } else { + assert(edge == (right_info.obj(), right_old)); + assert(prior.seen_at(right_info.obj()) == right_unlink); + } + }; + let tracked retired = auth.tracked_retire_node(old_info.obj(), prior); + + auth.tracked_register_node(reused_info, reused_retire); + let first_republication = auth.tracked_publish_link( + left_info.ptr(), + left_info.obj(), + reused_info.ptr(), + reused_info.obj(), + ); + assert(first_republication == 2); + let reused_unlink = auth.tracked_unlink( + left_info.ptr(), + left_info.obj(), + reused_info.ptr(), + reused_info.obj(), + ); + assert(reused_unlink == 3); + let second_republication = auth.tracked_publish_link( + left_info.ptr(), + left_info.obj(), + reused_info.ptr(), + reused_info.obj(), + ); + assert(second_republication == 4); + assert(auth.state().objects[old_info.obj()] == old_info.ptr()); + assert(auth.state().objects[reused_info.obj()] == reused_info.ptr()); + assert(old_info.ptr() == reused_info.ptr()); + (auth, retired) +} + +/// End-to-end writer example connecting successful native IRC11 CAS updates +/// to the traversal retire rule. +/// +/// Native timestamps are kept abstract. The proof relies only on the CAS +/// contract's successor timestamps and on `LinkedListAtomicLinkGhost`'s +/// explicit timestamp-to-index correspondence. +pub proof fn linked_list_native_cas_unlink_enables_retire( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked child_info: &RcuBlockInfo, + tracked child_retire: RcuBaseRetirePerm, + initial_history: Irc11History<*mut LinkedListNode>, + published_history: Irc11History<*mut LinkedListNode>, + unlinked_history: Irc11History<*mut LinkedListNode>, + initial_timestamp: nat, + published_timestamp: nat, + unlinked_timestamp: nat, + initial_view: Irc11ThreadView, + published_view: Irc11ThreadView, + unlinked_view: Irc11ThreadView, +) -> (tracked retired: RcuRetirePerm) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + child_info.wf(), + child_retire.wf(), + child_info.domain() == root_info.domain(), + child_retire.domain() == root_info.domain(), + child_retire.obj() == child_info.obj(), + child_retire.ptr() == child_info.ptr(), + child_info.ptr().addr() != 0, + child_info.ptr() != root_info.ptr(), + child_info.obj() != root_info.obj(), + initial_history.is_singleton(initial_timestamp, (core::ptr::null_mut(), initial_view)), + published_timestamp == initial_timestamp + 1, + published_history == initial_history.insert( + published_timestamp, + child_info.ptr(), + published_view, + ), + unlinked_timestamp == published_timestamp + 1, + unlinked_history == published_history.insert( + unlinked_timestamp, + core::ptr::null_mut(), + unlinked_view, + ), + ensures + retired.wf(), + retired.ready_to_retire(), + retired.domain() == root_info.domain(), + retired.obj() == child_info.obj(), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + auth.tracked_register_node(child_info, child_retire); + let initial_index = auth.tracked_initialize_null(root_info.ptr(), root_info.obj()); + assert(initial_index == 0); + let tracked mut link = LinkedListAtomicLinkGhost::tracked_initial_null( + initial_history, + initial_timestamp, + initial_view, + &auth, + root_info.ptr(), + root_info.obj(), + ); + + let published_index = link.tracked_cas_publish( + &mut auth, + initial_history, + published_history, + initial_timestamp, + published_timestamp, + child_info.ptr(), + child_info.obj(), + published_view, + ); + assert(published_index == 1); + let unlinked_index = link.tracked_cas_unlink( + &mut auth, + published_history, + unlinked_history, + published_timestamp, + unlinked_timestamp, + child_info.ptr(), + child_info.obj(), + unlinked_view, + ); + assert(unlinked_index == 2); + + let ghost prior = RcuSeenRemoved { + removed: Set::empty(), + link_view: RcuLinkView::empty().observe(root_info.obj(), unlinked_index), + }; + assert(auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut LinkedListNode, from_obj: nat| #[trigger] + auth.state().objects.contains_pair(from_obj, from) && prior.link_view.seen.contains_key( + from_obj, + ) implies { + &&& auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) < auth.state().successors[from_obj].len() + } by { + assert(from == root_info.ptr()); + assert(from_obj == root_info.obj()); + }; + } + assert(LinkedListTraversalSpec::seen_removed_sound(prior, auth.state())); + assert forall|edge: LinkEdge| #[trigger] + auth.state().incoming_all[child_info.obj()].contains(edge) implies prior.dead_edge( + edge, + ) by { + assert(edge == (root_info.obj(), published_index)); + assert(prior.seen_at(root_info.obj()) == unlinked_index); + }; + auth.tracked_retire_node(child_info.obj(), prior) +} + +/// Regression proof for one native link publishing multiple registered AIds. +/// +/// `first` and `second` deliberately name distinct allocation identities at +/// the same address. The native CAS values are therefore indistinguishable by +/// address, while the append-only traversal history still records the exact +/// AId selected by each publication. +pub proof fn linked_list_native_single_link_multiple_registered_aids( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked first_info: &RcuBlockInfo, + tracked first_retire: RcuBaseRetirePerm, + tracked second_info: &RcuBlockInfo, + tracked second_retire: RcuBaseRetirePerm, + initial_history: Irc11History<*mut LinkedListNode>, + initial_timestamp: nat, + initial_view: Irc11ThreadView, + first_publish_view: Irc11ThreadView, + first_unlink_view: Irc11ThreadView, + second_publish_view: Irc11ThreadView, + second_unlink_view: Irc11ThreadView, + first_republish_view: Irc11ThreadView, +) -> (tracked res: (LinkedListTraversalAuth, LinkedListAtomicLinkGhost)) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + first_info.wf(), + first_retire.wf(), + first_info.domain() == root_info.domain(), + first_retire.domain() == root_info.domain(), + first_retire.obj() == first_info.obj(), + first_retire.ptr() == first_info.ptr(), + second_info.wf(), + second_retire.wf(), + second_info.domain() == root_info.domain(), + second_retire.domain() == root_info.domain(), + second_retire.obj() == second_info.obj(), + second_retire.ptr() == second_info.ptr(), + first_info.obj() != root_info.obj(), + second_info.obj() != root_info.obj(), + first_info.obj() != second_info.obj(), + first_info.ptr() == second_info.ptr(), + initial_history.is_singleton(initial_timestamp, (core::ptr::null_mut(), initial_view)), + ensures + res.0.wf(), + res.0.removed() == Set::::empty(), + res.0.state().objects.contains_pair(first_info.obj(), first_info.ptr()), + res.0.state().objects.contains_pair(second_info.obj(), first_info.ptr()), + res.0.state().successors[root_info.obj()][0] is None, + res.0.state().successors[root_info.obj()][1] == Some((first_info.ptr(), first_info.obj())), + res.0.state().successors[root_info.obj()][2] is None, + res.0.state().successors[root_info.obj()][3] == Some( + (second_info.ptr(), second_info.obj()), + ), + res.0.state().successors[root_info.obj()][4] is None, + res.0.state().successors[root_info.obj()][5] == Some((first_info.ptr(), first_info.obj())), + res.0.state().incoming_all[first_info.obj()].contains((root_info.obj(), 1)), + res.0.state().incoming_all[first_info.obj()].contains((root_info.obj(), 5)), + res.0.state().incoming_all[second_info.obj()].contains((root_info.obj(), 3)), + res.1.current_timestamp() == initial_timestamp + 5, + res.1.wf( + initial_history.insert( + initial_timestamp + 1, + first_info.ptr(), + first_publish_view, + ).insert(initial_timestamp + 2, core::ptr::null_mut(), first_unlink_view).insert( + initial_timestamp + 3, + second_info.ptr(), + second_publish_view, + ).insert(initial_timestamp + 4, core::ptr::null_mut(), second_unlink_view).insert( + initial_timestamp + 5, + first_info.ptr(), + first_republish_view, + ), + res.0, + ), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + let initial_index = auth.tracked_initialize_null(root_info.ptr(), root_info.obj()); + assert(initial_index == 0); + let tracked mut link = LinkedListAtomicLinkGhost::tracked_initial_null( + initial_history, + initial_timestamp, + initial_view, + &auth, + root_info.ptr(), + root_info.obj(), + ); + link.tracked_register_target(initial_history, &mut auth, first_info, first_retire); + link.tracked_register_target(initial_history, &mut auth, second_info, second_retire); + + let ghost first_published = initial_history.insert( + initial_timestamp + 1, + first_info.ptr(), + first_publish_view, + ); + let first_index = link.tracked_cas_publish( + &mut auth, + initial_history, + first_published, + initial_timestamp, + initial_timestamp + 1, + first_info.ptr(), + first_info.obj(), + first_publish_view, + ); + assert(first_index == 1); + + let ghost first_unlinked = first_published.insert( + initial_timestamp + 2, + core::ptr::null_mut(), + first_unlink_view, + ); + let first_unlink_index = link.tracked_cas_unlink( + &mut auth, + first_published, + first_unlinked, + initial_timestamp + 1, + initial_timestamp + 2, + first_info.ptr(), + first_info.obj(), + first_unlink_view, + ); + assert(first_unlink_index == 2); + + let ghost second_published = first_unlinked.insert( + initial_timestamp + 3, + second_info.ptr(), + second_publish_view, + ); + let second_index = link.tracked_cas_publish( + &mut auth, + first_unlinked, + second_published, + initial_timestamp + 2, + initial_timestamp + 3, + second_info.ptr(), + second_info.obj(), + second_publish_view, + ); + assert(second_index == 3); + + let ghost second_unlinked = second_published.insert( + initial_timestamp + 4, + core::ptr::null_mut(), + second_unlink_view, + ); + let second_unlink_index = link.tracked_cas_unlink( + &mut auth, + second_published, + second_unlinked, + initial_timestamp + 3, + initial_timestamp + 4, + second_info.ptr(), + second_info.obj(), + second_unlink_view, + ); + assert(second_unlink_index == 4); + + let ghost first_republished = second_unlinked.insert( + initial_timestamp + 5, + first_info.ptr(), + first_republish_view, + ); + let first_republish_index = link.tracked_cas_publish( + &mut auth, + second_unlinked, + first_republished, + initial_timestamp + 4, + initial_timestamp + 5, + first_info.ptr(), + first_info.obj(), + first_republish_view, + ); + assert(first_republish_index == 5); + assert(second_info.ptr() == first_info.ptr()); + (auth, link) +} + +/// Uses an authoritative history snapshot to discharge the structural +/// premises of [`protect_link`]. The remaining `seen_removed_sound` premise is +/// the reader-side observation carried by the live guard. +pub proof fn linked_list_authorized_protect_next( + tracked auth: &LinkedListTraversalAuth, + tracked guard: &mut RcuReadGuardToken, + tracked root_info: &RcuBlockInfo, + tracked next_info: &RcuBlockInfo, + n: LinkIndex, +) -> (tracked next_protected: RcuProtectedPtr) + requires + auth.wf(), + old(guard).can_protect(*root_info), + old(guard).can_base_protect(*next_info), + root_info.domain() == auth.domain(), + next_info.domain() == auth.domain(), + root_info.ptr() == auth.state().root, + root_info.obj() == auth.state().root_obj, + auth.state().objects.contains_pair(next_info.obj(), next_info.ptr()), + n < auth.state().successors[root_info.obj()].len(), + auth.state().successors[root_info.obj()][n as int] == Some( + (next_info.ptr(), next_info.obj()), + ), + LinkedListTraversalSpec::seen_removed_sound(old(guard).seen_removed(), auth.state()), + old(guard).seen_at(root_info.obj()) <= n, + ensures + next_protected.ptr() == next_info.ptr(), + next_protected.obj() == next_info.obj(), + next_protected.domain() == auth.domain(), + next_protected.protected_by(*final(guard)), + final(guard).wf(), + LinkedListTraversalSpec::node_inv(next_info.ptr(), next_info.obj(), auth.state()), +{ + assert(LinkedListTraversalSpec::root_inv(root_info.ptr(), root_info.obj(), auth.state())); + assert(LinkedListTraversalSpec::link_inv( + root_info.ptr(), + root_info.obj(), + n, + next_info.ptr(), + next_info.obj(), + auth.state(), + )); + linked_list_protect_next_example( + guard, + root_info, + next_info, + root_info.ptr(), + n, + next_info.ptr(), + auth.state(), + ) +} + +/// Example: after protecting the root, following a non-stale successor-history +/// event protects the next node under the same guard. +pub proof fn linked_list_protect_next_example( + tracked guard: &mut RcuReadGuardToken, + tracked root_info: &RcuBlockInfo, + tracked next_info: &RcuBlockInfo, + root: *mut LinkedListNode, + n: LinkIndex, + next: *mut LinkedListNode, + g: LinkedListGhost, +) -> (tracked next_protected: RcuProtectedPtr) + requires + old(guard).can_protect(*root_info), + old(guard).can_base_protect(*next_info), + root_info.ptr() == root, + next_info.ptr() == next, + LinkedListTraversalSpec::root_inv(root, root_info.obj(), g), + LinkedListTraversalSpec::link_inv(root, root_info.obj(), n, next, next_info.obj(), g), + LinkedListTraversalSpec::seen_removed_sound(old(guard).seen_removed(), g), + old(guard).seen_at(root_info.obj()) <= n, + ensures + next_protected.ptr() == next, + next_protected.obj() == next_info.obj(), + next_protected.domain() == old(guard).domain(), + next_protected.protected_by(*final(guard)), + final(guard).wf(), + LinkedListTraversalSpec::node_inv(next, next_info.obj(), g), +{ + let tracked root_protected = protect_root::(guard, root_info, root, g); + protect_link::(guard, &root_protected, next_info, n, next, g) +} + +} // verus! diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs new file mode 100644 index 000000000..c3e339322 --- /dev/null +++ b/ostd/specs/sync/rcu_cpu.rs @@ -0,0 +1,3383 @@ +// SPDX-License-Identifier: MPL-2.0 +//! CPU-local grace-period participation for preemptible RCU. +//! +//! This is the proof component used to refine the paper's abstract reader guard +//! to Asterinas's per-CPU quiescent-state implementation. A +//! [`CpuRcuParticipant`] is intended to stay in one CPU core's tracked local +//! state across task switches. Starting a reader splits a fractional +//! [`CpuRcuReaderFragment`]. Reporting a quiescent state requires the whole +//! fraction, so it cannot race with a live reader from the generation being +//! closed. +//! +//! A report also creates an idempotent [`CpuRcuClosedGeneration`] resource. +//! Resource validity gives the two facts needed by the relaxed-memory proof: +//! +//! - a closed generation is strictly older than every reader that can coexist +//! with its report; +//! - the report's weak-memory view is included in every such later reader's +//! participant view, which [`CpuRcuParticipant::tracked_start_reader`] +//! requires the task's start view to include. +//! +//! # Refinement boundary +//! +//! A participant generation is not the global epoch from the paper's concrete +//! epoch-based implementation. It only names the interval between two +//! quiescent reports on one CPU. A complete grace period must separately prove +//! that every relevant CPU reported after the callback's retire point and that +//! each report view includes the callback's retire view. +//! +//! The end-to-end refinement must enforce all of the following: +//! +//! - preemption is disabled before splitting a reader fragment and remains +//! disabled until that same fragment is returned; +//! - the fragment is created before the first protected load and is retained by +//! the executable RCU guard until guard destruction; +//! - only the scheduler-owned CPU-local participant can issue a report; +//! - a task entering after a report imports the participant's persistent view +//! before starting a reader. +//! +//! The current refinement treats `online_cpus()` as stable for a grace period +//! and covers only readers represented by a task `RunningTaskContext`. +//! CPU-hotplug transitions and interrupt/NMI readers need separate +//! participants before they can be included in the end-to-end theorem. +//! +//! `RunningTaskContext` owns the scheduler-checked-out canonical participant. +//! Its older task/session generation remains a distinct preemption-session +//! counter and is not an authority for this persistent CPU generation. Reader +//! contexts obtain their CPU generation from [`CpuRcuReaderFragment`]. +use crate::specs::{ + mm::cpu::{CpuId, online_cpus}, + task::cpu_core::{CpuCoreLocalState, CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreRegistration}, +}; +use vstd::{ + modes::tracked_swap, + prelude::*, + resource::{ + Loc, + agree::AgreementRA, + algebra::{Resource, ResourceAlgebra}, + frac::FractionRA, + map::{GhostMapAuth, GhostPointsTo}, + product::ProductRA, + relations::frame_preserving_update_opt, + }, +}; + +use super::rcu::{ + RcuBlockInfo, RcuInactive, RcuProtectedPtr, RcuReadGuardToken, RcuReaderContext, + RcuRetiredFact, RcuRetiredFacts, RcuRetiredRecord, RcuSeenRemoved, +}; +use vstd_extra::atomic_irc11::{ThreadView as Irc11ThreadView, ThreadViewOrder}; +use vstd_extra::rcu_read_pool::{RcuTrackedReadLease, RcuTrackedReadPoolRegistry}; + +verus! { + +broadcast use {vstd::set::group_set_lemmas, vstd::thread_view::group_thread_view_axioms}; + +/// One CPU quiescent report retained by the participant PCM. +pub ghost struct CpuRcuReportView { + pub cpu: CpuId, + pub generation: nat, + pub view: Irc11ThreadView, + pub known_retired: Set, +} + +pub(super) ghost struct CpuRcuStateView { + pub(super) cpu: CpuId, + pub(super) generation: nat, + pub(super) view: Irc11ThreadView, + pub(super) known_retired: Set, +} + +pub(super) type CpuRcuState = ProductRA>; + +pub(super) ghost struct CpuRcuCarrier { + pub(super) state: Option, + pub(super) closed: Set, +} + +impl CpuRcuCarrier { + pub(super) open spec fn records_observed( + records: Set, + view: Irc11ThreadView, + ) -> bool { + forall|record: RcuRetiredRecord| #[trigger] + records.contains(record) ==> record.removal.observed_by(view) + } + + pub(super) open spec fn state( + cpu: CpuId, + generation: nat, + view: Irc11ThreadView, + known_retired: Set, + fraction: real, + ) -> Self { + CpuRcuCarrier { + state: Some( + ProductRA { + left: FractionRA::Frac(fraction), + right: AgreementRA::Agree( + CpuRcuStateView { cpu, generation, view, known_retired }, + ), + }, + ), + closed: Set::empty(), + } + } + + pub(super) open spec fn closed(report: CpuRcuReportView) -> Self { + CpuRcuCarrier { state: None, closed: Set::empty().insert(report) } + } + + pub(super) open spec fn reports_fit( + state: CpuRcuStateView, + reports: Set, + ) -> bool { + forall|report: CpuRcuReportView| #[trigger] + reports.contains(report) ==> { + &&& report.cpu == state.cpu + &&& report.generation < state.generation + &&& report.view.spec_le(state.view) + &&& report.known_retired.subset_of(state.known_retired) + &&& CpuRcuCarrier::records_observed(report.known_retired, report.view) + } + } + + pub(super) open spec fn state_view(self) -> CpuRcuStateView { + self.state.unwrap().right->Agree_0 + } + + pub(super) open spec fn fraction(self) -> real { + self.state.unwrap().left->Frac_0 + } + + pub(super) open spec fn has_valid_state(self) -> bool { + &&& self.state is Some + &&& self.state.unwrap().left is Frac + &&& self.state.unwrap().right is Agree + &&& self.state.valid() + } +} + +impl ResourceAlgebra for CpuRcuCarrier { + closed spec fn valid(self) -> bool { + match self.state { + Some(ProductRA { left: FractionRA::Frac(_), right: AgreementRA::Agree(state) }) => { + &&& self.state.valid() + &&& CpuRcuCarrier::reports_fit(state, self.closed) + &&& CpuRcuCarrier::records_observed(state.known_retired, state.view) + }, + None => forall|report: CpuRcuReportView| #[trigger] + self.closed.contains(report) ==> CpuRcuCarrier::records_observed( + report.known_retired, + report.view, + ), + _ => false, + } + } + + closed spec fn op(left: Self, right: Self) -> Self { + CpuRcuCarrier { + state: Option::::op(left.state, right.state), + closed: left.closed.union(right.closed), + } + } + + proof fn valid_op(left: Self, right: Self) { + Option::::valid_op(left.state, right.state); + match left.state { + None => { + let ghost combined = CpuRcuCarrier::op(left, right); + assert(combined.valid()); + assert(combined.closed == left.closed.union(right.closed)); + assert forall|report: CpuRcuReportView| #[trigger] + left.closed.contains(report) implies CpuRcuCarrier::records_observed( + report.known_retired, + report.view, + ) by { + assert(combined.closed.contains(report)); + match right.state { + None => { + assert(combined.state is None); + }, + Some( + ProductRA { + left: FractionRA::Frac(_), + right: AgreementRA::Agree(right_state), + }, + ) => { + assert(combined.state == right.state); + assert(combined.state_view() == right_state); + assert(CpuRcuCarrier::reports_fit(right_state, combined.closed)); + }, + _ => {}, + } + }; + }, + Some( + ProductRA { left: FractionRA::Frac(_), right: AgreementRA::Agree(left_state) }, + ) => { + assert forall|report: CpuRcuReportView| #[trigger] + left.closed.contains(report) implies { + &&& report.cpu == left_state.cpu + &&& report.generation < left_state.generation + &&& report.view.spec_le(left_state.view) + &&& report.known_retired.subset_of(left_state.known_retired) + &&& CpuRcuCarrier::records_observed(report.known_retired, report.view) + } by { + assert(left.closed.union(right.closed).contains(report)); + match right.state { + None => {}, + Some( + ProductRA { left: FractionRA::Frac(_), right: AgreementRA::Agree(_) }, + ) => {}, + _ => {}, + } + }; + }, + _ => {}, + } + } + + proof fn commutative(left: Self, right: Self) { + Option::::commutative(left.state, right.state); + assert(left.closed.union(right.closed) =~= right.closed.union(left.closed)); + } + + proof fn associative(left: Self, middle: Self, right: Self) { + Option::::associative(left.state, middle.state, right.state); + assert(left.closed.union(middle.closed.union(right.closed)) =~= left.closed.union( + middle.closed, + ).union(right.closed)); + } +} + +/// CPU-owned fractional authority for one RCU participation generation. +/// +/// This token belongs permanently to the CPU core named by [`Self::cpu`]. +/// It may cross task execution sessions, but it must never migrate to another +/// CPU's local-state aggregate. +pub tracked struct CpuRcuParticipant { + resource: Resource, + known_retired: RcuRetiredFacts, +} + +/// Generic CPU-core registration evidence specialized to the RCU local state. +pub type CpuRcuCoreBinding = CpuCoreOwnerBinding; + +/// Linear witness that one reader is live in a CPU participation generation. +pub tracked struct CpuRcuReaderFragment { + resource: Resource, + known_retired: RcuRetiredFacts, +} + +/// Idempotent proof that one CPU generation has passed a quiescent boundary. +/// +/// The resource can be split into two identical copies because its PCM element +/// is idempotent. It contains no executable state. +pub tracked struct CpuRcuClosedGeneration { + resource: Resource, + known_retired: RcuRetiredFacts, + binding: CpuRcuCoreBinding, +} + +/// Refinement of the paper guard with one live CPU reader fragment. +/// +/// The wrapped [`RcuReadGuardToken`] remains the reusable abstract +/// `Guard(tid, X, G)`. The fragment is the Asterinas implementation witness +/// that prevents this CPU from reporting a quiescent boundary until the guard +/// is destroyed. Keeping both resources in one linear token prevents the +/// executable guard from ending only the abstract critical section while +/// silently losing its CPU participation. +#[verifier::reject_recursive_types(T)] +pub tracked struct CpuRcuReadGuardToken { + paper_guard: RcuReadGuardToken, + reader: CpuRcuReaderFragment, + binding: CpuRcuCoreBinding, +} + +/// CPU-generation witness retained beside one active physical read lease. +/// +/// The executable guard keeps the other half of `reader`. The ghost snapshot +/// records which abstract guard protected the allocation without duplicating +/// that guard's linear `Guard(tid, X, G)` resource. +#[verifier::reject_recursive_types(T)] +pub tracked struct CpuRcuReadLeaseWitness { + reader: CpuRcuReaderFragment, + ghost paper_guard: RcuReadGuardToken, + binding: CpuRcuCoreBinding, + protected: RcuProtectedPtr, +} + +/// Physical-permission pools associated with one RCU root. +/// +/// A pool is indexed by the allocation identity from [`RcuBlockInfo`], rather +/// than by its address. Pools therefore survive root replacement and remain +/// distinguishable when a reclaimed address is later reused. Every active +/// lease record retains the CPU-generation witness needed by the monitor to +/// rule it out after a completed grace period. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRootPermissionState { + registry: RcuTrackedReadPoolRegistry>, + active_leases: GhostMapAuth, + reclaim_state: GhostMapAuth>, + unretired_claims: Map>>, + reclaimed: Map, + ghost scheduler: Loc, + ghost domain: Loc, + ghost root: Loc, + ghost retire_observation_registry: Loc, +} + +/// Authoritative metadata for one lease currently registered at an RCU root. +/// +/// The matching [`RcuRootReadLease`] owns the linear map points-to token. Its +/// agreement with the authority lets a later invariant opening recover the +/// exact active record created by the guarded load. +pub ghost struct RcuActiveLeaseBinding { + key: nat, + pool_id: Loc, + fraction: real, + participant_id: Loc, + reader_fraction: real, + domain: Loc, + root: Loc, + reader: RcuReaderContext, + start_view: Irc11ThreadView, + protected_addr: usize, +} + +/// Physical read lease together with proof that its active record still +/// belongs to the same RCU root. +#[verifier::reject_recursive_types(O)] +pub tracked struct RcuRootReadLease { + lease: RcuTrackedReadLease, + active: GhostPointsTo, +} + +/// Unique right to reclaim one retired allocation's physical permission. +/// +/// The matching authority remains in [`RcuRootPermissionState`]. Validation +/// against that authority proves that the allocation has not already been +/// reclaimed; consuming the claim changes its authoritative state exactly +/// once. +pub tracked struct RcuReclaimClaim { + points_to: GhostPointsTo>, +} + +/// Persistent grace-period evidence retained after one allocation is reclaimed. +/// +/// A later weak load may still select the allocation's old atomic-history +/// message. The closed generation for its CPU proves that such a coexisting +/// reader is newer and already carries `record` in its retired set. +pub tracked struct RcuReclaimedWitness { + ghost record: RcuRetiredRecord, + ghost scheduler: Loc, + retired: RcuRetiredFact, + closed_generations: Map, +} + +unsafe impl< + T, + O: vstd::thread_view::Objective, +> vstd::thread_view::Objective for RcuRootPermissionState { + +} + +unsafe impl vstd::thread_view::Objective for RcuReclaimClaim { + +} + +unsafe impl vstd::thread_view::Objective for RcuReclaimedWitness { + +} + +proof fn lemma_choose_singleton_report(report: CpuRcuReportView) + ensures + (choose|candidate: CpuRcuReportView| Set::empty().insert(report).contains(candidate)) + == report, +{ + let ghost reports = Set::empty().insert(report); + assert(reports.contains(report)); + let ghost chosen = choose|candidate: CpuRcuReportView| reports.contains(candidate); + assert(reports.contains(chosen)); + assert(chosen == report); +} + +impl CpuRcuParticipant { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().has_valid_state() + &&& self.resource.value().closed.is_empty() + &&& self.resource.value().state_view().known_retired == self.known_retired.records() + &&& CpuRcuCarrier::records_observed( + self.resource.value().state_view().known_retired, + self.resource.value().state_view().view, + ) + } + + /// Creates generation zero for one CPU. + pub proof fn new(cpu: CpuId, view: Irc11ThreadView) -> (tracked res: Self) + ensures + res.cpu() == cpu, + res.generation() == 0, + res.view() == view, + res.fraction() == 1real, + res.wf(), + { + let tracked known_retired = RcuRetiredFacts::empty(); + let tracked resource = Resource::alloc( + CpuRcuCarrier::state(cpu, 0, view, known_retired.records(), 1real), + ); + CpuRcuParticipant { resource, known_retired } + } + + /// Stable identity of this CPU's RCU participant. + pub closed spec fn id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn cpu(self) -> CpuId { + self.resource.value().state_view().cpu + } + + pub closed spec fn generation(self) -> nat { + self.resource.value().state_view().generation + } + + pub closed spec fn view(self) -> Irc11ThreadView { + self.resource.value().state_view().view + } + + /// Persistent retirement facts known before this CPU generation started. + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub closed spec fn fraction(self) -> real { + self.resource.value().fraction() + } + + pub open spec fn wf(self) -> bool { + 0real < self.fraction() <= 1real + } + + /// Starts a reader in the current CPU generation. + /// + /// `start_view` is the task view after it has imported the persistent CPU + /// view. The caller chooses a positive rational `reader_fraction`, so this + /// protocol imposes no fixed bound on the number of readers. + pub proof fn tracked_start_reader( + tracked self, + start_view: Irc11ThreadView, + reader_fraction: real, + ) -> (tracked res: (CpuRcuParticipant, CpuRcuReaderFragment)) + requires + self.wf(), + self.view().spec_le(start_view), + 0real < reader_fraction < self.fraction(), + ensures + res.0.id() == self.id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation(), + res.0.view() == self.view(), + res.0.known_retired() == self.known_retired(), + res.0.fraction() == self.fraction() - reader_fraction, + res.0.wf(), + res.1.participant_id() == self.id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + res.1.participant_view() == self.view(), + res.1.known_retired() == self.known_retired(), + res.1.fraction() == reader_fraction, + res.1.wf(), + { + use_type_invariant(&self); + let ghost participant = CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.view(), + self.known_retired(), + self.fraction() - reader_fraction, + ); + let ghost reader = CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.view(), + self.known_retired(), + reader_fraction, + ); + assert(self.resource.value() == CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.view(), + self.known_retired(), + self.fraction(), + )); + assert(0real < self.fraction() - reader_fraction <= 1real); + assert(0real < reader_fraction <= 1real); + assert(FractionRA::op( + FractionRA::Frac(self.fraction() - reader_fraction), + FractionRA::Frac(reader_fraction), + ) == FractionRA::Frac(self.fraction())); + assert(AgreementRA::op( + AgreementRA::Agree(self.resource.value().state_view()), + AgreementRA::Agree(self.resource.value().state_view()), + ) == AgreementRA::Agree(self.resource.value().state_view())); + assert(Option::::op(participant.state, reader.state) + == self.resource.value().state); + assert(participant.closed.union(reader.closed).is_empty()); + assert(self.resource.value() == CpuRcuCarrier::op(participant, reader)); + let tracked (participant_resource, reader_resource) = self.resource.split( + participant, + reader, + ); + let tracked reader_known_retired = self.known_retired.tracked_duplicate(); + ( + CpuRcuParticipant { resource: participant_resource, known_retired: self.known_retired }, + CpuRcuReaderFragment { resource: reader_resource, known_retired: reader_known_retired }, + ) + } + + /// Splits half of the participant's current rational fraction in place. + /// + /// Repeated nested reads therefore remain unbounded: each live reader gets + /// a positive fraction, while the participant retains a positive fraction + /// for subsequent splits. The complete fraction can be recovered only by + /// returning every fragment. + pub proof fn tracked_start_reader_in_place( + tracked &mut self, + start_view: Irc11ThreadView, + ) -> (tracked reader: CpuRcuReaderFragment) + requires + old(self).wf(), + old(self).view().spec_le(start_view), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation(), + final(self).view() == old(self).view(), + final(self).known_retired() == old(self).known_retired(), + final(self).fraction() == old(self).fraction() / 2real, + final(self).wf(), + reader.participant_id() == old(self).id(), + reader.cpu() == old(self).cpu(), + reader.generation() == old(self).generation(), + reader.participant_view() == old(self).view(), + reader.known_retired() == old(self).known_retired(), + reader.fraction() == old(self).fraction() / 2real, + reader.wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let ghost reader_fraction = self.fraction() / 2real; + assert(0real < reader_fraction < self.fraction()); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked (mut participant, reader) = owned.tracked_start_reader( + start_view, + reader_fraction, + ); + tracked_swap(self, &mut participant); + reader + } + + /// Returns a reader fragment to its CPU-local participant. + pub proof fn tracked_stop_reader( + tracked self, + tracked reader: CpuRcuReaderFragment, + ) -> (tracked res: CpuRcuParticipant) + requires + self.wf(), + reader.wf(), + self.id() == reader.participant_id(), + ensures + res.id() == self.id(), + res.cpu() == self.cpu(), + res.generation() == self.generation(), + res.view() == self.view(), + res.known_retired() == self.known_retired(), + res.fraction() == self.fraction() + reader.fraction(), + res.wf(), + { + use_type_invariant(&self); + use_type_invariant(&reader); + let tracked mut participant_resource = self.resource; + participant_resource.validate_2(&reader.resource); + let tracked resource = participant_resource.join(reader.resource); + CpuRcuParticipant { resource, known_retired: self.known_retired } + } + + /// Returns a reader fragment to this participant in place. + pub proof fn tracked_stop_reader_in_place( + tracked &mut self, + tracked reader: CpuRcuReaderFragment, + ) + requires + old(self).wf(), + reader.wf(), + old(self).id() == reader.participant_id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation(), + final(self).view() == old(self).view(), + final(self).known_retired() == old(self).known_retired(), + final(self).fraction() == old(self).fraction() + reader.fraction(), + final(self).wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked mut participant = owned.tracked_stop_reader(reader); + tracked_swap(self, &mut participant); + } + + /// Closes the current generation and imports retirement facts observed by + /// the report view. + /// + /// A whole fraction is incompatible with every live reader fragment. + /// The returned participant starts the next generation and retains the + /// report view and persistent retirement facts for task sessions that run + /// later on this CPU. + pub proof fn tracked_report_quiescent_with( + tracked self, + tracked binding: CpuRcuCoreBinding, + report_view: Irc11ThreadView, + tracked learned: &RcuRetiredFacts, + ) -> (tracked res: (CpuRcuParticipant, CpuRcuClosedGeneration)) + requires + self.wf(), + self.fraction() == 1real, + self.view().spec_le(report_view), + learned.observed_by(report_view), + binding.cpu() == self.cpu(), + binding.single_local_id() == self.id(), + ensures + res.0.id() == self.id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation() + 1, + res.0.view() == report_view, + res.0.known_retired() == self.known_retired().union(learned.records()), + res.0.fraction() == 1real, + res.0.wf(), + res.1.participant_id() == self.id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.generation(), + res.1.view() == report_view, + res.1.known_retired() == self.known_retired().union(learned.records()), + res.1.scheduler() == binding.registry(), + res.1.wf(), + { + use_type_invariant(&self); + let ghost old_cpu = self.cpu(); + let ghost old_generation = self.generation(); + let ghost old_view = self.view(); + let ghost old_known_retired = self.known_retired(); + let ghost merged_records = old_known_retired.union(learned.records()); + assert(self.resource.value() == CpuRcuCarrier::state( + old_cpu, + old_generation, + old_view, + old_known_retired, + 1real, + )); + assert(CpuRcuCarrier::records_observed(old_known_retired, old_view)); + assert(CpuRcuCarrier::records_observed(merged_records, report_view)) by { + assert forall|record: RcuRetiredRecord| #[trigger] + merged_records.contains(record) implies record.removal.observed_by(report_view) by { + if old_known_retired.contains(record) { + assert(record.removal.observed_by(old_view)); + old_view.lemma_spec_le_transitive(report_view, report_view); + } else { + assert(learned.records().contains(record)); + } + }; + }; + let ghost report = CpuRcuReportView { + cpu: old_cpu, + generation: old_generation, + view: report_view, + known_retired: merged_records, + }; + let ghost next_state = CpuRcuCarrier::state( + old_cpu, + old_generation + 1, + report_view, + merged_records, + 1real, + ); + let ghost next = CpuRcuCarrier { + state: next_state.state, + closed: Set::empty().insert(report), + }; + assert forall|frame: Option| + #![trigger Option::::op(Some(self.resource.value()), frame).valid()] + Option::::op( + Some(self.resource.value()), + frame, + ).valid() implies Option::::op(Some(next), frame).valid() by { + match frame { + Some(CpuRcuCarrier { state: None, closed }) => { + let ghost frame_carrier = CpuRcuCarrier { state: None, closed }; + let ghost combined = CpuRcuCarrier::op(self.resource.value(), frame_carrier); + assert(combined.valid()); + assert(Option::::op(self.resource.value().state, None) + == self.resource.value().state); + assert(self.resource.value().closed.is_empty()); + assert(self.resource.value().closed.union(closed) =~= closed); + assert(combined == CpuRcuCarrier { + state: self.resource.value().state, + closed, + }); + assert(combined.state_view() == CpuRcuStateView { + cpu: old_cpu, + generation: old_generation, + view: old_view, + known_retired: old_known_retired, + }); + assert(CpuRcuCarrier::reports_fit( + CpuRcuStateView { + cpu: old_cpu, + generation: old_generation, + view: old_view, + known_retired: old_known_retired, + }, + closed, + )); + assert forall|old_report: CpuRcuReportView| #[trigger] + closed.contains(old_report) implies { + &&& old_report.cpu == old_cpu + &&& old_report.generation < old_generation + 1 + &&& old_report.view.spec_le(report_view) + &&& old_report.known_retired.subset_of(merged_records) + } by { + assert(old_report.generation < old_generation); + old_report.view.lemma_spec_le_transitive(old_view, report_view); + assert(old_report.known_retired.subset_of(old_known_retired)); + assert(old_known_retired.subset_of(merged_records)); + }; + }, + None => { + assert(next.valid()); + }, + _ => {}, + } + }; + let tracked combined = self.resource.update(next); + let ghost participant = CpuRcuCarrier::state( + old_cpu, + old_generation + 1, + report_view, + merged_records, + 1real, + ); + let ghost closed = CpuRcuCarrier::closed(report); + assert(Option::::op(participant.state, closed.state) == participant.state); + assert(participant.closed.union(closed.closed) =~= Set::empty().insert(report)); + assert(next == CpuRcuCarrier::op(participant, closed)); + let tracked (participant_resource, closed_resource) = combined.split(participant, closed); + let tracked mut known_retired = self.known_retired; + known_retired.tracked_merge(learned); + let tracked closed_known_retired = known_retired.tracked_duplicate(); + assert(known_retired.records() == merged_records); + lemma_choose_singleton_report(report); + ( + CpuRcuParticipant { resource: participant_resource, known_retired }, + CpuRcuClosedGeneration { + resource: closed_resource, + known_retired: closed_known_retired, + binding, + }, + ) + } + + /// Closes the current generation without importing additional retirement + /// facts. + pub proof fn tracked_report_quiescent( + tracked self, + tracked binding: CpuRcuCoreBinding, + report_view: Irc11ThreadView, + ) -> (tracked res: (CpuRcuParticipant, CpuRcuClosedGeneration)) + requires + self.wf(), + self.fraction() == 1real, + self.view().spec_le(report_view), + binding.cpu() == self.cpu(), + binding.single_local_id() == self.id(), + ensures + res.0.id() == self.id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation() + 1, + res.0.view() == report_view, + res.0.known_retired() == self.known_retired(), + res.0.fraction() == 1real, + res.0.wf(), + res.1.participant_id() == self.id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.generation(), + res.1.view() == report_view, + res.1.known_retired() == self.known_retired(), + res.1.scheduler() == binding.registry(), + res.1.wf(), + { + let tracked empty = RcuRetiredFacts::empty(); + let tracked res = self.tracked_report_quiescent_with(binding, report_view, &empty); + assert(empty.records() == Set::::empty()); + assert(self.known_retired().union(empty.records()) =~= self.known_retired()); + res + } + + /// Closes the current generation while retaining this CPU's canonical + /// participant in place. + /// + /// The full-fraction requirement is the resource-level statement that no + /// reader fragment from the generation being closed remains live. + pub proof fn tracked_report_quiescent_in_place( + tracked &mut self, + tracked binding: CpuRcuCoreBinding, + report_view: Irc11ThreadView, + ) -> (tracked closed: CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).fraction() == 1real, + old(self).view().spec_le(report_view), + binding.cpu() == old(self).cpu(), + binding.single_local_id() == old(self).id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation() + 1, + final(self).view() == report_view, + final(self).known_retired() == old(self).known_retired(), + final(self).fraction() == 1real, + final(self).wf(), + closed.participant_id() == old(self).id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).generation(), + closed.view() == report_view, + closed.known_retired() == old(self).known_retired(), + closed.scheduler() == binding.registry(), + closed.wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked (mut participant, closed) = owned.tracked_report_quiescent( + binding, + report_view, + ); + tracked_swap(self, &mut participant); + closed + } + + /// In-place form of [`Self::tracked_report_quiescent_with`]. + pub proof fn tracked_report_quiescent_with_in_place( + tracked &mut self, + tracked binding: CpuRcuCoreBinding, + report_view: Irc11ThreadView, + tracked learned: &RcuRetiredFacts, + ) -> (tracked closed: CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).fraction() == 1real, + old(self).view().spec_le(report_view), + learned.observed_by(report_view), + binding.cpu() == old(self).cpu(), + binding.single_local_id() == old(self).id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation() + 1, + final(self).view() == report_view, + final(self).known_retired() == old(self).known_retired().union(learned.records()), + final(self).fraction() == 1real, + final(self).wf(), + closed.participant_id() == old(self).id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).generation(), + closed.view() == report_view, + closed.known_retired() == old(self).known_retired().union(learned.records()), + closed.scheduler() == binding.registry(), + closed.wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked (mut participant, closed) = owned.tracked_report_quiescent_with( + binding, + report_view, + learned, + ); + tracked_swap(self, &mut participant); + closed + } +} + +impl CpuCoreLocalState for CpuRcuParticipant { + open spec fn belongs_to_cpu(self, cpu: CpuId) -> bool { + self.cpu() == cpu + } + + open spec fn local_key(self) -> Seq { + seq![self.id()] + } +} + +impl CpuRcuParticipant { + /// Exposes the participant's generic CPU-local identity to scheduler + /// clients without requiring them to unfold this module's trait impl. + pub proof fn lemma_cpu_core_local_state(tracked &self) + ensures + self.belongs_to_cpu(self.cpu()), + self.local_key() == seq![self.id()], + { + } +} + +impl CpuRcuReaderFragment { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().has_valid_state() + &&& self.resource.value().closed.is_empty() + &&& self.resource.value().state_view().known_retired == self.known_retired.records() + &&& CpuRcuCarrier::records_observed( + self.resource.value().state_view().known_retired, + self.resource.value().state_view().view, + ) + } + + pub closed spec fn participant_id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn cpu(self) -> CpuId { + self.resource.value().state_view().cpu + } + + pub closed spec fn generation(self) -> nat { + self.resource.value().state_view().generation + } + + pub closed spec fn participant_view(self) -> Irc11ThreadView { + self.resource.value().state_view().view + } + + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub proof fn tracked_known_retired(tracked &self, record: RcuRetiredRecord) -> (tracked res: + &super::rcu::RcuRetiredFact) + requires + self.known_retired().contains(record), + ensures + res.wf(), + res.record() == record, + { + use_type_invariant(self); + reveal(CpuRcuReaderFragment::known_retired); + assert(self.known_retired() == self.known_retired.records()); + assert(self.known_retired.records().contains(record)); + self.known_retired.tracked_borrow(record) + } + + /// Borrows the persistent retirement facts known at this reader's + /// generation, after lifting their observations to `view`. + pub proof fn tracked_retired_facts_observed_by( + tracked &self, + view: Irc11ThreadView, + ) -> (tracked res: &RcuRetiredFacts) + requires + self.participant_view().spec_le(view), + ensures + res.records() == self.known_retired(), + res.observed_by(view), + { + use_type_invariant(self); + assert forall|record: RcuRetiredRecord| #[trigger] + self.known_retired.records().contains(record) implies record.removal.observed_by( + view, + ) by { + assert(self.resource.value().state_view().known_retired.contains(record)); + assert(record.removal.observed_by(self.participant_view())); + self.participant_view().lemma_spec_le_transitive(view, view); + }; + &self.known_retired + } + + pub closed spec fn fraction(self) -> real { + self.resource.value().fraction() + } + + pub open spec fn wf(self) -> bool { + 0real < self.fraction() <= 1real + } + + /// Splits this live-reader authority into two equal fragments. + /// + /// RCU uses the second fragment as the active-lease witness retained by + /// the root invariant. The first remains in the executable read guard. + /// Neither fragment alone permits the CPU participant to report a + /// quiescent state. + pub proof fn tracked_split(tracked self) -> (tracked res: (Self, Self)) + requires + self.wf(), + ensures + res.0.wf(), + res.1.wf(), + res.0.participant_id() == self.participant_id(), + res.1.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.1.cpu() == self.cpu(), + res.0.generation() == self.generation(), + res.1.generation() == self.generation(), + res.0.participant_view() == self.participant_view(), + res.1.participant_view() == self.participant_view(), + res.0.known_retired() == self.known_retired(), + res.1.known_retired() == self.known_retired(), + res.0.fraction() == self.fraction() / 2real, + res.1.fraction() == self.fraction() / 2real, + { + use_type_invariant(&self); + let ghost half = self.fraction() / 2real; + let ghost carrier = CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.participant_view(), + self.known_retired(), + half, + ); + assert(0real < half <= 1real); + assert(FractionRA::op(FractionRA::Frac(half), FractionRA::Frac(half)) == FractionRA::Frac( + self.fraction(), + )); + assert(AgreementRA::op( + AgreementRA::Agree(self.resource.value().state_view()), + AgreementRA::Agree(self.resource.value().state_view()), + ) == AgreementRA::Agree(self.resource.value().state_view())); + assert(self.resource.value() == CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.participant_view(), + self.known_retired(), + self.fraction(), + )); + assert(Option::::op(carrier.state, carrier.state) + == self.resource.value().state); + assert(carrier.closed.union(carrier.closed) == self.resource.value().closed); + assert(self.resource.value() == CpuRcuCarrier::op(carrier, carrier)); + let tracked duplicate_known_retired = self.known_retired.tracked_duplicate(); + let tracked (left, right) = self.resource.split(carrier, carrier); + ( + CpuRcuReaderFragment { resource: left, known_retired: self.known_retired }, + CpuRcuReaderFragment { resource: right, known_retired: duplicate_known_retired }, + ) + } + + /// Recombines two fragments from the same CPU participant generation. + pub proof fn tracked_join(tracked self, tracked other: Self) -> (tracked res: Self) + requires + self.wf(), + other.wf(), + self.participant_id() == other.participant_id(), + ensures + res.wf(), + res.participant_id() == self.participant_id(), + res.cpu() == self.cpu(), + res.generation() == self.generation(), + res.participant_view() == self.participant_view(), + res.known_retired() == self.known_retired(), + res.fraction() == self.fraction() + other.fraction(), + { + use_type_invariant(&self); + use_type_invariant(&other); + let tracked mut resource = self.resource; + resource.validate_2(&other.resource); + let tracked resource = resource.join(other.resource); + CpuRcuReaderFragment { resource, known_retired: self.known_retired } + } +} + +impl CpuRcuReadGuardToken { + pub closed spec fn paper_guard(self) -> RcuReadGuardToken { + self.paper_guard + } + + pub closed spec fn reader_fragment(self) -> CpuRcuReaderFragment { + self.reader + } + + pub closed spec fn binding(self) -> CpuRcuCoreBinding { + self.binding + } + + pub closed spec fn scheduler(self) -> Loc { + self.binding().registry() + } + + pub closed spec fn participant_id(self) -> Loc { + self.reader_fragment().participant_id() + } + + pub closed spec fn cpu(self) -> CpuId { + self.reader_fragment().cpu() + } + + pub closed spec fn generation(self) -> nat { + self.reader_fragment().generation() + } + + pub closed spec fn participant_view(self) -> Irc11ThreadView { + self.reader_fragment().participant_view() + } + + pub closed spec fn known_retired(self) -> Set { + self.reader_fragment().known_retired() + } + + pub closed spec fn domain(self) -> Loc { + self.paper_guard().domain() + } + + pub closed spec fn reader_context(self) -> RcuReaderContext { + self.paper_guard().reader() + } + + pub closed spec fn root(self) -> Loc { + self.paper_guard().root() + } + + pub closed spec fn start_view(self) -> Irc11ThreadView { + self.paper_guard().start_view() + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.paper_guard().retire_observation_registry() + } + + pub closed spec fn reader_registry(self) -> Loc { + self.paper_guard().reader_registry() + } + + pub closed spec fn expired(self) -> Set { + self.paper_guard().expired() + } + + pub closed spec fn protected(self) -> Map { + self.paper_guard().protected() + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.paper_guard().seen_removed() + } + + pub open spec fn protects(self, addr: usize, obj: nat) -> bool { + self.paper_guard().protects(addr, obj) + } + + pub open spec fn protects_pointer(self, ptr: RcuProtectedPtr) -> bool { + ptr.protected_by(self.paper_guard()) + } + + /// Instantiates the entry-time expiration guarantee for one known + /// retirement record belonging to this guard's RCU root. + pub proof fn lemma_known_retired_expired(tracked &self, record: RcuRetiredRecord) + requires + self.wf(), + self.known_retired().contains(record), + record.domain == self.domain(), + record.retire_observation_registry == self.retire_observation_registry(), + record.removal.root == self.root(), + ensures + self.expired().contains(record.obj), + { + } + + /// Forwards the paper guard's traversal-side expiration consequence. + pub proof fn lemma_expired_is_removed(tracked &self) + requires + self.wf(), + ensures + self.expired().subset_of(self.seen_removed().removed), + { + self.paper_guard.lemma_expired_is_removed(); + } + + /// A pointer protected by this guard cannot belong to its entry-time + /// expired set. + pub proof fn lemma_protected_not_expired(tracked &self, tracked protected: &RcuProtectedPtr) + requires + self.wf(), + protected.protected_by(self.paper_guard()), + ensures + !self.expired().contains(protected.obj()), + { + self.paper_guard.lemma_protected_not_expired(protected.ptr().addr(), protected.obj()); + } + + /// Agreement between the abstract guard and its concrete CPU participant. + pub open spec fn wf(self) -> bool { + &&& self.paper_guard().wf() + &&& self.reader_fragment().wf() + &&& self.binding().locals_key().len() == 1 + &&& self.binding().single_local_id() == self.participant_id() + &&& self.binding().cpu() == self.cpu() + &&& self.scheduler() == self.reader_context().scheduler + &&& self.reader_context().cpu == self.cpu() + &&& self.reader_context().generation == self.generation() + &&& self.participant_view().spec_le(self.start_view()) + &&& forall|record: RcuRetiredRecord| #[trigger] + self.known_retired().contains(record) && record.domain == self.domain() + && record.retire_observation_registry == self.retire_observation_registry() + && record.removal.root == self.root() ==> self.expired().contains(record.obj) + } + + /// Attaches the CPU implementation fragment to a freshly started paper + /// guard. + pub proof fn tracked_new( + tracked paper_guard: RcuReadGuardToken, + tracked reader: CpuRcuReaderFragment, + tracked binding: CpuRcuCoreBinding, + ) -> (tracked res: Self) + requires + paper_guard.wf(), + reader.wf(), + paper_guard.reader().cpu == reader.cpu(), + paper_guard.reader().generation == reader.generation(), + binding.registry() == paper_guard.reader().scheduler, + binding.cpu() == reader.cpu(), + binding.locals_key().len() == 1, + binding.single_local_id() == reader.participant_id(), + reader.participant_view().spec_le(paper_guard.start_view()), + forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry + == paper_guard.retire_observation_registry() && record.removal.root + == paper_guard.root() ==> paper_guard.expired().contains(record.obj), + ensures + res.wf(), + res.paper_guard() == paper_guard, + res.reader_fragment() == reader, + res.reader_context() == paper_guard.reader(), + res.scheduler() == binding.registry(), + res.binding() == binding, + res.binding().cpu() == binding.cpu(), + res.binding().single_local_id() == binding.single_local_id(), + res.participant_id() == reader.participant_id(), + res.cpu() == reader.cpu(), + res.generation() == reader.generation(), + res.participant_view() == reader.participant_view(), + res.known_retired() == reader.known_retired(), + res.domain() == paper_guard.domain(), + res.root() == paper_guard.root(), + res.start_view() == paper_guard.start_view(), + res.reader_registry() == paper_guard.reader_registry(), + res.retire_observation_registry() == paper_guard.retire_observation_registry(), + res.expired() == paper_guard.expired(), + res.seen_removed() == paper_guard.seen_removed(), + res.protected() == paper_guard.protected(), + { + CpuRcuReadGuardToken { paper_guard, reader, binding } + } + + /// Separates the implementation fragment from the abstract guard. + /// + /// This is intentionally consuming. The normal destruction path should use + /// [`Self::tracked_stop`] so the abstract `Guard -> Inactive` transition + /// cannot be forgotten. + pub proof fn tracked_into_parts(tracked self) -> (tracked res: ( + RcuReadGuardToken, + CpuRcuReaderFragment, + CpuRcuCoreBinding, + )) + requires + self.wf(), + ensures + res.0 == self.paper_guard(), + res.1 == self.reader_fragment(), + res.2 == self.binding(), + res.0.domain() == self.domain(), + res.0.reader_registry() == self.reader_registry(), + res.0.reader() == self.reader_context(), + res.0.root() == self.root(), + res.0.start_view() == self.start_view(), + res.0.retire_observation_registry() == self.retire_observation_registry(), + res.0.expired() == self.expired(), + res.0.seen_removed() == self.seen_removed(), + res.0.protected() == self.protected(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + res.1.participant_view() == self.participant_view(), + res.1.known_retired() == self.known_retired(), + res.2.registry() == self.scheduler(), + res.2.cpu() == self.cpu(), + res.2.locals_key().len() == 1, + res.2.single_local_id() == self.participant_id(), + res.0.wf(), + res.1.wf(), + res.0.reader().cpu == res.1.cpu(), + res.0.reader().generation == res.1.generation(), + res.1.participant_view().spec_le(res.0.start_view()), + forall|record: RcuRetiredRecord| #[trigger] + res.1.known_retired().contains(record) && record.domain == res.0.domain() + && record.retire_observation_registry == res.0.retire_observation_registry() + && record.removal.root == res.0.root() ==> res.0.expired().contains(record.obj), + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + let tracked CpuRcuReadGuardToken { paper_guard, reader, binding } = self; + assert(reader.known_retired() == known_retired); + assert(paper_guard.domain() == domain); + assert(paper_guard.retire_observation_registry() == retire_observation_registry); + assert(paper_guard.root() == root); + assert(paper_guard.expired() == expired); + assert forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry == paper_guard.retire_observation_registry() + && record.removal.root == paper_guard.root() implies paper_guard.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + (paper_guard, reader, binding) + } + + /// Ends the paper guard locally and returns the CPU fragment that Drop must + /// join back into the CPU-local participant. + pub proof fn tracked_stop(tracked self) -> (tracked res: (RcuInactive, CpuRcuReaderFragment)) + requires + self.wf(), + ensures + res.0.wf(), + res.0.domain() == self.domain(), + res.0.reader() == self.reader_context(), + res.1 == self.reader_fragment(), + res.1.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + { + let tracked (paper_guard, reader, _binding) = self.tracked_into_parts(); + let tracked base = paper_guard.tracked_into_base(); + let tracked inactive = base.tracked_stop(); + (inactive, reader) + } + + /// Splits out the CPU fragment retained with an active physical read + /// lease, while preserving a valid guard for the executable reader. + pub proof fn tracked_split_lease_fragment(tracked self) -> (tracked res: ( + Self, + CpuRcuReaderFragment, + )) + requires + self.wf(), + ensures + res.0.wf(), + res.0.paper_guard() == self.paper_guard(), + res.0.binding() == self.binding(), + res.0.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation(), + res.0.participant_view() == self.participant_view(), + res.0.known_retired() == self.known_retired(), + res.0.reader_fragment().fraction() == self.reader_fragment().fraction() / 2real, + res.1.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + res.1.participant_view() == self.participant_view(), + res.1.known_retired() == self.known_retired(), + res.1.fraction() == self.reader_fragment().fraction() / 2real, + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + let tracked CpuRcuReadGuardToken { paper_guard, reader, binding } = self; + let tracked (reader, lease_reader) = reader.tracked_split(); + assert forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry == paper_guard.retire_observation_registry() + && record.removal.root == paper_guard.root() implies paper_guard.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + let tracked guard = CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding); + (guard, lease_reader) + } + + /// Returns an active lease's CPU fragment to its executable guard before + /// the normal `Guard -> Inactive` transition. + pub proof fn tracked_join_lease_fragment( + tracked self, + tracked lease_reader: CpuRcuReaderFragment, + ) -> (tracked res: Self) + requires + self.wf(), + lease_reader.wf(), + self.participant_id() == lease_reader.participant_id(), + ensures + res.wf(), + res.paper_guard() == self.paper_guard(), + res.binding() == self.binding(), + res.participant_id() == self.participant_id(), + res.cpu() == self.cpu(), + res.generation() == self.generation(), + res.participant_view() == self.participant_view(), + res.known_retired() == self.known_retired(), + res.domain() == self.domain(), + res.root() == self.root(), + res.reader_registry() == self.reader_registry(), + res.retire_observation_registry() == self.retire_observation_registry(), + res.reader_context() == self.reader_context(), + res.start_view() == self.start_view(), + res.expired() == self.expired(), + res.seen_removed() == self.seen_removed(), + res.protected() == self.protected(), + res.reader_fragment().fraction() == self.reader_fragment().fraction() + + lease_reader.fraction(), + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + let tracked CpuRcuReadGuardToken { paper_guard, reader, binding } = self; + let tracked reader = reader.tracked_join(lease_reader); + assert forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry == paper_guard.retire_observation_registry() + && record.removal.root == paper_guard.root() implies paper_guard.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding) + } + + /// Applies the paper's `Guard-protect` rule without changing CPU + /// participation or the captured start view. + pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) + requires + old(self).wf(), + old(self).paper_guard().can_protect(*info), + ensures + final(self).wf(), + final(self).reader_fragment() == old(self).reader_fragment(), + final(self).participant_id() == old(self).participant_id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation(), + final(self).participant_view() == old(self).participant_view(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), + final(self).protects(info.addr(), info.obj()), + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + self.paper_guard.tracked_protect(info); + assert(self.known_retired() == known_retired); + assert(self.domain() == domain); + assert(self.retire_observation_registry() == retire_observation_registry); + assert(self.root() == root); + assert(self.expired() == expired); + assert forall|record: RcuRetiredRecord| #[trigger] + self.known_retired().contains(record) && record.domain == self.domain() + && record.retire_observation_registry == self.retire_observation_registry() + && record.removal.root == self.root() implies self.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + } +} + +impl CpuRcuReadLeaseWitness { + pub closed spec fn reader(self) -> CpuRcuReaderFragment { + self.reader + } + + pub closed spec fn paper_guard(self) -> RcuReadGuardToken { + self.paper_guard + } + + /// Persistent scheduler registration for the CPU participant retained by + /// this lease. This survives type erasure in the root registry so a grace + /// period report can recover the canonical participant for the same CPU. + pub closed spec fn binding(self) -> CpuRcuCoreBinding { + self.binding + } + + pub closed spec fn protected(self) -> RcuProtectedPtr { + self.protected + } + + pub open spec fn wf(self) -> bool { + &&& self.reader().wf() + &&& self.paper_guard().wf() + &&& online_cpus().contains(self.reader().cpu()) + &&& self.binding().registry() == self.paper_guard().reader().scheduler + &&& self.binding().cpu() == self.reader().cpu() + &&& self.binding().locals_key().len() == 1 + &&& self.binding().single_local_id() == self.reader().participant_id() + &&& self.paper_guard().reader().cpu == self.reader().cpu() + &&& self.paper_guard().reader().generation == self.reader().generation() + &&& self.reader().participant_view().spec_le(self.paper_guard().start_view()) + &&& self.paper_guard().expired().subset_of(self.paper_guard().seen_removed().removed) + &&& forall|record: RcuRetiredRecord| #[trigger] + self.reader().known_retired().contains(record) && record.domain + == self.paper_guard().domain() && record.retire_observation_registry + == self.paper_guard().retire_observation_registry() && record.removal.root + == self.paper_guard().root() ==> self.paper_guard().expired().contains(record.obj) + &&& self.protected().protected_by(self.paper_guard()) + } + + /// Relates this lease to the canonical participant closed by a grace-period + /// report for the same scheduler and CPU. + pub proof fn lemma_same_participant_as_closed( + tracked &self, + tracked closed: &CpuRcuClosedGeneration, + ) + requires + self.wf(), + closed.wf(), + closed.scheduler() == self.binding().registry(), + closed.cpu() == self.reader().cpu(), + ensures + self.reader().participant_id() == closed.participant_id(), + { + closed.lemma_same_participant_as_binding(&self.binding); + } + + /// Splits the registry fragment from a live executable guard. + pub proof fn tracked_from_guard( + tracked guard: CpuRcuReadGuardToken, + tracked protected: RcuProtectedPtr, + ) -> (tracked res: (CpuRcuReadGuardToken, Self)) + requires + guard.wf(), + protected.protected_by(guard.paper_guard()), + online_cpus().contains(guard.cpu()), + ensures + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.participant_id() == guard.participant_id(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.known_retired() == guard.known_retired(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.wf(), + res.1.reader().participant_id() == guard.participant_id(), + res.1.reader().cpu() == guard.cpu(), + res.1.reader().generation() == guard.generation(), + res.1.reader().participant_view() == guard.participant_view(), + res.1.reader().known_retired() == guard.known_retired(), + res.1.reader().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.paper_guard() == guard.paper_guard(), + res.1.binding().registry() == guard.scheduler(), + res.1.binding().cpu() == guard.cpu(), + res.1.binding().locals_key() == guard.binding().locals_key(), + res.1.binding().single_local_id() == guard.participant_id(), + res.1.protected() == protected, + { + guard.lemma_expired_is_removed(); + let ghost paper_guard = guard.paper_guard(); + let tracked binding = guard.binding.tracked_duplicate(); + let tracked (guard, reader) = guard.tracked_split_lease_fragment(); + let tracked witness = CpuRcuReadLeaseWitness { reader, paper_guard, binding, protected }; + (guard, witness) + } + + /// Builds the direct-root protection witness returned by a guarded atomic + /// load and splits off the registry's CPU-reader fragment in one step. + pub proof fn tracked_from_loaded_guard( + tracked guard: CpuRcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: (CpuRcuReadGuardToken, Self)) + requires + guard.wf(), + info.wf(), + info.domain() == guard.domain(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + online_cpus().contains(guard.cpu()), + ensures + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.wf(), + res.1.reader().participant_id() == guard.participant_id(), + res.1.reader().cpu() == guard.cpu(), + res.1.reader().generation() == guard.generation(), + res.1.reader().participant_view() == guard.participant_view(), + res.1.reader().known_retired() == guard.known_retired(), + res.1.reader().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.paper_guard() == guard.paper_guard(), + res.1.binding().registry() == guard.scheduler(), + res.1.binding().cpu() == guard.cpu(), + res.1.binding().locals_key() == guard.binding().locals_key(), + res.1.binding().single_local_id() == guard.participant_id(), + res.1.protected().domain() == info.domain(), + res.1.protected().obj() == info.obj(), + res.1.protected().ptr() == info.ptr(), + { + let tracked protected = RcuProtectedPtr::tracked_from_guard(&guard.paper_guard, info); + Self::tracked_from_guard(guard, protected) + } + + /// Returns the registry fragment to the corresponding executable guard. + pub proof fn tracked_return_to_guard( + tracked self, + tracked guard: CpuRcuReadGuardToken, + ) -> (tracked res: CpuRcuReadGuardToken) + requires + self.wf(), + guard.wf(), + self.reader().participant_id() == guard.participant_id(), + ensures + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.binding() == guard.binding(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.participant_view() == guard.participant_view(), + res.known_retired() == guard.known_retired(), + res.reader_fragment().fraction() == guard.reader_fragment().fraction() + + self.reader().fraction(), + { + guard.tracked_join_lease_fragment(self.reader) + } +} + +impl RcuReclaimClaim { + pub closed spec fn registry(self) -> Loc { + self.points_to.id() + } + + pub closed spec fn obj(self) -> nat { + self.points_to.key() + } + + pub closed spec fn is_pending(self) -> bool { + self.points_to.value() is Some + } + + pub closed spec fn ptr(self) -> *mut T + recommends + self.is_pending(), + { + self.points_to.value()->Some_0 + } +} + +impl RcuReclaimedWitness { + pub closed spec fn record(self) -> RcuRetiredRecord { + self.record + } + + pub closed spec fn closed_generations(self) -> Map { + self.closed_generations + } + + pub closed spec fn scheduler(self) -> Loc { + self.scheduler + } + + /// Persistent base-retirement fact retained after physical reclamation. + pub closed spec fn retired_fact(self) -> RcuRetiredFact { + self.retired + } + + /// Persistent retirement fact used to agree the callback summary with the + /// root domain's authoritative removal observation. + pub proof fn tracked_retired_fact(tracked &self) -> (tracked res: &RcuRetiredFact) + requires + self.wf(), + ensures + res.wf(), + res.record() == self.record(), + { + &self.retired + } + + pub proof fn tracked_closed_generation(tracked &self, cpu: CpuId) -> (tracked res: + &CpuRcuClosedGeneration) + requires + self.wf(), + online_cpus().contains(cpu), + ensures + *res == self.closed_generations()[cpu], + res.wf(), + res.cpu() == cpu, + res.scheduler() == self.scheduler(), + { + self.closed_generations.tracked_borrow(cpu) + } + + pub open spec fn wf(self) -> bool { + &&& self.retired_fact().wf() + &&& self.retired_fact().record() == self.record() + &&& self.closed_generations().dom() == online_cpus() + &&& forall|cpu: CpuId| #[trigger] + self.closed_generations().contains_key(cpu) ==> { + let closed = self.closed_generations()[cpu]; + &&& closed.wf() + &&& closed.cpu() == cpu + &&& closed.scheduler() == self.scheduler() + &&& closed.known_retired().contains(self.record()) + } + } + + pub proof fn tracked_new( + scheduler: Loc, + tracked retired: RcuRetiredFact, + tracked closed_generations: Map, + ) -> (tracked res: Self) + requires + retired.wf(), + closed_generations.dom() == online_cpus(), + forall|cpu: CpuId| #[trigger] + closed_generations.contains_key(cpu) ==> { + let closed = closed_generations[cpu]; + &&& closed.wf() + &&& closed.cpu() == cpu + &&& closed.scheduler() == scheduler + &&& closed.known_retired().contains(retired.record()) + }, + ensures + res.wf(), + res.record() == retired.record(), + res.scheduler() == scheduler, + { + let ghost record = retired.record(); + Self { record, scheduler, retired, closed_generations } + } + + /// Classifies a coexisting reader as later than the completed grace period. + pub proof fn tracked_later_reader( + tracked &self, + tracked reader: CpuRcuReaderFragment, + ) -> (tracked res: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + online_cpus().contains(reader.cpu()), + self.closed_generations()[reader.cpu()].participant_id() == reader.participant_id(), + ensures + res == reader, + res.wf(), + res.known_retired().contains(self.record()), + { + let tracked closed = self.closed_generations.tracked_borrow(reader.cpu()); + let tracked reader = closed.lemma_later_reader(reader); + assert(closed.known_retired().contains(self.record())); + reader + } +} + +impl RcuActiveLeaseBinding { + pub closed spec fn from_record( + record: vstd_extra::rcu_read_pool::RcuReadLeaseRecord>, + ) -> Self { + Self { + key: record.key(), + pool_id: record.pool_id(), + fraction: record.fraction(), + participant_id: record.witness().reader().participant_id(), + reader_fraction: record.witness().reader().fraction(), + domain: record.witness().paper_guard().domain(), + root: record.witness().paper_guard().root(), + reader: record.witness().paper_guard().reader(), + start_view: record.witness().paper_guard().start_view(), + protected_addr: record.witness().protected().ptr().addr(), + } + } +} + +impl RcuRootReadLease { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.active.key() == self.lease.lease_id() + &&& self.active.value().key == self.lease.key() + &&& self.active.value().pool_id == self.lease.pool_id() + &&& self.active.value().fraction == self.lease.fraction() + } + + pub closed spec fn lease_id(self) -> nat { + self.lease.lease_id() + } + + pub closed spec fn key(self) -> nat { + self.lease.key() + } + + pub closed spec fn pool_id(self) -> Loc { + self.lease.pool_id() + } + + pub closed spec fn fraction(self) -> real { + self.lease.fraction() + } + + pub closed spec fn resource(self) -> O { + self.lease.resource() + } + + pub closed spec fn active_registry(self) -> Loc { + self.active.id() + } + + pub closed spec fn participant_id(self) -> Loc { + self.active.value().participant_id + } + + pub closed spec fn reader_fraction(self) -> real { + self.active.value().reader_fraction + } + + pub closed spec fn domain(self) -> Loc { + self.active.value().domain + } + + pub closed spec fn root(self) -> Loc { + self.active.value().root + } + + pub closed spec fn reader_context(self) -> RcuReaderContext { + self.active.value().reader + } + + pub closed spec fn start_view(self) -> Irc11ThreadView { + self.active.value().start_view + } + + pub closed spec fn protected_addr(self) -> usize { + self.active.value().protected_addr + } + + pub proof fn borrow(tracked &self) -> (tracked resource: &O) + ensures + *resource == self.resource(), + { + self.lease.borrow() + } +} + +impl RcuRootPermissionState { + /// Creates an empty permission registry for one RCU root. + pub proof fn empty( + scheduler: Loc, + domain: Loc, + root: Loc, + retire_observation_registry: Loc, + ) -> (tracked res: Self) + ensures + res.wf(), + res.scheduler() == scheduler, + res.domain() == domain, + res.root() == root, + res.retire_observation_registry() == retire_observation_registry, + res.keys() == Set::::empty(), + res.allocations() == Set::::empty(), + res.reclaimed() == Map::::empty(), + res.active_ids() == Set::::empty(), + { + let tracked registry = RcuTrackedReadPoolRegistry::empty(); + let tracked (active_leases, _active) = GhostMapAuth::new(Map::empty()); + let tracked (reclaim_state, _claims) = GhostMapAuth::new(Map::empty()); + let tracked res = RcuRootPermissionState { + registry, + active_leases, + reclaim_state, + unretired_claims: Map::tracked_empty(), + reclaimed: Map::tracked_empty(), + scheduler, + domain, + root, + retire_observation_registry, + }; + assert(res.allocations() == Set::::empty()); + assert(res.unretired_claims().dom() == Set::::empty()); + assert(res.reclaimed().dom() == Set::::empty()); + assert(res.active_lease_bindings().dom() == Set::::empty()); + assert(res.reclaimed().dom() == res.allocations().difference(res.keys())); + assert forall|obj: nat| #[trigger] res.reclaimed().contains_key(obj) implies { + let completed = res.reclaimed()[obj]; + &&& completed.wf() + &&& completed.record().domain == res.domain() + &&& completed.record().obj == obj + &&& completed.record().retire_observation_registry == res.retire_observation_registry() + &&& completed.record().removal.root == res.root() + } by {}; + assert forall|obj: nat| #[trigger] res.allocations().contains(obj) implies { + res.keys().contains(obj) <==> res.reclaim_states()[obj] is Some + } by {}; + res + } + + pub closed spec fn registry(self) -> RcuTrackedReadPoolRegistry< + nat, + O, + CpuRcuReadLeaseWitness, + > { + self.registry + } + + pub proof fn tracked_registry_mut(tracked &mut self) -> (tracked res: + &mut RcuTrackedReadPoolRegistry>) + ensures + *res == old(self).registry(), + final(self).registry() == *final(res), + *final(res) == old(self).registry() ==> *final(self) == *old(self), + { + &mut self.registry + } + + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn scheduler(self) -> Loc { + self.scheduler + } + + pub closed spec fn root(self) -> Loc { + self.root + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retire_observation_registry + } + + pub closed spec fn keys(self) -> Set { + self.registry().keys() + } + + pub closed spec fn contains(self, obj: nat) -> bool { + self.registry().contains(obj) + } + + pub proof fn lemma_contains_iff_key(tracked &self, obj: nat) + ensures + self.contains(obj) <==> self.keys().contains(obj), + { + self.registry.lemma_contains_iff_key(obj); + } + + /// Opens pool membership for every allocation identity at once. + pub proof fn lemma_all_contains_iff_keys(tracked &self) + ensures + forall|obj: nat| #[trigger] self.contains(obj) <==> self.keys().contains(obj), + { + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + self.registry.lemma_all_contains_iff_keys(); + } + + /// Opens the allocation-state facts associated with a live permission pool. + pub proof fn lemma_live_reclaim_state(tracked &self, obj: nat) + requires + self.wf(), + self.contains(obj), + ensures + self.keys().contains(obj), + self.allocations().contains(obj), + self.reclaim_states().dom().contains(obj), + self.reclaim_states()[obj] is Some, + { + self.registry.lemma_contains_iff_key(obj); + assert(self.keys().contains(obj)); + assert(self.keys().subset_of(self.allocations())); + assert(self.allocations().contains(obj)); + } + + /// An unretired root claim always names a live permission pool. + pub proof fn lemma_unretired_is_live(tracked &self, obj: nat) + requires + self.wf(), + self.has_unretired_claim(obj), + ensures + self.contains(obj), + self.keys().contains(obj), + self.allocations().contains(obj), + self.reclaim_states().dom().contains(obj), + self.reclaim_states()[obj] is Some, + { + assert(self.unretired_claims().dom().contains(obj)); + assert(self.keys().contains(obj)); + self.registry.lemma_contains_iff_key(obj); + self.lemma_live_reclaim_state(obj); + } + + /// Opens the live allocation facts for every key in one quantifier. + pub proof fn lemma_all_live_reclaim_states(tracked &self) + requires + self.wf(), + ensures + forall|obj: nat| #[trigger] + self.keys().contains(obj) ==> { + &&& self.contains(obj) + &&& self.allocations().contains(obj) + &&& self.reclaim_states().dom().contains(obj) + &&& self.reclaim_states()[obj] is Some + }, + { + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + self.registry.lemma_all_contains_iff_keys(); + assert forall|obj: nat| #[trigger] self.keys().contains(obj) implies { + &&& self.contains(obj) + &&& self.allocations().contains(obj) + &&& self.reclaim_states().dom().contains(obj) + &&& self.reclaim_states()[obj] is Some + } by { + assert(self.contains(obj)); + assert(self.allocations().contains(obj)); + }; + } + + /// Every allocation identity ever registered by this root. + pub closed spec fn allocations(self) -> Set { + self.reclaim_state.dom() + } + + pub closed spec fn reclaim_states(self) -> Map> { + self.reclaim_state@ + } + + /// Every allocation retained by the append-only identity registry has a + /// corresponding reclaim-state cell, including after that cell changes + /// from `Some(ptr)` to `None` at reclamation. + pub proof fn lemma_allocation_has_reclaim_state(tracked &self, obj: nat) + requires + self.wf(), + self.allocations().contains(obj), + ensures + self.reclaim_states().dom().contains(obj), + { + } + + /// Opens every append-only allocation cell in one quantified fact. + pub proof fn lemma_all_allocations_have_reclaim_states(tracked &self) + requires + self.wf(), + ensures + forall|obj: nat| #[trigger] + self.allocations().contains(obj) ==> self.reclaim_states().dom().contains(obj), + { + } + + pub closed spec fn unretired_claims(self) -> Map>> { + self.unretired_claims + } + + pub closed spec fn reclaimed(self) -> Map { + self.reclaimed + } + + pub closed spec fn reclaim_registry(self) -> Loc { + self.reclaim_state.id() + } + + pub closed spec fn ownership(self, obj: nat) -> O + recommends + self.contains(obj), + { + self.registry().pool(obj).resource() + } + + pub closed spec fn has_unretired_claim(self, obj: nat) -> bool { + self.unretired_claims().contains_key(obj) + } + + /// Relates the claim predicate to membership in the claim map domain. + pub proof fn lemma_has_unretired_iff_domain(tracked &self, obj: nat) + ensures + self.has_unretired_claim(obj) <==> self.unretired_claims().dom().contains(obj), + { + } + + /// Opens claim-map membership for all allocation identities. + pub proof fn lemma_all_unretired_domains(tracked &self) + ensures + forall|obj: nat| #[trigger] + self.has_unretired_claim(obj) <==> self.unretired_claims().dom().contains(obj), + { + } + + pub closed spec fn active_ids(self) -> Set { + self.registry().active_ids() + } + + pub closed spec fn active_lease_bindings(self) -> Map { + self.active_leases@ + } + + pub closed spec fn active_lease_registry(self) -> Loc { + self.active_leases.id() + } + + pub open spec fn has_active(self, obj: nat) -> bool { + self.registry().has_active(obj) + } + + pub closed spec fn active_record( + self, + lease_id: nat, + ) -> vstd_extra::rcu_read_pool::RcuReadLeaseRecord> + recommends + self.active_ids().contains(lease_id), + { + self.registry().active_record(lease_id) + } + + /// Exposes the active-lease registry projections without revealing the + /// rest of the root permission state's representation. + pub proof fn lemma_active_registry_projection(tracked &self) + ensures + self.active_ids() == self.registry().active_ids(), + forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> self.active_record(lease_id) + == self.registry().active_record(lease_id), + { + } + + /// The registry accounting is valid and every active witness belongs to + /// this exact RCU root. + pub open spec fn wf(self) -> bool { + &&& self.registry().wf() + &&& self.active_lease_bindings().dom() == self.active_ids() + &&& forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> self.active_lease_bindings()[lease_id] + == RcuActiveLeaseBinding::from_record(self.active_record(lease_id)) + &&& self.registry().keys().subset_of(self.allocations()) + &&& forall|obj: nat| #[trigger] + self.allocations().contains(obj) ==> { + self.keys().contains(obj) <==> self.reclaim_states()[obj] is Some + } + &&& self.unretired_claims().dom().subset_of(self.registry().keys()) + &&& self.reclaimed().dom() == self.allocations().difference(self.keys()) + &&& forall|obj: nat| #[trigger] + self.reclaimed().contains_key(obj) ==> { + let completed = self.reclaimed()[obj]; + &&& completed.wf() + &&& completed.scheduler() == self.scheduler() + &&& completed.record().domain == self.domain() + &&& completed.record().obj == obj + &&& completed.record().retire_observation_registry + == self.retire_observation_registry() + &&& completed.record().removal.root == self.root() + } + &&& forall|obj: nat| #[trigger] + self.unretired_claims().contains_key(obj) ==> { + let claim = self.unretired_claims()[obj]; + &&& claim.id() == self.reclaim_registry() + &&& claim.key() == obj + &&& claim.value() == Some(self.reclaim_states()[obj]->Some_0) + } + &&& forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& witness.binding().registry() == self.scheduler() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } + } + + pub proof fn tracked_reclaimed(tracked &self, obj: nat) -> (tracked res: &RcuReclaimedWitness) + requires + self.wf(), + self.allocations().contains(obj), + !self.contains(obj), + ensures + self.reclaimed().contains_key(obj), + *res == self.reclaimed()[obj], + res.wf(), + res.record().domain == self.domain(), + res.record().obj == obj, + res.record().retire_observation_registry == self.retire_observation_registry(), + res.record().removal.root == self.root(), + { + self.registry.lemma_contains_iff_key(obj); + assert(self.reclaimed().contains_key(obj)); + self.reclaimed.tracked_borrow(obj) + } + + /// Registers the complete physical ownership of one published allocation. + pub proof fn tracked_insert( + tracked &mut self, + tracked info: &RcuBlockInfo, + tracked ownership: O, + ) + requires + old(self).wf(), + info.wf(), + info.domain() == old(self).domain(), + !old(self).allocations().contains(info.obj()), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys().insert(info.obj()), + final(self).allocations() == old(self).allocations().insert(info.obj()), + final(self).reclaim_states() == old(self).reclaim_states().insert( + info.obj(), + Some(info.ptr()), + ), + final(self).reclaimed() == old(self).reclaimed(), + final(self).has_unretired_claim(info.obj()), + final(self).unretired_claims().dom() == old(self).unretired_claims().dom().insert( + info.obj(), + ), + final(self).contains(info.obj()), + final(self).active_ids() == old(self).active_ids(), + final(self).registry().pool(info.obj()).resource() == ownership, + final(self).ownership(info.obj()) == ownership, + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + { + let ghost old_active = self.active_ids(); + let ghost new_obj = info.obj(); + self.registry.lemma_contains_iff_key(new_obj); + assert(!self.contains(new_obj)) by { + if self.contains(new_obj) { + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + } + }; + self.registry.insert(info.obj(), ownership); + let tracked claim = self.reclaim_state.insert(info.obj(), Some(info.ptr())); + self.unretired_claims.tracked_insert(info.obj(), claim); + assert(self.registry().keys().subset_of(self.allocations())) by { + assert forall|obj: nat| #[trigger] + self.registry().keys().contains(obj) implies self.allocations().contains(obj) by { + if obj != new_obj { + assert(old(self).keys().contains(obj)); + assert(old(self).allocations().contains(obj)); + } + }; + }; + assert forall|obj: nat| #[trigger] self.allocations().contains(obj) implies { + self.keys().contains(obj) <==> self.reclaim_states()[obj] is Some + } by { + if obj == new_obj { + assert(self.keys().contains(obj)); + assert(self.reclaim_states()[obj] == Some(info.ptr())); + } else { + assert(old(self).allocations().contains(obj)); + assert(self.reclaim_states()[obj] == old(self).reclaim_states()[obj]); + assert(self.keys().contains(obj) == old(self).keys().contains(obj)); + } + }; + assert(self.unretired_claims().dom().subset_of(self.registry().keys())); + assert(self.reclaimed().dom() == self.allocations().difference(self.keys())) by { + assert(self.reclaimed() == old(self).reclaimed()); + assert(old(self).reclaimed().dom() == old(self).allocations().difference( + old(self).keys(), + )); + }; + assert forall|obj: nat| #[trigger] self.reclaimed().contains_key(obj) implies { + let completed = self.reclaimed()[obj]; + &&& completed.wf() + &&& completed.record().domain == self.domain() + &&& completed.record().obj == obj + &&& completed.record().retire_observation_registry == self.retire_observation_registry() + &&& completed.record().removal.root == self.root() + } by { + assert(old(self).reclaimed().contains_key(obj)); + assert(self.reclaimed()[obj] == old(self).reclaimed()[obj]); + }; + assert forall|obj: nat| #[trigger] self.unretired_claims().contains_key(obj) implies { + let saved = self.unretired_claims()[obj]; + &&& saved.id() == self.reclaim_registry() + &&& saved.key() == obj + &&& saved.value() == Some(self.reclaim_states()[obj]->Some_0) + } by { + if obj == new_obj { + assert(self.unretired_claims()[obj] == claim); + } else { + assert(old(self).unretired_claims().contains_key(obj)); + assert(self.unretired_claims()[obj] == old(self).unretired_claims()[obj]); + } + }; + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(old_active.contains(lease_id)); + assert(old(self).active_ids().contains(lease_id)); + assert(self.registry().active_record(lease_id) == old(self).registry().active_record( + lease_id, + )); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + }; + } + + /// Moves the unique reclaim claim out when the root publication is retired. + pub proof fn tracked_retire(tracked &mut self, obj: nat) -> (tracked claim: RcuReclaimClaim) + requires + old(self).wf(), + old(self).has_unretired_claim(obj), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).registry() == old(self).registry(), + forall|candidate: nat| #[trigger] + old(self).contains(candidate) ==> final(self).ownership(candidate) == old( + self, + ).ownership(candidate), + final(self).active_ids() == old(self).active_ids(), + !final(self).has_unretired_claim(obj), + final(self).unretired_claims().dom() == old(self).unretired_claims().dom().remove(obj), + claim.registry() == old(self).reclaim_registry(), + claim.obj() == obj, + claim.is_pending(), + claim.ptr() == old(self).reclaim_states()[obj]->Some_0, + { + let tracked points_to = self.unretired_claims.tracked_remove(obj); + assert(self.registry() == old(self).registry()); + assert(self.reclaim_states() == old(self).reclaim_states()); + assert(self.reclaim_registry() == old(self).reclaim_registry()); + assert(self.unretired_claims().dom().subset_of(self.registry().keys())); + assert forall|other: nat| #[trigger] self.unretired_claims.contains_key(other) implies { + let claim = self.unretired_claims[other]; + &&& claim.id() == self.reclaim_state.id() + &&& claim.key() == other + &&& claim.value() == Some(self.reclaim_states()[other]->Some_0) + } by { + assert(old(self).unretired_claims.contains_key(other)); + assert(self.unretired_claims[other] == old(self).unretired_claims[other]); + }; + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + assert(self.domain() == old(self).domain()); + assert(self.root() == old(self).root()); + assert(self.retire_observation_registry() == old(self).retire_observation_registry()); + }; + RcuReclaimClaim { points_to } + } + + /// Splits a physical read lease for an allocation already protected by a + /// traversal-level guard. + /// + /// Unlike [`Self::tracked_split_loaded`], this transition borrows the + /// explicit [`RcuProtectedPtr`] minted by a traversal step. This lets an + /// internal-link load use the same AId-indexed physical pool as a direct + /// root load without reconstructing protection from the guard map. A + /// persistent copy is retained in the active registry record while the + /// caller keeps the original for further traversal. The CPU reader + /// fragment is split at the same time. + pub proof fn tracked_split_protected( + tracked &mut self, + tracked guard: CpuRcuReadGuardToken, + tracked protected: &RcuProtectedPtr, + ) -> (tracked res: (CpuRcuReadGuardToken, RcuRootReadLease)) + requires + old(self).wf(), + old(self).contains(protected.obj()), + guard.wf(), + guard.scheduler() == old(self).scheduler(), + protected.domain() == old(self).domain(), + protected.protected_by(guard.paper_guard()), + guard.domain() == old(self).domain(), + guard.root() == old(self).root(), + guard.retire_observation_registry() == old(self).retire_observation_registry(), + online_cpus().contains(guard.cpu()), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|candidate: nat| #[trigger] + final(self).has_unretired_claim(candidate) == old(self).has_unretired_claim( + candidate, + ), + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.scheduler() == guard.scheduler(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.domain() == guard.domain(), + res.0.root() == guard.root(), + res.0.reader_registry() == guard.reader_registry(), + res.0.retire_observation_registry() == guard.retire_observation_registry(), + res.0.reader_context() == guard.reader_context(), + res.0.start_view() == guard.start_view(), + res.0.expired() == guard.expired(), + res.0.seen_removed() == guard.seen_removed(), + res.0.protected() == guard.protected(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.key() == protected.obj(), + res.1.resource() == old(self).ownership(protected.obj()), + res.1.active_registry() == old(self).active_lease_registry(), + res.1.participant_id() == guard.participant_id(), + res.1.reader_fraction() == res.0.reader_fragment().fraction(), + res.1.domain() == guard.domain(), + res.1.root() == guard.root(), + res.1.reader_context() == guard.reader_context(), + res.1.start_view() == guard.start_view(), + res.1.protected_addr() == protected.ptr().addr(), + final(self).active_ids() == old(self).active_ids().insert(res.1.lease_id()), + final(self).active_record(res.1.lease_id()).witness().paper_guard() + == guard.paper_guard(), + final(self).active_record(res.1.lease_id()).witness().protected() == *protected, + { + let tracked saved_protected = protected.tracked_duplicate(); + let tracked (guard, witness) = CpuRcuReadLeaseWitness::tracked_from_guard( + guard, + saved_protected, + ); + let tracked lease = self.registry.split_lease(protected.obj(), witness); + let ghost binding = RcuActiveLeaseBinding::from_record( + self.active_record(lease.lease_id()), + ); + reveal(RcuActiveLeaseBinding::from_record); + assert(self.active_record(lease.lease_id()).witness() == witness); + assert(self.active_record(lease.lease_id()).witness().protected() == *protected); + assert(binding.protected_addr == protected.ptr().addr()); + let tracked active = self.active_leases.insert(lease.lease_id(), binding); + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + if lease_id == lease.lease_id() { + assert(self.active_record(lease_id).witness() == witness); + assert(self.active_record(lease_id).key() == protected.obj()); + } else { + assert(old(self).active_ids().contains(lease_id)); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + } + }; + let tracked lease = RcuRootReadLease { lease, active }; + (guard, lease) + } + + /// Splits a physical read lease for the object selected by a direct-root + /// guarded load. + /// + /// The direct-root adapter materializes the same traversal protection + /// witness and delegates to [`Self::tracked_split_protected`]. + pub proof fn tracked_split_loaded( + tracked &mut self, + tracked guard: CpuRcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: (CpuRcuReadGuardToken, RcuRootReadLease)) + requires + old(self).wf(), + old(self).contains(info.obj()), + guard.wf(), + guard.scheduler() == old(self).scheduler(), + info.wf(), + info.domain() == old(self).domain(), + guard.domain() == old(self).domain(), + guard.root() == old(self).root(), + guard.retire_observation_registry() == old(self).retire_observation_registry(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + online_cpus().contains(guard.cpu()), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|candidate: nat| #[trigger] + final(self).has_unretired_claim(candidate) == old(self).has_unretired_claim( + candidate, + ), + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.scheduler() == guard.scheduler(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.domain() == guard.domain(), + res.0.root() == guard.root(), + res.0.reader_registry() == guard.reader_registry(), + res.0.retire_observation_registry() == guard.retire_observation_registry(), + res.0.reader_context() == guard.reader_context(), + res.0.start_view() == guard.start_view(), + res.0.expired() == guard.expired(), + res.0.seen_removed() == guard.seen_removed(), + res.0.protected() == guard.protected(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.key() == info.obj(), + res.1.resource() == old(self).ownership(info.obj()), + res.1.active_registry() == old(self).active_lease_registry(), + res.1.participant_id() == guard.participant_id(), + res.1.reader_fraction() == res.0.reader_fragment().fraction(), + res.1.domain() == guard.domain(), + res.1.root() == guard.root(), + res.1.reader_context() == guard.reader_context(), + res.1.start_view() == guard.start_view(), + res.1.protected_addr() == info.addr(), + final(self).active_ids() == old(self).active_ids().insert(res.1.lease_id()), + final(self).active_record(res.1.lease_id()).witness().paper_guard() + == guard.paper_guard(), + final(self).active_record(res.1.lease_id()).witness().protected().obj() == info.obj(), + { + let tracked protected = RcuProtectedPtr::tracked_from_guard(&guard.paper_guard, info); + let tracked res = self.tracked_split_protected(guard, &protected); + info.lemma_wf_facts(); + assert(res.1.protected_addr() == info.ptr().addr()); + res + } + + /// Returns one physical lease and rejoins its CPU fragment with the + /// executable guard that originally issued it. + pub proof fn tracked_return_loaded( + tracked &mut self, + tracked lease: RcuRootReadLease, + tracked guard: CpuRcuReadGuardToken, + ) -> (tracked res: CpuRcuReadGuardToken) + requires + old(self).wf(), + lease.active_registry() == old(self).active_lease_registry(), + lease.participant_id() == guard.participant_id(), + lease.reader_fraction() == guard.reader_fragment().fraction(), + lease.domain() == guard.domain(), + lease.root() == guard.root(), + lease.reader_context() == guard.reader_context(), + lease.start_view() == guard.start_view(), + guard.protects(lease.protected_addr(), lease.key()), + guard.wf(), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|obj: nat| #[trigger] + final(self).has_unretired_claim(obj) == old(self).has_unretired_claim(obj), + forall|obj: nat| #[trigger] final(self).contains(obj) == old(self).contains(obj), + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + final(self).active_ids() == old(self).active_ids().remove(lease.lease_id()), + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.binding() == guard.binding(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.participant_view() == guard.participant_view(), + res.known_retired() == guard.known_retired(), + res.domain() == guard.domain(), + res.root() == guard.root(), + res.reader_registry() == guard.reader_registry(), + res.retire_observation_registry() == guard.retire_observation_registry(), + res.reader_context() == guard.reader_context(), + res.start_view() == guard.start_view(), + res.expired() == guard.expired(), + res.seen_removed() == guard.seen_removed(), + res.protected() == guard.protected(), + res.reader_fragment().fraction() == guard.reader_fragment().fraction() + old( + self, + ).active_record(lease.lease_id()).witness().reader().fraction(), + res.reader_fragment().fraction() == guard.reader_fragment().fraction() * 2real, + { + use_type_invariant(&lease); + let ghost protected_addr = lease.protected_addr(); + let tracked RcuRootReadLease { lease, active } = lease; + active.agree(&self.active_leases); + assert(self.active_ids().contains(lease.lease_id())); + assert(self.active_lease_bindings()[lease.lease_id()] == RcuActiveLeaseBinding::from_record( + self.active_record(lease.lease_id()), + )); + assert(self.active_record(lease.lease_id()).key() == lease.key()); + assert(self.active_record(lease.lease_id()).pool_id() == lease.pool_id()); + assert(self.active_record(lease.lease_id()).fraction() == lease.fraction()); + assert(self.active_record(lease.lease_id()).witness().reader().participant_id() + == guard.participant_id()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().domain() + == guard.domain()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().root() == guard.root()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().reader() + == guard.reader_context()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().start_view() + == guard.start_view()); + assert(self.active_record(lease.lease_id()).witness().protected().ptr().addr() + == protected_addr); + let ghost returned_id = lease.lease_id(); + let ghost returned_key = lease.key(); + self.active_leases.delete_points_to(active); + let tracked witness = self.registry.return_lease(lease); + let tracked guard = witness.tracked_return_to_guard(guard); + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(lease_id != returned_id); + assert(old(self).active_ids().contains(lease_id)); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + }; + assert forall|obj: nat| #[trigger] old(self).contains(obj) implies self.ownership(obj) + == old(self).ownership(obj) by { + if obj == returned_key { + assert(self.registry().pool(obj).resource() == old(self).registry().pool( + obj, + ).resource()); + } else { + assert(self.registry().pool(obj) == old(self).registry().pool(obj)); + } + }; + assert forall|obj: nat| #[trigger] + self.has_unretired_claim(obj) == old(self).has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] self.contains(obj) == old(self).contains(obj) by {}; + guard + } + + /// A completed grace period rules out every still-active lease for the + /// retired allocation. + /// + /// Any lease coexisting with the persistent closed-generation report must + /// be a later reader. Such a reader already knows the retirement record, + /// so its paper guard marks the allocation expired. That contradicts the + /// same lease's protection witness. + pub proof fn lemma_completed_excludes_active( + tracked &mut self, + tracked completed: &RcuReclaimedWitness, + obj: nat, + ) + requires + old(self).wf(), + completed.wf(), + completed.scheduler() == old(self).scheduler(), + completed.record().domain == old(self).domain(), + completed.record().obj == obj, + completed.record().retire_observation_registry == old( + self, + ).retire_observation_registry(), + completed.record().removal.root == old(self).root(), + ensures + *final(self) == *old(self), + !final(self).has_active(obj), + { + if self.has_active(obj) { + assert(exists|lease_id: nat| + #![auto] + self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == obj); + let ghost lease_id = choose|lease_id: nat| + #![auto] + self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == obj; + let ghost record = self.active_record(lease_id); + assert(record.key() == obj); + assert(record.witness().wf()); + assert(record.witness().binding().registry() == self.scheduler()); + assert(record.witness().protected().obj() == obj); + assert(record.witness().protected().domain() == self.domain()); + assert(record.witness().paper_guard().domain() == self.domain()); + assert(record.witness().paper_guard().root() == self.root()); + assert(record.witness().paper_guard().retire_observation_registry() + == self.retire_observation_registry()); + let tracked witness = self.registry.tracked_borrow_active_witness_mut(lease_id); + assert(*witness == record.witness()); + let tracked closed = completed.tracked_closed_generation(witness.reader().cpu()); + witness.lemma_same_participant_as_closed(closed); + assert(closed.participant_id() == witness.reader().participant_id()); + closed.lemma_later_lease_witness_ref(witness); + assert(closed.known_retired().contains(completed.record())); + assert(completed.record().domain == witness.paper_guard().domain()); + assert(completed.record().retire_observation_registry + == witness.paper_guard().retire_observation_registry()); + assert(completed.record().removal.root == witness.paper_guard().root()); + assert(witness.reader().known_retired().contains(completed.record())); + assert(witness.paper_guard().expired().contains(obj)); + assert(witness.protected().protected_by(witness.paper_guard())); + assert(!witness.paper_guard().seen_removed().removed.contains(obj)); + assert(witness.paper_guard().expired().subset_of( + witness.paper_guard().seen_removed().removed, + )); + assert(false); + } + } + + /// Recovers one allocation after completion has ruled out every active + /// lease for its identity. + pub proof fn tracked_reclaim( + tracked &mut self, + tracked claim: RcuReclaimClaim, + tracked completed: RcuReclaimedWitness, + ) -> (tracked ownership: O) + requires + old(self).wf(), + claim.registry() == old(self).reclaim_registry(), + claim.is_pending(), + !old(self).has_active(claim.obj()), + completed.wf(), + completed.scheduler() == old(self).scheduler(), + completed.record().domain == old(self).domain(), + completed.record().obj == claim.obj(), + completed.record().retire_observation_registry == old( + self, + ).retire_observation_registry(), + completed.record().removal.root == old(self).root(), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys().remove(claim.obj()), + final(self).allocations() == old(self).allocations(), + final(self).active_ids() == old(self).active_ids(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|candidate: nat| #[trigger] + final(self).has_unretired_claim(candidate) == old(self).has_unretired_claim( + candidate, + ), + final(self).reclaimed() == old(self).reclaimed().insert(claim.obj(), completed), + forall|candidate: nat| #[trigger] + final(self).keys().contains(candidate) ==> { + &&& old(self).keys().contains(candidate) + &&& old(self).contains(candidate) + &&& final(self).ownership(candidate) == old(self).ownership(candidate) + }, + forall|candidate: nat| #[trigger] + old(self).allocations().contains(candidate) && candidate != claim.obj() + ==> final(self).reclaim_states()[candidate] == old( + self, + ).reclaim_states()[candidate], + old(self).contains(claim.obj()), + claim.ptr() == old(self).reclaim_states()[claim.obj()]->Some_0, + ownership == old(self).registry().pool(claim.obj()).resource(), + ownership == old(self).ownership(claim.obj()), + final(self).reclaimed().contains_pair(claim.obj(), completed), + { + let ghost obj = claim.obj(); + self.registry.lemma_all_contains_iff_keys(); + let tracked RcuReclaimClaim { mut points_to } = claim; + points_to.agree(&self.reclaim_state); + assert(self.reclaim_state@[obj] == Some(points_to.value()->Some_0)); + assert(self.allocations().contains(obj)); + assert(self.keys().contains(obj)); + self.registry.lemma_contains_iff_key(obj); + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + assert(self.contains(obj)); + if self.unretired_claims.contains_key(obj) { + let tracked existing = self.unretired_claims.tracked_borrow_mut(obj); + points_to.disjoint(existing); + assert(false); + } + let ghost old_reclaim_states = self.reclaim_states(); + points_to.update(&mut self.reclaim_state, None); + let tracked ownership = self.registry.reclaim(obj); + self.reclaimed.tracked_insert(obj, completed); + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& witness.binding().registry() == self.scheduler() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).active_record(lease_id).key() != obj); + assert(self.registry().active_records() == old(self).registry().active_records()); + assert(self.registry().active_record(lease_id) == old(self).registry().active_record( + lease_id, + )); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + }; + assert(self.registry().keys().subset_of(self.allocations())); + assert forall|candidate: nat| #[trigger] self.allocations().contains(candidate) implies { + self.keys().contains(candidate) <==> self.reclaim_states()[candidate] is Some + } by { + if candidate == obj { + assert(!self.keys().contains(candidate)); + assert(self.reclaim_states()[candidate] is None); + } else { + assert(self.reclaim_states()[candidate] == old_reclaim_states[candidate]); + assert(self.keys().contains(candidate) == old(self).keys().contains(candidate)); + } + }; + assert(self.unretired_claims().dom().subset_of(self.registry().keys())); + assert(self.reclaimed().dom() == self.allocations().difference(self.keys())) by { + assert(old(self).reclaimed().dom() == old(self).allocations().difference( + old(self).keys(), + )); + }; + assert forall|candidate: nat| #[trigger] self.reclaimed().contains_key(candidate) implies { + let saved = self.reclaimed()[candidate]; + &&& saved.wf() + &&& saved.scheduler() == self.scheduler() + &&& saved.record().domain == self.domain() + &&& saved.record().obj == candidate + &&& saved.record().retire_observation_registry == self.retire_observation_registry() + &&& saved.record().removal.root == self.root() + } by { + if candidate == obj { + assert(self.reclaimed()[candidate] == completed); + } else { + assert(old(self).reclaimed().contains_key(candidate)); + assert(self.reclaimed()[candidate] == old(self).reclaimed()[candidate]); + } + }; + self.registry.lemma_all_contains_iff_keys(); + assert forall|candidate: nat| #[trigger] self.keys().contains(candidate) implies { + &&& old(self).keys().contains(candidate) + &&& old(self).contains(candidate) + &&& self.ownership(candidate) == old(self).ownership(candidate) + } by { + assert(candidate != obj); + assert(self.contains(candidate)); + assert(old(self).keys().contains(candidate)); + assert(old(self).contains(candidate)); + assert(self.registry().pool(candidate) == old(self).registry().pool(candidate)); + }; + ownership + } +} + +impl CpuRcuClosedGeneration { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().state is None + &&& self.resource.value().closed =~= Set::empty().insert(self.report()) + &&& self.report().known_retired == self.known_retired.records() + &&& self.binding.single_local_id() == self.resource.loc() + &&& self.binding.cpu() == self.report().cpu + &&& CpuRcuCarrier::records_observed(self.report().known_retired, self.report().view) + } + + closed spec fn report(self) -> CpuRcuReportView { + choose|report: CpuRcuReportView| self.resource.value().closed.contains(report) + } + + pub closed spec fn participant_id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn binding(self) -> CpuRcuCoreBinding { + self.binding + } + + pub closed spec fn scheduler(self) -> Loc { + self.binding().registry() + } + + pub closed spec fn cpu(self) -> CpuId { + self.report().cpu + } + + pub closed spec fn closed_generation(self) -> nat { + self.report().generation + } + + pub closed spec fn view(self) -> Irc11ThreadView { + self.report().view + } + + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub closed spec fn wf(self) -> bool { + &&& self.resource.value() == CpuRcuCarrier::closed(self.report()) + &&& self.binding().single_local_id() == self.participant_id() + &&& self.binding().cpu() == self.cpu() + } + + /// Relates this report to another client of the scheduler's canonical + /// CPU-local registration. Agreement of the registry entry rules out a + /// second RCU participant identity for the same CPU. + pub proof fn lemma_same_participant_as_binding(tracked &self, tracked other: &CpuRcuCoreBinding) + requires + self.wf(), + other.registry() == self.scheduler(), + other.cpu() == self.cpu(), + other.locals_key().len() == 1, + ensures + other.single_local_id() == self.participant_id(), + { + self.binding.lemma_same_cpu_agree(other); + } + + /// Splits the idempotent closed-generation fact. + pub proof fn tracked_duplicate(tracked self) -> (tracked res: ( + CpuRcuClosedGeneration, + CpuRcuClosedGeneration, + )) + requires + self.wf(), + ensures + res.0.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.0.closed_generation() == self.closed_generation(), + res.0.view() == self.view(), + res.0.known_retired() == self.known_retired(), + res.0.scheduler() == self.scheduler(), + res.0.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.closed_generation(), + res.1.view() == self.view(), + res.1.known_retired() == self.known_retired(), + res.1.scheduler() == self.scheduler(), + res.1.wf(), + { + use_type_invariant(&self); + let ghost report = self.report(); + let ghost records = self.known_retired.records(); + let ghost carrier = CpuRcuCarrier::closed(self.report()); + assert(carrier.closed.union(carrier.closed) =~= carrier.closed); + assert(carrier == CpuRcuCarrier::op(carrier, carrier)); + let tracked (left, right) = self.resource.split(carrier, carrier); + let tracked right_known_retired = self.known_retired.tracked_duplicate(); + let tracked right_binding = self.binding.tracked_duplicate(); + assert(left.value() == carrier); + assert(right.value() == carrier); + lemma_choose_singleton_report(report); + assert((choose|candidate: CpuRcuReportView| left.value().closed.contains(candidate)) + == report); + assert((choose|candidate: CpuRcuReportView| right.value().closed.contains(candidate)) + == report); + assert(report.known_retired == records); + assert(right_known_retired.records() == records); + ( + CpuRcuClosedGeneration { + resource: left, + known_retired: self.known_retired, + binding: self.binding, + }, + CpuRcuClosedGeneration { + resource: right, + known_retired: right_known_retired, + binding: right_binding, + }, + ) + } + + /// Creates another copy of this persistent closed-generation fact. + /// + /// This is admissible because composing the idempotent carrier with itself + /// leaves the carrier unchanged. The monitor uses this operation when one + /// completed grace period authorizes multiple callbacks. + pub proof fn tracked_duplicate_from_ref(tracked &self) -> (tracked duplicate: + CpuRcuClosedGeneration) + requires + self.wf(), + ensures + duplicate.participant_id() == self.participant_id(), + duplicate.cpu() == self.cpu(), + duplicate.closed_generation() == self.closed_generation(), + duplicate.view() == self.view(), + duplicate.known_retired() == self.known_retired(), + duplicate.scheduler() == self.scheduler(), + duplicate.wf(), + { + use_type_invariant(self); + let ghost report = self.report(); + let ghost records = self.known_retired.records(); + let ghost carrier = CpuRcuCarrier::closed(self.report()); + assert(carrier.closed.union(carrier.closed) =~= carrier.closed); + assert(CpuRcuCarrier::op(carrier, carrier) == carrier); + assert(frame_preserving_update_opt::( + carrier, + CpuRcuCarrier::op(carrier, carrier), + )) by { + assert forall|frame: Option| + #![trigger Option::::op(Some(carrier), frame), + Option::::op( + Some(CpuRcuCarrier::op(carrier, carrier)), + frame, + )] + Option::::op(Some(carrier), frame).valid() implies Option::< + CpuRcuCarrier, + >::op(Some(CpuRcuCarrier::op(carrier, carrier)), frame).valid() by { + assert(CpuRcuCarrier::op(carrier, carrier) == carrier); + }; + }; + let tracked resource = self.resource.duplicate_previous(carrier); + let tracked known_retired = self.known_retired.tracked_duplicate(); + let tracked binding = self.binding.tracked_duplicate(); + assert(resource.value() == carrier); + lemma_choose_singleton_report(report); + assert((choose|candidate: CpuRcuReportView| resource.value().closed.contains(candidate)) + == report); + assert(report.known_retired == records); + assert(known_retired.records() == records); + CpuRcuClosedGeneration { resource, known_retired, binding } + } + + /// Any reader coexisting with this report started in a later generation + /// and carries a participant view that includes the report view. + pub proof fn lemma_later_reader( + tracked &self, + tracked mut reader: CpuRcuReaderFragment, + ) -> (tracked res: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + self.participant_id() == reader.participant_id(), + ensures + res == reader, + res.wf(), + self.closed_generation() < res.generation(), + self.view().spec_le(res.participant_view()), + self.known_retired().subset_of(res.known_retired()), + { + use_type_invariant(&reader); + use_type_invariant(self); + assert(reader.resource.value().state_view().known_retired + == reader.known_retired.records()); + assert(self.report().known_retired == self.known_retired.records()); + reader.resource.validate_2(&self.resource); + let ghost report = self.report(); + let ghost reader_state = reader.resource.value().state_view(); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()).valid()); + assert(Option::::op(reader.resource.value().state, None) + == reader.resource.value().state); + assert(reader.resource.value().closed.is_empty()); + assert(reader.resource.value().closed.union(Set::empty().insert(report)) + =~= Set::empty().insert(report)); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()) == CpuRcuCarrier { + state: reader.resource.value().state, + closed: Set::empty().insert(report), + }); + assert(CpuRcuCarrier::reports_fit(reader_state, Set::empty().insert(report))); + assert(Set::empty().insert(report).contains(report)); + assert(report.generation < reader_state.generation); + assert(self.view().spec_le(reader.participant_view())); + assert(report.known_retired.subset_of(reader_state.known_retired)); + assert(self.known_retired().subset_of(reader.known_retired())); + reader + } + + /// Reference-preserving form of [`Self::lemma_later_reader`]. + /// + /// This form is used when a reader fragment is retained as the witness of + /// an active read lease. Validating it against the persistent closed- + /// generation resource does not consume either token. + pub proof fn lemma_later_reader_ref(tracked &self, tracked reader: &mut CpuRcuReaderFragment) + requires + self.wf(), + old(reader).wf(), + self.participant_id() == old(reader).participant_id(), + ensures + *final(reader) == *old(reader), + self.closed_generation() < final(reader).generation(), + self.view().spec_le(final(reader).participant_view()), + self.known_retired().subset_of(final(reader).known_retired()), + { + use_type_invariant(&*reader); + use_type_invariant(self); + assert(reader.resource.value().state_view().known_retired + == reader.known_retired.records()); + assert(self.report().known_retired == self.known_retired.records()); + reader.resource.validate_2(&self.resource); + let ghost report = self.report(); + let ghost reader_state = reader.resource.value().state_view(); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()).valid()); + assert(Option::::op(reader.resource.value().state, None) + == reader.resource.value().state); + assert(reader.resource.value().closed.is_empty()); + assert(reader.resource.value().closed.union(Set::empty().insert(report)) + =~= Set::empty().insert(report)); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()) == CpuRcuCarrier { + state: reader.resource.value().state, + closed: Set::empty().insert(report), + }); + assert(CpuRcuCarrier::reports_fit(reader_state, Set::empty().insert(report))); + assert(Set::empty().insert(report).contains(report)); + assert(report.generation < reader_state.generation); + assert(self.view().spec_le(reader.participant_view())); + assert(report.known_retired.subset_of(reader_state.known_retired)); + assert(self.known_retired().subset_of(reader.known_retired())); + } + + /// Classifies an active physical-lease witness as a later reader without + /// consuming the witness retained by the root registry. + pub proof fn lemma_later_lease_witness_ref( + tracked &self, + tracked witness: &mut CpuRcuReadLeaseWitness, + ) + requires + self.wf(), + old(witness).wf(), + self.participant_id() == old(witness).reader().participant_id(), + ensures + *final(witness) == *old(witness), + self.closed_generation() < final(witness).reader().generation(), + self.view().spec_le(final(witness).reader().participant_view()), + self.known_retired().subset_of(final(witness).reader().known_retired()), + { + self.lemma_later_reader_ref(&mut witness.reader); + } + + /// Lifts [`Self::lemma_later_reader`] to the task view used to start a + /// reader. + /// + /// This is the paper's later-reader branch: once the task imports the + /// persistent CPU view, it also observes every detachment observation + /// carried by an earlier report. + pub proof fn lemma_later_reader_start_view( + tracked &self, + tracked reader: CpuRcuReaderFragment, + start_view: Irc11ThreadView, + ) -> (tracked res: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + self.participant_id() == reader.participant_id(), + reader.participant_view().spec_le(start_view), + ensures + res == reader, + res.wf(), + self.closed_generation() < res.generation(), + self.view().spec_le(start_view), + self.known_retired().subset_of(res.known_retired()), + { + let tracked reader = self.lemma_later_reader(reader); + self.view().lemma_spec_le_transitive(reader.participant_view(), start_view); + reader + } + + /// A reader from a generation covered by this report cannot remain live. + pub proof fn lemma_excludes_old_reader(tracked &self, tracked reader: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + self.participant_id() == reader.participant_id(), + reader.generation() <= self.closed_generation(), + ensures + false, + { + let tracked _reader = self.lemma_later_reader(reader); + } + + /// A complete refined guard coexisting with this report necessarily + /// started after the generation closed by the report. + pub proof fn lemma_later_guard( + tracked &self, + tracked guard: CpuRcuReadGuardToken, + ) -> (tracked res: CpuRcuReadGuardToken) + requires + self.wf(), + guard.wf(), + self.participant_id() == guard.participant_id(), + self.cpu() == guard.cpu(), + ensures + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.reader_fragment() == guard.reader_fragment(), + res.scheduler() == guard.scheduler(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.domain() == guard.domain(), + res.root() == guard.root(), + res.retire_observation_registry() == guard.retire_observation_registry(), + res.start_view() == guard.start_view(), + res.expired() == guard.expired(), + res.seen_removed() == guard.seen_removed(), + self.closed_generation() < res.generation(), + self.view().spec_le(res.start_view()), + self.known_retired().subset_of(res.known_retired()), + { + let tracked (paper_guard, reader, binding) = guard.tracked_into_parts(); + let tracked reader = self.lemma_later_reader_start_view(reader, paper_guard.start_view()); + CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding) + } + + /// Reference-preserving classification of a live refined guard. + pub proof fn lemma_later_guard_ref( + tracked &self, + tracked guard: &mut CpuRcuReadGuardToken, + ) + requires + self.wf(), + old(guard).wf(), + self.participant_id() == old(guard).participant_id(), + self.cpu() == old(guard).cpu(), + ensures + *final(guard) == *old(guard), + self.closed_generation() < final(guard).generation(), + self.view().spec_le(final(guard).start_view()), + self.known_retired().subset_of(final(guard).known_retired()), + { + self.lemma_later_reader_ref(&mut guard.reader); + self.view().lemma_spec_le_transitive(guard.start_view(), guard.start_view()); + } + + /// The old-reader branch of the paper proof: a guard from a generation + /// covered by this report cannot still own its CPU reader fragment. + pub proof fn lemma_excludes_old_guard(tracked &self, tracked guard: CpuRcuReadGuardToken) + requires + self.wf(), + guard.wf(), + self.scheduler() == guard.scheduler(), + self.cpu() == guard.cpu(), + guard.generation() <= self.closed_generation(), + ensures + false, + { + self.binding.lemma_same_cpu_agree(&guard.binding); + assert(self.participant_id() == guard.participant_id()); + let tracked (_paper_guard, reader, _binding) = guard.tracked_into_parts(); + self.lemma_excludes_old_reader(reader); + } +} + +/// Regression proof for both sides of the grace-period reader dichotomy. +proof fn cpu_rcu_generation_smoke_test(cpu: CpuId, initial: Irc11ThreadView, later: Irc11ThreadView) + requires + initial.spec_le(later), +{ + let tracked participant = CpuRcuParticipant::new(cpu, initial); + let tracked core = CpuCoreOwner::new(cpu, participant); + let ghost registration = core.registration(); + let tracked (mut registry, _entries) = GhostMapAuth::new( + Map::::empty(), + ); + let tracked entry = registry.insert(cpu, registration); + let tracked binding = CpuCoreOwnerBinding::tracked_new(entry, &core); + let tracked (handle, participant) = core.tracked_open(); + let tracked (participant, reader) = participant.tracked_start_reader(initial, 0.5real); + let tracked participant = participant.tracked_stop_reader(reader); + assert(participant.fraction() == 1real); + let tracked (participant, closed) = participant.tracked_report_quiescent(binding, later); + let tracked (participant, new_reader) = participant.tracked_start_reader(later, 0.5real); + let tracked new_reader = closed.lemma_later_reader_start_view(new_reader, later); + assert(closed.closed_generation() < new_reader.generation()); + assert(closed.view().spec_le(later)); + let tracked participant = participant.tracked_stop_reader(new_reader); + let tracked _core = handle.tracked_restore(participant); +} + +} // verus! diff --git a/ostd/specs/sync/sc_model.rs b/ostd/specs/sync/sc_model.rs new file mode 100644 index 000000000..bf926e364 --- /dev/null +++ b/ostd/specs/sync/sc_model.rs @@ -0,0 +1,11 @@ +//! This module models the semantics for the SC fences and global SC views +//! coupled with relaxed accesses. It is based on the semantics of the C11 +//! memory model but we exclude the SC access mode which is unnecessary. +//! +//! This is important to verifying some sync primitives like raw RCU. +use vstd::prelude::*; + +verus! { + + +} // verus! diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs new file mode 100644 index 000000000..5689aabaa --- /dev/null +++ b/ostd/specs/sync/weak_memory.rs @@ -0,0 +1,6240 @@ +// SPDX-License-Identifier: MPL-2.0 +//! OSTD-specific adapters for Verus' native IRC11 weak-memory atomics. +//! +//! This module contains only transitions coupled to the RCU root and monitor +//! ghost state. Generic native primitives are re-exported by +//! [`vstd_extra::atomic_irc11`]. +use core::{marker::PhantomData, sync::atomic::Ordering}; + +use super::rcu::RcuTraversalSafety; +use super::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec}; +use crate::specs::mm::cpu::online_cpus; +use vstd::invariant::{AtomicInvariant, InvariantPredicate}; +use vstd::modes::tracked_static_ref; +use vstd::prelude::*; +use vstd::resource::Loc; +use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; +use vstd::thread_view::Objective; +use vstd_extra::atomic_irc11::{ + AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, PAtomicWeakPtr, + ReleaseViewSeen, ThreadView as Irc11ThreadView, ThreadViewOrder as Irc11ThreadViewOrder, + Timestamp, ViewSeen, +}; + +verus! { + +broadcast use {vstd::atomic_weak::group_view_history, vstd::thread_view::group_thread_view_axioms}; + +/// Complete ghost state protected by one RCU root atomic invariant. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRootAtomicState { + pub(crate) points_to: AtomicPointsTo<*mut T>, + pub(crate) root: rcu_spec::RcuRootOwnedGhost, + pub(crate) permissions: rcu_cpu_spec::RcuRootPermissionState, +} + +unsafe impl Objective for RcuRootAtomicState { + +} + +impl RcuRootAtomicState { + pub closed spec fn points_to(self) -> AtomicPointsTo<*mut T> { + self.points_to + } + + pub closed spec fn root(self) -> rcu_spec::RcuRootOwnedGhost { + self.root + } + + pub closed spec fn permissions(self) -> rcu_cpu_spec::RcuRootPermissionState { + self.permissions + } +} + +/// Invariant tying native IRC11 history to paper identities and physical +/// permissions for every allocation that has not yet been reclaimed. +pub struct RcuRootAtomicInv { + _marker: PhantomData, +} + +impl InvariantPredicate< + (rcu_spec::RcuRootKey, Irc11AtomicId), + RcuRootAtomicState, +> for RcuRootAtomicInv where OwnPred: rcu_spec::RcuRootOwnershipPredicate { + open spec fn inv( + key_loc: (rcu_spec::RcuRootKey, Irc11AtomicId), + state: RcuRootAtomicState, + ) -> bool { + let (key, loc) = key_loc; + let g = state.root(); + let permissions = state.permissions(); + &&& rcu_spec::RcuOwnedWeakAtomicInv::::inv( + key_loc, + (state.points_to(), g), + ) + &&& permissions.wf() + &&& permissions.scheduler() == key.scheduler + &&& permissions.domain() == key.domain + &&& permissions.root() == key.domain + &&& permissions.retire_observation_registry() == key.retire_observation_registry + &&& permissions.reclaim_registry() == key.reclaim_registry + &&& permissions.active_lease_registry() == key.active_lease_registry + &&& permissions.allocations() == g.infos().dom() + &&& forall|obj: nat| #[trigger] + permissions.keys().contains(obj) ==> { + &&& permissions.contains(obj) + &&& permissions.allocations().contains(obj) + &&& permissions.reclaim_states().dom().contains(obj) + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } + &&& permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + } + &&& forall|obj: nat| #[trigger] + g.removals().contains_key(obj) ==> !permissions.has_unretired_claim(obj) + &&& forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) ==> { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } + } +} + +pub type RcuRootAtomicInvariant = AtomicInvariant< + (rcu_spec::RcuRootKey, Irc11AtomicId), + RcuRootAtomicState, + RcuRootAtomicInv, +>; + +/// Exposes the permission-state facts carried by the RCU root invariant. +/// +/// Keeping this unfolding lemma beside [`RcuRootAtomicInv`] avoids making +/// executable callback code depend on the invariant's concrete conjunction. +pub(crate) proof fn lemma_root_atomic_permission_facts( + key_loc: (rcu_spec::RcuRootKey, Irc11AtomicId), + tracked state: &RcuRootAtomicState, +) where OwnPred: rcu_spec::RcuRootOwnershipPredicate + requires + RcuRootAtomicInv::::inv(key_loc, *state), + ensures + rcu_spec::RcuOwnedWeakAtomicInv::::inv( + key_loc, + (state.points_to, state.root), + ), + state.root.root().domain_wf(), + state.root.domain() == key_loc.0.domain, + state.root.retire_observation_registry() == key_loc.0.retire_observation_registry, + state.root.removals() == state.root.root().domain_auth().retire_observations(), + state.permissions.wf(), + state.permissions.scheduler() == key_loc.0.scheduler, + state.permissions.domain() == key_loc.0.domain, + state.permissions.root() == key_loc.0.domain, + state.permissions.retire_observation_registry() == key_loc.0.retire_observation_registry, + state.permissions.reclaim_registry() == key_loc.0.reclaim_registry, + state.permissions.active_lease_registry() == key_loc.0.active_lease_registry, + state.permissions.allocations() == state.root.infos().dom(), + state.permissions.unretired_claims().dom() == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }, + forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) ==> { + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states().dom().contains(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 == state.root.infos()[obj].ptr() + &&& OwnPred::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + }, + forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) ==> !state.permissions.has_unretired_claim(obj), + forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) ==> { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + }, +{ +} + +/// Re-folds the root invariant after a proof-only permission-state update. +pub(crate) proof fn lemma_build_root_atomic_inv( + key_loc: (rcu_spec::RcuRootKey, Irc11AtomicId), + tracked state: &RcuRootAtomicState, +) where OwnPred: rcu_spec::RcuRootOwnershipPredicate + requires + rcu_spec::RcuOwnedWeakAtomicInv::::inv( + key_loc, + (state.points_to, state.root), + ), + state.permissions.wf(), + state.permissions.scheduler() == key_loc.0.scheduler, + state.permissions.domain() == key_loc.0.domain, + state.permissions.root() == key_loc.0.domain, + state.permissions.retire_observation_registry() == key_loc.0.retire_observation_registry, + state.permissions.reclaim_registry() == key_loc.0.reclaim_registry, + state.permissions.active_lease_registry() == key_loc.0.active_lease_registry, + state.permissions.allocations() == state.root.infos().dom(), + forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) ==> { + &&& state.permissions.contains(obj) + &&& state.permissions.allocations().contains(obj) + &&& state.permissions.reclaim_states().dom().contains(obj) + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 == state.root.infos()[obj].ptr() + &&& OwnPred::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + }, + state.permissions.unretired_claims().dom() == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }, + forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) ==> !state.permissions.has_unretired_claim(obj), + forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) ==> { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + }, + ensures + RcuRootAtomicInv::::inv(key_loc, *state), +{ +} + +/// Retired root metadata paired with the unique claim for its permission pool. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetiredRootObject { + detached: rcu_spec::RcuRetiredOwnedObject, + claim: rcu_cpu_spec::RcuReclaimClaim, +} + +impl RcuRetiredRootObject { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + &&& self.detached.object().wf() + &&& self.claim.obj() == self.detached.object().obj() + &&& self.claim.is_pending() + &&& equal(self.claim.ptr(), self.detached.object().ptr()) + } + + pub closed spec fn object(self) -> rcu_spec::RcuObjectId { + self.detached.object() + } + + pub closed spec fn retired(self) -> rcu_spec::RcuRetired { + self.detached.retired() + } + + pub closed spec fn ptr(self) -> *mut T { + self.detached.ptr() + } + + pub closed spec fn obj(self) -> nat { + self.detached.obj() + } + + pub closed spec fn claim(self) -> rcu_cpu_spec::RcuReclaimClaim { + self.claim + } + + pub proof fn tracked_into_parts(tracked self) -> (tracked res: ( + rcu_spec::RcuObjectId, + rcu_spec::RcuRetired, + rcu_cpu_spec::RcuReclaimClaim, + )) + ensures + res.0 == self.object(), + res.1 == self.retired(), + res.2 == self.claim(), + res.0.domain() == res.1.domain(), + res.0.obj() == res.1.obj(), + res.0.ptr() == res.1.ptr(), + equal(res.0.ptr(), self.object().ptr()), + res.0.obj() == res.2.obj(), + res.0.wf(), + res.2.is_pending(), + equal(res.2.ptr(), res.0.ptr()), + { + use_type_invariant(&self); + assert(self.claim.obj() == self.detached.object().obj()); + let tracked (object, retired, _unit) = self.detached.tracked_into_parts(); + assert(object == self.detached.object()); + assert(self.claim.obj() == object.obj()); + (object, retired, self.claim) + } +} + +/// Writer resources obtained after the traversal proof has certified that the +/// fixed child has no live incoming edge. +/// +/// This is the handoff between the linked-list layer and base RCU: `retire` +/// authorizes the paper's `rcu-retire` transition, while `claim` is the unique +/// right to recover the physical ownership after a completed grace period. +pub tracked struct LinkedListDetachedChild { + object: rcu_spec::RcuObjectId, + retire: rcu_spec::RcuRetirePerm, + claim: rcu_cpu_spec::RcuReclaimClaim, + ghost removal: rcu_spec::RcuRemovalObservation, +} + +impl LinkedListDetachedChild { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + &&& self.object().wf() + &&& self.retire().wf() + &&& self.retire().ready_to_retire() + &&& self.object().domain() == self.retire().domain() + &&& self.object().obj() == self.retire().obj() + &&& self.object().ptr() == self.retire().ptr() + &&& self.claim().obj() == self.object().obj() + &&& self.claim().is_pending() + &&& equal(self.claim().ptr(), self.object().ptr()) + } + + pub closed spec fn object(self) -> rcu_spec::RcuObjectId { + self.object + } + + pub closed spec fn retire(self) -> rcu_spec::RcuRetirePerm { + self.retire + } + + pub closed spec fn claim(self) -> rcu_cpu_spec::RcuReclaimClaim { + self.claim + } + + pub closed spec fn removal(self) -> rcu_spec::RcuRemovalObservation { + self.removal + } + + /// Performs the base-RCU retirement transition after traversal retirement. + pub proof fn tracked_retire( + tracked self, + tracked domain: &mut rcu_spec::RcuDomainAuth, + ) -> (tracked res: LinkedListRetiredChild) + requires + old(domain).wf(), + self.retire().belongs_to(*old(domain)), + self.object().domain() == old(domain).id(), + ensures + final(domain).wf(), + final(domain).id() == old(domain).id(), + res.object() == self.object(), + res.claim() == self.claim(), + res.retired().domain() == self.object().domain(), + res.retired().obj() == self.object().obj(), + res.retired().ptr() == self.object().ptr(), + res.retired().removal() == self.removal(), + { + use_type_invariant(&self); + let ghost object_value = self.object(); + let ghost claim_value = self.claim(); + assert(self.object().domain() == old(domain).id()); + let tracked LinkedListDetachedChild { object, retire, claim, removal } = self; + let tracked retired = domain.tracked_retire(retire, removal); + assert(object == object_value); + assert(claim == claim_value); + assert(object.wf()); + assert(retired.wf()); + assert(retired.domain() == old(domain).id()); + assert(object.domain() == retired.domain()); + assert(object.obj() == retired.obj()); + assert(object.ptr() == retired.ptr()); + assert(claim.obj() == object.obj()); + assert(claim.is_pending()); + assert(equal(claim.ptr(), object.ptr())); + LinkedListRetiredChild { object, retired, claim } + } +} + +/// A retired linked-list child paired with the claim that will eventually +/// recover its physical ownership. +pub tracked struct LinkedListRetiredChild { + object: rcu_spec::RcuObjectId, + retired: rcu_spec::RcuRetired, + claim: rcu_cpu_spec::RcuReclaimClaim, +} + +impl LinkedListRetiredChild { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + &&& self.object().wf() + &&& self.retired().wf() + &&& self.object().domain() == self.retired().domain() + &&& self.object().obj() == self.retired().obj() + &&& self.object().ptr() == self.retired().ptr() + &&& self.claim().obj() == self.object().obj() + &&& self.claim().is_pending() + &&& equal(self.claim().ptr(), self.object().ptr()) + } + + pub closed spec fn object(self) -> rcu_spec::RcuObjectId { + self.object + } + + pub closed spec fn retired(self) -> rcu_spec::RcuRetired { + self.retired + } + + pub closed spec fn claim(self) -> rcu_cpu_spec::RcuReclaimClaim { + self.claim + } + + /// Compresses the typed traversal retirement into the type-erased safety + /// certificate consumed by the existing callback monitor. + pub proof fn tracked_certify_callback(tracked self) -> (tracked res: ( + rcu_spec::RcuObjectId, + rcu_spec::RcuCallbackSafety, + rcu_cpu_spec::RcuReclaimClaim, + )) + ensures + res.0 == self.object(), + res.2 == self.claim(), + res.1.domain() == res.0.domain(), + res.1.obj() == res.0.obj(), + res.1.removal() == self.retired().removal(), + res.1.retire_observation_registry() == self.retired().retire_observation_registry(), + rcu_spec::callback_safety_from_traversal(res.1, res.0), + { + use_type_invariant(&self); + let tracked LinkedListRetiredChild { object, retired, claim } = self; + let tracked cert = rcu_spec::certify_callback_from_retired(&object, retired); + (object, cert, claim) + } +} + +/// Immutable identities carried by a native linked-list link whose target is +/// selected from an extensible allocation-ID registry. +pub ghost struct RegisteredLinkedListAtomicKey { + pub scheduler: Loc, + pub domain: Loc, + pub root: Loc, + pub retire_observation_registry: Loc, + pub reclaim_registry: Loc, + pub active_lease_registry: Loc, + pub registry: Loc, + pub current: Loc, + pub lifecycle: Loc, + pub timestamp_registry: Loc, + pub native_observation_registry: Loc, + pub source: *mut rcu_spec::LinkedListNode, + pub source_obj: nat, +} + +/// Complete proof state for one native link with arbitrary registered targets. +/// +/// `registry` is append-only in the current API. `current` records the AId of +/// the latest non-null successor independently of its address, which is what +/// makes a successful address-based CAS safe when a reclaimed address is later +/// reused by a fresh allocation identity. +pub tracked struct RegisteredLinkedListAtomicState { + pub(crate) points_to: AtomicPointsTo<*mut rcu_spec::LinkedListNode>, + pub(crate) link: rcu_spec::LinkedListAtomicLinkGhost, + pub(crate) auth: rcu_spec::LinkedListTraversalAuth, + pub(crate) registry: GhostVarAuth>, + pub(crate) current: GhostVarAuth>, + pub(crate) permissions: rcu_cpu_spec::RcuRootPermissionState, + pub(crate) lifecycle: GhostVarAuth>, +} + +unsafe impl Objective for RegisteredLinkedListAtomicState { + +} + +impl RegisteredLinkedListAtomicState { + pub closed spec fn points_to(self) -> AtomicPointsTo<*mut rcu_spec::LinkedListNode> { + self.points_to + } + + pub closed spec fn link(self) -> rcu_spec::LinkedListAtomicLinkGhost { + self.link + } + + pub closed spec fn auth(self) -> rcu_spec::LinkedListTraversalAuth { + self.auth + } + + pub closed spec fn registry(self) -> GhostVarAuth> { + self.registry + } + + pub closed spec fn current(self) -> GhostVarAuth> { + self.current + } + + pub closed spec fn permissions(self) -> rcu_cpu_spec::RcuRootPermissionState< + rcu_spec::LinkedListNode, + O, + > { + self.permissions + } + + pub closed spec fn lifecycle(self) -> GhostVarAuth> { + self.lifecycle + } +} + +/// Per-allocation part of the registered-link invariant. Keeping this fact +/// separate lets operations preserve unaffected AIds without unfolding the +/// complete native atomic protocol. +pub open spec fn registered_linked_list_target_inv( + key: RegisteredLinkedListAtomicKey, + state: RegisteredLinkedListAtomicState, + obj: nat, +) -> bool where OwnPred: rcu_spec::RcuRootOwnershipPredicate { + let phase = state.lifecycle()@[obj]; + &&& obj != key.source_obj + &&& state.registry()@.contains_key(obj) + &&& (state.auth().removed().contains(obj) <==> phase.is_retired() || phase.is_reclaimed()) + &&& (state.auth().has_retire_perm(obj) <==> !phase.is_retired() && !phase.is_reclaimed()) + &&& (state.permissions().contains(obj) <==> !phase.is_reclaimed()) + &&& state.permissions().reclaim_states()[obj] == if phase.is_reclaimed() { + None + } else { + Some(state.registry()@[obj]) + } + &&& (state.permissions().has_unretired_claim(obj) <==> !phase.is_retired() + && !phase.is_reclaimed()) + &&& (!phase.is_reclaimed() ==> OwnPred::owns( + state.registry()@[obj], + state.permissions().ownership(obj), + )) + &&& (phase.is_linked() <==> state.current()@ == Some(obj)) + &&& match phase { + LinkedListChildPhase::Unpublished => { + state.auth().state().incoming_all[obj] == Set::::empty() + }, + LinkedListChildPhase::Linked { index, timestamp } => { + &&& state.link().current_timestamp() == timestamp + &&& state.link().index_at(timestamp) == index + &&& index + 1 == state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj].last() == Some( + (state.registry()@[obj], obj), + ) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth().state().incoming_all[obj].contains(edge) ==> { + &&& edge.0 == key.source_obj + &&& edge.1 <= index + } + }, + LinkedListChildPhase::Unlinked { index, removal } + | LinkedListChildPhase::Retired { index, removal } => { + &&& removal.root == key.root + &&& state.points_to().hist().contains_timestamp(removal.timestamp) + &&& state.link().index_at(removal.timestamp) == index + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& index < state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj][index as int] is None + &&& index > 0 + &&& state.auth().state().successors[key.source_obj][(index - 1) as int] == Some( + (state.registry()@[obj], obj), + ) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth().state().incoming_all[obj].contains(edge) ==> { + &&& edge.0 == key.source_obj + &&& edge.1 < index + } + }, + LinkedListChildPhase::Reclaimed { index, removal } => { + &&& state.permissions().reclaimed().contains_key(obj) + &&& state.permissions().reclaimed()[obj].record().removal == removal + &&& removal.root == key.root + &&& state.points_to().hist().contains_timestamp(removal.timestamp) + &&& state.link().index_at(removal.timestamp) == index + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& index < state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj][index as int] is None + &&& index > 0 + &&& state.auth().state().successors[key.source_obj][(index - 1) as int] == Some( + (state.registry()@[obj], obj), + ) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth().state().incoming_all[obj].contains(edge) ==> { + &&& edge.0 == key.source_obj + &&& edge.1 < index + } + }, + } +} + +/// Native IRC11 invariant for one link ranging over any registered AId. +pub struct RegisteredLinkedListAtomicInv { + _marker: PhantomData, +} + +impl InvariantPredicate< + (RegisteredLinkedListAtomicKey, Irc11AtomicId), + RegisteredLinkedListAtomicState, +> for RegisteredLinkedListAtomicInv where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + open spec fn inv( + key_loc: (RegisteredLinkedListAtomicKey, Irc11AtomicId), + state: RegisteredLinkedListAtomicState, + ) -> bool { + let (key, loc) = key_loc; + let target_ids = state.registry()@.dom().remove(key.source_obj); + &&& state.points_to().loc() == loc + &&& key.source.addr() != 0 + &&& state.auth().wf() + &&& state.auth().domain() == key.domain + &&& state.auth().state().root == key.source + &&& state.auth().state().root_obj == key.source_obj + &&& state.auth().removed().subset_of(target_ids) + &&& state.registry().id() == key.registry + &&& state.registry()@ == state.auth().state().objects + &&& state.current().id() == key.current + &&& state.lifecycle().id() == key.lifecycle + &&& state.lifecycle()@.dom() == target_ids + &&& state.permissions().wf() + &&& state.permissions().scheduler() == key.scheduler + &&& state.permissions().domain() == key.domain + &&& state.permissions().root() == key.root + &&& state.permissions().retire_observation_registry() == key.retire_observation_registry + &&& state.permissions().reclaim_registry() == key.reclaim_registry + &&& state.permissions().active_lease_registry() == key.active_lease_registry + &&& state.permissions().allocations() == target_ids + &&& forall|obj: nat| #[trigger] + target_ids.contains(obj) ==> registered_linked_list_target_inv::( + key, + state, + obj, + ) + &&& state.link().source() == key.source + &&& state.link().source_obj() == key.source_obj + &&& state.link().timestamp_registry() == key.timestamp_registry + &&& state.link().native_observation_registry() == key.native_observation_registry + &&& state.link().wf(state.points_to().hist(), state.auth()) + &&& state.link().native_observations_wf(state.points_to()) + &&& forall|n: rcu_spec::LinkIndex| + n < state.auth().state().successors[key.source_obj].len() + && state.auth().state().successors[key.source_obj][n as int] is Some + ==> #[trigger] state.auth().state().successors[key.source_obj][n as int]->Some_0.1 + != key.source_obj + &&& match state.current()@ { + None => state.auth().state().successors[key.source_obj].last() is None, + Some(obj) => { + &&& obj != key.source_obj + &&& state.registry()@.contains_key(obj) + &&& state.auth().state().successors[key.source_obj].last() == Some( + (state.registry()@[obj], obj), + ) + }, + } + } +} + +pub type RegisteredLinkedListAtomicInvariant = AtomicInvariant< + (RegisteredLinkedListAtomicKey, Irc11AtomicId), + RegisteredLinkedListAtomicState, + RegisteredLinkedListAtomicInv, +>; + +/// A real weak atomic link whose non-null values are selected by AId from an +/// extensible registry rather than fixed by the wrapper's constructor. +pub struct RegisteredLinkedListWeakAtomicLink { + atomic: PAtomicWeakPtr, + tracked_atomic_inv: Tracked<&'static RegisteredLinkedListAtomicInvariant>, + tracked_registry: Tracked>>, + tracked_current: Tracked>>, + tracked_lifecycle: Tracked>>, +} + +impl RegisteredLinkedListWeakAtomicLink { + pub closed spec fn constant(&self) -> RegisteredLinkedListAtomicKey { + self.tracked_atomic_inv@.constant().0 + } + + pub closed spec fn native_loc(&self) -> Irc11AtomicId { + self.atomic.loc() + } + + pub closed spec fn invariant_namespace(&self) -> int { + self.tracked_atomic_inv@.namespace() + } + + pub closed spec fn registered_targets(&self) -> Map { + self.tracked_registry@.view() + } + + pub closed spec fn current_target(&self) -> Option { + self.tracked_current@.view() + } + + pub closed spec fn target_lifecycles(&self) -> Map { + self.tracked_lifecycle@.view() + } + + pub closed spec fn target_phase(&self, obj: nat) -> LinkedListChildPhase + recommends + self.target_lifecycles().contains_key(obj), + { + self.target_lifecycles()[obj] + } + + pub open spec fn no_reclaimed_targets(&self) -> bool { + forall|obj: nat| #[trigger] + self.registered_targets().dom().remove(self.constant().source_obj).contains(obj) + ==> self.target_lifecycles().contains_key(obj) && !self.target_phase( + obj, + ).is_reclaimed() + } + + pub closed spec fn well_formed(&self) -> bool { + &&& self.tracked_atomic_inv@.constant().1 == self.native_loc() + &&& self.tracked_registry@.id() == self.constant().registry + &&& self.tracked_current@.id() == self.constant().current + &&& self.tracked_lifecycle@.id() == self.constant().lifecycle + &&& self.registered_targets().contains_pair( + self.constant().source_obj, + self.constant().source, + ) + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } +} + +impl RegisteredLinkedListWeakAtomicLink where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + /// Creates a null native link. Additional target AIds can be registered + /// after construction and then selected by the publication CAS. + pub const fn new( + Ghost(scheduler): Ghost, + Ghost(root): Ghost, + Ghost(retire_observation_registry): Ghost, + Tracked(source_info): Tracked<&rcu_spec::RcuBlockInfo>, + Tracked(source_retire): Tracked>, + ) -> (res: Self) + requires + source_info.wf(), + source_retire.wf(), + source_retire.domain() == source_info.domain(), + source_retire.obj() == source_info.obj(), + source_retire.ptr() == source_info.ptr(), + ensures + res.well_formed(), + res.constant().scheduler == scheduler, + res.constant().domain == source_info.domain(), + res.constant().root == root, + res.constant().retire_observation_registry == retire_observation_registry, + res.constant().source == source_info.ptr(), + res.constant().source_obj == source_info.obj(), + res.registered_targets() == Map::empty().insert(source_info.obj(), source_info.ptr()), + res.current_target() is None, + res.target_lifecycles() == Map::::empty(), + { + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + PAtomicWeakPtr::new(core::ptr::null_mut()); + let tracked mut auth = rcu_spec::LinkedListTraversalAuth::tracked_new( + source_info, + source_retire, + ); + let ghost initial_index = auth.tracked_initialize_null( + source_info.ptr(), + source_info.obj(), + ); + let tracked link = rcu_spec::LinkedListAtomicLinkGhost::tracked_initial_null( + points_to.hist(), + timestamp, + initial_view@, + &auth, + source_info.ptr(), + source_info.obj(), + ); + let tracked (registry, registry_peer) = GhostVarAuth::new( + Map::empty().insert(source_info.obj(), source_info.ptr()), + ); + let tracked (current, current_peer) = GhostVarAuth::new(None); + let tracked permissions = rcu_cpu_spec::RcuRootPermissionState::empty( + scheduler, + source_info.domain(), + root, + retire_observation_registry, + ); + let tracked (lifecycle, lifecycle_peer) = GhostVarAuth::new(Map::empty()); + let tracked state = RegisteredLinkedListAtomicState { + points_to, + link, + auth, + registry, + current, + permissions, + lifecycle, + }; + let ghost key = RegisteredLinkedListAtomicKey { + scheduler, + domain: source_info.domain(), + root, + retire_observation_registry, + reclaim_registry: state.permissions().reclaim_registry(), + active_lease_registry: state.permissions().active_lease_registry(), + registry: state.registry().id(), + current: state.current().id(), + lifecycle: state.lifecycle().id(), + timestamp_registry: state.link().timestamp_registry(), + native_observation_registry: state.link().native_observation_registry(), + source: source_info.ptr(), + source_obj: source_info.obj(), + }; + proof { + source_info.lemma_wf_facts(); + assert(initial_index == 0); + assert(state.points_to().loc() == atomic.loc()); + assert(state.auth().state().objects == Map::empty().insert(key.source_obj, key.source)); + assert(state.auth().removed() == Set::::empty()); + assert(state.registry()@ == state.auth().state().objects); + assert(state.registry()@.dom().remove(key.source_obj) == Set::::empty()); + assert(state.lifecycle()@ == Map::::empty()); + assert(state.permissions().allocations() == Set::::empty()); + assert(state.permissions().keys() == Set::::empty()); + assert(state.permissions().unretired_claims().dom() == Set::::empty()); + assert(state.link().native_observations() == Map::empty()); + assert(state.link().native_observations_wf(state.points_to())) by { + assert forall|observation_id: nat| #[trigger] + state.link().native_observations().contains_key(observation_id) implies { + let observation = state.link().native_observations()[observation_id]; + &&& observation.0 == state.points_to().loc() + &&& state.points_to().get_timestamp(observation.1) == Some(observation.2) + } by {}; + }; + assert(RegisteredLinkedListAtomicInv::::inv((key, atomic.loc()), state)); + } + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); + let tracked atomic_inv = tracked_static_ref(atomic_inv); + RegisteredLinkedListWeakAtomicLink { + atomic, + tracked_atomic_inv: Tracked(atomic_inv), + tracked_registry: Tracked(registry_peer), + tracked_current: Tracked(current_peer), + tracked_lifecycle: Tracked(lifecycle_peer), + } + } + + fn raw_atomic(&self) -> (res: &PAtomicWeakPtr) + requires + self.well_formed(), + ensures + res.loc() == self.native_loc(), + opens_invariants none + no_unwind + { + &self.atomic + } + + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: + &'static RegisteredLinkedListAtomicInvariant) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.native_loc()), + { + self.tracked_atomic_inv.get() + } + + /// Adds a fresh allocation identity to this link's target registry under + /// the caller's writer-side invariant-opening authority. + pub proof fn tracked_register_target( + tracked &mut self, + target: *mut rcu_spec::LinkedListNode, + tracked info: &rcu_spec::RcuBlockInfo, + tracked retire: rcu_spec::RcuBaseRetirePerm, + tracked ownership: O, + tracked credit: vstd::invariant::OpenInvariantCredit, + ) + requires + old(self).well_formed(), + info.wf(), + retire.wf(), + info.domain() == old(self).constant().domain, + retire.domain() == old(self).constant().domain, + retire.obj() == info.obj(), + retire.ptr() == info.ptr(), + equal(target, info.ptr()), + info.obj() != old(self).constant().source_obj, + !old(self).registered_targets().contains_key(info.obj()), + OwnPred::owns(target, ownership), + ensures + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets().insert( + info.obj(), + target, + ), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + info.obj(), + LinkedListChildPhase::Unpublished, + ), + opens_invariants [self.invariant_namespace()] + { + use_type_invariant(&*self); + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + let ghost state_before = state; + let ghost old_auth = state.auth; + let ghost old_objects = state.auth.state().objects; + let ghost old_registry = state.registry@; + let ghost old_lifecycle = state.lifecycle@; + let ghost old_permissions = state.permissions; + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(old_registry == self.registered_targets()); + assert(!old_objects.contains_key(info.obj())); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_live_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + state.auth.lemma_registry_domains(); + assert(state.auth.state().successors.contains_key(key.source_obj)); + state.auth.lemma_unregistered_has_no_resources(info.obj()); + state.link.tracked_register_target( + state.points_to.hist(), + &mut state.auth, + info, + retire, + ); + state.registry.update( + self.tracked_registry.borrow_mut(), + old_registry.insert(info.obj(), target), + ); + state.permissions.tracked_insert(info, ownership); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert(info.obj(), LinkedListChildPhase::Unpublished), + ); + assert(state.registry@ == state.auth.state().objects); + state.auth.lemma_registry_domains(); + match state.current@ { + None => { + assert(state.auth.state().successors[key.source_obj] + == old_auth.state().successors[key.source_obj]); + }, + Some(obj) => { + assert(obj != info.obj()); + assert(old_registry.contains_key(obj)); + assert(state.registry@.contains_key(obj)); + assert(state.registry@[obj] == old_registry[obj]); + assert(state.auth.state().successors[key.source_obj] + == old_auth.state().successors[key.source_obj]); + }, + } + assert(state.registry@.dom().remove(key.source_obj) + == old_registry.dom().remove(key.source_obj).insert(info.obj())); + assert(state.permissions.allocations() + == old_permissions.allocations().insert(info.obj())); + assert(state.permissions.keys() == old_permissions.keys().insert(info.obj())); + assert(state.permissions.unretired_claims().dom() + == old_permissions.unretired_claims().dom().insert(info.obj())); + assert(state.lifecycle@.dom() == old_lifecycle.dom().insert(info.obj())); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + state.permissions.lemma_all_live_reclaim_states(); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(obj != key.source_obj); + assert(state.registry@.contains_key(obj)); + assert(state.permissions.allocations().contains(obj)); + if obj == info.obj() { + assert(state.registry@[obj] == target); + assert(state.permissions.reclaim_states()[obj] == Some(target)); + assert(state.permissions.ownership(obj) == ownership); + assert(state.permissions.unretired_claims().dom().contains(obj)); + assert(state.permissions.has_unretired_claim(obj)); + assert(state.lifecycle@[obj] is Unpublished); + assert(state.current@ != Some(obj)); + assert(state.auth.state().incoming_all[obj] + == Set::::empty()); + assert(!state.auth.removed().contains(obj)); + assert(state.auth.has_retire_perm(obj)); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + assert(old_registry.dom().remove(key.source_obj).contains(obj)); + assert(old_permissions.allocations().contains(obj)); + assert(state.permissions.allocations().contains(obj)); + assert(state.permissions.reclaim_states().dom().contains(obj)); + assert(state.registry@[obj] == old_registry[obj]); + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.current == state_before.current); + assert(state.auth.removed() == state_before.auth.removed()); + assert(state.auth.has_retire_perm(obj) + == state_before.auth.has_retire_perm(obj)); + assert(state.auth.state().successors[key.source_obj] + == state_before.auth.state().successors[key.source_obj]); + assert(state.auth.state().incoming_all[obj] + == state_before.auth.state().incoming_all[obj]); + assert(state.permissions.reclaim_states()[obj] + == old_permissions.reclaim_states()[obj]); + assert(state.permissions.has_unretired_claim(obj) + == old_permissions.has_unretired_claim(obj)); + assert(state.permissions.reclaimed() == old_permissions.reclaimed()); + if !state.lifecycle@[obj].is_reclaimed() { + assert(old_permissions.contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == old_permissions.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + }); + } + + /// Acquire-loads the native link and resolves a non-null message to the + /// exact registered AId selected by that historical message. + #[inline(always)] + pub fn load_acquire_and_protect( + &self, + Tracked(guard): Tracked<&mut rcu_spec::RcuReadGuardToken>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + old(guard).wf(), + old(guard).domain() == self.constant().domain, + old(guard).seen_removed().removed == Set::::empty(), + match previous { + None => old(guard).seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& old(guard).seen_at(self.constant().source_obj) == observation.index() + }, + }, + old(from).protected_by(*old(guard)), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).seen_removed().removed == old(guard).seen_removed().removed, + final(guard).seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.4@.registry() == self.constant().timestamp_registry, + res.4@.native_registry() == self.constant().native_observation_registry, + res.4@.loc() == self.native_loc(), + res.4@.timestamp() == res.1@, + res.4@.index() == res.2@, + res.4@.view() == final(tv)@, + (res.3@ is Some) == (res.0.addr() != 0), + match res.3@ { + None => res.0.addr() == 0, + Some(child) => { + &&& equal(child.ptr(), res.0) + &&& child.domain() == self.constant().domain + &&& child.protected_by(*final(guard)) + &&& self.registered_targets().contains_pair(child.obj(), child.ptr()) + }, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(&*self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + match previous { + None => {}, + Some(observation) => { + use_type_invariant(observation); + state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + match previous { + None => {}, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.index() <= index); + }, + } + assert(guard.seen_at(from.obj()) <= index); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + old(guard).seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + old(guard).seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> old(guard).seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + guard, + from, + timestamp, + ); + let tracked observation = state.link.tracked_observation_at( + &state.points_to, + &state.auth, + timestamp, + tv@, + ); + } + proof { + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + match &protected { + None => {}, + Some(child) => { + assert(state.auth.state().objects.contains_pair( + child.obj(), + child.ptr(), + )); + assert(state.registry@.contains_pair(child.obj(), child.ptr())); + assert(self.registered_targets().contains_pair( + child.obj(), + child.ptr(), + )); + }, + } + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(protected), + Tracked(observation), + ); + }); + result + } + + /// Acquire-loads any registered target and splits that target's AId-keyed + /// physical permission pool in the same native atomic invariant opening. + #[inline(always)] + pub fn load_acquire_and_lease_cpu( + &self, + Tracked(guard): Tracked>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>, + Tracked>>, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + online_cpus().contains(guard.cpu()), + guard.seen_removed().removed == Set::::empty(), + match previous { + None => guard.paper_guard().seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& guard.paper_guard().seen_at(self.constant().source_obj) + == observation.index() + }, + }, + old(from).protected_by(guard.paper_guard()), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + res.3@.wf(), + res.3@.binding() == guard.binding(), + res.3@.participant_id() == guard.participant_id(), + res.3@.cpu() == guard.cpu(), + res.3@.generation() == guard.generation(), + res.3@.participant_view() == guard.participant_view(), + res.3@.known_retired() == guard.known_retired(), + res.3@.scheduler() == guard.scheduler(), + res.3@.domain() == guard.domain(), + res.3@.root() == guard.root(), + res.3@.reader_registry() == guard.reader_registry(), + res.3@.retire_observation_registry() == guard.retire_observation_registry(), + res.3@.reader_context() == guard.reader_context(), + res.3@.start_view() == guard.start_view(), + res.3@.expired() == guard.expired(), + res.3@.seen_removed().removed == guard.seen_removed().removed, + res.3@.paper_guard().seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.6@.registry() == self.constant().timestamp_registry, + res.6@.native_registry() == self.constant().native_observation_registry, + res.6@.loc() == self.native_loc(), + res.6@.timestamp() == res.1@, + res.6@.index() == res.2@, + res.6@.view() == final(tv)@, + (res.4@ is Some) == (res.0.addr() != 0), + res.5@ is Some ==> res.4@ is Some, + match (res.4@, res.5@) { + (None, None) => { + &&& res.0.addr() == 0 + &&& res.3@.reader_fragment() == guard.reader_fragment() + }, + (Some(child), Some(lease)) => { + &&& equal(child.ptr(), res.0) + &&& child.domain() == self.constant().domain + &&& child.protected_by(res.3@.paper_guard()) + &&& child.obj() != self.constant().source_obj + &&& self.registered_targets().contains_pair(child.obj(), child.ptr()) + &&& self.target_lifecycles().contains_key(child.obj()) + &&& !self.target_phase(child.obj()).is_reclaimed() + &&& res.3@.reader_fragment().fraction() == guard.reader_fragment().fraction() + / 2real + &&& lease.key() == child.obj() + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == res.3@.participant_id() + &&& lease.reader_fraction() == res.3@.reader_fragment().fraction() + &&& lease.domain() == res.3@.domain() + &&& lease.root() == res.3@.root() + &&& lease.reader_context() == res.3@.reader_context() + &&& lease.start_view() == res.3@.start_view() + &&& lease.protected_addr() == child.ptr().addr() + &&& OwnPred::owns(child.ptr(), lease.resource()) + }, + (Some(child), None) => { + &&& equal(child.ptr(), res.0) + &&& child.domain() == self.constant().domain + &&& child.protected_by(res.3@.paper_guard()) + &&& child.obj() != self.constant().source_obj + &&& self.registered_targets().contains_pair(child.obj(), child.ptr()) + &&& self.target_lifecycles().contains_key(child.obj()) + &&& self.target_phase(child.obj()).is_reclaimed() + &&& res.3@.reader_fragment() == guard.reader_fragment() + }, + (None, Some(_)) => false, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(self); + } + proof_decl! { + let tracked (mut paper_guard, cpu_reader, binding) = guard.tracked_into_parts(); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost state_before = state; + let ghost permissions_before = state.permissions; + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + match previous { + None => {}, + Some(observation) => { + use_type_invariant(observation); + state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + match previous { + None => {}, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.index() <= index); + }, + } + assert(paper_guard.seen_at(from.obj()) <= index); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + paper_guard.seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + paper_guard.seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> paper_guard.seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + &mut paper_guard, + from, + timestamp, + ); + let tracked cpu_guard = rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new( + paper_guard, + cpu_reader, + binding, + ); + let tracked final_guard; + let tracked lease; + let tracked observation; + } + proof { + observation = state.link.tracked_observation_at( + &state.points_to, + &state.auth, + timestamp, + tv@, + ); + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + state.permissions.lemma_all_contains_iff_keys(); + match &protected { + None => { + final_guard = cpu_guard; + lease = None; + }, + Some(child) => { + assert(rcu_spec::LinkedListTraversalSpec::link_inv( + self.constant().source, + self.constant().source_obj, + index, + child.ptr(), + child.obj(), + state.auth.state(), + )); + assert(state.auth.state().successors[ + self.constant().source_obj + ][index as int] == Some((child.ptr(), child.obj()))); + assert(state.registry@.contains_pair(child.obj(), child.ptr())); + assert(child.obj() != self.constant().source_obj); + assert(state.registry@.dom().remove( + self.constant().source_obj, + ).contains(child.obj())); + assert(registered_linked_list_target_inv::( + self.constant(), + state, + child.obj(), + )); + if state.lifecycle@[child.obj()].is_reclaimed() { + final_guard = cpu_guard; + lease = None; + } else { + assert(state.permissions.contains(child.obj())); + let ghost ownership = state.permissions.ownership(child.obj()); + assert(OwnPred::owns(child.ptr(), ownership)); + let tracked split = state.permissions.tracked_split_protected( + cpu_guard, + child, + ); + final_guard = split.0; + lease = Some(split.1); + assert(split.1.resource() == ownership); + assert(OwnPred::owns(child.ptr(), split.1.resource())); + } + }, + } + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() == permissions_before.reclaim_states()); + assert(state.permissions.reclaimed() == permissions_before.reclaimed()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + assert(state.points_to == state_before.points_to); + assert(state.link.source() == state_before.link.source()); + assert(state.link.source_obj() == state_before.link.source_obj()); + assert(state.link.timestamps() == state_before.link.timestamps()); + assert(state.link.current_timestamp() + == state_before.link.current_timestamp()); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle == state_before.lifecycle); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(self.constant().source_obj).contains(obj) implies + registered_linked_list_target_inv::( + self.constant(), + state, + obj, + ) by { + assert(registered_linked_list_target_inv::( + self.constant(), + state_before, + obj, + )); + if !state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.contains(obj)); + assert(permissions_before.keys().contains(obj)); + assert(state.permissions.keys().contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + }; + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(final_guard), + Tracked(protected), + Tracked(lease), + Tracked(observation), + ); + }); + result + } + + /// Returns an AId-keyed traversal lease and rejoins its saved CPU reader + /// fraction with the live guard. + #[verifier::atomic] + pub fn return_registered_lease_cpu( + &self, + Tracked(lease): Tracked>>, + Tracked(guard): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked>) + requires + self.well_formed(), + match lease { + None => true, + Some(lease) => { + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + }, + }, + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + old(tv)@.spec_le(final(tv)@), + res@.wf(), + res@.paper_guard() == guard.paper_guard(), + res@.binding() == guard.binding(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + res@.participant_view() == guard.participant_view(), + res@.known_retired() == guard.known_retired(), + res@.domain() == guard.domain(), + res@.root() == guard.root(), + res@.reader_registry() == guard.reader_registry(), + res@.retire_observation_registry() == guard.retire_observation_registry(), + res@.reader_context() == guard.reader_context(), + res@.start_view() == guard.start_view(), + res@.expired() == guard.expired(), + res@.seen_removed() == guard.seen_removed(), + res@.protected() == guard.protected(), + res@.reader_fragment().fraction() == match lease { + None => guard.reader_fragment().fraction(), + Some(_) => guard.reader_fragment().fraction() * 2real, + }, + no_unwind + { + let raw_atomic = &self.atomic; + proof_decl! { + let tracked final_guard; + } + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost state_before = state; + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + state.permissions.lemma_all_contains_iff_keys(); + match lease { + None => { + final_guard = guard; + }, + Some(lease) => { + final_guard = state.permissions.tracked_return_loaded(lease, guard); + }, + } + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() == permissions_before.reclaim_states()); + assert(state.permissions.reclaimed() == permissions_before.reclaimed()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + assert(state.points_to == state_before.points_to); + assert(state.link.source() == state_before.link.source()); + assert(state.link.source_obj() == state_before.link.source_obj()); + assert(state.link.timestamps() == state_before.link.timestamps()); + assert(state.link.current_timestamp() + == state_before.link.current_timestamp()); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle == state_before.lifecycle); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(self.constant().source_obj).contains(obj) implies + registered_linked_list_target_inv::( + self.constant(), + state, + obj, + ) by { + assert(registered_linked_list_target_inv::( + self.constant(), + state_before, + obj, + )); + if !state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.contains(obj)); + assert(permissions_before.keys().contains(obj)); + assert(state.permissions.keys().contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + }; + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + Tracked(final_guard) + } + + /// Publishes any registered target from a null link. + #[inline(always)] + pub fn compare_exchange_publish( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).current_target() is None, + target_obj != old(self).constant().source_obj, + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_unpublished() || old(self).target_phase( + target_obj, + ).is_unlinked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> final(self).current_target() == Some(target_obj), + res.0 is Ok ==> final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Linked { + index: res.1@->Some_0, + timestamp: final(self).target_phase(target_obj)->Linked_timestamp, + }, + ), + res.0 is Ok ==> final(self).target_phase(target_obj).is_linked(), + res.0 is Ok ==> final(self).target_phase(target_obj)->Linked_index == res.1@->Some_0, + res.0 is Err ==> final(self).current_target() is None, + res.0 is Err ==> final(self).target_lifecycles() == old(self).target_lifecycles(), + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.current@ is None); + assert(state.auth.state().objects.contains_pair(target_obj, target)); + state.auth.lemma_registry_domains(); + assert(state.auth.state().incoming_all.contains_key(target_obj)); + state.auth.lemma_has_info_for_object(target_obj); + let tracked target_info = state.auth.tracked_info_for(target_obj); + assert(target_info.ptr() == target); + target_info.lemma_wf_facts(); + assert(target.addr() != 0); + assert(state.auth.state().successors.contains_key(key.source_obj)); + } + let ghost prev = state.points_to.hist(); + let cas = raw_atomic.compare_exchange( + null, + target, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost published_index: Option; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp == current_timestamp); + assert(equal(prev.value(current_timestamp), value)); + assert(value.addr() == 0); + assert(state.auth.state().successors[key.source_obj].last() is None); + assert(next == prev.insert( + update.load_timestamp + 1, + target, + update.store_message_view, + )); + let index = state.link.tracked_cas_publish( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + target, + target_obj, + update.store_message_view, + ); + state.current.update( + self.tracked_current.borrow_mut(), + Some(target_obj), + ); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Linked { + index, + timestamp: update.load_timestamp + 1, + }, + ), + ); + published_index = Some(index); + }, + Result::Err(_) => { + published_index = None; + }, + } + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + match cas.0 { + Result::Ok(_) => { + if obj == target_obj { + assert(state.lifecycle@[obj] is Linked); + assert(state.current@ == Some(obj)); + assert(state.auth.state().incoming_all[obj] + == state_before.auth.state().incoming_all[obj].insert(( + key.source_obj, + published_index->Some_0, + ))); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) implies { + &&& edge.0 == key.source_obj + &&& edge.1 <= published_index->Some_0 + } by { + if edge != (key.source_obj, published_index->Some_0) { + assert(state_before.auth.state().incoming_all[obj].contains( + edge, + )); + match old_lifecycle[obj] { + LinkedListChildPhase::Unpublished => assert(false), + LinkedListChildPhase::Unlinked { + index: old_index, + removal: _, + } => { + assert(edge.1 < old_index); + assert(old_index + < state_before.auth.state().successors[ + key.source_obj + ].len()); + }, + _ => assert(false), + } + } + }; + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(!old_lifecycle[obj].is_linked()); + assert(state.current@ != Some(obj)); + } + }, + Result::Err(_) => { + assert(state.lifecycle@ == old_lifecycle); + assert(state.current@ is None); + }, + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + state.auth.lemma_registry_domains(); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + } + result = (cas.0, Ghost(published_index)); + }); + result + } + + /// Unlinks the currently selected AId. The AId precondition, rather than + /// pointer-address equality, identifies which logical allocation loses its + /// incoming edge. + #[inline(always)] + pub fn compare_exchange_unlink( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).current_target() == Some(target_obj), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_linked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> final(self).current_target() is None, + res.0 is Ok ==> res.1@->Some_0.1.root == final(self).constant().root, + res.0 is Ok ==> res.1@->Some_0.1.observed_by(final(tv)@), + res.0 is Ok ==> final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Unlinked { + index: res.1@->Some_0.0, + removal: res.1@->Some_0.1, + }, + ), + res.0 is Ok ==> final(self).target_phase(target_obj).is_unlinked(), + res.0 is Err ==> final(self).current_target() == Some(target_obj), + res.0 is Err ==> final(self).target_lifecycles() == old(self).target_lifecycles(), + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.current@ == Some(target_obj)); + state.auth.lemma_registry_domains(); + assert(state.auth.state().successors.contains_key(key.source_obj)); + assert(state.auth.state().successors[key.source_obj].last() + == Some((target, target_obj))); + } + let ghost prev = state.points_to.hist(); + let cas = raw_atomic.compare_exchange( + target, + null, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost unlinked: Option<( + rcu_spec::LinkIndex, + rcu_spec::RcuRemovalObservation, + )>; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp == current_timestamp); + assert(equal(prev.value(current_timestamp), value)); + assert(value.addr() == target.addr()); + assert(next == prev.insert( + update.load_timestamp + 1, + null, + update.store_message_view, + )); + let index = state.link.tracked_cas_unlink( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + target, + target_obj, + update.store_message_view, + ); + let ghost removal = rcu_spec::RcuRemovalObservation { + root: key.root, + timestamp: update.load_timestamp + 1, + message_view: update.store_message_view, + }; + state.current.update(self.tracked_current.borrow_mut(), None); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Unlinked { index, removal }, + ), + ); + unlinked = Some((index, removal)); + }, + Result::Err(_) => { + unlinked = None; + }, + } + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + match cas.0 { + Result::Ok(_) => { + if obj == target_obj { + assert(state.lifecycle@[obj] is Unlinked); + assert(state.auth.state().incoming_all[obj] + == state_before.auth.state().incoming_all[obj]); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) implies { + &&& edge.0 == key.source_obj + &&& edge.1 < unlinked->Some_0.0 + } by { + assert(edge.1 + <= old_lifecycle[obj]->Linked_index); + assert(old_lifecycle[obj]->Linked_index + 1 + == unlinked->Some_0.0); + }; + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(obj != target_obj); + } + assert(!state.lifecycle@[obj].is_linked()); + assert(state.current@ is None); + }, + Result::Err(_) => { + assert(state.lifecycle@ == old_lifecycle); + assert(state.current@ == Some(target_obj)); + }, + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + state.auth.lemma_registry_domains(); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + } + result = (cas.0, Ghost(unlinked)); + }); + result + } + + /// Converts one unlinked registered AId into the traversal-retire and + /// physical-reclaim resources needed by the existing RCU monitor. + #[verifier::atomic] + pub fn retire_unlinked_target( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked) + requires + old(self).well_formed(), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_unlinked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Retired { + index: old(self).target_phase(target_obj)->Unlinked_index, + removal: old(self).target_phase(target_obj)->Unlinked_removal, + }, + ), + final(self).target_phase(target_obj).is_retired(), + res@.object().wf(), + res@.object().domain() == final(self).constant().domain, + res@.object().obj() == target_obj, + res@.object().ptr() == target, + res@.retire().wf(), + res@.retire().ready_to_retire(), + res@.retire().domain() == final(self).constant().domain, + res@.retire().obj() == target_obj, + res@.retire().ptr() == target, + res@.claim().registry() == final(self).constant().reclaim_registry, + res@.claim().obj() == target_obj, + res@.claim().is_pending(), + res@.claim().ptr() == target, + res@.removal() == old(self).target_phase(target_obj)->Unlinked_removal, + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked detached; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(registered_linked_list_target_inv::( + key, + state, + target_obj, + )); + state.auth.lemma_registry_domains(); + state.permissions.lemma_all_unretired_domains(); + assert(state.auth.state().successors.contains_key(key.source_obj)); + assert(state.auth.state().incoming_all.contains_key(target_obj)); + assert(state.lifecycle@[target_obj] is Unlinked); + let ghost index = state.lifecycle@[target_obj]->Unlinked_index; + let ghost removal = state.lifecycle@[target_obj]->Unlinked_removal; + let ghost latest = (state.auth.state().successors[key.source_obj].len() - 1) + as nat; + let ghost prior = rcu_spec::RcuSeenRemoved { + removed: state.auth.removed(), + link_view: rcu_spec::RcuLinkView::empty().observe(key.source_obj, latest), + }; + assert(state.auth.state().successors[key.source_obj].len() > 0); + assert(latest + 1 == state.auth.state().successors[key.source_obj].len()); + assert(state.auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut rcu_spec::LinkedListNode, from_obj: nat| #[trigger] + state.auth.state().objects.contains_pair(from_obj, from) + && prior.link_view.seen.contains_key(from_obj) implies { + &&& state.auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) + < state.auth.state().successors[from_obj].len() + } by { + assert(from_obj == key.source_obj); + assert(state.auth.state().successors.contains_key(from_obj)); + assert(prior.link_view.seen_at(from_obj) == latest); + }; + } + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + prior, + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + prior.removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> prior.dead_edge(edge) + } by { + assert(state.auth.removed().contains(obj)); + assert(state.registry@.dom().remove(key.source_obj).contains(obj)); + assert(registered_linked_list_target_inv::(key, state, obj)); + assert(state.registry@.contains_key(obj)); + assert(state.auth.state().objects.contains_key(obj)); + assert(state.auth.state().incoming_all.contains_key(obj)); + assert(state.lifecycle@[obj].is_retired() + || state.lifecycle@[obj].is_reclaimed()); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) implies + prior.dead_edge(edge) by { + match state.lifecycle@[obj] { + LinkedListChildPhase::Retired { + index: removed_index, + removal: _, + } + | LinkedListChildPhase::Reclaimed { + index: removed_index, + removal: _, + } => { + assert(edge.0 == key.source_obj); + assert(edge.1 < removed_index); + assert(removed_index + < state.auth.state().successors[key.source_obj].len()); + assert(removed_index <= latest); + }, + _ => assert(false), + } + }; + }; + }; + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[target_obj].contains(edge) implies + prior.dead_edge(edge) by { + assert(edge.0 == key.source_obj); + assert(edge.1 < index); + assert(index < state.auth.state().successors[key.source_obj].len()); + }; + assert(state.auth.state().incoming_all[target_obj].len() > 0) by { + assert(state.auth.state().successors[key.source_obj][(index - 1) as int] + == Some((target, target_obj))); + assert(rcu_spec::LinkedListTraversalSpec::link_inv( + key.source, + key.source_obj, + (index - 1) as nat, + target, + target_obj, + state.auth.state(), + )); + assert(state.auth.state().incoming_all[target_obj].contains(( + key.source_obj, + (index - 1) as nat, + ))); + }; + state.auth.lemma_has_info_for_object(target_obj); + let tracked object = state.auth.tracked_info_for(target_obj); + let tracked retire = state.auth.tracked_retire_node(target_obj, prior); + let tracked claim = state.permissions.tracked_retire(target_obj); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Retired { index, removal }, + ), + ); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + if obj == target_obj { + assert(state.lifecycle@[obj] is Retired); + assert(state.auth.removed().contains(obj)); + assert(!state.auth.has_retire_perm(obj)); + assert(state.permissions.contains(obj)); + assert(!state.permissions.has_unretired_claim(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.auth.removed().contains(obj) + == state_before.auth.removed().contains(obj)); + assert(state.auth.has_retire_perm(obj) + == state_before.auth.has_retire_perm(obj)); + assert(state.permissions.has_unretired_claim(obj) + == permissions_before.has_unretired_claim(obj)); + if !state.lifecycle@[obj].is_reclaimed() { + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(state.auth.removed().subset_of( + state.registry@.dom().remove(key.source_obj), + )); + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == permissions_before.allocations()); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + assert(state.registry@ == state.auth.state().objects); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.auth.state() == state_before.auth.state()); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle.id() == state_before.lifecycle.id()); + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(state.link.wf(state.points_to.hist(), state.auth)); + assert(state.link.native_observations_wf(state.points_to)); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth.state().successors[key.source_obj].len() + && state.auth.state().successors[key.source_obj][n as int] is Some + implies #[trigger] + state.auth.state().successors[key.source_obj][n as int]->Some_0.1 + != key.source_obj by {}; + assert(match state.current@ { + None => state.auth.state().successors[key.source_obj].last() is None, + Some(obj) => { + &&& obj != key.source_obj + &&& state.registry@.contains_key(obj) + &&& state.auth.state().successors[key.source_obj].last() == Some(( + state.registry@[obj], + obj, + )) + }, + }); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(object.ptr() == target); + assert(retire.ptr() == target); + assert(claim.ptr() == target); + detached = LinkedListDetachedChild { object, retire, claim, removal }; + } + }); + Tracked(detached) + } + + /// Recovers one retired registered allocation after a completed grace + /// period has excluded all outstanding leases for that AId. + #[verifier::atomic] + pub fn reclaim_retired_target( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(claim): Tracked>, + Tracked(completed): Tracked, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked) + requires + old(self).well_formed(), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == target_obj, + claim.is_pending(), + claim.ptr() == target, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == target_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).target_phase(target_obj)->Retired_removal, + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Reclaimed { + index: old(self).target_phase(target_obj)->Retired_index, + removal: old(self).target_phase(target_obj)->Retired_removal, + }, + ), + final(self).target_phase(target_obj).is_reclaimed(), + OwnPred::owns(target, res@), + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked ownership; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(registered_linked_list_target_inv::( + key, + state, + target_obj, + )); + assert(state.lifecycle@[target_obj] is Retired); + let ghost index = state.lifecycle@[target_obj]->Retired_index; + let ghost removal = state.lifecycle@[target_obj]->Retired_removal; + assert(completed.record().removal == removal); + assert(OwnPred::owns( + target, + permissions_before.ownership(target_obj), + )); + state.permissions.lemma_completed_excludes_active(&completed, target_obj); + assert(!state.permissions.has_active(target_obj)); + ownership = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Reclaimed { index, removal }, + ), + ); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + if obj == target_obj { + assert(state.lifecycle@[obj] is Reclaimed); + assert(!state.permissions.contains(obj)); + assert(state.permissions.reclaim_states()[obj] is None); + assert(state.permissions.reclaimed().contains_pair(obj, completed)); + assert(state.permissions.reclaimed()[obj].record().removal == removal); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.permissions.reclaim_states()[obj] + == permissions_before.reclaim_states()[obj]); + assert(state.permissions.has_unretired_claim(obj) + == permissions_before.has_unretired_claim(obj)); + if state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed()[obj] + == permissions_before.reclaimed()[obj]); + } + if !state.lifecycle@[obj].is_reclaimed() { + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == permissions_before.allocations()); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle.id() == state_before.lifecycle.id()); + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(ownership == permissions_before.ownership(target_obj)); + assert(OwnPred::owns(target, ownership)); + } + }); + Tracked(ownership) + } + + /// Proof-mode reclaim entry used by a type-erased callback that already + /// owns an invariant-opening credit from the monitor. + pub proof fn tracked_reclaim_retired_target( + tracked &mut self, + target: *mut rcu_spec::LinkedListNode, + target_obj: nat, + tracked claim: rcu_cpu_spec::RcuReclaimClaim, + tracked completed: rcu_cpu_spec::RcuReclaimedWitness, + tracked credit: vstd::invariant::OpenInvariantCredit, + ) -> (tracked ownership: O) + requires + old(self).well_formed(), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == target_obj, + claim.is_pending(), + claim.ptr() == target, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == target_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).target_phase(target_obj)->Retired_removal, + ensures + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Reclaimed { + index: old(self).target_phase(target_obj)->Retired_index, + removal: old(self).target_phase(target_obj)->Retired_removal, + }, + ), + final(self).target_phase(target_obj).is_reclaimed(), + OwnPred::owns(target, ownership), + opens_invariants [self.invariant_namespace()] + { + use_type_invariant(&*self); + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked mut recovered; + vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + let ghost permissions_before = state.permissions; + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(registered_linked_list_target_inv::( + key, + state, + target_obj, + )); + assert(state.lifecycle@[target_obj] is Retired); + let ghost index = state.lifecycle@[target_obj]->Retired_index; + let ghost removal = state.lifecycle@[target_obj]->Retired_removal; + assert(completed.record().removal == removal); + assert(OwnPred::owns(target, permissions_before.ownership(target_obj))); + state.permissions.lemma_completed_excludes_active(&completed, target_obj); + assert(!state.permissions.has_active(target_obj)); + recovered = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Reclaimed { index, removal }, + ), + ); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + if obj == target_obj { + assert(state.lifecycle@[obj] is Reclaimed); + assert(!state.permissions.contains(obj)); + assert(state.permissions.reclaim_states()[obj] is None); + assert(state.permissions.reclaimed().contains_pair(obj, completed)); + assert(state.permissions.reclaimed()[obj].record().removal == removal); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.permissions.reclaim_states()[obj] + == permissions_before.reclaim_states()[obj]); + assert(state.permissions.has_unretired_claim(obj) + == permissions_before.has_unretired_claim(obj)); + if state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed()[obj] + == permissions_before.reclaimed()[obj]); + } + if !state.lifecycle@[obj].is_reclaimed() { + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle.id() == state_before.lifecycle.id()); + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(recovered == permissions_before.ownership(target_obj)); + assert(OwnPred::owns(target, recovered)); + }); + recovered + } +} + +/// Immutable identities carried by the native atomic invariant for the +/// two-node linked-list traversal example. +pub ghost struct LinkedListAtomicKey { + pub scheduler: Loc, + pub domain: Loc, + pub root: Loc, + pub retire_observation_registry: Loc, + pub reclaim_registry: Loc, + pub active_lease_registry: Loc, + pub lifecycle: Loc, + pub timestamp_registry: Loc, + pub native_observation_registry: Loc, + pub source: *mut rcu_spec::LinkedListNode, + pub source_obj: nat, + pub child: *mut rcu_spec::LinkedListNode, + pub child_obj: nat, +} + +/// Writer-visible lifecycle of the fixed child managed by one concrete link. +/// +/// The two halves of a [`GhostVar`] keep this phase synchronized between the +/// executable wrapper and its atomic invariant. In particular, a writer that +/// has moved the child to `Retired` can no longer call the publication rule, +/// while stale readers may continue returning leases until `Reclaimed`. +pub ghost enum LinkedListChildPhase { + /// The child is registered but has never been published by this link. + Unpublished, + /// The child is the latest value of the link. + Linked { index: rcu_spec::LinkIndex, timestamp: Timestamp }, + /// The child was removed by the recorded native atomic message. + Unlinked { index: rcu_spec::LinkIndex, removal: rcu_spec::RcuRemovalObservation }, + /// Traversal retirement and the physical reclaim claim left the invariant. + Retired { index: rcu_spec::LinkIndex, removal: rcu_spec::RcuRemovalObservation }, + /// Grace-period completion recovered the physical ownership resource. + Reclaimed { index: rcu_spec::LinkIndex, removal: rcu_spec::RcuRemovalObservation }, +} + +impl LinkedListChildPhase { + pub open spec fn is_reclaimed(self) -> bool { + self is Reclaimed + } + + pub open spec fn is_unpublished(self) -> bool { + self is Unpublished + } + + pub open spec fn is_linked(self) -> bool { + self is Linked + } + + pub open spec fn is_unlinked(self) -> bool { + self is Unlinked + } + + pub open spec fn is_retired(self) -> bool { + self is Retired + } +} + +/// Complete state protected by one native linked-list link invariant. +pub tracked struct LinkedListAtomicState { + pub(crate) points_to: AtomicPointsTo<*mut rcu_spec::LinkedListNode>, + pub(crate) link: rcu_spec::LinkedListAtomicLinkGhost, + pub(crate) auth: rcu_spec::LinkedListTraversalAuth, + pub(crate) permissions: rcu_cpu_spec::RcuRootPermissionState, + pub(crate) lifecycle: GhostVarAuth, +} + +unsafe impl Objective for LinkedListAtomicState { + +} + +impl LinkedListAtomicState { + pub closed spec fn points_to(self) -> AtomicPointsTo<*mut rcu_spec::LinkedListNode> { + self.points_to + } + + pub closed spec fn link(self) -> rcu_spec::LinkedListAtomicLinkGhost { + self.link + } + + pub closed spec fn auth(self) -> rcu_spec::LinkedListTraversalAuth { + self.auth + } + + pub closed spec fn permissions(self) -> rcu_cpu_spec::RcuRootPermissionState< + rcu_spec::LinkedListNode, + O, + > { + self.permissions + } + + pub closed spec fn lifecycle(self) -> GhostVarAuth { + self.lifecycle + } +} + +/// Native IRC11 invariant for a link whose only non-null value is one +/// pre-registered child. +/// +/// Restricting the first executable-style wrapper to two nodes keeps this +/// acceptance protocol closed while a production data-structure adapter and +/// its public node-registration API are still being selected. +pub struct LinkedListAtomicInv { + _marker: PhantomData, +} + +impl InvariantPredicate< + (LinkedListAtomicKey, Irc11AtomicId), + LinkedListAtomicState, +> for LinkedListAtomicInv where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + open spec fn inv( + key_loc: (LinkedListAtomicKey, Irc11AtomicId), + state: LinkedListAtomicState, + ) -> bool { + let (key, loc) = key_loc; + let permissions = state.permissions(); + &&& state.points_to().loc() == loc + &&& key.source.addr() != 0 + &&& key.child.addr() != 0 + &&& key.source.addr() != key.child.addr() + &&& key.source_obj != key.child_obj + &&& state.auth().wf() + &&& state.auth().domain() == key.domain + &&& state.auth().state().root == key.source + &&& state.auth().state().root_obj == key.source_obj + &&& state.auth().state().objects == Map::empty().insert(key.source_obj, key.source).insert( + key.child_obj, + key.child, + ) + &&& state.auth().state().incoming_all.contains_key(key.child_obj) + &&& state.auth().state().successors.contains_key(key.source_obj) + &&& permissions.wf() + &&& permissions.scheduler() == key.scheduler + &&& permissions.domain() == key.domain + &&& permissions.root() == key.root + &&& permissions.retire_observation_registry() == key.retire_observation_registry + &&& permissions.reclaim_registry() == key.reclaim_registry + &&& permissions.active_lease_registry() == key.active_lease_registry + &&& permissions.allocations() == Set::empty().insert(key.child_obj) + &&& state.lifecycle().id() == key.lifecycle + &&& state.link().source() == key.source + &&& state.link().source_obj() == key.source_obj + &&& state.link().timestamp_registry() == key.timestamp_registry + &&& state.link().native_observation_registry() == key.native_observation_registry + &&& state.link().wf(state.points_to().hist(), state.auth()) + &&& state.link().native_observations_wf(state.points_to()) + &&& forall|n: rcu_spec::LinkIndex| + n < state.auth().state().successors[key.source_obj].len() + && state.auth().state().successors[key.source_obj][n as int] is Some + ==> #[trigger] state.auth().state().successors[key.source_obj][n as int] == Some( + (key.child, key.child_obj), + ) + &&& match state.lifecycle()@ { + LinkedListChildPhase::Unpublished => { + &&& state.auth().removed() == Set::::empty() + &&& state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& state.auth().state().successors[key.source_obj].len() == 1 + &&& state.auth().state().successors[key.source_obj].last() is None + &&& state.auth().state().incoming_all[key.child_obj] == Set::< + rcu_spec::LinkEdge, + >::empty() + }, + LinkedListChildPhase::Linked { index, timestamp } => { + &&& state.auth().removed() == Set::::empty() + &&& state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& state.link().current_timestamp() == timestamp + &&& state.link().index_at(timestamp) == index + &&& index + 1 == state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj].last() == Some( + (key.child, key.child_obj), + ) + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, index), + ) + }, + LinkedListChildPhase::Unlinked { index, removal } => { + &&& state.auth().removed() == Set::::empty() + &&& state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& removal.root == key.root + &&& state.link().current_timestamp() == removal.timestamp + &&& state.link().index_at(removal.timestamp) == index + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& index + 1 == state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj].last() is None + &&& index > 0 + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, (index - 1) as nat), + ) + }, + LinkedListChildPhase::Retired { index, removal } => { + &&& state.auth().removed() == Set::::empty().insert(key.child_obj) + &&& !state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& !permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& removal.root == key.root + &&& state.link().current_timestamp() == removal.timestamp + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& state.auth().state().successors[key.source_obj].last() is None + &&& index > 0 + &&& state.link().index_at(removal.timestamp) == index + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, (index - 1) as nat), + ) + }, + LinkedListChildPhase::Reclaimed { index, removal } => { + &&& state.auth().removed() == Set::::empty().insert(key.child_obj) + &&& !state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::::empty() + &&& !permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] is None + &&& !permissions.has_unretired_claim(key.child_obj) + &&& permissions.reclaimed().contains_key(key.child_obj) + &&& permissions.reclaimed()[key.child_obj].record().removal == removal + &&& removal.root == key.root + &&& state.link().current_timestamp() == removal.timestamp + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& state.auth().state().successors[key.source_obj].last() is None + &&& index > 0 + &&& state.link().index_at(removal.timestamp) == index + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, (index - 1) as nat), + ) + }, + } + } +} + +pub type LinkedListAtomicInvariant = AtomicInvariant< + (LinkedListAtomicKey, Irc11AtomicId), + LinkedListAtomicState, + LinkedListAtomicInv, +>; + +/// Executable-style native weak atomic used to close the linked-list +/// traversal invariant before generalizing it to arbitrary data structures. +pub struct LinkedListWeakAtomicLink { + atomic: PAtomicWeakPtr, + child: *mut rcu_spec::LinkedListNode, + tracked_atomic_inv: Tracked<&'static LinkedListAtomicInvariant>, + tracked_child_phase: Tracked>, +} + +impl LinkedListWeakAtomicLink { + pub closed spec fn constant(&self) -> LinkedListAtomicKey { + self.tracked_atomic_inv@.constant().0 + } + + pub closed spec fn native_loc(&self) -> Irc11AtomicId { + self.atomic.loc() + } + + /// Namespace of the atomic invariant owned by this wrapper. + pub closed spec fn invariant_namespace(&self) -> int { + self.tracked_atomic_inv@.namespace() + } + + pub closed spec fn child_ptr(&self) -> *mut rcu_spec::LinkedListNode { + self.child + } + + /// Runtime child pointer retained by this one-link wrapper. + #[inline(always)] + pub(crate) fn child_raw(&self) -> (res: *mut rcu_spec::LinkedListNode) + ensures + equal(res, self.child_ptr()), + { + self.child + } + + pub closed spec fn child_phase(&self) -> LinkedListChildPhase { + self.tracked_child_phase@.view() + } + + pub closed spec fn well_formed(&self) -> bool { + &&& self.tracked_atomic_inv@.constant().1 == self.native_loc() + &&& self.child_ptr() == self.constant().child + &&& self.child_ptr().addr() != 0 + &&& self.tracked_child_phase@.id() == self.constant().lifecycle + } + + /// Exposes the structural consequences of the closed wrapper invariant to + /// executable adapters without exposing its proof-resource representation. + pub proof fn lemma_well_formed_facts(&self) + requires + self.well_formed(), + ensures + self.child_ptr() == self.constant().child, + self.child_ptr().addr() != 0, + { + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } +} + +impl LinkedListWeakAtomicLink where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + /// Creates a null link with a pre-registered source and child. + pub const fn new( + Ghost(scheduler): Ghost, + Ghost(root): Ghost, + Ghost(retire_observation_registry): Ghost, + child: *mut rcu_spec::LinkedListNode, + Tracked(source_info): Tracked<&rcu_spec::RcuBlockInfo>, + Tracked(source_retire): Tracked>, + Tracked(child_info): Tracked<&rcu_spec::RcuBlockInfo>, + Tracked(child_retire): Tracked>, + Tracked(child_ownership): Tracked, + ) -> (res: Self) + requires + source_info.wf(), + source_retire.wf(), + source_retire.domain() == source_info.domain(), + source_retire.obj() == source_info.obj(), + source_retire.ptr() == source_info.ptr(), + child_info.wf(), + child_retire.wf(), + child_info.domain() == source_info.domain(), + child_retire.domain() == source_info.domain(), + child_retire.obj() == child_info.obj(), + child_retire.ptr() == child_info.ptr(), + child == child_info.ptr(), + source_info.addr() != 0, + child_info.addr() != 0, + source_info.addr() != child_info.addr(), + source_info.ptr() != child_info.ptr(), + source_info.obj() != child_info.obj(), + OwnPred::owns(child, child_ownership), + ensures + res.well_formed(), + res.constant().scheduler == scheduler, + res.constant().domain == source_info.domain(), + res.constant().root == root, + res.constant().retire_observation_registry == retire_observation_registry, + res.constant().source == source_info.ptr(), + res.constant().source_obj == source_info.obj(), + res.constant().child == child_info.ptr(), + res.constant().child_obj == child_info.obj(), + res.child_ptr() == child, + res.child_phase() is Unpublished, + { + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + PAtomicWeakPtr::new(core::ptr::null_mut()); + let tracked mut auth = rcu_spec::LinkedListTraversalAuth::tracked_new( + source_info, + source_retire, + ); + proof { + assert(auth.has_retire_perm(source_info.obj())); + assert(!auth.has_retire_perm(child_info.obj())); + auth.tracked_register_node(child_info, child_retire); + } + let ghost initial_index = auth.tracked_initialize_null( + source_info.ptr(), + source_info.obj(), + ); + let tracked link = rcu_spec::LinkedListAtomicLinkGhost::tracked_initial_null( + points_to.hist(), + timestamp, + initial_view@, + &auth, + source_info.ptr(), + source_info.obj(), + ); + proof_decl! { + let tracked mut permissions: + rcu_cpu_spec::RcuRootPermissionState; + let ghost reclaim_registry: Loc; + let ghost active_lease_registry: Loc; + let ghost child_ownership_value: O; + } + proof { + permissions = + rcu_cpu_spec::RcuRootPermissionState::empty( + scheduler, + source_info.domain(), + root, + retire_observation_registry, + ); + reclaim_registry = permissions.reclaim_registry(); + active_lease_registry = permissions.active_lease_registry(); + child_ownership_value = child_ownership; + assert(permissions.allocations() == Set::::empty()); + permissions.tracked_insert(child_info, child_ownership); + } + let tracked (lifecycle, lifecycle_peer) = GhostVarAuth::new( + LinkedListChildPhase::Unpublished, + ); + let tracked state = LinkedListAtomicState { points_to, link, auth, permissions, lifecycle }; + let ghost key = LinkedListAtomicKey { + scheduler, + domain: source_info.domain(), + root, + retire_observation_registry, + reclaim_registry, + active_lease_registry, + lifecycle: state.lifecycle().id(), + timestamp_registry: state.link().timestamp_registry(), + native_observation_registry: state.link().native_observation_registry(), + source: source_info.ptr(), + source_obj: source_info.obj(), + child: child_info.ptr(), + child_obj: child_info.obj(), + }; + proof { + source_info.lemma_wf_facts(); + child_info.lemma_wf_facts(); + assert(initial_index == 0); + assert(state.points_to().loc() == atomic.loc()); + assert(key.source.addr() == source_info.addr()); + assert(key.child.addr() == child_info.addr()); + assert(state.auth().wf()); + assert(state.auth().domain() == key.domain); + assert(state.auth().removed() == Set::::empty()); + assert(state.auth().state().root == key.source); + assert(state.auth().state().root_obj == key.source_obj); + assert(state.auth().state().objects == Map::empty().insert( + key.source_obj, + key.source, + ).insert(key.child_obj, key.child)); + assert(state.permissions().wf()); + assert(state.permissions().scheduler() == key.scheduler); + assert(state.permissions().domain() == key.domain); + assert(state.permissions().root() == key.root); + assert(state.permissions().retire_observation_registry() + == key.retire_observation_registry); + assert(state.permissions().reclaim_registry() == key.reclaim_registry); + assert(state.permissions().active_lease_registry() == key.active_lease_registry); + assert(state.permissions().allocations() == Set::empty().insert(key.child_obj)); + assert(state.permissions().keys() == Set::empty().insert(key.child_obj)); + assert(state.permissions().contains(key.child_obj)); + assert(state.permissions().reclaim_states()[key.child_obj] == Some(key.child)); + assert(state.permissions().has_unretired_claim(key.child_obj)); + assert(state.permissions().ownership(key.child_obj) == child_ownership_value); + assert(OwnPred::owns(key.child, state.permissions().ownership(key.child_obj))); + assert(state.lifecycle()@ is Unpublished); + assert(state.link().source() == key.source); + assert(state.link().source_obj() == key.source_obj); + assert(state.link().wf(state.points_to().hist(), state.auth())); + assert(state.link().native_observations() == Map::empty()); + assert(state.link().native_observations_wf(state.points_to())) by { + assert forall|observation_id: nat| #[trigger] + state.link().native_observations().contains_key(observation_id) implies { + let observation = state.link().native_observations()[observation_id]; + &&& observation.0 == state.points_to().loc() + &&& state.points_to().get_timestamp(observation.1) == Some(observation.2) + } by {}; + }; + assert forall|n: rcu_spec::LinkIndex| + n < state.auth().state().successors[key.source_obj].len() + && state.auth().state().successors[key.source_obj][n as int] is Some implies #[trigger] state.auth().state().successors[key.source_obj][n as int] + == Some((key.child, key.child_obj)) by { + assert(n == 0); + assert(state.auth().state().successors[key.source_obj][n as int] is None); + assert(false); + }; + assert(LinkedListAtomicInv::::inv((key, atomic.loc()), state)); + } + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); + let tracked atomic_inv = tracked_static_ref(atomic_inv); + LinkedListWeakAtomicLink { + atomic, + child, + tracked_atomic_inv: Tracked(atomic_inv), + tracked_child_phase: Tracked(lifecycle_peer), + } + } + + fn raw_atomic(&self) -> (res: &PAtomicWeakPtr) + requires + self.well_formed(), + ensures + res.loc() == self.native_loc(), + opens_invariants none + no_unwind + { + &self.atomic + } + + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: + &'static LinkedListAtomicInvariant) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.native_loc()), + { + self.tracked_atomic_inv.get() + } + + /// Acquire-loads the native link and applies the paper's guarded traversal + /// rule in the same atomic-invariant opening. + /// + /// `previous` is absent for the first observation and otherwise carries + /// the persistent timestamp/index fact returned by the preceding load. + /// Its native view lower bound prevents a later load from moving the + /// guard's dense traversal position backwards. + #[inline(always)] + pub fn load_acquire_and_protect( + &self, + Tracked(guard): Tracked<&mut rcu_spec::RcuReadGuardToken>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + !self.child_phase().is_reclaimed(), + old(guard).wf(), + old(guard).domain() == self.constant().domain, + old(guard).seen_removed().removed == Set::::empty(), + match previous { + None => old(guard).seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& old(guard).seen_at(self.constant().source_obj) == observation.index() + }, + }, + old(from).protected_by(*old(guard)), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).seen_removed().removed == old(guard).seen_removed().removed, + final(guard).seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.4@.registry() == self.constant().timestamp_registry, + res.4@.native_registry() == self.constant().native_observation_registry, + res.4@.loc() == self.native_loc(), + res.4@.timestamp() == res.1@, + res.4@.index() == res.2@, + res.4@.view() == final(tv)@, + (res.3@ is Some) == (res.0.addr() != 0), + match res.3@ { + None => res.0.addr() == 0, + Some(child) => { + &&& equal(child.ptr(), res.0) + &&& child.ptr() == self.constant().child + &&& child.obj() == self.constant().child_obj + &&& child.domain() == self.constant().domain + &&& child.protected_by(*final(guard)) + }, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(&*self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.lifecycle@ == self.child_phase()); + match state.lifecycle@ { + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + _ => {}, + } + assert(state.permissions.contains(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(state.points_to.loc() == self.native_loc()); + match previous { + None => { + assert(guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + use_type_invariant(observation); + assert(observation.loc() == self.native_loc()); + state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(state.points_to.hist().contains_timestamp( + observation.timestamp(), + )); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + assert(state.points_to.hist().contains_timestamp(timestamp)); + match previous { + None => { + assert(guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.timestamp() <= timestamp); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(observation.index() <= index); + assert(guard.seen_at(from.obj()) == observation.index()); + }, + } + assert(guard.seen_at(from.obj()) <= index); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + old(guard).seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + old(guard).seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> old(guard).seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + guard, + from, + timestamp, + ); + let tracked observation; + } + proof { + observation = state.link.tracked_observation_at( + &state.points_to, + &state.auth, + timestamp, + tv@, + ); + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + match &protected { + None => { + assert(loaded.0.addr() == 0); + }, + Some(child) => { + assert(state.auth.state().successors[self.constant().source_obj][index as int] + is Some); + assert(state.auth.state().successors[self.constant().source_obj][index as int] + == Some((self.constant().child, self.constant().child_obj))); + assert(child.ptr() == self.constant().child); + assert(child.obj() == self.constant().child_obj); + }, + } + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(protected), + Tracked(observation), + ); + }); + result + } + + /// Acquire-loads an internal link, applies the paper traversal rule, and + /// splits a physical read lease for the observed child in one atomic + /// invariant opening. + /// + /// A non-null result retains the child protection witness for subsequent + /// traversal while the lease supplies the physical ownership resource + /// used to derive a concrete `RefPermission`. A null result leaves the + /// CPU reader fraction unchanged. + #[inline(always)] + pub fn load_acquire_and_lease_cpu( + &self, + Tracked(guard): Tracked>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>, + Tracked>>, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + !self.child_phase().is_reclaimed(), + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + online_cpus().contains(guard.cpu()), + guard.seen_removed().removed == Set::::empty(), + match previous { + None => guard.paper_guard().seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& guard.paper_guard().seen_at(self.constant().source_obj) + == observation.index() + }, + }, + old(from).protected_by(guard.paper_guard()), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + res.3@.wf(), + res.3@.binding() == guard.binding(), + res.3@.participant_id() == guard.participant_id(), + res.3@.cpu() == guard.cpu(), + res.3@.generation() == guard.generation(), + res.3@.participant_view() == guard.participant_view(), + res.3@.known_retired() == guard.known_retired(), + res.3@.scheduler() == guard.scheduler(), + res.3@.domain() == guard.domain(), + res.3@.root() == guard.root(), + res.3@.reader_registry() == guard.reader_registry(), + res.3@.retire_observation_registry() == guard.retire_observation_registry(), + res.3@.reader_context() == guard.reader_context(), + res.3@.start_view() == guard.start_view(), + res.3@.expired() == guard.expired(), + res.3@.seen_removed().removed == guard.seen_removed().removed, + res.3@.paper_guard().seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.6@.registry() == self.constant().timestamp_registry, + res.6@.native_registry() == self.constant().native_observation_registry, + res.6@.loc() == self.native_loc(), + res.6@.timestamp() == res.1@, + res.6@.index() == res.2@, + res.6@.view() == final(tv)@, + (res.4@ is Some) == (res.0.addr() != 0), + (res.5@ is Some) == (res.0.addr() != 0), + match (res.4@, res.5@) { + (None, None) => { + &&& res.0.addr() == 0 + &&& res.3@.reader_fragment() == guard.reader_fragment() + }, + (Some(child), Some(lease)) => { + &&& equal(child.ptr(), res.0) + &&& child.ptr() == self.constant().child + &&& child.obj() == self.constant().child_obj + &&& child.domain() == self.constant().domain + &&& child.protected_by(res.3@.paper_guard()) + &&& res.3@.reader_fragment().fraction() == guard.reader_fragment().fraction() + / 2real + &&& lease.key() == child.obj() + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == res.3@.participant_id() + &&& lease.reader_fraction() == res.3@.reader_fragment().fraction() + &&& lease.domain() == res.3@.domain() + &&& lease.root() == res.3@.root() + &&& lease.reader_context() == res.3@.reader_context() + &&& lease.start_view() == res.3@.start_view() + &&& lease.protected_addr() == child.ptr().addr() + &&& OwnPred::owns(child.ptr(), lease.resource()) + }, + _ => false, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(self); + } + proof_decl! { + let tracked (mut paper_guard, cpu_reader, binding) = guard.tracked_into_parts(); + } + proof { + assert(paper_guard == guard.paper_guard()); + assert(cpu_reader == guard.reader_fragment()); + assert(binding == guard.binding()); + assert(paper_guard.domain() == self.constant().domain); + assert(paper_guard.seen_removed().removed == Set::::empty()); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost permissions_before = state.permissions; + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.lifecycle@ == self.child_phase()); + match state.lifecycle@ { + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + _ => {}, + } + assert(state.permissions.contains(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(state.points_to.loc() == self.native_loc()); + match previous { + None => { + assert(paper_guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + use_type_invariant(observation); + assert(observation.loc() == self.native_loc()); + state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(state.points_to.hist().contains_timestamp( + observation.timestamp(), + )); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + assert(state.points_to.hist().contains_timestamp(timestamp)); + match previous { + None => { + assert(paper_guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.timestamp() <= timestamp); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(observation.index() <= index); + assert(paper_guard.seen_at(from.obj()) == observation.index()); + }, + } + assert(paper_guard.seen_at(from.obj()) <= index); + assert(paper_guard.domain() == state.auth.domain()); + assert(state.auth.state().successors.contains_key(self.constant().source_obj)); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + paper_guard.seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + paper_guard.seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> paper_guard.seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + &mut paper_guard, + from, + timestamp, + ); + let tracked cpu_guard = rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new( + paper_guard, + cpu_reader, + binding, + ); + let tracked final_guard; + let tracked lease; + let tracked observation; + } + proof { + observation = state.link.tracked_observation_at( + &state.points_to, + &state.auth, + timestamp, + tv@, + ); + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + match &protected { + None => { + assert(loaded.0.addr() == 0); + final_guard = cpu_guard; + lease = None; + }, + Some(child) => { + assert(state.auth.state().successors[ + self.constant().source_obj + ][index as int] is Some); + assert(state.auth.state().successors[ + self.constant().source_obj + ][index as int] == Some(( + self.constant().child, + self.constant().child_obj, + ))); + assert(child.ptr() == self.constant().child); + assert(child.obj() == self.constant().child_obj); + assert(state.permissions.contains(child.obj())); + let ghost child_ownership = state.permissions.ownership(child.obj()); + assert(OwnPred::owns(child.ptr(), child_ownership)); + let tracked split = state.permissions.tracked_split_protected( + cpu_guard, + child, + ); + final_guard = split.0; + lease = Some(split.1); + assert(split.1.resource() == child_ownership); + assert(OwnPred::owns(child.ptr(), split.1.resource())); + assert(state.permissions.ownership(child.obj()) == child_ownership); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + }, + } + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() + == permissions_before.reclaim_states()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + state.permissions.lemma_contains_iff_key(self.constant().child_obj); + assert(state.permissions.contains(self.constant().child_obj)); + assert(state.permissions.reclaim_states()[self.constant().child_obj] + == Some(self.constant().child)); + match state.lifecycle@ { + LinkedListChildPhase::Unpublished + | LinkedListChildPhase::Linked { index: _, timestamp: _ } + | LinkedListChildPhase::Unlinked { index: _, removal: _ } => { + assert(state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Retired { index: _, removal: _ } => { + assert(!state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + } + assert(OwnPred::owns( + self.constant().child, + permissions_before.ownership(self.constant().child_obj), + )); + assert(state.permissions.ownership(self.constant().child_obj) + == permissions_before.ownership(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(final_guard), + Tracked(protected), + Tracked(lease), + Tracked(observation), + ); + }); + result + } + + /// Returns a child traversal lease and rejoins its saved CPU reader + /// fraction with the live guard. + #[verifier::atomic] + pub fn return_child_lease_cpu( + &self, + Tracked(lease): Tracked>>, + Tracked(guard): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked>) + requires + self.well_formed(), + !self.child_phase().is_reclaimed(), + match lease { + None => true, + Some(lease) => { + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + }, + }, + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + old(tv)@.spec_le(final(tv)@), + res@.wf(), + res@.paper_guard() == guard.paper_guard(), + res@.binding() == guard.binding(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + res@.participant_view() == guard.participant_view(), + res@.known_retired() == guard.known_retired(), + res@.domain() == guard.domain(), + res@.root() == guard.root(), + res@.reader_registry() == guard.reader_registry(), + res@.retire_observation_registry() == guard.retire_observation_registry(), + res@.reader_context() == guard.reader_context(), + res@.start_view() == guard.start_view(), + res@.expired() == guard.expired(), + res@.seen_removed() == guard.seen_removed(), + res@.protected() == guard.protected(), + res@.reader_fragment().fraction() == match lease { + None => guard.reader_fragment().fraction(), + Some(_) => guard.reader_fragment().fraction() * 2real, + }, + no_unwind + { + let raw_atomic = &self.atomic; + proof_decl! { + let tracked final_guard; + } + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.lifecycle@ == self.child_phase()); + match state.lifecycle@ { + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + _ => {}, + } + match lease { + None => { + final_guard = guard; + }, + Some(lease) => { + final_guard = state.permissions.tracked_return_loaded(lease, guard); + }, + } + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() + == permissions_before.reclaim_states()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + state.permissions.lemma_contains_iff_key(self.constant().child_obj); + assert(state.permissions.contains(self.constant().child_obj)); + assert(state.permissions.reclaim_states()[self.constant().child_obj] + == Some(self.constant().child)); + match state.lifecycle@ { + LinkedListChildPhase::Unpublished + | LinkedListChildPhase::Linked { index: _, timestamp: _ } + | LinkedListChildPhase::Unlinked { index: _, removal: _ } => { + assert(state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Retired { index: _, removal: _ } => { + assert(!state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + } + assert(OwnPred::owns( + self.constant().child, + permissions_before.ownership(self.constant().child_obj), + )); + assert(state.permissions.ownership(self.constant().child_obj) + == permissions_before.ownership(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + Tracked(final_guard) + } + + /// Publishes the pre-registered child with a native AcqRel/Acquire CAS. + /// The successful branch appends the matching traversal event before the + /// atomic invariant is closed. + #[inline(always)] + pub fn compare_exchange_publish_child(&mut self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).child_phase().is_unpublished(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> res.0->Ok_0.addr() == 0, + res.0 is Ok ==> final(self).child_phase() is Linked, + res.0 is Ok ==> final(self).child_phase()->Linked_index == res.1@->Some_0, + res.0 is Err ==> final(self).child_phase() is Unpublished, + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let child = self.child; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + assert(state.lifecycle@ is Unpublished); + assert(state.points_to.loc() == native_loc); + } + let ghost prev = state.points_to.hist(); + let ghost old_successors = state.auth.state().successors[key.source_obj]; + let cas = raw_atomic.compare_exchange( + null, + child, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost published_index: Option; + } + proof { + match cas.0 { + Result::Ok(_) => { + assert(next == prev.insert( + update.load_timestamp + 1, + child, + update.store_message_view, + )); + let index = state.link.tracked_cas_publish( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + child, + key.child_obj, + update.store_message_view, + ); + published_index = Some(index); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Linked { + index, + timestamp: update.load_timestamp + 1, + }, + ); + assert(state.auth.state().incoming_all[key.child_obj].contains(( + key.source_obj, + index, + ))); + assert(state.auth.state().incoming_all[key.child_obj].len() > 0); + assert(state.auth.removed() == Set::::empty()); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth.state().successors[key.source_obj].len() + && state.auth.state().successors[key.source_obj][n as int] is Some implies + #[trigger] state.auth.state().successors[key.source_obj][n as int] == Some(( + key.child, + key.child_obj, + )) by { + if n == index { + } else { + assert(n < old_successors.len()); + assert(state.auth.state().successors[key.source_obj][n as int] == old_successors[n as int]); + } + }; + }, + Result::Err(_) => { + published_index = None; + }, + } + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + } + result = (cas.0, Ghost(published_index)); + }); + result + } + + /// Unlinks the pre-registered child with a native AcqRel/Acquire CAS. + /// A successful address comparison is resolved through the invariant to + /// the child's persistent object identity before the null traversal event + /// is appended. + #[inline(always)] + pub fn compare_exchange_unlink_child(&mut self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).child_phase().is_linked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> res.0->Ok_0.addr() == final(self).child_ptr().addr(), + res.0 is Ok ==> final(self).child_phase() is Unlinked, + res.0 is Ok ==> final(self).child_phase()->Unlinked_index == res.1@->Some_0, + res.0 is Ok ==> final(self).child_phase()->Unlinked_removal.root + == final(self).constant().root, + res.0 is Ok ==> final(self).child_phase()->Unlinked_removal.observed_by(final(tv)@), + res.0 is Err ==> final(self).child_phase() is Linked, + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let child = self.child; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + assert(state.lifecycle@ is Linked); + assert(state.points_to.loc() == native_loc); + } + let ghost prev = state.points_to.hist(); + let ghost old_successors = state.auth.state().successors[key.source_obj]; + let cas = raw_atomic.compare_exchange( + child, + null, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost unlinked_index: Option; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + let ghost current_index = state.link.index_at(current_timestamp); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp <= current_timestamp); + assert(current_timestamp <= update.load_timestamp); + assert(current_timestamp == update.load_timestamp); + assert(state.lifecycle@->Linked_timestamp == current_timestamp); + assert(state.lifecycle@->Linked_index == current_index); + assert(equal(prev.value(update.load_timestamp), value)); + assert(value.addr() == child.addr()); + assert(prev.value(current_timestamp).addr() == child.addr()); + assert(current_index + 1 == old_successors.len()); + assert(old_successors[current_index as int] is Some) by { + if old_successors[current_index as int] is None { + assert(prev.value(current_timestamp).addr() == 0); + assert(false); + } + }; + assert(old_successors[current_index as int] == Some(( + key.child, + key.child_obj, + ))); + assert(old_successors.last() == Some(( + key.child, + key.child_obj, + ))); + assert(state.auth.state().objects.contains_pair( + key.source_obj, + key.source, + )); + assert(state.auth.state().objects.contains_pair( + key.child_obj, + key.child, + )); + assert(state.auth.state().incoming_all.contains_key(key.child_obj)); + assert(current_index < old_successors.len()); + assert(state.auth.state().successors[key.source_obj][current_index as int] + == Some((key.child, key.child_obj))); + assert(state.auth.state().incoming_all[key.child_obj].len() > 0); + assert(next == prev.insert( + update.load_timestamp + 1, + null, + update.store_message_view, + )); + let index = state.link.tracked_cas_unlink( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + key.child, + key.child_obj, + update.store_message_view, + ); + unlinked_index = Some(index); + let ghost removal = rcu_spec::RcuRemovalObservation { + root: key.root, + timestamp: update.load_timestamp + 1, + message_view: update.store_message_view, + }; + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Unlinked { index, removal }, + ); + assert(state.auth.removed() == Set::::empty()); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth.state().successors[key.source_obj].len() + && state.auth.state().successors[key.source_obj][n as int] is Some implies + #[trigger] state.auth.state().successors[key.source_obj][n as int] == Some(( + key.child, + key.child_obj, + )) by { + assert(n < old_successors.len()); + assert(state.auth.state().successors[key.source_obj][n as int] == old_successors[n as int]); + }; + assert(state.lifecycle@ is Unlinked); + assert(state.link.current_timestamp() == removal.timestamp); + assert(state.link.index_at(removal.timestamp) == index); + assert(state.points_to.hist().thread_view(removal.timestamp) + == removal.message_view); + assert(index + 1 + == state.auth.state().successors[key.source_obj].len()); + assert(state.auth.state().successors[key.source_obj].last() is None); + assert(state.auth.state().incoming_all[key.child_obj].len() > 0); + }, + Result::Err(_) => { + unlinked_index = None; + }, + } + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + } + result = (cas.0, Ghost(unlinked_index)); + }); + result + } + + /// Converts a successfully unlinked child into the writer resources + /// required by base RCU and by eventual physical reclamation. + /// + /// The traversal observation records the unlink event itself. Since the + /// only historical incoming edge is the immediately preceding publish, + /// this proves that every incoming edge is dead and consumes the child's + /// unique traversal retire permission. + #[verifier::atomic] + pub fn retire_unlinked_child(&mut self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: Tracked< + LinkedListDetachedChild, + >) + requires + old(self).well_formed(), + old(self).child_phase().is_unlinked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).child_phase() is Retired, + final(self).child_phase()->Retired_index == old(self).child_phase()->Unlinked_index, + final(self).child_phase()->Retired_removal == old(self).child_phase()->Unlinked_removal, + final(self).child_phase()->Retired_removal.root == final(self).constant().root, + res@.object().wf(), + res@.object().domain() == final(self).constant().domain, + res@.object().obj() == final(self).constant().child_obj, + res@.object().ptr() == final(self).constant().child, + res@.retire().wf(), + res@.retire().ready_to_retire(), + res@.retire().domain() == final(self).constant().domain, + res@.retire().obj() == final(self).constant().child_obj, + res@.retire().ptr() == final(self).constant().child, + res@.claim().registry() == final(self).constant().reclaim_registry, + res@.claim().obj() == final(self).constant().child_obj, + res@.claim().is_pending(), + res@.claim().ptr() == final(self).constant().child, + res@.removal() == final(self).child_phase()->Retired_removal, + res@.removal().root == final(self).constant().root, + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked detached; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.lifecycle@ is Unlinked); + let ghost index = state.lifecycle@->Unlinked_index; + let ghost removal = state.lifecycle@->Unlinked_removal; + let ghost prior = rcu_spec::RcuSeenRemoved { + removed: Set::empty(), + link_view: rcu_spec::RcuLinkView::empty().observe(key.source_obj, index), + }; + assert(state.auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut rcu_spec::LinkedListNode, from_obj: nat| #[trigger] + state.auth.state().objects.contains_pair(from_obj, from) + && prior.link_view.seen.contains_key(from_obj) implies { + &&& state.auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) + < state.auth.state().successors[from_obj].len() + } by { + assert(from_obj == key.source_obj); + assert(from == key.source); + assert(prior.link_view.seen_at(from_obj) == index); + assert(index + 1 == state.auth.state().successors[from_obj].len()); + }; + } + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + prior, + state.auth.state(), + )); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[key.child_obj].contains(edge) implies + prior.dead_edge(edge) by { + assert(edge == (key.source_obj, (index - 1) as nat)); + assert(prior.seen_at(key.source_obj) == index); + }; + state.auth.lemma_has_info_for_object(key.child_obj); + assert(state.auth.has_info(key.child_obj)); + assert(state.permissions.contains(key.child_obj)); + let ghost auth_before = state.auth; + let ghost permissions_before = state.permissions; + let ghost child_ownership = state.permissions.ownership(key.child_obj); + let tracked object = state.auth.tracked_info_for(key.child_obj); + let tracked retire = state.auth.tracked_retire_node(key.child_obj, prior); + assert(auth_before.state().objects.contains_pair(key.child_obj, retire.ptr())); + assert(retire.ptr() == key.child); + let tracked claim = state.permissions.tracked_retire(key.child_obj); + assert(claim.ptr() == key.child); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Retired { index, removal }, + ); + assert(state.auth.removed() == Set::::empty().insert(key.child_obj)); + assert(!state.auth.has_retire_perm(key.child_obj)); + assert(state.permissions.keys() == Set::empty().insert(key.child_obj)); + state.permissions.lemma_contains_iff_key(key.child_obj); + assert(state.permissions.contains(key.child_obj)); + assert(state.permissions.allocations().contains(key.child_obj)); + assert(state.permissions.reclaim_states()[key.child_obj] == Some(key.child)); + assert(!state.permissions.has_unretired_claim(key.child_obj)); + assert(state.permissions.ownership(key.child_obj) == child_ownership); + assert(OwnPred::owns( + key.child, + state.permissions.ownership(key.child_obj), + )); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(object.ptr() == key.child); + assert(retire.ptr() == key.child); + assert(claim.ptr() == key.child); + detached = LinkedListDetachedChild { object, retire, claim, removal }; + } + }); + Tracked(detached) + } + + /// Reclaims the retired child's physical ownership after the existing RCU + /// monitor has produced a completed grace-period witness. + #[verifier::atomic] + pub fn reclaim_retired_child( + &mut self, + Tracked(claim): Tracked>, + Tracked(completed): Tracked, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked) + requires + old(self).well_formed(), + old(self).child_phase().is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == old(self).constant().child_obj, + claim.is_pending(), + claim.ptr() == old(self).constant().child, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == old(self).constant().child_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).child_phase()->Retired_removal, + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).child_phase() is Reclaimed, + final(self).child_phase()->Reclaimed_index == old(self).child_phase()->Retired_index, + final(self).child_phase()->Reclaimed_removal == old( + self, + ).child_phase()->Retired_removal, + OwnPred::owns(final(self).constant().child, res@), + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked ownership; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.lifecycle@ is Retired); + let ghost index = state.lifecycle@->Retired_index; + let ghost removal = state.lifecycle@->Retired_removal; + assert(completed.record().removal == removal); + assert(completed.record().removal.root == key.root); + state.permissions.lemma_completed_excludes_active( + &completed, + key.child_obj, + ); + assert(!state.permissions.has_active(key.child_obj)); + ownership = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Reclaimed { index, removal }, + ); + assert(state.permissions.keys() == Set::::empty()); + state.permissions.lemma_contains_iff_key(key.child_obj); + assert(!state.permissions.contains(key.child_obj)); + assert(state.permissions.allocations().contains(key.child_obj)); + assert(state.permissions.reclaim_states()[key.child_obj] is None); + assert(!state.permissions.has_unretired_claim(key.child_obj)); + assert(state.permissions.reclaimed().contains_key(key.child_obj)); + assert(state.permissions.reclaimed()[key.child_obj].record().removal == removal); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(OwnPred::owns(key.child, ownership)); + } + }); + Tracked(ownership) + } + + /// Proof-mode form used by a type-erased callback after its executable + /// monitor has supplied an open-invariant credit and a completed grace + /// period. No additional atomic access is needed at runtime: the callback + /// owns this link and only consumes proof resources before deallocating the + /// recovered smart pointer. + pub proof fn tracked_reclaim_retired_child( + tracked &mut self, + tracked claim: rcu_cpu_spec::RcuReclaimClaim, + tracked completed: rcu_cpu_spec::RcuReclaimedWitness, + tracked credit: vstd::invariant::OpenInvariantCredit, + ) -> (tracked ownership: O) + requires + old(self).well_formed(), + old(self).child_phase().is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == old(self).constant().child_obj, + claim.is_pending(), + claim.ptr() == old(self).constant().child, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == old(self).constant().child_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).child_phase()->Retired_removal, + ensures + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).child_phase() is Reclaimed, + final(self).child_phase()->Reclaimed_index == old(self).child_phase()->Retired_index, + final(self).child_phase()->Reclaimed_removal == old( + self, + ).child_phase()->Retired_removal, + OwnPred::owns(final(self).constant().child, ownership), + opens_invariants [self.invariant_namespace()] + { + use_type_invariant(&*self); + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked mut recovered; + vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.lifecycle@ is Retired); + let ghost index = state.lifecycle@->Retired_index; + let ghost removal = state.lifecycle@->Retired_removal; + assert(completed.record().removal == removal); + assert(completed.record().removal.root == key.root); + state.permissions.lemma_completed_excludes_active( + &completed, + key.child_obj, + ); + assert(!state.permissions.has_active(key.child_obj)); + recovered = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Reclaimed { index, removal }, + ); + assert(state.permissions.keys() == Set::::empty()); + state.permissions.lemma_contains_iff_key(key.child_obj); + assert(!state.permissions.contains(key.child_obj)); + assert(state.permissions.allocations().contains(key.child_obj)); + state.permissions.lemma_allocation_has_reclaim_state(key.child_obj); + assert(state.permissions.reclaim_states()[key.child_obj] is None); + assert(!state.permissions.has_unretired_claim(key.child_obj)); + assert(state.permissions.reclaimed().contains_key(key.child_obj)); + assert(state.permissions.reclaimed()[key.child_obj].record().removal == removal); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(OwnPred::owns(key.child, recovered)); + }); + recovered + } +} + +/// OSTD's RCU-specific specialization of the generic weak pointer atomic. +/// +/// This is an RCU client of Verus' native IRC11 protocol. The only local TCB +/// component is `PAtomicWeakPtr`, needed because upstream does not yet expose +/// a native weak-memory `AtomicPtr`. +#[verifier::reject_recursive_types(T)] +pub struct RcuWeakAtomicPtr { + atomic: PAtomicWeakPtr, + tracked_atomic_inv: Tracked<&'static RcuRootAtomicInvariant>, +} + +impl RcuWeakAtomicPtr { + pub closed spec fn constant(&self) -> rcu_spec::RcuRootKey { + self.tracked_atomic_inv@.constant().0 + } + + pub closed spec fn id(&self) -> Loc { + self.constant().domain + } + + pub closed spec fn native_loc(&self) -> Irc11AtomicId { + self.atomic.loc() + } + + pub closed spec fn well_formed(&self) -> bool { + self.tracked_atomic_inv@.constant().1 == self.native_loc() + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } +} + +impl RcuWeakAtomicPtr where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + pub const fn new( + Ghost(nullable): Ghost, + Ghost(scheduler): Ghost, + init: *mut T, + Tracked(ownership): Tracked>, + ) -> (res: Self) + requires + nullable || !init.is_null(), + (ownership is Some) == !init.is_null(), + ownership is Some ==> OwnPred::owns(init, ownership->Some_0), + ensures + res.well_formed(), + res.constant().nullable == nullable, + res.constant().scheduler == scheduler, + { + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + PAtomicWeakPtr::new(init); + proof_decl! { + let tracked unit_ownership: Option<()>; + let tracked physical_ownership: Option; + } + proof { + match ownership { + Some(ownership) => { + unit_ownership = Some(()); + physical_ownership = Some(ownership); + }, + None => { + unit_ownership = None; + physical_ownership = None; + }, + } + } + let tracked g = rcu_spec::RcuRootOwnedGhost::tracked_initial( + init, + unit_ownership, + points_to.hist(), + timestamp, + initial_view@, + ); + proof_decl! { + let tracked state: RcuRootAtomicState; + } + proof { + let tracked mut permissions = rcu_cpu_spec::RcuRootPermissionState::empty( + scheduler, + g.domain(), + g.domain(), + g.retire_observation_registry(), + ); + assert(permissions.allocations() == Set::::empty()); + if physical_ownership is Some { + let tracked info = g.tracked_info_at(points_to.hist(), timestamp).tracked_unwrap(); + let ghost initial_obj = info.obj(); + let ghost initial_ownership = physical_ownership->Some_0; + assert(!permissions.allocations().contains(info.obj())); + permissions.tracked_insert(&info, physical_ownership.tracked_unwrap()); + assert(g.current_registration() is Some); + assert(info.obj() == g.current_registration()->Some_0.0.obj()); + assert(permissions.has_unretired_claim(info.obj())); + assert(permissions.keys() == Set::::empty().insert(initial_obj)); + assert(equal(info.ptr(), init)); + assert(OwnPred::owns(info.ptr(), initial_ownership)); + permissions.lemma_live_reclaim_state(initial_obj); + assert forall|obj: nat| #[trigger] permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(obj == initial_obj); + assert(g.infos().contains_key(initial_obj)); + assert(equal(info.ptr(), g.infos()[initial_obj].ptr())); + assert(permissions.reclaim_states()[initial_obj] == Some(info.ptr())); + assert(permissions.ownership(initial_obj) == initial_ownership); + }; + } else { + assert(g.current_registration() is None); + assert(permissions.keys() == Set::::empty()); + assert forall|obj: nat| #[trigger] permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by {}; + } + assert(permissions.allocations() == g.infos().dom()); + assert(permissions.scheduler() == scheduler); + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by {}; + state = RcuRootAtomicState { points_to, root: g, permissions }; + } + let ghost key = rcu_spec::RcuRootKey { + nullable, + scheduler, + domain: state.root.domain(), + reader_registry: state.root.reader_registry(), + retire_observation_registry: state.root.retire_observation_registry(), + reclaim_registry: state.permissions.reclaim_registry(), + active_lease_registry: state.permissions.active_lease_registry(), + }; + proof { + assert(rcu_spec::rcu_history_inv(nullable, state.points_to.hist())) by { + assert(!state.points_to.hist().dom().is_empty()); + if !nullable { + assert forall|ts: nat| + state.points_to.hist().contains_timestamp( + ts, + ) implies #[trigger] state.points_to.hist().value(ts).addr() != 0 by { + assert(ts == timestamp); + assert(equal(state.points_to.hist().value(ts), init)); + }; + } + }; + assert forall|obj: nat| state.root.removals().contains_key(obj) implies { + let removal = #[trigger] state.root.removals()[obj]; + state.points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + assert(state.root.removals() == Map::empty()); + }; + assert(RcuRootAtomicInv::::inv((key, atomic.loc()), state)); + } + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); + let tracked atomic_inv = tracked_static_ref(atomic_inv); + Self { atomic, tracked_atomic_inv: Tracked(atomic_inv) } + } + + fn raw_atomic(&self) -> (res: &PAtomicWeakPtr) + requires + self.well_formed(), + ensures + res.loc() == self.native_loc(), + opens_invariants none + no_unwind + { + &self.atomic + } + + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &'static RcuRootAtomicInvariant< + T, + O, + OwnPred, + >) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.native_loc()), + { + self.tracked_atomic_inv.get() + } + + /// Acquire-load helper for RCU root pointers. + #[inline(always)] + pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + )) + requires + self.well_formed(), + ensures + old(tv)@.spec_le(final(tv)@), + !self.constant().nullable ==> !res.0.is_null(), + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + }, + _ => false, + }, + { + let result; + let ghost start_view = tv@; + proof { + use_type_invariant(self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { points_to, root: g, permissions } = state; + proof { + assert(points_to.loc() == self.native_loc()); + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&points_to), + ); + proof { + assert(rcu_spec::rcu_owned_root_history_inv(points_to.hist(), g)); + } + proof_decl! { + let ghost timestamp = loaded.2@.timestamp; + let ghost published = g.published_at(timestamp); + let tracked loaded_info; + } + proof { + loaded_info = g.tracked_info_at(points_to.hist(), timestamp); + match (published, &loaded_info) { + (Some(object), Some(info)) => { + assert(equal(points_to.hist().value(timestamp), loaded.0)); + assert(equal(info.ptr(), loaded.0)); + }, + (None, None) => { + assert(loaded.0.addr() == 0); + }, + _ => assert(false), + }; + if !self.constant().nullable { + rcu_spec::rcu_history_inv_read_nonnull::(points_to.hist(), timestamp); + assert(!loaded.0.is_null()); + } + } + result = (loaded.0, Ghost(timestamp), Ghost(published), Tracked(loaded_info)); + proof { + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + result + } + + /// Acquire-load an RCU root while starting a paper read-side guard. + /// + /// The ghost reader transition occurs in the same invariant opening as the + /// real acquire load. Executably this is identical to `load_acquire_rcu`. + #[inline(always)] + pub fn load_acquire_rcu_guarded_with_retired( + &self, + Ghost(reader): Ghost, + Tracked(retired_facts): Tracked<&rcu_spec::RcuRetiredFacts>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + Tracked>, + )) + requires + self.well_formed(), + retired_facts.observed_by(old(tv)@), + ensures + old(tv)@.spec_le(final(tv)@), + !self.constant().nullable ==> !res.0.is_null(), + res.4@.wf(), + res.4@.domain() == self.constant().domain, + res.4@.reader_registry() == self.constant().reader_registry, + res.4@.retire_observation_registry() == self.constant().retire_observation_registry, + res.4@.reader() == reader, + res.4@.root() == self.id(), + res.4@.start_view() == old(tv)@, + forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + retired_facts.records().contains(record) && record.domain == res.4@.domain() + && record.retire_observation_registry == res.4@.retire_observation_registry() + && record.removal.root == res.4@.root() ==> res.4@.expired().contains( + record.obj, + ), + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == res.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + &&& !res.4@.expired().contains(info.obj()) + &&& !res.4@.seen_removed().removed.contains(info.obj()) + &&& res.4@.protects(info.addr(), info.obj()) + }, + _ => false, + }, + { + let result; + proof { + use_type_invariant(self); + } + let ghost start_view = tv@; + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { points_to, root: mut g, permissions } = state; + let ghost root_before_reader = g; + proof { + assert(points_to.loc() == self.native_loc()); + assert(g.retire_observation_registry() + == self.constant().retire_observation_registry); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + } + proof_decl! { + let tracked base_guard = + g.tracked_start_reader(points_to.hist(), self.id(), start_view, reader); + } + proof { + g.lemma_retired_facts_observed( + points_to.hist(), + &retired_facts, + self.id(), + start_view, + ); + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + proof { + assert(rcu_spec::rcu_owned_root_history_inv(points_to.hist(), g)); + } + proof_decl! { + let tracked loaded_info; + } + proof { + loaded_info = g.tracked_info_at(points_to.hist(), timestamp); + } + proof_decl! { + let ghost published = g.published_at(timestamp); + } + proof { + match (published, &loaded_info) { + (Some(object), Some(info)) => { + assert(equal(points_to.hist().value(timestamp), loaded.0)); + assert(equal(info.ptr(), loaded.0)); + assert(info.domain() == g.domain()); + assert(info.domain() == base_guard.domain()); + }, + (None, None) => { + assert(loaded.0.addr() == 0); + }, + _ => assert(false), + }; + if !self.constant().nullable { + rcu_spec::rcu_history_inv_read_nonnull::(points_to.hist(), timestamp); + assert(!loaded.0.is_null()); + } + assert(base_guard.domain() == self.constant().domain); + assert(base_guard.reader_registry() == self.constant().reader_registry); + assert(base_guard.retire_observation_registry() + == g.retire_observation_registry()); + assert(g.retire_observation_registry() + == self.constant().retire_observation_registry); + assert(base_guard.retire_observation_registry() + == self.constant().retire_observation_registry); + } + proof_decl! { + let tracked mut guard = + rcu_spec::RcuReadGuardToken::tracked_from_base(base_guard); + } + proof { + assert(guard.expired() + == g.root().domain_auth().observed_retired(self.id(), start_view)); + match &loaded_info { + Some(info) => { + if guard.expired().contains(info.obj()) { + assert(g.root().domain_auth().observed_retired( + self.id(), + start_view, + ).contains(info.obj())); + g.lemma_observed_retired( + points_to.hist(), + self.id(), + start_view, + info.obj(), + ); + let ghost removal = g.removals()[info.obj()]; + assert(removal.root == self.id()); + assert(removal.observed_by(start_view)); + assert(points_to.get_timestamp(removal.message_view) + == Some(removal.timestamp)); + points_to.get_timestamp_monotonic(start_view, removal.message_view); + assert(points_to.get_timestamp(start_view) is Some); + assert(removal.timestamp + <= points_to.get_timestamp(start_view)->Some_0); + assert(points_to.get_timestamp(start_view)->Some_0 <= timestamp); + assert(removal.timestamp <= timestamp); + assert(g.removals_wf(points_to.hist())); + assert(g.publications()[timestamp] != Some(info.obj())); + assert(published == Some(rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + })); + g.lemma_published_object_id( + points_to.hist(), + timestamp, + rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + }, + ); + assert(g.publications()[timestamp] == Some(info.obj())); + assert(false); + } + assert(guard.can_protect(*info)); + guard.tracked_protect(info); + }, + None => {}, + } + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(published), + Tracked(loaded_info), + Tracked(guard), + ); + proof { + assert(g.current_owned() == root_before_reader.current_owned()); + assert(g.domain() == root_before_reader.domain()); + assert(g.reader_registry() == root_before_reader.reader_registry()); + assert(g.retire_observation_registry() + == root_before_reader.retire_observation_registry()); + assert(g.publications() == root_before_reader.publications()); + assert(g.infos() == root_before_reader.infos()); + assert(g.removals() == root_before_reader.removals()); + assert(rcu_spec::rcu_current_ownership_inv::< + T, + (), + rcu_spec::UnitRcuRootOwnership, + >(g)); + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + assert(root_before_reader.removals().contains_key(obj)); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + assert(permissions.allocations() == g.infos().dom()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by {}; + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by {}; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + result + } + + /// Acquire-load an RCU root while starting a paper read-side guard. + /// + /// This compatibility entry point has no CPU-generation retirement + /// history, so it starts the guard with only the root invariant's directly + /// observed retirements. + #[inline(always)] + pub fn load_acquire_rcu_guarded( + &self, + Ghost(reader): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + Tracked>, + )) + requires + self.well_formed(), + ensures + old(tv)@.spec_le(final(tv)@), + !self.constant().nullable ==> !res.0.is_null(), + res.4@.wf(), + res.4@.domain() == self.constant().domain, + res.4@.reader_registry() == self.constant().reader_registry, + res.4@.retire_observation_registry() == self.constant().retire_observation_registry, + res.4@.reader() == reader, + res.4@.root() == self.id(), + res.4@.start_view() == old(tv)@, + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == res.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + &&& !res.4@.expired().contains(info.obj()) + &&& !res.4@.seen_removed().removed.contains(info.obj()) + &&& res.4@.protects(info.addr(), info.obj()) + }, + _ => false, + }, + { + proof_decl! { + let tracked retired_facts = rcu_spec::RcuRetiredFacts::empty(); + } + self.load_acquire_rcu_guarded_with_retired( + Ghost(reader), + Tracked(&retired_facts), + Tracked(tv), + ) + } + + /// Acquire-load an RCU root while retaining the CPU implementation + /// fragment in the returned guard. + /// + /// The caller must split `cpu_reader` after disabling preemption and before + /// calling this method. The fragment is therefore live before the first + /// protected load, while the participant view bound ensures that the paper + /// guard starts no earlier than the CPU state from which it was split. + #[inline(always)] + pub fn load_acquire_rcu_guarded_cpu( + &self, + Ghost(reader): Ghost, + Tracked(cpu_reader): Tracked, + Tracked(binding): Tracked, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + Tracked>, + Tracked>>, + )) + requires + self.well_formed(), + cpu_reader.wf(), + reader.cpu == cpu_reader.cpu(), + reader.generation == cpu_reader.generation(), + binding.registry() == reader.scheduler, + reader.scheduler == self.constant().scheduler, + binding.cpu() == cpu_reader.cpu(), + binding.locals_key().len() == 1, + binding.single_local_id() == cpu_reader.participant_id(), + online_cpus().contains(cpu_reader.cpu()), + cpu_reader.participant_view().spec_le(old(tv)@), + ensures + old(tv)@.spec_le(final(tv)@), + !self.constant().nullable ==> !res.0.is_null(), + res.4@.wf(), + res.4@.participant_id() == cpu_reader.participant_id(), + res.4@.cpu() == cpu_reader.cpu(), + res.4@.generation() == cpu_reader.generation(), + res.4@.participant_view() == cpu_reader.participant_view(), + res.4@.scheduler() == binding.registry(), + res.4@.domain() == self.constant().domain, + res.4@.reader_registry() == self.constant().reader_registry, + res.4@.retire_observation_registry() == self.constant().retire_observation_registry, + res.4@.reader_context() == reader, + res.4@.root() == self.id(), + res.4@.start_view() == old(tv)@, + match (res.2@, res.3@, res.5@) { + (None, None, None) => { + &&& res.0.addr() == 0 + &&& res.4@.reader_fragment() == cpu_reader + }, + (Some(object), Some(info), Some(lease)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == res.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + &&& !res.4@.expired().contains(info.obj()) + &&& !res.4@.seen_removed().removed.contains(info.obj()) + &&& res.4@.protects(info.addr(), info.obj()) + &&& res.4@.reader_fragment().fraction() == cpu_reader.fraction() / 2real + &&& lease.key() == info.obj() + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == res.4@.participant_id() + &&& lease.reader_fraction() == res.4@.reader_fragment().fraction() + &&& lease.domain() == res.4@.domain() + &&& lease.root() == res.4@.root() + &&& lease.reader_context() == res.4@.reader_context() + &&& lease.start_view() == res.4@.start_view() + &&& lease.protected_addr() == info.addr() + &&& OwnPred::owns(res.0, lease.resource()) + }, + _ => false, + }, + { + let result; + proof { + use_type_invariant(self); + } + let ghost start_view = tv@; + let ghost cpu_reader_at_entry = cpu_reader; + proof_decl! { + let tracked retired_facts_ref = + cpu_reader.tracked_retired_facts_observed_by(start_view); + let tracked retired_facts = retired_facts_ref.tracked_duplicate(); + let tracked mut cpu_reader = cpu_reader; + } + proof { + assert(retired_facts.observed_by(start_view)); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { + points_to, + root: mut g, + permissions: mut permissions, + } = state; + let ghost root_before_cpu_reader = g; + let ghost permissions_before_cpu_reader = permissions; + proof { + assert(points_to.loc() == self.native_loc()); + assert(g.retire_observation_registry() + == self.constant().retire_observation_registry); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + } + proof_decl! { + let tracked base_guard = + g.tracked_start_reader(points_to.hist(), self.id(), start_view, reader); + } + proof { + g.lemma_retired_facts_observed( + points_to.hist(), + &retired_facts, + self.id(), + start_view, + ); + } + let loaded = raw_atomic.load(Ordering::Acquire, Tracked(tv), Tracked(&points_to)); + let ghost timestamp = loaded.2@.timestamp; + proof_decl! { + let tracked loaded_info; + } + proof { + assert(rcu_spec::rcu_owned_root_history_inv(points_to.hist(), g)); + loaded_info = g.tracked_info_at(points_to.hist(), timestamp); + assert(g.publications().contains_key(timestamp)); + } + proof_decl! { + let ghost published = g.published_at(timestamp); + } + proof { + match (published, &loaded_info) { + (Some(object), Some(info)) => { + assert(equal(points_to.hist().value(timestamp), loaded.0)); + assert(equal(info.ptr(), loaded.0)); + assert(info.domain() == g.domain()); + assert(info.domain() == base_guard.domain()); + }, + (None, None) => assert(loaded.0.addr() == 0), + _ => assert(false), + } + if !self.constant().nullable { + rcu_spec::rcu_history_inv_read_nonnull::(points_to.hist(), timestamp); + assert(!loaded.0.is_null()); + } + assert(base_guard.domain() == self.constant().domain); + assert(base_guard.reader_registry() == self.constant().reader_registry); + assert(base_guard.retire_observation_registry() + == self.constant().retire_observation_registry); + } + proof_decl! { + let tracked mut paper_guard = + rcu_spec::RcuReadGuardToken::tracked_from_base(base_guard); + } + proof { + assert(paper_guard.expired() + == g.root().domain_auth().observed_retired(self.id(), start_view)); + match &loaded_info { + Some(info) => { + assert(g.infos().contains_key(info.obj())); + assert(permissions.allocations().contains(info.obj())); + if !permissions.contains(info.obj()) { + let tracked completed = permissions.tracked_reclaimed(info.obj()); + assert(permissions.reclaimed().contains_key(info.obj())); + assert(g.removals().contains_key(info.obj())); + assert(completed.record().removal == g.removals()[info.obj()]); + let tracked closed = completed.tracked_closed_generation( + cpu_reader.cpu(), + ); + assert(closed.scheduler() == permissions.scheduler()); + assert(permissions.scheduler() == self.constant().scheduler); + assert(binding.registry() == self.constant().scheduler); + closed.lemma_same_participant_as_binding(&binding); + assert(completed.closed_generations()[cpu_reader.cpu()].participant_id() + == cpu_reader.participant_id()); + cpu_reader = completed.tracked_later_reader(cpu_reader); + assert(retired_facts.records().contains(completed.record())); + } + if paper_guard.expired().contains(info.obj()) { + assert(g.root().domain_auth().observed_retired( + self.id(), + start_view, + ).contains(info.obj())); + g.lemma_observed_retired( + points_to.hist(), + self.id(), + start_view, + info.obj(), + ); + let ghost removal = g.removals()[info.obj()]; + assert(removal.root == self.id()); + assert(removal.observed_by(start_view)); + assert(points_to.get_timestamp(removal.message_view) + == Some(removal.timestamp)); + points_to.get_timestamp_monotonic(start_view, removal.message_view); + assert(points_to.get_timestamp(start_view) is Some); + assert(removal.timestamp + <= points_to.get_timestamp(start_view)->Some_0); + assert(points_to.get_timestamp(start_view)->Some_0 <= timestamp); + assert(removal.timestamp <= timestamp); + assert(g.removals_wf(points_to.hist())); + assert(g.publications()[timestamp] != Some(info.obj())); + assert(published == Some(rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + })); + g.lemma_published_object_id( + points_to.hist(), + timestamp, + rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + }, + ); + assert(g.publications()[timestamp] == Some(info.obj())); + assert(false); + } + assert(permissions.contains(info.obj())); + assert(paper_guard.can_protect(*info)); + paper_guard.tracked_protect(info); + }, + None => {}, + } + assert(cpu_reader == cpu_reader_at_entry); + } + proof_decl! { + let tracked cpu_guard = rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new( + paper_guard, + cpu_reader, + binding, + ); + let tracked final_guard; + let tracked lease; + } + proof { + assert(cpu_guard.reader_context() == reader); + match &loaded_info { + Some(info) => { + assert(permissions.contains(info.obj())); + permissions.lemma_live_reclaim_state(info.obj()); + assert(g.infos().contains_key(info.obj())); + let ghost loaded_ownership = permissions.ownership(info.obj()); + assert(permissions.reclaim_states()[info.obj()] is Some); + assert(permissions.reclaim_states()[info.obj()]->Some_0 == info.ptr()); + assert(equal(info.ptr(), loaded.0)); + assert(OwnPred::owns(loaded.0, loaded_ownership)); + let tracked split = permissions.tracked_split_loaded( + cpu_guard, + info, + ); + final_guard = split.0; + lease = Some(split.1); + assert(split.1.resource() == loaded_ownership); + assert(OwnPred::owns(loaded.0, split.1.resource())); + assert(final_guard.scheduler() == binding.registry()); + assert(final_guard.domain() == self.constant().domain); + }, + None => { + final_guard = cpu_guard; + lease = None; + assert(final_guard.scheduler() == binding.registry()); + assert(final_guard.domain() == self.constant().domain); + }, + } + assert(final_guard.reader_context() == reader); + assert(final_guard.start_view() == start_view); + match (&loaded_info, &lease) { + (None, None) => { + assert(final_guard.reader_fragment() == cpu_reader_at_entry); + }, + (Some(info), Some(lease)) => { + assert(lease.key() == info.obj()); + assert(final_guard.reader_fragment().fraction() + == cpu_reader_at_entry.fraction() / 2real); + }, + _ => assert(false), + } + assert(g.current_owned() == root_before_cpu_reader.current_owned()); + assert(g.domain() == root_before_cpu_reader.domain()); + assert(g.reader_registry() == root_before_cpu_reader.reader_registry()); + assert(g.retire_observation_registry() + == root_before_cpu_reader.retire_observation_registry()); + assert(g.publications() == root_before_cpu_reader.publications()); + assert(g.infos() == root_before_cpu_reader.infos()); + assert(g.removals() == root_before_cpu_reader.removals()); + assert(rcu_spec::rcu_current_ownership_inv::< + T, + (), + rcu_spec::UnitRcuRootOwnership, + >(g)); + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + assert(root_before_cpu_reader.removals().contains_key(obj)); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + assert(permissions.allocations() + == permissions_before_cpu_reader.allocations()); + assert(permissions.reclaim_states() + == permissions_before_cpu_reader.reclaim_states()); + assert(permissions.reclaimed() == permissions_before_cpu_reader.reclaimed()); + assert(permissions.unretired_claims() + == permissions_before_cpu_reader.unretired_claims()); + assert(permissions.wf()); + assert(permissions.domain() == self.constant().domain); + assert(permissions.root() == self.constant().domain); + assert(permissions.retire_observation_registry() + == self.constant().retire_observation_registry); + assert(permissions.reclaim_registry() == self.constant().reclaim_registry); + assert(permissions.allocations() == g.infos().dom()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(permissions_before_cpu_reader.keys().contains(obj)); + assert(permissions.ownership(obj) + == permissions_before_cpu_reader.ownership(obj)); + }; + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by {}; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(published), + Tracked(loaded_info), + Tracked(final_guard), + Tracked(lease), + ); + }); + result + } + + /// Return a guarded load's physical lease to this root. + /// + /// The lease's linear membership receipt identifies the active registry + /// entry after the atomic invariant is reopened. Returning it also rejoins + /// the CPU fragment retained by that entry with the executable guard. + #[verifier::atomic] + pub fn return_cpu_rcu_read_lease( + &self, + Tracked(lease): Tracked>>, + Tracked(guard): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked>) + requires + self.well_formed(), + match lease { + None => true, + Some(lease) => { + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + }, + }, + guard.wf(), + guard.domain() == self.constant().domain, + guard.root() == self.id(), + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + old(tv)@.spec_le(final(tv)@), + res@.wf(), + res@.paper_guard() == guard.paper_guard(), + res@.binding() == guard.binding(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + res@.participant_view() == guard.participant_view(), + res@.known_retired() == guard.known_retired(), + res@.domain() == guard.domain(), + res@.root() == guard.root(), + res@.reader_registry() == guard.reader_registry(), + res@.retire_observation_registry() == guard.retire_observation_registry(), + res@.reader_context() == guard.reader_context(), + res@.start_view() == guard.start_view(), + res@.expired() == guard.expired(), + res@.seen_removed() == guard.seen_removed(), + res@.protected() == guard.protected(), + res@.reader_fragment().fraction() == match lease { + None => guard.reader_fragment().fraction(), + Some(_) => guard.reader_fragment().fraction() * 2real, + }, + no_unwind + { + let raw_atomic = &self.atomic; + proof_decl! { + let tracked final_guard; + } + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.permissions.active_lease_registry() + == self.constant().active_lease_registry); + match lease { + None => { + final_guard = guard; + }, + Some(lease) => { + let ghost permissions_before = state.permissions; + final_guard = state.permissions.tracked_return_loaded(lease, guard); + assert(state.permissions.allocations() + == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() + == permissions_before.reclaim_states()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + assert(state.permissions.reclaimed() + == permissions_before.reclaimed()); + assert forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) implies { + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 + == state.root.infos()[obj].ptr() + &&& OwnPred::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + } by { + assert(permissions_before.keys().contains(obj)); + assert(permissions_before.contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.allocations().contains(obj)); + assert(state.permissions.reclaim_states().dom().contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + }; + assert forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) implies + !state.permissions.has_unretired_claim(obj) by { + assert(!permissions_before.has_unretired_claim(obj)); + }; + assert forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) implies { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + } by { + assert(permissions_before.reclaimed().contains_key(obj)); + }; + }, + } + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + Tracked(final_guard) + } + + /// End a paper read-side guard without executing another atomic operation. + #[inline(always)] + pub fn stop_rcu_reader(&self, Tracked(guard): Tracked>) + requires + self.well_formed(), + guard.wf(), + guard.domain() == self.constant().domain, + guard.reader_registry() == self.constant().reader_registry, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + { + proof_decl! { + let tracked base_guard = guard.tracked_into_base(); + let tracked _inactive = base_guard.tracked_stop(); + } + proof { + use_type_invariant(self); + } + } + + /// Ends a CPU-refined reader and returns its linear CPU fragment. + /// + /// The fragment is intentionally returned instead of dropped. The standard + /// guard destruction path must join it back into the current CPU's + /// participant before executable preemption is re-enabled. + #[inline(always)] + pub fn stop_cpu_rcu_reader( + &self, + Tracked(guard): Tracked>, + ) -> (res: Tracked) + requires + self.well_formed(), + guard.wf(), + guard.domain() == self.constant().domain, + guard.root() == self.id(), + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + res@.wf(), + res@ == guard.reader_fragment(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + opens_invariants none + no_unwind + { + proof_decl! { + let tracked (_inactive, reader) = guard.tracked_stop(); + } + proof { + use_type_invariant(self); + } + Tracked(reader) + } + + /// Release-swap helper for a freshly introduced RCU root pointer. + /// + /// The new registration remains owned by the atomic invariant. The return + /// value contains the previous root's retired ownership, if any. Root + /// removal and the base retire transition happen while the same atomic + /// invariant is open. + #[inline(always)] + pub fn swap_release_rcu( + &self, + value: *mut T, + Tracked(ownership): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: (*mut T, Tracked>>)) + requires + self.well_formed(), + self.constant().nullable || !value.is_null(), + match ownership { + Some(ownership) => { + &&& !value.is_null() + &&& OwnPred::owns(value, ownership) + }, + None => value.is_null(), + }, + ensures + old(tv)@.spec_le(final(tv)@), + (res.1@ is Some) == !res.0.is_null(), + res.1@ is Some ==> res.1@->Some_0.object().wf(), + res.1@ is Some ==> res.1@->Some_0.object().domain() == self.constant().domain, + res.1@ is Some ==> equal(res.1@->Some_0.object().ptr(), res.0), + res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), + res.1@ is Some ==> res.1@->Some_0.retired().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> res.1@->Some_0.retired().removal().root == self.id(), + res.1@ is Some ==> res.1@->Some_0.retired().removal().root == self.constant().domain, + res.1@ is Some ==> res.1@->Some_0.retired().retire_observation_registry() + == self.constant().retire_observation_registry, + res.1@ is Some ==> res.1@->Some_0.retired().removal().observed_by(final(tv)@), + res.1@ is Some ==> res.1@->Some_0.claim().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> res.1@->Some_0.claim().registry() + == self.constant().reclaim_registry, + { + let result; + let ghost start_view = tv@; + proof_decl! { + let tracked retired_ownership; + let tracked unit_ownership: Option<()>; + let tracked physical_ownership: Option; + } + proof { + use_type_invariant(self); + match ownership { + Some(ownership) => { + unit_ownership = Some(()); + physical_ownership = Some(ownership); + }, + None => { + unit_ownership = None; + physical_ownership = None; + }, + } + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { + points_to: mut points_to, + root: mut g, + permissions: mut permissions, + } = state; + let ghost root_before_update = g; + let ghost permissions_before_update = permissions; + proof { + assert(points_to.loc() == self.native_loc()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + match g.current_registration() { + Some(registration) => { + assert(permissions.has_unretired_claim(registration.0.obj())); + permissions.lemma_unretired_is_live(registration.0.obj()); + }, + None => {}, + } + } + let ghost prev = points_to.hist(); + let ghost previous_removals = g.removals(); + let swap = raw_atomic.swap_release(value, Tracked(tv), Tracked(&mut points_to)); + result = swap.0; + let ghost update = swap.2@; + let ghost next = points_to.hist(); + proof { + assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); + if !self.constant().nullable { + assert(!value.is_null()); + } + rcu_spec::preserve_rcu_history_inv_on_push( + self.constant().nullable, + prev, + next, + update.load_timestamp + 1, + value, + update.store_message_view, + ); + let tracked detached = g.tracked_push_fresh::( + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + value, + update.store_message_view, + self.id(), + unit_ownership, + ); + assert(detached is Some ==> detached->Some_0.object().wf()); + assert(detached is Some ==> equal(detached->Some_0.ptr(), result)); + assert(detached is Some ==> detached->Some_0.retired().removal().root + == self.id()); + assert(detached is Some ==> detached->Some_0.retired().removal().timestamp + == update.load_timestamp + 1); + assert(detached is Some ==> detached->Some_0.retired().removal().observed_by( + tv@, + )); + assert(match detached { + Some(detached) => { + &&& root_before_update.current_registration() is Some + &&& detached.object() + == root_before_update.current_registration()->Some_0.0 + }, + None => root_before_update.current_registration() is None, + }); + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + match detached { + Some(detached) => { + if obj == detached.obj() { + assert(g.removals()[obj] == detached.retired().removal()); + } else { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + } + }, + None => { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + }, + } + }; + retired_ownership = match detached { + Some(detached) => { + assert(permissions.has_unretired_claim(detached.obj())); + assert(permissions.keys().contains(detached.obj())); + assert(permissions.reclaim_states()[detached.obj()] is Some); + assert(permissions.reclaim_states()[detached.obj()]->Some_0 + == root_before_update.infos()[detached.obj()].ptr()); + assert(equal( + root_before_update.infos()[detached.obj()].ptr(), + detached.object().ptr(), + )); + let tracked claim = permissions.tracked_retire(detached.obj()); + assert(claim.obj() == detached.object().obj()); + assert(equal(claim.ptr(), detached.object().ptr())); + Some(RcuRetiredRootObject { detached, claim }) + }, + None => None, + }; + permissions.lemma_all_live_reclaim_states(); + let ghost permissions_after_retire = permissions; + if physical_ownership is Some { + let tracked info = g.tracked_info_at( + points_to.hist(), + update.load_timestamp + 1, + ).tracked_unwrap(); + let ghost inserted_obj = info.obj(); + let ghost inserted_ownership = physical_ownership->Some_0; + permissions.tracked_insert(&info, physical_ownership.tracked_unwrap()); + assert(permissions.contains(inserted_obj)); + permissions.lemma_live_reclaim_state(inserted_obj); + permissions.lemma_all_live_reclaim_states(); + assert(permissions.ownership(inserted_obj) == inserted_ownership); + assert(equal(info.ptr(), value)); + assert(OwnPred::owns(info.ptr(), inserted_ownership)); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + if obj == inserted_obj { + assert(equal(info.ptr(), g.infos()[obj].ptr())); + } else { + assert(permissions_before_update.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(permissions_after_retire.contains(obj)); + assert(root_before_update.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_update.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_update.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_update.ownership(obj)); + } + }; + } else { + permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(permissions_before_update.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(root_before_update.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_update.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_update.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_update.ownership(obj)); + }; + } + assert(permissions.allocations() == g.infos().dom()); + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by { + assert(permissions_before_update.reclaimed().contains_key(obj)); + assert(root_before_update.removals().contains_key(obj)); + assert(g.removals()[obj] == root_before_update.removals()[obj]); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + permissions.lemma_all_unretired_domains(); + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by { + if !root_before_update.removals().contains_key(obj) { + assert(retired_ownership is Some); + assert(retired_ownership->Some_0.obj() == obj); + } + }; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + (result, Tracked(retired_ownership)) + } + + /// Strong AcqRel/Acquire CAS helper for a freshly introduced RCU pointer. + /// + /// Registration occurs only in the successful CAS branch. A successful + /// CAS returns the previous root registration; a failed CAS leaves the + /// ownership state unchanged and returns no detached registration. + #[inline(always)] + pub fn compare_exchange_acqrel_acquire_rcu( + &self, + current: *mut T, + new: *mut T, + Tracked(new_ownership): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + Result<*mut T, *mut T>, + Ghost, + Tracked<(Option>, Option)>, + )) + requires + self.well_formed(), + self.constant().nullable || !new.is_null(), + match new_ownership { + Some(ownership) => { + &&& !new.is_null() + &&& OwnPred::owns(new, ownership) + }, + None => new.is_null(), + }, + ensures + old(tv)@.spec_le(final(tv)@), + res.0 is Err ==> res.2@.0 is None, + res.0 is Err ==> res.2@.1 == new_ownership, + res.0 is Ok ==> res.2@.1 is None, + res.0 is Ok ==> ((res.2@.0 is Some) == !res.0->Ok_0.is_null()), + res.2@.0 is Some ==> res.2@.0->Some_0.object().wf(), + res.2@.0 is Some ==> res.2@.0->Some_0.object().domain() == self.constant().domain, + res.2@.0 is Some ==> equal(res.2@.0->Some_0.object().ptr(), res.0->Ok_0), + res.2@.0 is Some ==> equal(res.2@.0->Some_0.ptr(), res.0->Ok_0), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().obj() == res.2@.0->Some_0.obj(), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().root == self.id(), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().root + == self.constant().domain, + res.2@.0 is Some ==> res.2@.0->Some_0.retired().retire_observation_registry() + == self.constant().retire_observation_registry, + res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().observed_by(final(tv)@), + res.2@.0 is Some ==> res.2@.0->Some_0.claim().obj() == res.2@.0->Some_0.obj(), + res.2@.0 is Some ==> res.2@.0->Some_0.claim().registry() + == self.constant().reclaim_registry, + { + let result; + let ghost start_view = tv@; + proof_decl! { + let tracked retired_ownership; + let tracked unit_ownership: Option<()>; + let tracked physical_ownership: Option; + } + proof { + use_type_invariant(self); + match new_ownership { + Some(ownership) => { + unit_ownership = Some(()); + physical_ownership = Some(ownership); + }, + None => { + unit_ownership = None; + physical_ownership = None; + }, + } + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { + points_to: mut points_to, + root: mut g, + permissions: mut permissions, + } = state; + let ghost root_before_cas = g; + let ghost permissions_before_cas = permissions; + proof { + assert(points_to.loc() == self.native_loc()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + match g.current_registration() { + Some(registration) => { + assert(permissions.has_unretired_claim(registration.0.obj())); + permissions.lemma_unretired_is_live(registration.0.obj()); + }, + None => {}, + } + } + let ghost prev = points_to.hist(); + let ghost previous_removals = g.removals(); + proof_decl! { + let tracked release_view = vstd::thread_view::ReleaseViewSeen::new(); + } + let cas_result = raw_atomic.compare_exchange( + current, + new, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut points_to), + ); + result = (cas_result.0, Ghost(cas_result.2@.load_timestamp)); + let ghost update = cas_result.2@; + let ghost next = points_to.hist(); + proof { + assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); + match cas_result.0 { + Result::Ok(_) => { + rcu_spec::preserve_rcu_history_inv_on_push( + self.constant().nullable, + prev, + next, + update.load_timestamp + 1, + new, + update.store_message_view, + ); + let tracked detached = g.tracked_push_fresh::< + rcu_spec::UnitRcuRootOwnership, + >( + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + new, + update.store_message_view, + self.id(), + unit_ownership, + ); + assert(detached is Some ==> detached->Some_0.object().wf()); + assert(detached is Some ==> equal( + detached->Some_0.ptr(), + cas_result.0->Ok_0, + )); + assert(detached is Some ==> detached->Some_0.retired().removal().root + == self.id()); + assert(detached is Some ==> + detached->Some_0.retired().removal().observed_by(tv@)); + assert(match detached { + Some(detached) => { + &&& root_before_cas.current_registration() is Some + &&& detached.object() + == root_before_cas.current_registration()->Some_0.0 + }, + None => root_before_cas.current_registration() is None, + }); + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) + == Some(removal.timestamp) + } by { + match detached { + Some(detached) => { + if obj == detached.obj() { + assert(g.removals()[obj] == detached.retired().removal()); + } else { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + } + }, + None => { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + }, + } + }; + let tracked detached = match detached { + Some(detached) => { + assert(permissions.has_unretired_claim(detached.obj())); + assert(permissions.keys().contains(detached.obj())); + assert(permissions.reclaim_states()[detached.obj()] is Some); + assert(permissions.reclaim_states()[detached.obj()]->Some_0 + == root_before_cas.infos()[detached.obj()].ptr()); + assert(equal( + root_before_cas.infos()[detached.obj()].ptr(), + detached.object().ptr(), + )); + let tracked claim = permissions.tracked_retire(detached.obj()); + assert(claim.obj() == detached.object().obj()); + assert(equal(claim.ptr(), detached.object().ptr())); + Some(RcuRetiredRootObject { detached, claim }) + }, + None => None, + }; + permissions.lemma_all_live_reclaim_states(); + let ghost permissions_after_retire = permissions; + if physical_ownership is Some { + let tracked info = g.tracked_info_at( + points_to.hist(), + update.load_timestamp + 1, + ).tracked_unwrap(); + let ghost inserted_obj = info.obj(); + let ghost inserted_ownership = physical_ownership->Some_0; + permissions.tracked_insert( + &info, + physical_ownership.tracked_unwrap(), + ); + assert(permissions.contains(inserted_obj)); + permissions.lemma_live_reclaim_state(inserted_obj); + permissions.lemma_all_live_reclaim_states(); + assert(permissions.ownership(inserted_obj) == inserted_ownership); + assert(equal(info.ptr(), new)); + assert(OwnPred::owns(info.ptr(), inserted_ownership)); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 + == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + if obj == inserted_obj { + assert(equal(info.ptr(), g.infos()[obj].ptr())); + } else { + assert(permissions_before_cas.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(permissions_after_retire.contains(obj)); + assert(root_before_cas.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_cas.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_cas.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_cas.ownership(obj)); + } + }; + } else { + permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 + == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(permissions_before_cas.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(root_before_cas.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_cas.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_cas.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_cas.ownership(obj)); + }; + } + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + permissions.lemma_all_unretired_domains(); + assert(permissions.unretired_claims().dom() + == match g.current_registration() { + Some(registration) => { + Set::empty().insert(registration.0.obj()) + }, + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim( + obj, + ) by { + if !root_before_cas.removals().contains_key(obj) { + assert(detached is Some); + assert(detached->Some_0.obj() == obj); + } + }; + retired_ownership = (detached, None); + }, + Result::Err(_) => { + retired_ownership = (None, physical_ownership); + assert(next == prev); + assert(permissions == permissions_before_cas); + assert(g == root_before_cas); + }, + } + assert(permissions.allocations() == g.infos().dom()); + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by { + assert(permissions_before_cas.reclaimed().contains_key(obj)); + match cas_result.0 { + Result::Ok(_) => { + assert(root_before_cas.removals().contains_key(obj)); + assert(g.removals()[obj] == root_before_cas.removals()[obj]); + }, + Result::Err(_) => {}, + } + }; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + (result.0, result.1, Tracked(retired_ownership)) + } +} + +/// Native IRC11 weak boolean atomic specialized for the RCU monitor flag. +pub struct RcuMonitorWeakAtomicBool { + atomic: Irc11AtomicBool, + tracked_atomic_inv: Tracked< + AtomicInvariant< + Irc11AtomicId, + (AtomicPointsTo, rcu_spec::RcuMonitorFlagGhost), + rcu_spec::RcuMonitorFlagInv, + >, + >, +} + +impl RcuMonitorWeakAtomicBool { + pub closed spec fn id(&self) -> Irc11AtomicId { + self.atomic.loc() + } + + pub closed spec fn well_formed(&self) -> bool { + self.tracked_atomic_inv@.constant() == self.id() + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } + + pub const fn new() -> (res: Self) + ensures + res.well_formed(), + { + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + Irc11AtomicBool::new(false); + let tracked flag_ghost = rcu_spec::RcuMonitorFlagGhost::tracked_initial(timestamp); + proof { + rcu_spec::rcu_monitor_flag_initial_inv(points_to.hist(), timestamp, initial_view@); + assert(rcu_spec::RcuMonitorFlagInv::inv(atomic.loc(), (points_to, flag_ghost))); + } + let tracked pair = (points_to, flag_ghost); + let tracked atomic_inv = AtomicInvariant::new(atomic.loc(), pair, 0); + Self { atomic, tracked_atomic_inv: Tracked(atomic_inv) } + } + + pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: (bool, Ghost)) + requires + self.well_formed(), + ensures + old(tv)@.spec_le(final(tv)@), + { + let result; + proof { + use_type_invariant(self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { + let tracked (points_to, flag_ghost) = pair; + let loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&points_to), + ); + result = (loaded.0, Ghost(loaded.2@.timestamp)); + proof { + pair = (points_to, flag_ghost); + } + }); + result + } + + fn raw_atomic(&self) -> (res: &Irc11AtomicBool) + requires + self.well_formed(), + ensures + res.loc() == self.id(), + { + &self.atomic + } + + proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &vstd::invariant::AtomicInvariant< + Irc11AtomicId, + (AtomicPointsTo, rcu_spec::RcuMonitorFlagGhost), + rcu_spec::RcuMonitorFlagInv, + >) + requires + self.well_formed(), + ensures + res.constant() == self.id(), + { + self.tracked_atomic_inv.borrow() + } + + /// Relaxed-store helper for the RCU monitor flag. + /// + /// The executable flag remains a relaxed atomic flag, matching the old + /// monitor protocol. The proof-side effect is stronger: each stored flag + /// message inserts the lock-protected monitor-state snapshot supplied by + /// the writer. + #[inline(always)] + pub fn store_relaxed_rcu_monitor( + &self, + value: bool, + Ghost(state): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) + requires + self.well_formed(), + state.wf(), + !value ==> state.no_pending_work(), + ensures + old(tv)@.spec_le(final(tv)@), + { + proof { + use_type_invariant(self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { + let tracked (mut points_to, mut flag_ghost) = pair; + proof { + assert(points_to.loc() == self.id()); + assert(raw_atomic.loc() == self.id()); + } + let ghost prev = points_to.hist(); + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let store = raw_atomic.store( + value, + Ordering::Relaxed, + Tracked(tv), + Tracked(release_view), + Tracked(&mut points_to), + ); + let ghost next = points_to.hist(); + proof { + rcu_spec::preserve_rcu_monitor_flag_inv_on_insert( + prev, + next, + store@.timestamp, + value, + store@.message_view, + flag_ghost, + flag_ghost.insert(store@.timestamp, state), + state, + ); + flag_ghost = flag_ghost.tracked_insert(store@.timestamp, state); + pair = (points_to, flag_ghost); + } + }); + } +} + +} // verus! diff --git a/ostd/specs/task/cpu_core.rs b/ostd/specs/task/cpu_core.rs index dec7241e0..ff85d4935 100644 --- a/ostd/specs/task/cpu_core.rs +++ b/ostd/specs/task/cpu_core.rs @@ -8,7 +8,13 @@ //! state, then restores that state before returning the owner to the scheduler. use core::marker::PhantomData; -use vstd::{prelude::*, resource::Loc}; +use vstd::{ + prelude::*, + resource::{ + Loc, + map::{GhostPersistentPointsTo, GhostPointsTo}, + }, +}; use vstd_extra::resource::ghost_resource::excl::ExclusiveGhost; use crate::specs::mm::cpu::CpuId; @@ -26,6 +32,17 @@ pub ghost struct CpuCoreOwnerView { pub locals_key: Seq, } +/// Stable identity of one scheduler-owned CPU core and its local aggregate. +/// +/// A scheduler registry stores this value under the corresponding [`CpuId`]. +/// The core ID distinguishes independently allocated owners, while +/// `locals_key` identifies the exact ordered CPU-local resources installed in +/// the owner. +pub ghost struct CpuCoreRegistration { + pub owner_id: Loc, + pub locals_key: Seq, +} + /// A typed collection of resources that belongs permanently to one CPU. /// /// Implementations may aggregate any number of differently typed CPU-local @@ -84,6 +101,17 @@ pub tracked struct CpuCoreOwner { locals: L, } +/// Persistent evidence for the canonical core registered for one CPU. +/// +/// This is deliberately defined in the generic CPU-core model rather than in +/// an individual CPU-local client. A reader, scheduler transition, or +/// quiescent report may duplicate the evidence and later establish that they +/// refer to the same core and the same local-resource aggregate. +pub tracked struct CpuCoreOwnerBinding { + entry: GhostPersistentPointsTo, + marker: PhantomData, +} + impl View for CpuCoreOwnerHandle { type V = CpuCoreOwnerView; @@ -100,6 +128,80 @@ impl View for CpuCoreOwner { } } +impl CpuCoreOwnerBinding { + /// Persists one entry from the scheduler's authoritative core registry. + pub proof fn tracked_new( + tracked entry: GhostPointsTo, + tracked core: &CpuCoreOwner, + ) -> (tracked res: Self) + requires + core.wf(), + entry.key() == core.cpu(), + entry.value() == core.registration(), + ensures + res.registry() == entry.id(), + res.cpu() == core.cpu(), + res.owner_id() == core.id(), + res.locals_key() == core.locals_key(), + { + let tracked entry = entry.persist(); + CpuCoreOwnerBinding { entry, marker: PhantomData } + } + + /// Identity of the authoritative scheduler core registry. + pub closed spec fn registry(self) -> Loc { + self.entry.id() + } + + /// CPU whose canonical core is recorded by this entry. + pub closed spec fn cpu(self) -> CpuId { + self.entry.key() + } + + /// Stable identity of the registered [`CpuCoreOwner`]. + pub closed spec fn owner_id(self) -> Loc { + self.entry.value().owner_id + } + + /// Ordered identities of the registered core's CPU-local resources. + pub closed spec fn locals_key(self) -> Seq { + self.entry.value().locals_key + } + + /// Identity of the only local resource in a singleton aggregate. + pub open spec fn single_local_id(self) -> Loc + recommends + self.locals_key().len() == 1, + { + self.locals_key()[0] + } + + /// Creates another persistent copy for a CPU-local client. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.registry() == self.registry(), + res.cpu() == self.cpu(), + res.owner_id() == self.owner_id(), + res.locals_key() == self.locals_key(), + { + CpuCoreOwnerBinding { entry: self.entry.duplicate(), marker: PhantomData } + } + + /// Entries for the same CPU in one registry agree on the complete core + /// registration. + pub proof fn lemma_same_cpu_agree(tracked &self, tracked other: &Self) + requires + self.registry() == other.registry(), + self.cpu() == other.cpu(), + ensures + self.owner_id() == other.owner_id(), + self.locals_key() == other.locals_key(), + { + let tracked mut duplicate = self.entry.duplicate(); + duplicate.intersection_agrees(&other.entry); + } +} + impl CpuCoreOwnerHandle { /// Unique identity of this core resource. pub closed spec fn id(&self) -> Loc { @@ -140,6 +242,13 @@ impl CpuCoreOwnerHandle { ensures res.id() == self.id(), res@ == self@, + res.cpu() == self.cpu(), + res.current_task() == self.current_task(), + res.locals_key() == self.expected_locals_key(), + res.registration() == (CpuCoreRegistration { + owner_id: self.id(), + locals_key: self.expected_locals_key(), + }), res.wf(), res.locals() == locals, res.locals().local_key() == self.expected_locals_key(), @@ -158,6 +267,9 @@ impl CpuCoreOwner { res.is_idle(), res.wf(), res.locals() == locals, + res.locals_key() == locals.local_key(), + res.registration().owner_id == res.id(), + res.registration().locals_key == locals.local_key(), { let ghost locals_key = locals.local_key(); let tracked state = ExclusiveGhost::alloc( @@ -197,6 +309,11 @@ impl CpuCoreOwner { self.handle.expected_locals_key() } + /// Stable value stored in the scheduler's canonical CPU-core registry. + pub closed spec fn registration(&self) -> CpuCoreRegistration { + CpuCoreRegistration { owner_id: self.id(), locals_key: self.locals_key() } + } + /// The core identity is valid and every local resource belongs to its CPU. pub closed spec fn wf(&self) -> bool { &&& self.handle.wf() @@ -215,6 +332,7 @@ impl CpuCoreOwner { final(self).current_task() == Some(task), final(self).locals() == old(self).locals(), final(self).locals_key() == old(self).locals_key(), + final(self).registration() == old(self).registration(), final(self).wf(), { let ghost next = CpuCoreOwnerView { @@ -237,6 +355,7 @@ impl CpuCoreOwner { final(self).is_idle(), final(self).locals() == old(self).locals(), final(self).locals_key() == old(self).locals_key(), + final(self).registration() == old(self).registration(), final(self).wf(), { let task = self.current_task()->0; @@ -261,6 +380,8 @@ impl CpuCoreOwner { res.0.id() == self.id(), res.0@ == self@, res.0.wf(), + res.0.cpu() == self.cpu(), + res.0.current_task() == self.current_task(), res.0.expected_locals_key() == self.locals_key(), res.1 == self.locals(), res.1.belongs_to_cpu(res.0.cpu()), diff --git a/ostd/src/mm/frame/segment.rs b/ostd/src/mm/frame/segment.rs index db94ef001..e665f8e00 100644 --- a/ostd/src/mm/frame/segment.rs +++ b/ostd/src/mm/frame/segment.rs @@ -1097,7 +1097,7 @@ impl<'a, M: AnyFrameMeta + Repr + OwnerOf> SegmentIterator<'a, let item = (frame, Tracked(from_raw_obl)); proof { remaining.resolve_cons(item); - broadcast use vstd::seq::group_seq_lemmas; + broadcast use vstd::seq::group_seq_axioms; assert(remaining.seq() == old_remaining.drop_first()); assert(item == old_remaining[0]); diff --git a/ostd/src/sync/mod.rs b/ostd/src/sync/mod.rs index 4908e75f4..0e84af234 100644 --- a/ostd/src/sync/mod.rs +++ b/ostd/src/sync/mod.rs @@ -10,20 +10,20 @@ mod rwlock; mod rwmutex; mod spin; mod wait; -//pub(crate) use self::rcu::finish_grace_period; + +pub(crate) use self::rcu::finish_grace_period; pub use self::{ atomic_data::*, guard::{GuardTransfer, LocalIrqDisabled, PreemptDisabled, SpinGuardian, /*WriteIrqDisabled*/}, mutex::{Mutex, MutexGuard}, once::{Once, OnceImpl, TrivialPred}, - rcu::{non_null /*, Rcu, RcuDrop, RcuOption, RcuOptionReadGuard, RcuReadGuard*/}, + rcu::{Rcu, RcuDrop, RcuOption, RcuOptionReadGuard, RcuReadGuard, non_null}, rwarc::{RoArc, RwArc}, rwlock::{RwLock, RwLockReadGuard, RwLockUpgradeableGuard, RwLockWriteGuard}, rwmutex::{RwMutex, RwMutexReadGuard, RwMutexUpgradeableGuard, RwMutexWriteGuard}, spin::{SpinLock, SpinLockGuard}, wait::{WaitQueue, Waiter, Waker}, }; -/* pub(crate) fn init() { rcu::init(); -}*/ +} diff --git a/ostd/src/sync/once.rs b/ostd/src/sync/once.rs index 1e7f54245..aaf173a1a 100644 --- a/ostd/src/sync/once.rs +++ b/ostd/src/sync/once.rs @@ -3,6 +3,7 @@ use vstd::{ cell::pcell::{PCell, PointsTo}, modes::tracked_static_ref, prelude::*, + thread_view::Objective, }; use super::AtomicDataWithOwner; @@ -27,6 +28,10 @@ pub tracked enum OnceState { Init(&'static PointsTo>), } +unsafe impl Objective for OnceState { + +} + /// A [`Predicate`] is something you're gonna preserve during the lifetime /// of any synchronization primitives like [`Once`]. pub trait Predicate { diff --git a/ostd/src/sync/rcu/__mod.rs b/ostd/src/sync/rcu/__mod.rs new file mode 100644 index 000000000..e5b454780 --- /dev/null +++ b/ostd/src/sync/rcu/__mod.rs @@ -0,0 +1,1128 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Read-copy update (RCU). +//! +//! # Note +//! +//! Currently this RCU model assumes a sequential consistency (SC) memory model. +//! We may explore weak memory models in the future. +use vstd::{ + atomic_ghost::AtomicPtr, atomic_with_ghost, map::Map, modes::tracked_static_ref, prelude::*, + resource::Loc, +}; + +use vstd_extra::{ + prelude::*, + resource::ghost_resource::{count::Count, tokens::CountResource}, +}; + +use core::{ + marker::PhantomData, mem::ManuallyDrop, ops::Deref, + ptr::NonNull, + /* + sync::atomic::{ + AtomicPtr, + Ordering::{AcqRel, Acquire}, + }, + */ +}; + +use non_null::{NonNullPtr, NonNullPtrRef}; +// use spin::once::Once; +use super::Once; + +use self::monitor::{RcuMonitor, RcuMonitorOwner, RcuMonitorPred}; +use crate::task::{ + DisabledPreemptGuard, + //atomic_mode::{AsAtomicModeGuard, InAtomicMode}, + disable_preempt, +}; + +mod monitor; +pub mod non_null; + +use crate::specs::task::InAtomicMode; + +verus! { + +broadcast use vstd_extra::external::nonnull::group_nonull_axioms; +// Verification-only budget for splitting read-side ghost tokens. +// +// This is not a runtime reader counter and does not model an overflow condition +// in the RCU implementation. It is a temporary bounded approximation needed by +// `CountResource`; the final RCU proof should discharge the admission assumption +// with an unbounded ghost registry or a CPU/epoch-based sharding model. + +const RCU_READER_SLOTS: u64 = 1u64 << 60; + +type RcuReadPool

= CountResource<

::Permission, RCU_READER_SLOTS>; + +type RcuReadToken

= Count<

::Permission, RCU_READER_SLOTS>; + +type RcuRetiredEntry

= (Ghost<*mut

::Target>, RcuReadPool

); + +/// Called by `drop` of the read guard to track the retired read permissions. +type RcuRetiredPools

= Map>; + +type RcuReturnedTokens

= Map>; + +tracked struct RcuPtrGhost { + tracked current: Option>, + tracked retired: RcuRetiredPools

, + tracked returned: RcuReturnedTokens

, +} + +closed spec fn retired_pools_inv(retired: RcuRetiredPools

) -> bool { + forall|id: Loc| #[trigger] + retired.contains_key(id) ==> { + let entry = retired[id]; + &&& !(entry.0@).is_null() + &&& entry.1.id() == id + &&& P::ptr_perm_match(entry.0@, entry.1@) + &&& entry.1@.inv() + &&& entry.1.wf() + &&& entry.1.not_empty() + } +} + +closed spec fn returned_tokens_inv(returned: RcuReturnedTokens

) -> bool { + forall|id: Loc| #[trigger] + returned.contains_key(id) ==> { + let token = returned[id]; + &&& token.id() == id + &&& token.resource().inv() + &&& token.frac() > 0 + } +} + +/// A Read-Copy Update (RCU) cell for sharing a pointer between threads. +/// +/// The pointer should be a non-null pointer with type `P`, which implements +/// [`NonNullPtr`]. For example, `P` can be `Box` or `Arc`. +/// +/// # Overview +/// +/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high- +/// performance, low-latency read operations in concurrent systems. It allows +/// multiple readers to access shared data simultaneously without contention, +/// while writers can update the data safely in a way that does not disrupt +/// ongoing reads. RCU is particularly suited for situations where reads are +/// far more frequent than writes. +/// +/// The original design and implementation of RCU is described in paper _The +/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared- +/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal +/// 47.2 (2008). +/// +/// # Examples +/// +/// ``` +/// use ostd::sync::Rcu; +/// +/// let rcu = Rcu::new(Box::new(42)); +/// +/// let rcu_guard = rcu.read(); +/// +/// assert_eq!(*rcu_guard, Some(&42)); +/// +/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); +/// +/// let rcu_guard = rcu.read(); +/// +/// assert_eq!(*rcu_guard, Some(&43)); +/// ``` +pub struct Rcu(RcuInner

); + +/// A guard that allows access to the pointed data protected by a [`Rcu`]. +#[clippy::has_significant_drop] +#[must_use] +pub struct RcuReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); + +/// A Read-Copy Update (RCU) cell for sharing a _ghost_nullable_ pointer. +/// +/// This is a variant of [`Rcu`] that allows the contained pointer to be null. +/// So that it can implement `Rcu>` where `P` is not a ghost_nullable +/// pointer. It is the same as [`Rcu`] in other aspects. +/// +/// # Examples +/// +/// ``` +/// use ostd::sync::RcuOption; +/// +/// static RCU: RcuOption> = RcuOption::new_none(); +/// +/// assert!(RCU.read().is_none()); +/// +/// RCU.update(Box::new(42)); +/// +/// // Read the data protected by RCU +/// { +/// let rcu_guard = RCU.read().try_get().unwrap(); +/// assert_eq!(*rcu_guard, 42); +/// } +/// +/// // Update the data protected by RCU +/// { +/// let rcu_guard = RCU.read().try_get().unwrap(); +/// +/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); +/// +/// let rcu_guard = RCU.read().try_get().unwrap(); +/// assert_eq!(*rcu_guard, 43); +/// } +/// ``` +pub struct RcuOption(RcuInner

); + +/// A guard that allows access to the pointed data protected by a [`RcuOption`]. +#[clippy::has_significant_drop] +#[must_use] +pub struct RcuOptionReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); + +struct_with_invariants! { +/// The inner implementation of both [`Rcu`] and [`RcuOption`]. +struct RcuInner { + ptr: AtomicPtr<

::Target,_,RcuPtrGhost

,_>, + // We want to implement Send and Sync explicitly. + // Having a pointer field prevents them from being implemented + // automatically by the compiler. + _marker: PhantomData<*const

::Target>, + ghost_nullable: Ghost, +} + +closed spec fn wf(self) -> bool { + invariant on ptr with (ghost_nullable, _marker) is ( + v: *mut

::Target, + g: RcuPtrGhost

, + ) { + &&& retired_pools_inv::

(g.retired) + &&& returned_tokens_inv::

(g.returned) + &&& match g.current { + Some(perm) => { + &&& !v.is_null() + &&& P::ptr_perm_match(v, perm@) + &&& perm@.inv() + &&& perm.wf() + &&& perm.not_empty() + }, + None => ghost_nullable@ && v.is_null(), + } + } +} +} +// SAFETY: It is apparent that if `P` is `Send`, then `Rcu

` is `Send`. + + +#[verifier::external] +unsafe impl Send for RcuInner

where P: Send { + +} + +// SAFETY: To implement `Sync` for `Rcu

`, we need to meet two conditions: +// 1. `P` must be `Sync` because `Rcu::get` allows concurrent access. +// 2. `P` must be `Send` because `Rcu::update` may obtain an object +// of `P` created on another thread. +#[verifier::external] +unsafe impl Sync for RcuInner

where P: Send + Sync { + +} + +impl RcuInner

{ + /// Whether the contained pointer can be null. Used to distinguish `Rcu` and `RcuOption`. + pub closed spec fn is_nullable(self) -> bool { + self.ghost_nullable@ + } +} + +#[verus_verify] +impl RcuInner

{ + #[inline(always)] + const fn new_none() -> (res: Self) + ensures + res.is_nullable(), + res.wf(), + { + proof_decl! { + let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { + current: None, + retired: Map::tracked_empty(), + returned: Map::tracked_empty(), + }; + } + Self { + ptr: AtomicPtr::new( + Ghost((Ghost(true), PhantomData::<*const

::Target>)), + core::ptr::null_mut(), + Tracked(ptr_ghost), + ), + _marker: PhantomData::<*const

::Target>, + ghost_nullable: Ghost(true), + } + } + + /// Creates a new RCU primitive with the given pointer `pointer`. + #[inline(always)] + #[verus_spec(r => + with + Ghost(ghost_nullable): Ghost, + ensures + r.type_inv(), + r.is_nullable() == ghost_nullable, + )] + fn new(pointer: P) -> Self { + // let ptr =

::into_raw(pointer).as_ptr(); + let (ptr, Tracked(ptr_perm)) =

::into_raw(pointer); + let ptr = ptr.as_ptr(); + proof_decl! { + let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { + current: Some(CountResource::alloc(ptr_perm)), + retired: Map::tracked_empty(), + returned: Map::tracked_empty(), + }; + } + + let ptr = AtomicPtr::new( + Ghost((Ghost(ghost_nullable), PhantomData)), + ptr, + Tracked(ptr_ghost), + ); + Self { ptr, _marker: PhantomData, ghost_nullable: Ghost(ghost_nullable) } + } + + #[verus_spec( + requires + self.is_nullable() || new_ptr is Some, + )] + fn update(&self, new_ptr: Option

) { + let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { + //

::into_raw(new_ptr).as_ptr() + let (ptr, Tracked(perm)) =

::into_raw(new_ptr); + let ptr = ptr.as_ptr(); + (ptr, Tracked(Some(perm))) + } else { + (core::ptr::null_mut(), Tracked(None)) + }; + + proof_decl! { + let tracked mut old_perm: Option> = None; + } + proof { + use_type_invariant(self); + } + let old_raw_ptr = + atomic_with_ghost! { + self.ptr => swap(new_ptr); + update prev -> next; + ghost g => { + old_perm = g.current; + if old_perm is Some { + let tracked mut pool = old_perm.tracked_unwrap(); + let ghost id = pool.id(); + if g.retired.contains_key(id) { + // Use tracked_borrow_mut instead + let tracked entry = g.retired.tracked_remove(id); + let tracked mut retired_pool = entry.1; + let tracked pool_token = pool.split(pool.frac()); + retired_pool.validate_with_frac(&pool_token); + retired_pool.combine(pool_token); + g.retired.tracked_insert(id, (entry.0, retired_pool)); + } else { + g.retired.tracked_insert(id, (Ghost(prev), pool)); + } + } + g.current = match new_perm { + Some(perm) => Some(CountResource::alloc(perm)), + None => None, + }; + assert(retired_pools_inv::

(g.retired)); + } + }; + + if let Some(p) = NonNull::new(old_raw_ptr) { + // SAFETY: + // 1. The pointer was previously returned by `into_raw`. + // 2. The pointer is removed from the RCU slot so that no one will + // use it after the end of the current grace period. The removal + // is done atomically, so it will only be dropped once. + // unsafe { delay_drop::

(p) }; + } + } + + #[verus_spec(obj_ptr => + with + -> tracked_ref_perm: Tracked>>, + ensures + !self.is_nullable() ==> tracked_ref_perm@ is Some, + match tracked_ref_perm@ { + Some(perm) => { + &&& !obj_ptr.is_null() + &&& P::ptr_perm_match(obj_ptr, perm.resource()) + &&& perm.resource().inv() + &&& perm.frac() == 1 + }, + None => obj_ptr.is_null(), + }, + )] + fn load_read_token(&self) -> *mut

::Target { + proof_decl! { + let tracked mut tracked_ref_perm: Option> = None; + } + proof { + use_type_invariant(self); + } + let obj_ptr = + atomic_with_ghost! { + self.ptr => load(); + update prev -> _next; + returning loaded; + ghost g => { + if g.current is Some { + let tracked mut perm = g.current.tracked_unwrap(); + assert(loaded == prev); + assert(!loaded.is_null()); + assert(P::ptr_perm_match(loaded, perm@)); + assert(perm@.inv()); + let ghost perm_snapshot = perm@; + + // Verification-only admission for the bounded read-token pool. + // This is not a runtime reader limit; it only reflects that + // `CountResource` uses a Rust const-generic `u64` budget rather + // than an unbounded mathematical `nat`. + assume(perm.not_empty()); + assume(1 < perm.frac()); + let tracked token = perm.split_one(); + assert(perm@ == perm_snapshot); + assert(token.frac() == 1); + tracked_ref_perm = Some(token); + g.current = Some(perm); + } else { + } + assert(retired_pools_inv::

(g.retired)); + } + }; + proof_with! { |= Tracked(tracked_ref_perm) } + obj_ptr + } + + #[verus_spec(r => + ensures + r.type_inv(), + r.rcu.is_nullable() == self.is_nullable(), + !self.is_nullable() ==> r.tracked_ref_perm@ is Some, + )] + fn read(&self) -> RcuReadGuardInner<'_, P> { + let guard = disable_preempt(); + proof_decl! { + let tracked mut tracked_ref_perm: Option> = None; + } + let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] + self.load_read_token(); + RcuReadGuardInner { + obj_ptr, + rcu: self, + _inner_guard: guard, + tracked_ref_perm: Tracked(tracked_ref_perm), + } + } + + #[verus_spec] + pub fn read_with<'a, A: InAtomicMode>( + &'a self, + _guard: &'a A, // &'a dyn InAtomicMode is not well-supported in Verus. + ) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + proof_decl! { + let tracked mut tracked_ref_perm: Option> = None; + } + let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] + self.load_read_token(); + if obj_ptr.is_null() { + return None; + } + proof_decl! { + // `read_with` returns only the reference and has no guard object to + // store the read token. For this temporary skeleton, leak the + // verification-only token so the returned ref can borrow it for + // `'a`. The final RCU proof should attach this token to the + // atomic-mode/CPU epoch state instead. + let tracked tracked_ref_perm = tracked_ref_perm.tracked_unwrap(); + let tracked tracked_ref_perm = tracked_static_ref(tracked_ref_perm); + let tracked tracked_ref_perm:

>::RefPermission = + P::borrow_perm_as_ref_perm(tracked_ref_perm.borrow()); + } + // SAFETY: + // 1. This pointer is not NULL. + // 2. The `_guard` guarantees atomic mode for the duration of lifetime + // `'a`, the pointer is valid because other writers won't release the + // allocation until this task passes the quiescent state. + NonNull::new(obj_ptr).map( + |ptr| + requires + P::ptr_perm_match( + ptr.view_ptr_mut(), + P::ref_perm_view_permission(tracked_ref_perm), + ), + { + unsafe { P::raw_as_ref(ptr, Tracked(tracked_ref_perm)) } + }, + ) + } +} + +/* +impl Drop for RcuInner

{ + fn drop(&mut self) { + let ptr = self.ptr.load(Acquire); + if let Some(p) = NonNull::new(ptr) { + // SAFETY: It was previously returned by `into_raw` when creating + // the RCU primitive. + let pointer = unsafe {

::from_raw(p) }; + // It is OK not to delay the drop because the RCU primitive is + // owned by nobody else. + drop(pointer); + } + } +} +*/ + +/// The inner implementation of both [`RcuReadGuard`] and [`RcuOptionReadGuard`]. +struct RcuReadGuardInner<'a, P: NonNullPtr> { + obj_ptr: *mut

::Target, + rcu: &'a RcuInner

, + _inner_guard: DisabledPreemptGuard, + tracked_ref_perm: Tracked>>, +} + +#[verus_verify] +impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { + #[inline] + #[verus_spec(r => + ensures + self.tracked_ref_perm@ is Some ==> r is Some, + )] + fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> { + proof { + use_type_invariant(self); + } + + // SAFETY: The guard ensures that `P` will not be dropped. Thus, `P` + // outlives the lifetime of `&self`. Additionally, during this period, + // it is impossible to create a mutable reference to `P`. + NonNull::new(self.obj_ptr).map( + |ptr| + requires + self.tracked_ref_perm@ is Some, + P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_ref_perm->0.resource()), + { + unsafe { + P::raw_as_ref( + ptr, + Tracked( + P::borrow_perm_as_ref_perm( + self.tracked_ref_perm.tracked_borrow().borrow(), + ), + ), + ) + } + }, + ) + } + + #[verus_spec(r => + requires + self.rcu.is_nullable() || new_ptr is Some, + ensures + new_ptr is Some && r is Err ==> r->Err_0 is Some, + )] + fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { + let obj_ptr = self.obj_ptr; + proof { + use_type_invariant(&self); + use_type_invariant(self.rcu); + } + proof_decl! { + let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + let ghost new_ptr_is_some = new_ptr is Some; + let tracked mut old_perm: Option> = None; + let tracked mut err_new_perm: Option::Permission>> = None; + } + let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { + //

::into_raw(new_ptr).as_ptr() + let (ptr, Tracked(perm)) =

::into_raw(new_ptr); + (ptr.as_ptr(), Tracked(Some(perm))) + } else { + (core::ptr::null_mut(), Tracked(None)) + }; + let res = + atomic_with_ghost! { + self.rcu.ptr => compare_exchange(obj_ptr, new_ptr); + update _prev -> next; + returning res; + ghost g => { + if res is Ok { + old_perm = g.current; + if old_perm is Some { + let tracked mut pool = old_perm.tracked_unwrap(); + let ghost id = pool.id(); + if g.retired.contains_key(id) { + // use tracked_borrow_mut instead + let tracked entry = g.retired.tracked_remove(id); + let tracked mut retired_pool = entry.1; + let tracked pool_token = pool.split(pool.frac()); + retired_pool.validate_with_frac(&pool_token); + retired_pool.combine(pool_token); + g.retired.tracked_insert(id, (entry.0, retired_pool)); + } else { + g.retired.tracked_insert(id, (Ghost(_prev), pool)); + } + } + g.current = match new_perm { + Some(perm) => Some(CountResource::alloc(perm)), + None => None, + }; + } else { + err_new_perm = Some(new_perm); + } + if tracked_ref_perm is Some { + let tracked token = tracked_ref_perm.tracked_unwrap(); + let ghost id = token.id(); + if g.retired.contains_key(id) { + let tracked entry = g.retired.tracked_remove(id); + let tracked mut pool = entry.1; + pool.combine(token); + g.retired.tracked_insert(id, (entry.0, pool)); + } else if g.current is Some { + let tracked mut pool = g.current.tracked_unwrap(); + if pool.id() == id { + pool.combine(token); + g.current = Some(pool); + } else { + g.current = Some(pool); + assume(false); + } + } else { + assume(false); + } + } + assert(retired_pools_inv::

(g.retired)); + } + }; + if res.is_ok() { + if let Some(p) = NonNull::new(obj_ptr) { + // SAFETY: + // 1. The pointer was previously returned by `into_raw`. + // 2. The pointer is removed from the RCU slot so that no one will + // use it after the end of the current grace period. The removal + // is done atomically, so it will only be dropped once. + // unsafe { delay_drop::

(p) }; + } + Ok(()) + } else { + let Some(new_nonnull) = NonNull::new(new_ptr) else { + return Err(None); + }; + proof_decl! { + let tracked new_perm = err_new_perm.tracked_unwrap().tracked_unwrap(); + } + // SAFETY: + // 1. It was previously returned by `into_raw`. + // 2. The `compare_exchange` fails so the pointer will not + // be used by other threads via reading the RCU primitive. + Err(Some(unsafe {

::from_raw(new_nonnull, Tracked(new_perm)) })) + } + } + + /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. + #[inline] + #[verus_spec] + fn drop(self) { + let rcu = self.rcu; + let obj_ptr = self.obj_ptr; + proof { + use_type_invariant(&self); + use_type_invariant(rcu); + } + proof_decl! { + let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + } + atomic_with_ghost! { + rcu.ptr => load(); + update prev -> _next; + returning _loaded; + ghost g => { + if tracked_ref_perm is Some { + let tracked token = tracked_ref_perm.tracked_unwrap(); + let ghost id = token.id(); + if g.current is Some { + let tracked mut pool = g.current.tracked_unwrap(); + if prev == obj_ptr && pool.id() == id { + assert(!obj_ptr.is_null()); + assert(P::ptr_perm_match(obj_ptr, token.resource())); + assert(token.resource().inv()); + assert(token.frac() == 1); + assert(P::ptr_perm_match(prev, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + pool.combine(token); + assert(P::ptr_perm_match(prev, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + g.current = Some(pool); + } else { + g.current = Some(pool); + if g.retired.contains_key(id) { + let tracked entry = g.retired.tracked_remove(id); + let tracked mut pool = entry.1; + assert(pool.id() == id); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + pool.combine(token); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + g.retired.tracked_insert(id, (entry.0, pool)); + } else { + assume(false); + } + } + } else { + if g.retired.contains_key(id) { + let tracked entry = g.retired.tracked_remove(id); + let tracked mut pool = entry.1; + assert(pool.id() == id); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + pool.combine(token); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + g.retired.tracked_insert(id, (entry.0, pool)); + } else { + assume(false); + } + } + } + match &g.current { + Some(pool) => { + assert(!prev.is_null()); + assert(P::ptr_perm_match(prev, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + }, + None => { + assert(rcu.ghost_nullable@); + assert(prev.is_null()); + }, + } + assert(retired_pools_inv::

(g.retired)); + } + }; + } +} + +#[verus_verify] +impl Rcu

{ + /// Creates a new RCU primitive with the given pointer `pointer`. + #[verus_spec] + pub fn new(pointer: P) -> Self { + Self( + #[verus_spec(with Ghost(false))] + RcuInner::new(pointer), + ) + } + + /// Replaces the current pointer with a null pointer. + /// + /// This function updates the pointer to the new pointer regardless of the + /// original pointer. The original pointer will be dropped after the grace + /// period. + /// + /// Oftentimes this function is not recommended unless you have serialized + /// writes with locks. Otherwise, you can use [`Self::read`] and then + /// [`RcuReadGuard::compare_exchange`] to update the pointer. + #[inline] + pub fn update(&self, new_ptr: P) { + self.0.update(Some(new_ptr)); + } + + /// Retrieves a read guard for the RCU primitive. + /// + /// The guard allows read access to the data protected by RCU, as well + /// as the ability to do compare-and-exchange. + #[inline] + pub fn read(&self) -> RcuReadGuard<'_, P> { + proof { + use_type_invariant(self); + } + RcuReadGuard(self.0.read()) + } + // #[inline] + // pub fn read_with<'a, G: AsAtomicModeGuard + ?Sized>(&'a self, guard: &'a G) -> P::Ref<'a> where + // P: NonNullPtrRef<'a>, + // { + // self.0.read_with(guard.as_atomic_mode_guard()).unwrap() + // } + +} + +#[verus_verify] +impl RcuOption

{ + /// Creates a new RCU primitive with the given pointer. + #[verus_spec] + pub fn new(pointer: Option

) -> Self { + if let Some(pointer) = pointer { + Self( + #[verus_spec(with Ghost(true))] + RcuInner::new(pointer), + ) + } else { + Self(RcuInner::new_none()) + } + } + + /// Creates a new RCU primitive that contains nothing. + /// + /// This is a constant equivalence to [`RcuOption::new(None)`]. + #[inline(always)] + pub const fn new_none() -> Self { + Self(RcuInner::new_none()) + } + + /// Replaces the current pointer with a null pointer. + /// + /// This function updates the pointer to the new pointer regardless of the + /// original pointer. If the original pointer is not NULL, it will be + /// dropped after the grace period. + /// + /// Oftentimes this function is not recommended unless you have + /// synchronized writes with locks. Otherwise, you can use [`Self::read`] + /// and then [`RcuOptionReadGuard::compare_exchange`] to update the pointer. + #[inline] + pub fn update(&self, new_ptr: Option

) { + proof { + use_type_invariant(self); + } + self.0.update(new_ptr); + } + + /// Retrieves a read guard for the RCU primitive. + /// + /// The guard allows read access to the data protected by RCU, as well + /// as the ability to do compare-and-exchange. + /// + /// The contained pointer can be NULL and you can only get a reference + /// (if checked non-NULL) via [`RcuOptionReadGuard::get`]. + #[inline] + pub fn read(&self) -> RcuOptionReadGuard<'_, P> { + proof { + use_type_invariant(self); + } + RcuOptionReadGuard(self.0.read()) + } +} + +#[verus_verify] +impl RcuReadGuard<'_, P> { + /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. + #[inline] + pub fn drop(self) { + self.0.drop(); + } + + /// Gets the reference of the protected data. + #[inline] + pub fn get<'a>(&'a self) ->

>::Ref where P: NonNullPtrRef<'a> { + proof { + use_type_invariant(self); + } + self.0.get().unwrap() + } + + /// Tries to replace the already read pointer with a new pointer. + /// + /// If another thread has updated the pointer after the read, this + /// function will fail, and returns the given pointer back. Otherwise, + /// it will replace the pointer with the new one and drop the old pointer + /// after the grace period. + /// + /// If spinning on [`Rcu::read`] and this function, it is recommended + /// to relax the CPU or yield the task on failure. Otherwise contention + /// will occur. + /// + /// This API does not help to avoid + /// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem). + #[inline] + pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { + self.0.compare_exchange(Some(new_ptr)).map_err( + |err| + requires + err is Some, + { err.unwrap() }, + ) + } +} + +/* +impl AsAtomicModeGuard for RcuReadGuard<'_, P> { + fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { + self.0.inner_guard.as_atomic_mode_guard() + } +}*/ + +#[verus_verify] +impl RcuOptionReadGuard<'_, P> { + /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. + #[inline] + pub fn drop(self) { + self.0.drop(); + } + + #[inline] + pub fn get<'a>(&'a self) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + self.0.get() + } + + #[inline] + pub fn is_none(&self) -> bool { + self.0.obj_ptr.is_null() + } + + #[inline] + pub fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { + proof { + use_type_invariant(&self); + } + self.0.compare_exchange(new_ptr) + } +} + +/* +impl AsAtomicModeGuard for RcuOptionReadGuard<'_, P> { + fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { + self.0.inner_guard.as_atomic_mode_guard() + } +} +*/ + +/* +/// Delays the dropping of a [`NonNullPtr`] after the RCU grace period. +/// +/// This is internally needed for implementing [`Rcu`] and [`RcuOption`] +/// because we cannot alias a [`Box`]. Restoring `P` and use [`RcuDrop`] for it +/// can lead to multiple [`Box`]es simultaneously pointing to the same +/// content. +/// +/// # Safety +/// +/// The pointer must be previously returned by `into_raw`, will not be used +/// after the end of the current grace period, and will only be dropped once. +/// +/// [`Box`]: alloc::boxed::Box +unsafe fn delay_drop(pointer: NonNull<

::Target>) { + struct ForceSend(NonNull<

::Target>); + // SAFETY: Sending a raw pointer to another task is safe as long as + // the pointer access in another task is safe (guaranteed by the trait + // bound `P: Send`). + unsafe impl Send for ForceSend

{} + + let pointer: ForceSend

= ForceSend(pointer); + + let rcu_monitor = RCU_MONITOR.get().unwrap(); + rcu_monitor.after_grace_period(move || { + // This is necessary to make the Rust compiler to move the entire + // `ForceSend` structure into the closure. + let pointer = pointer; + + // SAFETY: + // 1. The pointer was previously returned by `into_raw`. + // 2. The pointer won't be used anymore since the grace period has + // finished and this is the only time the pointer gets dropped. + let p = unsafe {

::from_raw(pointer.0) }; + drop(p); + }); +} */ + +/// A wrapper to delay calling destructor of `T` after the RCU grace period. +/// +/// Upon dropping this structure, a callback will be registered to the global +/// RCU monitor and the destructor of `T` will be delayed until the callback. +/// +/// [`RcuDrop`] is guaranteed to have the same layout as `T`. You can also +/// access the inner value safely via [`RcuDrop`]. +#[repr(transparent)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RcuDrop { + value: ManuallyDrop, +} + +impl View for RcuDrop { + type V = T; + + closed spec fn view(&self) -> T { + self.value@ + } +} + +#[verus_verify] +impl RcuDrop { + /// Creates a new [`RcuDrop`] that wraps the given value. + #[inline] + #[verus_spec(r => + ensures + r@ == value, + )] + pub fn new(value: T) -> Self { + Self { value: ManuallyDrop::new(value) } + }/* + /// Extracts the value from the `RcuDrop` container. + /// + /// # Safety + /// + /// The caller must ensure that the returned value will be dropped after + /// all the threads cannot access it anymore. Specifically, dropping it + /// after the RCU grace period is guaranteed to be safe. + /// + /// Note that panic unwinding may cause the returned value to be dropped + /// immediately, which is not sound. Therefore, the caller must forget the + /// [`PanicGuard`] after it ensures that the value will be dropped at the + /// correct time. + pub(crate) unsafe fn into_inner(slot: RcuDrop) -> (T, PanicGuard) { + let mut slot = ManuallyDrop::new(slot); + let panic_guard = PanicGuard::new(); + // SAFETY: The `slot` will not be used after this point. + let val = unsafe { ManuallyDrop::take(&mut slot.value) }; + (val, panic_guard) + } + */ + +} + +#[verus_verify] +impl Deref for RcuDrop { + type Target = T; + + #[verus_spec(r => + ensures + *r == self@, + )] + #[inline] + fn deref(&self) -> &Self::Target { + &self.value + } +} + +/* +impl Drop for RcuDrop { + fn drop(&mut self) { + // SAFETY: The `ManuallyDrop` will not be used after this point. + let taken = unsafe { ManuallyDrop::take(&mut self.value) }; + let rcu_monitor = RCU_MONITOR.get().unwrap(); + rcu_monitor.after_grace_period(|| { + drop(taken); + }); + } +} + +/// Finishes the current grace period. +/// +/// This function is called when the current grace period on current CPU is +/// finished. If this CPU is the last CPU to finish the current grace period, +/// it takes all the current callbacks and invokes them. +/// +/// # Safety +/// +/// The caller must ensure that this CPU is not executing in a RCU read-side +/// critical section. +pub unsafe fn finish_grace_period() { + let rcu_monitor = RCU_MONITOR.get().unwrap(); + // SAFETY: The caller ensures safety. + unsafe { + rcu_monitor.finish_grace_period(); + } +} + +*/ + +exec static RCU_MONITOR: Once + ensures + RCU_MONITOR.wf(), + RCU_MONITOR.inv() == RcuMonitorPred, +{ + Once::new(Ghost(RcuMonitorPred)) +} + +pub fn init() { + RCU_MONITOR.init(RcuMonitor::new_data()); +} + +} // verus! +verus! { + +impl RcuInner

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +impl RcuOption

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& self.0.is_nullable() + } +} + +impl Rcu

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& !self.0.is_nullable() + } +} + +impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& !self.0.rcu.is_nullable() + &&& self.0.tracked_ref_perm@ is Some + } +} + +impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& self.0.rcu.is_nullable() + } +} + +impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + match self.tracked_ref_perm@ { + Some(perm) => { + &&& !self.obj_ptr.is_null() + &&& P::ptr_perm_match(self.obj_ptr, perm.resource()) + &&& perm.resource().inv() + &&& perm.frac() == 1 + }, + None => self.obj_ptr.is_null(), + } + } +} + +impl Inv for Rcu

{ + closed spec fn inv(self) -> bool { + self.type_inv() + } +} + +} // verus! diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 555431cda..5a1aa1d71 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -1,736 +1,1971 @@ // SPDX-License-Identifier: MPL-2.0 //! Read-copy update (RCU). //! -//! # Note +//! This is the new weak-memory RCU skeleton. The previous SC proof-oriented +//! implementation is kept in `__mod.rs` as reference and is not compiled. //! -//! Currently this RCU model assumes a sequential consistency (SC) memory model. -//! We may explore weak memory models in the future. -use vstd::{ - atomic_ghost::AtomicPtr, atomic_with_ghost, map::Map, modes::tracked_static_ref, prelude::*, - resource::Loc, -}; - -use vstd_extra::{ - prelude::*, - resource::ghost_resource::count_auth::{Count, CountResource}, -}; - -use core::{ - marker::PhantomData, mem::ManuallyDrop, ops::Deref, - ptr::NonNull, - /* - sync::atomic::{ - AtomicPtr, - Ordering::{AcqRel, Acquire}, +//! # Verification model +//! +//! The executable RCU API is built on Verus' native IRC11 weak-memory model. +//! The atomic root pointer is a trusted executable `AtomicPtr` adapter, while +//! proofs use native `AtomicPointsTo`, `AtomicHistory`, and operation relations. +//! Weak atomic operations borrow the unique native `ViewSeen` from the current +//! task's `RunningTaskContext`; RCU never mints a fresh per-operation view and +//! therefore preserves observations across RCU operations and publication. +//! +//! The root-pointer invariant keeps publication metadata for the complete +//! atomic history. Each non-null message has a domain-local allocation ID, so +//! stale messages remain distinguishable even if a physical address is later +//! reused. Multiple messages may refer to one registration and therefore carry +//! the same allocation ID; an atomic timestamp is never used as an allocation +//! identity. The owned root invariant retains a persistent typed `BlockInfo` +//! for every registered AId, including retired historical entries. An acquire +//! load therefore returns proof of the exact typed pointer and AId it observed, +//! rather than reconstructing identity from the address. `Rcu` roots are +//! non-null in every message, while `RcuOption` roots may contain null messages +//! without allocation IDs. Physical `P::Permission`, reader permissions, +//! traversal snapshots, and reclamation are modeled separately in +//! [`specs::sync::rcu`] and are being connected incrementally. +//! +//! The traversal layer follows the paper's shape: +//! +//! - [`RcuReadGuardToken`] represents a read-side critical section together +//! with its base `Guard(tid, X, G)` state and `SeenRemoved(D, LV)` observation. +//! - [`RcuProtectedPtr`] records an AId/address pair installed in the live +//! guard's mutable protection map `G`. +//! - [`RcuBaseRetirePerm`] becomes [`RcuRetirePerm`] only after the caller has +//! observed enough traversal state to prove the allocation ID is in the +//! removed set. The domain's base `rcu-retire` transition then records it in +//! `RcuState.R` as `RcuRetired`. +//! - `RcuCallbackSafety` compresses that recorded retire proof into an erased +//! summary containing the domain, AId, removal observation, +//! observation-registry identity, retire epoch, and retire view, +//! which is what the monitor stores next to a type-erased executable +//! callback. `removal` is the paper's `Retired(a, Q)` detachment observation: +//! it records the root atomic and the first timestamp after the object was +//! removed. The retire view observes that timestamp and records the +//! observations that every quiescent report must cover before physical +//! reclamation. +//! +//! # Callback boundary +//! +//! Executable callbacks use +//! `vstd_extra::raw_callback::RawCallbackWithProof`. The raw +//! representation stores a thin data pointer plus a monomorphized runner +//! pointer, while its type requires the monitor's linear reclaim permit at +//! invocation. The RCU monitor wraps it in `monitor::RcuCallback`, which can +//! only be constructed from a `RcuCallbackSafety` certificate. This prevents +//! the proof layer from treating an arbitrary type-erased callback as a safe +//! reclamation callback or dropping the completion proof at the erasure +//! boundary. +//! +//! The monitor also has a weak-memory `is_monitoring` flag with an RCU-specific +//! invariant: every flag-history message records a snapshot of the +//! lock-protected monitor state (`specs::sync::rcu::MonitorStateView`), and a +//! `false` message certifies that its snapshot has no pending callbacks and no +//! incomplete grace period. `finish_grace_period` removes the completed batch +//! under the monitor lock and produces a private `CompletedGracePeriod` +//! certificate. For each callback, the monitor combines that certificate with +//! the callback's traversal-retire safety token to produce a linear +//! object-level reclaim permit, then executes exactly that batch outside the +//! lock. The monitor lock carries a linear release view: enqueue publishes the +//! callback's `retire_view`, and each CPU report is created only after an +//! acquire imports that view. A completed certificate therefore proves that +//! every online CPU's report view covers every callback in the batch. +//! +//! # Usage outline +//! +//! Use `Rcu

` when the root pointer is always non-null, and `RcuOption

` +//! when the root may be null. `P` must implement `NonNullPtr`; the common cases +//! are sized thin-pointer owners such as `Box` and `Arc`. Readers call +//! `read()` to obtain a guard and then use `get()` while the guard is live. +//! Writers install a new pointer with `update()` or use the read guard's +//! `compare_exchange()` to replace the value they observed. +//! +//! Verified callers carry one `RunningTaskContext` for the current task. RCU +//! operations receive a mutable borrow of that context through erased +//! `#[verus_spec(with ...)]` arguments. Starting a read-side critical section +//! increments its modeled preemption depth and removes one session fraction; +//! guard destruction reverses both changes. The scheduler can check the +//! updated view back in only after the context is quiescent. +//! +//! Delayed reclamation is connected to the weak-memory proof. The weak atomic +//! invariant retains the current registration together with +//! `P::Permission`; release swap and successful CAS establish root removal, +//! return the old raw pointer and matching ownership, and route a certified +//! callback into the monitor. Scheduler handoff now preserves a per-CPU +//! `ThreadView`: schedule-out joins the departing task's observations into the +//! CPU view, and schedule-in imports that view into the incoming task together +//! with the CPU's canonical `CpuRcuParticipant`. +//! +//! An executable `read()` now performs the paper's `Inactive -> Guard` +//! transition while opening the root weak-atomic invariant. The resulting +//! `CpuRcuReadGuardToken`, its fractional CPU reader fragment, and the exact +//! historical `BlockInfo` remain in the executable read guard until +//! destruction or consuming CAS performs `Guard -> Inactive` and returns the +//! fragment before re-enabling preemption. +//! +//! A guarded weak load now installs the loaded root's exact `BlockInfo` in the +//! guard's protection map. The proof derives the guard's expired set from the +//! entering task's view and the recorded root-removal observations. If the +//! loaded AId were expired, weak-memory coherence and the root history's +//! removal invariant would contradict the load timestamp. The guarded load +//! also splits a physical read lease from `P::Permission`; `get()` borrows that +//! lease to derive `P::RefPermission`, with no pointer-permission assumption. +//! +//! The proof-only `rcu_cpu` module now defines the required persistent +//! `CpuRcuParticipant`: a reader splits a fractional fragment, and a quiescent +//! report requires the full fraction before advancing the CPU generation and +//! view. Monitor completion retains one `CpuRcuClosedGeneration` for every +//! online CPU and duplicates those persistent resources into each callback's +//! `RcuReclaimPermit`. A live guard that coexists with such a permit is +//! necessarily from a later CPU generation and its start view includes the +//! callback's removal observation. The matching retirement record therefore +//! belongs to the guard's expired set. Since traversal well-formedness embeds +//! expired objects in `SeenRemoved`, a callback permit and a guard-protected +//! pointer to the same object are proved mutually exclusive. +//! +//! Before invoking a callback, its reclaim permit excludes every active lease +//! for the retired allocation. Reclamation then recovers the complete +//! `P::Permission` from the root invariant and passes it to the typed callback. +//! Two language-integration boundaries remain explicit: the legacy +//! `read_with()` compatibility API uses `assume_shared_ref`, while new callers +//! can use `read_with_guard()` to retain the physical lease in a verified read +//! guard. Verified callers use the consuming guard `drop()` method because +//! Verus cannot yet attach this invariant-opening transition to Rust's implicit +//! `Drop::drop(&mut self)`. Runtime destruction still restores the executable +//! preemption counter through `DisabledPreemptGuard`. +use alloc::boxed::Box; +use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; + +use vstd::invariant::InvariantPredicate; +use vstd::prelude::*; +use vstd::resource::Loc; +use vstd_extra::prelude::*; +use vstd_extra::raw_callback::{RawCallbackContextWithProof, RawCallbackWithProof}; + +use crate::{ + specs::{ + mm::cpu::online_cpus, + sync::{ + rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, + weak_memory::{ + LinkedListRetiredChild, RcuRetiredRootObject, RcuRootAtomicInv, + RcuRootAtomicInvariant, RcuRootAtomicState, RcuWeakAtomicPtr, + RegisteredLinkedListWeakAtomicLink, + }, }, - */ + task::InAtomicMode, + }, + sync::Once, + task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, }; +use vstd_extra::atomic_irc11::{ThreadViewOrder, ViewSeen}; use non_null::{NonNullPtr, NonNullPtrRef}; -// use spin::once::Once; -use super::Once; - -use self::monitor::{RcuMonitor, RcuMonitorOwner, RcuMonitorPred}; -use crate::task::{ - DisabledPreemptGuard, - //atomic_mode::{AsAtomicModeGuard, InAtomicMode}, - disable_preempt, -}; +use rcu_spec::RcuRootOwnershipPredicate; -mod monitor; +pub mod monitor; pub mod non_null; -use crate::specs::task::InAtomicMode; - verus! { broadcast use vstd_extra::external::nonnull::group_nonull_axioms; -// Verification-only budget for splitting read-side ghost tokens. -// -// This is not a runtime reader counter and does not model an overflow condition -// in the RCU implementation. It is a temporary bounded approximation needed by -// `CountResource`; the final RCU proof should discharge the admission assumption -// with an unbounded ghost registry or a CPU/epoch-based sharding model. -const RCU_READER_SLOTS: usize = 1usize << 60; +exec static RCU_MONITOR: Once< + monitor::RcuMonitor, + monitor::RcuMonitorOwner, + monitor::RcuMonitorPred, +> + ensures + RCU_MONITOR.wf(), + RCU_MONITOR.inv() == monitor::RcuMonitorPred, +{ + Once::new(Ghost(monitor::RcuMonitorPred)) +} -type RcuReadPool

= CountResource<

::Permission, RCU_READER_SLOTS>; +struct RcuPointerOwnership { + _marker: PhantomData

, +} -type RcuReadToken

= Count<

::Permission, RCU_READER_SLOTS>; +impl rcu_spec::RcuRootOwnershipPredicate< +

::Target, +

::Permission, +> for RcuPointerOwnership

{ + open spec fn owns( + ptr: *mut

::Target, + ownership:

::Permission, + ) -> bool { + &&& P::ptr_perm_match(ptr, ownership) + &&& ownership.inv() + } +} -type RcuRetiredEntry

= (Ghost<*mut

::Target>, RcuReadPool

); +/// Concrete linked-list atomic whose pointee ownership is a real smart-pointer +/// permission understood by [`NonNullPtrRef`]. +type RcuLinkedListAtomicLink

= RegisteredLinkedListWeakAtomicLink< +

::Permission, + RcuPointerOwnership

, +>; -/// Called by `drop` of the read guard to track the retired read permissions. -type RcuRetiredPools

= Map>; +/// One loaded internal child together with its physical RCU read lease. +/// +/// This is the linked-list counterpart of [`RcuReadGuardInner`]. In +/// particular, [`Self::get`] derives `P::RefPermission` from the lease and +/// invokes the same verified `raw_as_ref` boundary as a direct-root read. +struct LinkedListChildReadGuard<'a, P> where P: NonNullPtr { + obj_ptr: *mut rcu_spec::LinkedListNode, + link: &'a RcuLinkedListAtomicLink

, + proof_active: bool, + tracked_guard: Tracked>>, + tracked_child: Tracked>>, + tracked_lease: Tracked>>, + tracked_observation: Tracked>, +} -type RcuReturnedTokens

= Map>; +impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr { + #[verifier::type_invariant] + closed spec fn type_inv(&self) -> bool { + &&& self.link.well_formed() + &&& self.link.no_reclaimed_targets() + &&& self.proof_active ==> { + &&& self.tracked_guard@ is Some + &&& self.tracked_observation@ is Some + &&& self.tracked_guard@->Some_0.wf() + &&& self.tracked_guard@->Some_0.scheduler() == self.link.constant().scheduler + &&& self.tracked_guard@->Some_0.domain() == self.link.constant().domain + &&& self.tracked_guard@->Some_0.root() == self.link.constant().root + &&& self.tracked_guard@->Some_0.retire_observation_registry() + == self.link.constant().retire_observation_registry + &&& self.tracked_observation@->Some_0.registry() + == self.link.constant().timestamp_registry + &&& self.tracked_observation@->Some_0.native_registry() + == self.link.constant().native_observation_registry + &&& self.tracked_observation@->Some_0.loc() == self.link.native_loc() + &&& self.tracked_guard@->Some_0.paper_guard().seen_at(self.link.constant().source_obj) + == self.tracked_observation@->Some_0.index() + &&& (self.tracked_child@ is Some) == (self.obj_ptr.addr() != 0) + &&& (self.tracked_lease@ is Some) == (self.obj_ptr.addr() != 0) + &&& match (self.tracked_child@, self.tracked_lease@) { + (None, None) => self.obj_ptr.addr() == 0, + (Some(child), Some(lease)) => { + &&& equal(child.ptr(), self.obj_ptr) + &&& self.link.registered_targets().contains_pair(child.obj(), child.ptr()) + &&& self.link.target_lifecycles().contains_key(child.obj()) + &&& !self.link.target_phase(child.obj()).is_reclaimed() + &&& child.protected_by(self.tracked_guard@->Some_0.paper_guard()) + &&& lease.key() == child.obj() + &&& lease.active_registry() == self.link.constant().active_lease_registry + &&& lease.participant_id() == self.tracked_guard@->Some_0.participant_id() + &&& lease.reader_fraction() + == self.tracked_guard@->Some_0.reader_fragment().fraction() + &&& lease.domain() == self.tracked_guard@->Some_0.domain() + &&& lease.root() == self.tracked_guard@->Some_0.root() + &&& lease.reader_context() == self.tracked_guard@->Some_0.reader_context() + &&& lease.start_view() == self.tracked_guard@->Some_0.start_view() + &&& lease.protected_addr() == child.ptr().addr() + &&& RcuPointerOwnership::

::owns(child.ptr(), lease.resource()) + }, + _ => false, + } + } + } -tracked struct RcuPtrGhost { - tracked current: Option>, - tracked retired: RcuRetiredPools

, - tracked returned: RcuReturnedTokens

, -} + /// Loads an internal child and retains both its traversal witness and its + /// physical lease. `previous` may be supplied to repeat a load from the + /// same source without resetting the dense traversal view. The retained + /// observation is certified by the link's native-view registry and stays + /// valid as the atomic history grows. + fn load( + link: &'a RcuLinkedListAtomicLink

, + Tracked(guard): Tracked>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Self) + requires + link.well_formed(), + link.no_reclaimed_targets(), + guard.wf(), + guard.scheduler() == link.constant().scheduler, + guard.domain() == link.constant().domain, + guard.root() == link.constant().root, + guard.retire_observation_registry() == link.constant().retire_observation_registry, + online_cpus().contains(guard.cpu()), + guard.seen_removed().removed == Set::::empty(), + match previous { + None => guard.paper_guard().seen_at(link.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == link.constant().timestamp_registry + &&& observation.native_registry() == link.constant().native_observation_registry + &&& observation.loc() == link.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& guard.paper_guard().seen_at(link.constant().source_obj) + == observation.index() + }, + }, + old(from).protected_by(guard.paper_guard()), + old(from).ptr() == link.constant().source, + old(from).obj() == link.constant().source_obj, + ensures + res.type_inv(), + res.proof_active, + old(tv)@.spec_le(final(tv)@), + final(from).ptr() == link.constant().source, + final(from).obj() == link.constant().source_obj, + { + let ( + obj_ptr, + _timestamp, + _index, + Tracked(guard), + Tracked(child), + Tracked(lease), + Tracked(observation), + ) = link.load_acquire_and_lease_cpu( + Tracked(guard), + Tracked(from), + Tracked(previous), + Tracked(tv), + ); + proof { + match (&child, &lease) { + (Some(loaded_child), None) => { + assert(loaded_child.obj() != link.constant().source_obj); + assert(link.registered_targets().dom().remove( + link.constant().source_obj, + ).contains(loaded_child.obj())); + assert(!link.target_phase(loaded_child.obj()).is_reclaimed()); + assert(false); + }, + _ => {}, + } + } + let res = Self { + obj_ptr, + link, + proof_active: true, + tracked_guard: Tracked(Some(guard)), + tracked_child: Tracked(child), + tracked_lease: Tracked(lease), + tracked_observation: Tracked(Some(observation)), + }; + proof { + assert(res.type_inv()); + } + res + } -closed spec fn retired_pools_inv(retired: RcuRetiredPools

) -> bool { - forall|id: Loc| #[trigger] - retired.contains_key(id) ==> { - let entry = retired[id]; - &&& !(entry.0@).is_null() - &&& entry.1.id() == id - &&& P::ptr_perm_match(entry.0@, entry.1@) - &&& entry.1@.inv() - &&& entry.1.wf() - &&& entry.1.not_empty() + /// Obtains the smart pointer's real shared-reference representation from + /// the internal child's physical lease. + fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> + requires + self.proof_active, + { + proof { + use_type_invariant(self); + reveal(LinkedListChildReadGuard::type_inv); + if self.obj_ptr.addr() != 0 { + assert(self.tracked_lease@ is Some); + assert(RcuPointerOwnership::

::owns( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + )); + assert(P::ptr_perm_match(self.obj_ptr, self.tracked_lease@->Some_0.resource())); + assert(self.tracked_lease@->Some_0.resource().inv()); + } } -} + NonNull::new(self.obj_ptr).map( + |ptr| + requires + self.tracked_lease@ is Some, + P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_lease@->Some_0.resource()), + { + proof_decl! { + let tracked lease = self.tracked_lease.tracked_borrow(); + let tracked ref_perm = P::borrow_perm_as_ref_perm(lease.borrow()); + } + unsafe { P::raw_as_ref(ptr, Tracked(ref_perm)) } + }, + ) + } -closed spec fn returned_tokens_inv(returned: RcuReturnedTokens

) -> bool { - forall|id: Loc| #[trigger] - returned.contains_key(id) ==> { - let token = returned[id]; - &&& token.id() == id - &&& token.resource().inv() - &&& token.frac() > 0 + /// Returns the physical lease, the traversal witness, and the updated CPU + /// guard so the caller may continue traversing. + fn finish(self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: Tracked< + ( + rcu_cpu_spec::CpuRcuReadGuardToken, + Option>, + rcu_spec::LinkedListLinkObservation, + ), + >) + requires + self.type_inv(), + self.proof_active, + self.link.no_reclaimed_targets(), + ensures + old(tv)@.spec_le(final(tv)@), + res@.0.wf(), + res@.0.paper_guard().seen_at(self.link.constant().source_obj) == res@.2.index(), + res@.2.registry() == self.link.constant().timestamp_registry, + res@.2.native_registry() == self.link.constant().native_observation_registry, + res@.2.loc() == self.link.native_loc(), + (res@.1 is Some) == (self.obj_ptr.addr() != 0), + { + let mut this = self; + proof { + use_type_invariant(&this); + reveal(LinkedListChildReadGuard::type_inv); + } + this.proof_active = false; + proof_decl! { + let tracked guard = this.tracked_guard.borrow_mut().tracked_take(); + let tracked mut child = None; + vstd::modes::tracked_swap(this.tracked_child.borrow_mut(), &mut child); + let tracked mut lease = None; + vstd::modes::tracked_swap(this.tracked_lease.borrow_mut(), &mut lease); + let tracked observation = this.tracked_observation.borrow_mut().tracked_take(); } + let Tracked(guard) = this.link.return_registered_lease_cpu( + Tracked(lease), + Tracked(guard), + Tracked(tv), + ); + Tracked((guard, child, observation)) + } } -/// A Read-Copy Update (RCU) cell for sharing a pointer between threads. -/// -/// The pointer should be a non-null pointer with type `P`, which implements -/// [`NonNullPtr`]. For example, `P` can be `Box` or `Arc`. -/// -/// # Overview -/// -/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high- -/// performance, low-latency read operations in concurrent systems. It allows -/// multiple readers to access shared data simultaneously without contention, -/// while writers can update the data safely in a way that does not disrupt -/// ongoing reads. RCU is particularly suited for situations where reads are -/// far more frequent than writes. -/// -/// The original design and implementation of RCU is described in paper _The -/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared- -/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal -/// 47.2 (2008). -/// -/// # Examples -/// -/// ``` -/// use ostd::sync::Rcu; -/// -/// let rcu = Rcu::new(Box::new(42)); -/// -/// let rcu_guard = rcu.read(); -/// -/// assert_eq!(*rcu_guard, Some(&42)); -/// -/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); -/// -/// let rcu_guard = rcu.read(); +impl<'a> LinkedListChildReadGuard<'a, Box> { + /// Concrete acceptance path: turn the boxed child's leased + /// `RefPermission` into an actual Rust shared reference and dereference + /// the node allocation. + fn deref_box_child<'b>(&'b self) -> Option<&'b rcu_spec::LinkedListNode> + requires + self.proof_active, + { + self.get().map(|child| child.deref_target()) + } +} + +/// The weak-memory atomic slot used by RCU. /// -/// assert_eq!(*rcu_guard, Some(&43)); -/// ``` +/// `bool` is the constant key: `true` means the public cell may contain null +/// (`RcuOption`), and `false` means the public cell is non-null (`Rcu`). The +/// RCU-specific predicate requires non-null `Rcu` cells to contain only +/// non-null history messages. Its publication registry also assigns every +/// non-null message a domain-local allocation identity, matching the paper's +/// distinction between physical addresses and allocation IDs. +type RcuAtomicPtr

= RcuWeakAtomicPtr< +

::Target, +

::Permission, + RcuPointerOwnership

, +>; + +/// A Read-Copy Update cell for sharing a non-null pointer. pub struct Rcu(RcuInner

); -/// A guard that allows access to the pointed data protected by a [`Rcu`]. +/// A read-side guard for [`Rcu`]. #[clippy::has_significant_drop] #[must_use] pub struct RcuReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); -/// A Read-Copy Update (RCU) cell for sharing a _ghost_nullable_ pointer. -/// -/// This is a variant of [`Rcu`] that allows the contained pointer to be null. -/// So that it can implement `Rcu>` where `P` is not a ghost_nullable -/// pointer. It is the same as [`Rcu`] in other aspects. -/// -/// # Examples -/// -/// ``` -/// use ostd::sync::RcuOption; -/// -/// static RCU: RcuOption> = RcuOption::new_none(); -/// -/// assert!(RCU.read().is_none()); -/// -/// RCU.update(Box::new(42)); -/// -/// // Read the data protected by RCU -/// { -/// let rcu_guard = RCU.read().try_get().unwrap(); -/// assert_eq!(*rcu_guard, 42); -/// } -/// -/// // Update the data protected by RCU -/// { -/// let rcu_guard = RCU.read().try_get().unwrap(); -/// -/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); -/// -/// let rcu_guard = RCU.read().try_get().unwrap(); -/// assert_eq!(*rcu_guard, 43); -/// } -/// ``` +/// A Read-Copy Update cell that may contain null. pub struct RcuOption(RcuInner

); -/// A guard that allows access to the pointed data protected by a [`RcuOption`]. +/// A read-side guard for [`RcuOption`]. #[clippy::has_significant_drop] #[must_use] pub struct RcuOptionReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); -struct_with_invariants! { -/// The inner implementation of both [`Rcu`] and [`RcuOption`]. -struct RcuInner { - ptr: AtomicPtr<

::Target,_,RcuPtrGhost

,_>, - // We want to implement Send and Sync explicitly. - // Having a pointer field prevents them from being implemented - // automatically by the compiler. - _marker: PhantomData<*const

::Target>, +pub struct RcuInner { + ptr: RcuAtomicPtr

, ghost_nullable: Ghost, + _marker: PhantomData<*const

::Target>, } -closed spec fn wf(self) -> bool { - invariant on ptr with (ghost_nullable, _marker) is ( - v: *mut

::Target, - g: RcuPtrGhost

, - ) { - &&& retired_pools_inv::

(g.retired) - &&& returned_tokens_inv::

(g.returned) - &&& match g.current { - Some(perm) => { - &&& !v.is_null() - &&& P::ptr_perm_match(v, perm@) - &&& perm@.inv() - &&& perm.wf() - &&& perm.not_empty() - }, - None => ghost_nullable@ && v.is_null(), - } - } +struct RcuReadGuardInner<'a, P: NonNullPtr> { + obj_ptr: *mut

::Target, + rcu: &'a RcuInner

, + proof_active: bool, + _inner_guard: DisabledPreemptGuard, + tracked_info: Tracked::Target>>>, + tracked_guard: Tracked::Target>>>, + tracked_lease: Tracked::Permission>>>, + tracked_session: Tracked>, } + +/// Sized callback payload that retains the physical ownership of one detached +/// RCU object until the monitor executes its callback. +struct RcuDropCallbackContext { + pointer: NonNull<

::Target>, + tracked_object: Tracked::Target>>, + tracked_claim: Tracked::Target>>, + ghost_removal: Ghost, + ghost_retire_observation_registry: Ghost, + ghost_scheduler: Ghost, + tracked_root_inv: Tracked< + &'static RcuRootAtomicInvariant< +

::Target, +

::Permission, + RcuPointerOwnership

, + >, + >, } -// SAFETY: It is apparent that if `P` is `Send`, then `Rcu

` is `Send`. +/// Type-erased callback payload for an internal linked-list child. +/// +/// The callback owns the entire link wrapper after unlink/retire. This gives +/// it exclusive access to the link's phase token and physical permission pool +/// when the monitor eventually supplies a reclaim permit. +struct LinkedListDropCallbackContext

where + P: NonNullPtr + Send, + { + pointer: NonNull, + link: RcuLinkedListAtomicLink

, + ghost_target_obj: Ghost, + tracked_object: Tracked>, + tracked_claim: Tracked>, + ghost_removal: Ghost, + ghost_retire_observation_registry: Ghost, + ghost_scheduler: Ghost, +} +// SAFETY: the context owns the detached `P` allocation and the unique link +// wrapper that protects its proof-only permission pool. No borrowed runtime +// state crosses into the monitor queue. #[verifier::external] -unsafe impl Send for RcuInner

where P: Send { +unsafe impl

Send for LinkedListDropCallbackContext

where + P: NonNullPtr + Send, + { + +} + +impl

LinkedListDropCallbackContext

where + P: NonNullPtr + Send, + { + pub closed spec fn permit_matches(&self, permit: monitor::RcuReclaimPermit) -> bool { + &&& permit.wf() + &&& permit.callback().domain == self.tracked_object@.domain() + &&& permit.callback().obj == self.tracked_object@.obj() + &&& permit.callback().removal == self.ghost_removal@ + &&& permit.callback().retire_observation_registry == self.ghost_retire_observation_registry@ + &&& permit.callback().scheduler == self.ghost_scheduler@ + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.link.well_formed() + &&& self.link.registered_targets().contains_pair( + self.ghost_target_obj@, + self.pointer.view_ptr_mut(), + ) + &&& self.link.target_lifecycles().contains_key(self.ghost_target_obj@) + &&& (self.link.target_phase(self.ghost_target_obj@).is_retired() || self.link.target_phase( + self.ghost_target_obj@, + ).is_reclaimed()) + &&& self.tracked_object@.wf() + &&& equal(self.tracked_object@.ptr(), self.pointer.view_ptr_mut()) + &&& self.tracked_object@.domain() == self.link.constant().domain + &&& self.tracked_object@.obj() == self.ghost_target_obj@ + &&& self.ghost_scheduler@ == self.link.constant().scheduler + &&& match self.link.target_phase(self.ghost_target_obj@) { + crate::specs::sync::weak_memory::LinkedListChildPhase::Retired { index: _, removal } + | crate::specs::sync::weak_memory::LinkedListChildPhase::Reclaimed { + index: _, + removal, + } => self.ghost_removal@ == removal, + _ => false, + } + &&& self.ghost_removal@.root == self.link.constant().root + &&& self.ghost_retire_observation_registry@ + == self.link.constant().retire_observation_registry + &&& self.link.target_phase(self.ghost_target_obj@).is_retired() + &&& self.tracked_claim@.registry() == self.link.constant().reclaim_registry + &&& self.tracked_claim@.obj() == self.tracked_object@.obj() + &&& self.tracked_claim@.is_pending() + &&& equal(self.tracked_claim@.ptr(), self.pointer.view_ptr_mut()) + } +} +impl

RawCallbackContextWithProof for LinkedListDropCallbackContext< + P, +> where P: NonNullPtr + Send { + open spec fn call_requires(&self, permit: monitor::RcuReclaimPermit) -> bool { + self.permit_matches(permit) + } + + fn run(self, Tracked(permit): Tracked) { + let Tracked(credit) = vstd::invariant::create_open_invariant_credit(); + proof_decl! { + let tracked permission; + let tracked completed; + } + proof { + use_type_invariant(&self); + use_type_invariant(&permit); + reveal(LinkedListDropCallbackContext::type_inv); + permit.lemma_authorizes_callback(); + let ghost callback = permit.callback(); + assert(self.permit_matches(permit)); + assert(permit.authorizes(callback)); + assert(self.link.target_phase(self.ghost_target_obj@).is_retired()); + completed = permit.tracked_into_reclaimed_witness(callback); + assert(completed.wf()); + assert(completed.scheduler() == self.link.constant().scheduler); + assert(completed.record() == callback.retired_record()); + assert(completed.record().domain == self.link.constant().domain); + assert(completed.record().obj == self.ghost_target_obj@); + assert(completed.record().retire_observation_registry + == self.link.constant().retire_observation_registry); + assert(completed.record().removal == self.link.target_phase( + self.ghost_target_obj@, + )->Retired_removal); + } + let LinkedListDropCallbackContext { + pointer, + mut link, + ghost_target_obj, + tracked_object: _, + tracked_claim, + ghost_removal: _, + ghost_retire_observation_registry: _, + ghost_scheduler: _, + } = self; + proof { + permission = + link.tracked_reclaim_retired_target( + pointer.as_ptr(), + ghost_target_obj@, + tracked_claim.get(), + completed, + credit, + ); + assert(RcuPointerOwnership::

::owns(pointer.as_ptr(), permission)); + assert(P::ptr_perm_match(pointer.as_ptr(), permission)); + assert(permission.inv()); + } + let _pointer = unsafe { P::from_raw(pointer, Tracked(permission)) }; + } } -// SAFETY: To implement `Sync` for `Rcu

`, we need to meet two conditions: -// 1. `P` must be `Sync` because `Rcu::get` allows concurrent access. -// 2. `P` must be `Send` because `Rcu::update` may obtain an object -// of `P` created on another thread. +// SAFETY: the callback consumes the same owning pointer type `P` that was +// accepted by the RCU cell. The tracked permission has no runtime payload. #[verifier::external] -unsafe impl Sync for RcuInner

where P: Send + Sync { +unsafe impl Send for RcuDropCallbackContext

{ + +} + +impl RawCallbackContextWithProof< + monitor::RcuReclaimPermit, +> for RcuDropCallbackContext

{ + open spec fn call_requires(&self, permit: monitor::RcuReclaimPermit) -> bool { + self.permit_matches(permit) + } + + fn run(self, Tracked(permit): Tracked) { + let pointer = self.pointer; + let Tracked(credit) = vstd::invariant::create_open_invariant_credit(); + proof_decl! { + let tracked permission; + } + proof { + use_type_invariant(&self); + use_type_invariant(&permit); + reveal(RcuDropCallbackContext::type_inv); + permit.lemma_authorizes_callback(); + let tracked root_inv = self.tracked_root_inv.get(); + let ghost callback = permit.callback(); + assert(self.permit_matches(permit)); + assert(permit.authorizes(callback)); + vstd::invariant::open_atomic_invariant_in_proof!(credit => root_inv => state => { + assert(RcuRootAtomicInv::>::inv( + root_inv.constant(), + state, + )); + crate::specs::sync::weak_memory::lemma_root_atomic_permission_facts::< +

::Target, +

::Permission, + RcuPointerOwnership

, + >(root_inv.constant(), &state); + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv(root_inv.constant(), (state.points_to, state.root))); + assert(state.permissions.wf()); + assert(callback.scheduler == state.permissions.scheduler()); + assert(callback.domain == state.permissions.domain()); + assert(callback.retire_observation_registry + == state.permissions.retire_observation_registry()); + assert(callback.removal.root == state.permissions.root()); + state.permissions.lemma_active_registry_projection(); + let ghost permissions_before_exclusion = state.permissions; + let ghost registry_before_exclusion = state.permissions.registry(); + assert forall|lease_id: nat| #![auto] + state.permissions.active_ids().contains(lease_id) + && state.permissions.active_record(lease_id).key() == callback.obj + implies { + let witness = state.permissions.active_record(lease_id).witness(); + &&& witness.wf() + &&& witness.protected().obj() == callback.obj + &&& witness.reader().cpu() == witness.paper_guard().reader().cpu + &&& permit.reports().contains_key(witness.reader().cpu()) + &&& callback.scheduler == witness.binding().registry() + &&& callback.domain == witness.paper_guard().domain() + &&& callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry() + &&& callback.removal.root == witness.paper_guard().root() + } by { + assert(state.permissions.active_ids().contains(lease_id)); + assert(state.permissions.wf()); + let witness = state.permissions.active_record(lease_id).witness(); + assert(witness.wf()); + assert(witness.reader().cpu() == witness.paper_guard().reader().cpu); + assert(online_cpus().contains(witness.reader().cpu())); + assert(permit.reports().dom() == online_cpus()); + assert(permit.reports().contains_key(witness.reader().cpu())); + assert(state.permissions.active_record(lease_id).key() + == witness.protected().obj()); + assert(witness.protected().obj() == callback.obj); + assert(callback.scheduler == witness.binding().registry()); + assert(callback.domain == witness.paper_guard().domain()); + assert(callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry()); + assert(callback.removal.root == witness.paper_guard().root()); + }; + { + let tracked registry = state.permissions.tracked_registry_mut(); + assert(*registry == registry_before_exclusion); + assert forall|lease_id: nat| #![auto] + (*registry).active_ids().contains(lease_id) + && (*registry).active_record(lease_id).key() == callback.obj + implies { + let witness = (*registry).active_record(lease_id).witness(); + &&& witness.wf() + &&& witness.protected().obj() == callback.obj + &&& witness.reader().cpu() == witness.paper_guard().reader().cpu + &&& permit.reports().contains_key(witness.reader().cpu()) + &&& callback.scheduler == witness.binding().registry() + &&& callback.domain == witness.paper_guard().domain() + &&& callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry() + &&& callback.removal.root == witness.paper_guard().root() + } by {}; + permit.tracked_excludes_active_leases(callback, registry); + } + assert(state.permissions == permissions_before_exclusion); + assert(state.permissions.wf()); + let tracked completed = permit.tracked_into_reclaimed_witness(callback); + assert(completed.scheduler() == state.permissions.scheduler()); + assert(completed.record() == callback.retired_record()); + assert(completed.record().domain == state.permissions.domain()); + assert(completed.record().retire_observation_registry + == state.permissions.retire_observation_registry()); + assert(completed.record().removal.root == state.permissions.root()); + { + let tracked retired_fact = completed.tracked_retired_fact(); + state.root.lemma_retired_fact_agrees(retired_fact); + } + assert(state.root.removals().contains_pair( + callback.obj, + callback.removal, + )); + state.permissions.lemma_all_unretired_domains(); + let ghost before = state.permissions; + assert(before.reclaim_registry() == root_inv.constant().0.reclaim_registry); + assert(before.unretired_claims().dom() + == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) implies + !before.has_unretired_claim(obj) by {}; + state.permissions.lemma_contains_iff_key(self.tracked_claim@.obj()); + permission = state.permissions.tracked_reclaim( + self.tracked_claim.get(), + completed, + ); + assert(before.contains(callback.obj)); + assert(before.keys().contains(callback.obj)); + assert(before.reclaim_states()[callback.obj] is Some); + assert(before.reclaim_states()[callback.obj]->Some_0 == pointer.as_ptr()); + assert(RcuPointerOwnership::

::owns(pointer.as_ptr(), permission)); + assert(P::ptr_perm_match(pointer.as_ptr(), permission)); + assert(permission.inv()); + state.permissions.lemma_all_live_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert(state.permissions.allocations() == state.root.infos().dom()); + assert forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) implies { + &&& state.permissions.contains(obj) + &&& state.permissions.allocations().contains(obj) + &&& state.permissions.reclaim_states().dom().contains(obj) + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 + == state.root.infos()[obj].ptr() + &&& RcuPointerOwnership::

::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + } by { + assert(obj != callback.obj); + assert(before.keys().contains(obj)); + assert(before.contains(obj)); + assert(state.permissions.ownership(obj) == before.ownership(obj)); + assert(state.permissions.reclaim_states()[obj] + == before.reclaim_states()[obj]); + }; + assert(state.permissions.unretired_claims() == before.unretired_claims()); + assert(state.permissions.unretired_claims().dom() + == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) implies + !state.permissions.has_unretired_claim(obj) by { + assert(!before.has_unretired_claim(obj)); + assert(before.has_unretired_claim(obj) + == before.unretired_claims().dom().contains(obj)); + assert(state.permissions.has_unretired_claim(obj) + == state.permissions.unretired_claims().dom().contains(obj)); + assert(state.permissions.has_unretired_claim(obj) + == before.has_unretired_claim(obj)); + }; + assert forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) implies { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + } by { + if obj == callback.obj { + assert(state.permissions.reclaimed()[obj].record() + == callback.retired_record()); + assert(state.root.removals()[obj] == callback.removal); + } else { + assert(before.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed()[obj] == before.reclaimed()[obj]); + } + }; + assert(state.permissions.scheduler() == root_inv.constant().0.scheduler); + assert(state.permissions.domain() == root_inv.constant().0.domain); + assert(state.permissions.root() == root_inv.constant().0.domain); + assert(state.permissions.retire_observation_registry() + == root_inv.constant().0.retire_observation_registry); + assert(state.permissions.reclaim_registry() == before.reclaim_registry()); + assert(state.permissions.reclaim_registry() + == root_inv.constant().0.reclaim_registry); + assert(state.permissions.active_lease_registry() + == root_inv.constant().0.active_lease_registry); + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv(root_inv.constant(), (state.points_to, state.root))); + crate::specs::sync::weak_memory::lemma_build_root_atomic_inv::< +

::Target, +

::Permission, + RcuPointerOwnership

, + >( + root_inv.constant(), + &state, + ); + }); + } + let _pointer = unsafe { P::from_raw(pointer, Tracked(permission)) }; + } +} + +impl RcuDropCallbackContext

{ + pub closed spec fn permit_matches(&self, permit: monitor::RcuReclaimPermit) -> bool { + &&& permit.wf() + &&& permit.callback().domain == self.tracked_object@.domain() + &&& permit.callback().obj == self.tracked_object@.obj() + &&& permit.callback().removal == self.ghost_removal@ + &&& permit.callback().retire_observation_registry == self.ghost_retire_observation_registry@ + &&& permit.callback().scheduler == self.ghost_scheduler@ + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.tracked_object@.wf() + &&& equal(self.tracked_object@.ptr(), self.pointer.view_ptr_mut()) + &&& self.tracked_claim@.obj() == self.tracked_object@.obj() + &&& self.tracked_claim@.is_pending() + &&& equal(self.tracked_claim@.ptr(), self.pointer.view_ptr_mut()) + &&& self.tracked_object@.domain() == self.tracked_root_inv@.constant().0.domain + &&& self.tracked_claim@.registry() == self.tracked_root_inv@.constant().0.reclaim_registry + &&& self.ghost_scheduler@ == self.tracked_root_inv@.constant().0.scheduler + &&& self.ghost_removal@.root == self.tracked_root_inv@.constant().0.domain + &&& self.ghost_retire_observation_registry@ + == self.tracked_root_inv@.constant().0.retire_observation_registry + } +} + +/// Erases a detached owned object into an executable callback payload. +/// +/// This function does not certify or enqueue the callback. Those operations +/// still require `RcuRetired` and a monitor grace-period certificate. +fn callback_from_detached( + pointer: *mut

::Target, + Tracked(owned): Tracked::Target>>, + Tracked(root_inv): Tracked< + &'static RcuRootAtomicInvariant< +

::Target, +

::Permission, + RcuPointerOwnership

, + >, + >, +) -> (res: (RawCallbackWithProof, Tracked)) + requires + !pointer.is_null(), + equal(owned.ptr(), pointer), + equal(owned.object().ptr(), pointer), + owned.object().domain() == root_inv.constant().0.domain, + owned.claim().registry() == root_inv.constant().0.reclaim_registry, + owned.retired().removal().root == root_inv.constant().0.domain, + owned.retired().retire_observation_registry() + == root_inv.constant().0.retire_observation_registry, + ensures + res.1@.removal() == owned.retired().removal(), + forall|permit: monitor::RcuReclaimPermit| + #![auto] + permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj + == res.1@.obj() && permit.callback().removal == res.1@.removal() + && permit.callback().retire_observation_registry + == res.1@.retire_observation_registry() && permit.callback().scheduler + == root_inv.constant().0.scheduler ==> res.0.call_requires(permit), +{ + proof { + use_type_invariant(&owned); + } + proof_decl! { + let tracked (object, retired, claim) = owned.tracked_into_parts(); + let tracked cert = rcu_spec::certify_callback_from_retired(&object, retired); + } + proof { + assert(object.domain() == root_inv.constant().0.domain); + assert(claim.registry() == root_inv.constant().0.reclaim_registry); + assert(cert.removal().root == root_inv.constant().0.domain); + assert(cert.retire_observation_registry() + == root_inv.constant().0.retire_observation_registry); + } + let pointer = unsafe { NonNull::new_unchecked(pointer) }; + proof { + assert(object.wf()); + assert(equal(object.ptr(), pointer.view_ptr_mut())); + assert(claim.obj() == object.obj()); + assert(claim.is_pending()); + assert(equal(claim.ptr(), pointer.view_ptr_mut())); + assert(object.domain() == root_inv.constant().0.domain); + assert(claim.registry() == root_inv.constant().0.reclaim_registry); + assert(root_inv.constant().0.scheduler == root_inv.constant().0.scheduler); + assert(cert.removal().root == root_inv.constant().0.domain); + assert(cert.retire_observation_registry() + == root_inv.constant().0.retire_observation_registry); + } + let context = RcuDropCallbackContext::

{ + pointer, + tracked_object: Tracked(object), + tracked_claim: Tracked(claim), + ghost_removal: Ghost(cert.removal()), + ghost_retire_observation_registry: Ghost(cert.retire_observation_registry()), + ghost_scheduler: Ghost(root_inv.constant().0.scheduler), + tracked_root_inv: Tracked(root_inv), + }; + proof { + use_type_invariant(&context); + } + (RawCallbackWithProof::new(context), Tracked(cert)) +} + +/// Erases a retired internal child into the real monitor callback pipeline. +/// The link moves into the callback context, so the successful callback is the +/// only code that can change its phase to `Reclaimed` and recover `P`'s full +/// physical permission. +fn callback_from_linked_list_child

( + link: RcuLinkedListAtomicLink

, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(retired): Tracked, +) -> (res: ( + RawCallbackWithProof, + Tracked, +)) where P: NonNullPtr + Send + requires + link.well_formed(), + link.registered_targets().contains_pair(target_obj, target), + link.target_lifecycles().contains_key(target_obj), + link.target_phase(target_obj).is_retired(), + retired.object().wf(), + retired.object().domain() == link.constant().domain, + retired.object().obj() == target_obj, + equal(retired.object().ptr(), target), + retired.claim().registry() == link.constant().reclaim_registry, + retired.claim().obj() == target_obj, + retired.claim().is_pending(), + equal(retired.claim().ptr(), target), + retired.retired().removal() == link.target_phase(target_obj)->Retired_removal, + link.target_phase(target_obj)->Retired_removal.root == link.constant().root, + retired.retired().retire_observation_registry() + == link.constant().retire_observation_registry, + ensures + res.1@.domain() == link.constant().domain, + res.1@.obj() == target_obj, + res.1@.removal() == link.target_phase(target_obj)->Retired_removal, + res.1@.retire_observation_registry() == link.constant().retire_observation_registry, + forall|permit: monitor::RcuReclaimPermit| + #![auto] + permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj + == res.1@.obj() && permit.callback().removal == res.1@.removal() + && permit.callback().retire_observation_registry + == res.1@.retire_observation_registry() && permit.callback().scheduler + == link.constant().scheduler ==> res.0.call_requires(permit), +{ + proof_decl! { + let tracked (object, cert, claim) = retired.tracked_certify_callback(); + } + proof { + object.lemma_wf_facts(); + assert(link.well_formed()); + assert(target.addr() != 0); + } + let pointer = unsafe { NonNull::new_unchecked(target) }; + proof { + assert(equal(pointer.view_ptr_mut(), target)); + assert(object.wf()); + assert(link.target_phase(target_obj) is Retired); + assert(equal(object.ptr(), pointer.view_ptr_mut())); + assert(object.domain() == link.constant().domain); + assert(object.obj() == target_obj); + assert(claim.registry() == link.constant().reclaim_registry); + assert(claim.obj() == object.obj()); + assert(claim.is_pending()); + assert(equal(claim.ptr(), pointer.view_ptr_mut())); + assert(cert.removal() == link.target_phase(target_obj)->Retired_removal); + assert(cert.removal().root == link.constant().root); + assert(cert.retire_observation_registry() == link.constant().retire_observation_registry); + } + let ghost scheduler = link.constant().scheduler; + let context = LinkedListDropCallbackContext::

{ + pointer, + link, + ghost_target_obj: Ghost(target_obj), + tracked_object: Tracked(object), + tracked_claim: Tracked(claim), + ghost_removal: Ghost(cert.removal()), + ghost_retire_observation_registry: Ghost(cert.retire_observation_registry()), + ghost_scheduler: Ghost(scheduler), + }; + proof { + use_type_invariant(&context); + } + (RawCallbackWithProof::new(context), Tracked(cert)) +} +/// Schedules one retired internal child on the existing `call_rcu` monitor +/// path. This is intentionally kept private to the linked-list acceptance +/// case until a production data-structure adapter chooses its public API. +fn after_grace_period_linked_list_child

( + link: RcuLinkedListAtomicLink

, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(retired): Tracked, + Tracked(session): Tracked<&mut RunningTaskContext>, +) where P: NonNullPtr + Send + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + link.well_formed(), + link.constant().scheduler == old(session).scheduler(), + link.registered_targets().contains_pair(target_obj, target), + link.target_lifecycles().contains_key(target_obj), + link.target_phase(target_obj).is_retired(), + retired.object().wf(), + retired.object().domain() == link.constant().domain, + retired.object().obj() == target_obj, + equal(retired.object().ptr(), target), + retired.claim().registry() == link.constant().reclaim_registry, + retired.claim().obj() == target_obj, + retired.claim().is_pending(), + equal(retired.claim().ptr(), target), + retired.retired().removal() == link.target_phase(target_obj)->Retired_removal, + link.target_phase(target_obj)->Retired_removal.root == link.constant().root, + retired.retired().removal().observed_by(old(session).irc11_view()), + retired.retired().retire_observation_registry() + == link.constant().retire_observation_registry, + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + final(session).rcu_participant_id() == old(session).rcu_participant_id(), + final(session).rcu_generation() == old(session).rcu_generation(), + final(session).rcu_participant_view() == old(session).rcu_participant_view(), + final(session).rcu_fraction() == old(session).rcu_fraction(), +{ + let (callback, cert) = callback_from_linked_list_child::

( + link, + target, + Ghost(target_obj), + Tracked(retired), + ); + if let Some(monitor) = RCU_MONITOR.get() { + #[verus_spec(with Tracked(session))] + monitor.after_grace_period(callback, cert); + } } impl RcuInner

{ - /// Whether the contained pointer can be null. Used to distinguish `Rcu` and `RcuOption`. - pub closed spec fn is_nullable(self) -> bool { + closed spec fn is_nullable(self) -> bool { self.ghost_nullable@ } + + closed spec fn wf(self) -> bool { + &&& self.ptr.well_formed() + &&& self.ptr.constant().nullable == self.ghost_nullable@ + &&& self.ptr.constant().scheduler == rcu_spec::rcu_scheduler() + } +} + +// SAFETY: `RcuInner` only shares a raw pointer through an atomic slot. Sending +// the cell follows the same requirement as sending the managed pointer wrapper. +#[verifier::external] +unsafe impl Send for RcuInner

where P: Send { + } +// SAFETY: Readers may obtain shared references, so `P` must be `Sync`; writers +// may install pointers created on another thread, so `P` must be `Send`. +#[verifier::external] +unsafe impl Sync for RcuInner

where P: Send + Sync { + +} + +#[verus_verify] impl RcuInner

{ #[inline(always)] const fn new_none() -> (res: Self) ensures + res.type_inv(), res.is_nullable(), - res.wf(), + res.ptr.constant().scheduler == rcu_spec::rcu_scheduler(), { + let ptr = RcuAtomicPtr::

::new( + Ghost(true), + Ghost(rcu_spec::rcu_scheduler()), + core::ptr::null_mut(), + Tracked(None), + ); Self { - ptr: AtomicPtr::new( - Ghost((Ghost(true), PhantomData::<*const

::Target>)), - core::ptr::null_mut(), - Tracked( - RcuPtrGhost { - current: None, - retired: Map::tracked_empty(), - returned: Map::tracked_empty(), - }, - ), - ), - _marker: PhantomData::<*const

::Target>, + ptr, ghost_nullable: Ghost(true), + _marker: PhantomData::<*const

::Target>, } } -} -#[verus_verify] -impl RcuInner

{ - /// Creates a new RCU primitive with the given pointer `pointer`. #[inline(always)] - #[verus_spec(r => + #[verus_spec(res => with - Ghost(ghost_nullable): Ghost, + Ghost(nullable): Ghost, ensures - r.type_inv(), - r.is_nullable() == ghost_nullable, + res.type_inv(), + res.is_nullable() == nullable, + res.ptr.constant().scheduler == rcu_spec::rcu_scheduler(), )] fn new(pointer: P) -> Self { - // let ptr =

::into_raw(pointer).as_ptr(); - let (ptr, Tracked(ptr_perm)) =

::into_raw(pointer); - let ptr = ptr.as_ptr(); - proof_decl! { - let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { - current: Some(CountResource::alloc(ptr_perm)), - retired: Map::tracked_empty(), - returned: Map::tracked_empty(), - }; + let (raw, Tracked(perm)) = P::into_raw(pointer); + let raw_ptr = raw.as_ptr(); + proof { + assert(!raw_ptr.is_null()); } - - let ptr = AtomicPtr::new( - Ghost((Ghost(ghost_nullable), PhantomData)), + let ptr = RcuAtomicPtr::

::new( + Ghost(nullable), + Ghost(rcu_spec::rcu_scheduler()), + raw_ptr, + Tracked(Some(perm)), + ); + Self { ptr, - Tracked(ptr_ghost), + ghost_nullable: Ghost(nullable), + _marker: PhantomData::<*const

::Target>, + } + } + + #[inline(always)] + fn load_ptr_acquire(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( + *mut

::Target, + Tracked::Target>>>, + )) + requires + self.type_inv(), + ensures + old(tv)@.spec_le(final(tv)@), + !self.is_nullable() ==> !res.0.is_null(), + match res.1@ { + None => res.0.is_null(), + Some(info) => { + &&& !res.0.is_null() + &&& info.wf() + &&& equal(info.ptr(), res.0) + }, + }, + { + proof { + assert(self.ptr.constant().nullable == self.is_nullable()); + } + let res = self.ptr.load_acquire_rcu(Tracked(tv)); + proof { + if !self.is_nullable() { + assert(!self.ptr.constant().nullable); + assert(!res.0.is_null()); + } + } + (res.0, res.3) + } + + #[inline(always)] + fn load_ptr_acquire_guarded( + &self, + Ghost(reader): Ghost, + Tracked(cpu_reader): Tracked, + Tracked(binding): Tracked, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut

::Target, + Tracked::Target>>>, + Tracked::Target>>, + Tracked::Permission>>>, + )) + requires + self.type_inv(), + cpu_reader.wf(), + online_cpus().contains(cpu_reader.cpu()), + reader.cpu == cpu_reader.cpu(), + reader.generation == cpu_reader.generation(), + binding.registry() == reader.scheduler, + reader.scheduler == self.ptr.constant().scheduler, + binding.cpu() == cpu_reader.cpu(), + binding.locals_key().len() == 1, + binding.single_local_id() == cpu_reader.participant_id(), + cpu_reader.participant_view().spec_le(old(tv)@), + ensures + old(tv)@.spec_le(final(tv)@), + !self.is_nullable() ==> !res.0.is_null(), + res.2@.wf(), + res.2@.participant_id() == cpu_reader.participant_id(), + res.2@.cpu() == cpu_reader.cpu(), + res.2@.generation() == cpu_reader.generation(), + res.2@.participant_view() == cpu_reader.participant_view(), + res.2@.scheduler() == binding.registry(), + res.2@.domain() == self.ptr.constant().domain, + res.2@.reader_registry() == self.ptr.constant().reader_registry, + res.2@.retire_observation_registry() == self.ptr.constant().retire_observation_registry, + res.2@.root() == self.ptr.id(), + res.2@.reader_context() == reader, + match (res.1@, res.3@) { + (None, None) => { + &&& res.0.is_null() + &&& res.2@.reader_fragment() == cpu_reader + }, + (Some(info), Some(lease)) => { + &&& !res.0.is_null() + &&& info.wf() + &&& info.domain() == res.2@.domain() + &&& equal(info.ptr(), res.0) + &&& !res.2@.expired().contains(info.obj()) + &&& !res.2@.seen_removed().removed.contains(info.obj()) + &&& res.2@.protects(info.addr(), info.obj()) + &&& res.2@.reader_fragment().fraction() == cpu_reader.fraction() / 2real + &&& lease.key() == info.obj() + &&& lease.active_registry() == self.ptr.constant().active_lease_registry + &&& lease.participant_id() == res.2@.participant_id() + &&& lease.reader_fraction() == res.2@.reader_fragment().fraction() + &&& lease.domain() == res.2@.domain() + &&& lease.root() == res.2@.root() + &&& lease.reader_context() == res.2@.reader_context() + &&& lease.start_view() == res.2@.start_view() + &&& lease.protected_addr() == info.addr() + &&& RcuPointerOwnership::

::owns(res.0, lease.resource()) + }, + _ => false, + }, + { + proof { + assert(self.ptr.constant().nullable == self.is_nullable()); + } + let res = self.ptr.load_acquire_rcu_guarded_cpu( + Ghost(reader), + Tracked(cpu_reader), + Tracked(binding), + Tracked(tv), ); - Self { ptr, _marker: PhantomData, ghost_nullable: Ghost(ghost_nullable) } + proof { + if !self.is_nullable() { + assert(!self.ptr.constant().nullable); + assert(!res.0.is_null()); + } + } + (res.0, res.3, res.4, res.5) } - #[verus_spec( + #[inline(always)] + fn swap_ptr_release( + &self, + new_ptr: *mut

::Target, + Tracked(ownership): Tracked::Permission>>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut

::Target, + Tracked::Target>>>, + )) + requires + self.type_inv(), + self.is_nullable() || !new_ptr.is_null(), + match ownership { + Some(ownership) => { + &&& !new_ptr.is_null() + &&& P::ptr_perm_match(new_ptr, ownership) + &&& ownership.inv() + }, + None => new_ptr.is_null(), + }, + ensures + old(tv)@.spec_le(final(tv)@), + (res.1@ is Some) == !res.0.is_null(), + res.1@ is Some ==> res.1@->Some_0.object().wf(), + res.1@ is Some ==> res.1@->Some_0.object().domain() == self.ptr.constant().domain, + res.1@ is Some ==> equal(res.1@->Some_0.object().ptr(), res.0), + res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), + res.1@ is Some ==> res.1@->Some_0.retired().removal().observed_by(final(tv)@), + res.1@ is Some ==> res.1@->Some_0.claim().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> res.1@->Some_0.claim().registry() + == self.ptr.constant().reclaim_registry, + res.1@ is Some ==> res.1@->Some_0.retired().removal().root + == self.ptr.constant().domain, + res.1@ is Some ==> res.1@->Some_0.retired().retire_observation_registry() + == self.ptr.constant().retire_observation_registry, + { + proof { + assert(self.ptr.constant().nullable == self.is_nullable()); + assert(self.ptr.constant().nullable || !new_ptr.is_null()); + } + let res = self.ptr.swap_release_rcu(new_ptr, Tracked(ownership), Tracked(tv)); + proof { + if res.1@ is Some { + assert(res.1@->Some_0.object().domain() == self.ptr.constant().domain); + assert(res.1@->Some_0.retired().removal().root == self.ptr.constant().domain); + } + } + res + } + + fn update(&self, new_ptr: Option

, Tracked(session): Tracked<&mut RunningTaskContext>) requires + self.type_inv(), self.is_nullable() || new_ptr is Some, - )] - fn update(&self, new_ptr: Option

) { - let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { - //

::into_raw(new_ptr).as_ptr() - let (ptr, Tracked(perm)) =

::into_raw(new_ptr); - let ptr = ptr.as_ptr(); - (ptr, Tracked(Some(perm))) + old(session).wf(), + old(session).scheduler() == self.ptr.constant().scheduler, + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + { + proof_decl! { + let ghost new_ptr_is_some = new_ptr is Some; + } + let (raw, Tracked(perm)) = if let Some(new_ptr) = new_ptr { + let (ptr, Tracked(perm)) = P::into_raw(new_ptr); + (ptr.as_ptr(), Tracked(Some(perm))) } else { (core::ptr::null_mut(), Tracked(None)) }; - proof_decl! { - let tracked mut old_perm: Option> = None; - } proof { - use_type_invariant(self); + if !self.is_nullable() { + assert(new_ptr_is_some); + } + assert(self.is_nullable() || !raw.is_null()); } - let old_raw_ptr = - atomic_with_ghost! { - self.ptr => swap(new_ptr); - update prev -> next; - ghost g => { - old_perm = g.current; - if old_perm is Some { - let tracked mut pool = old_perm.tracked_unwrap(); - let ghost id = pool.id(); - if g.retired.contains_key(id) { - // Use tracked_borrow_mut instead - let tracked entry = g.retired.tracked_remove(id); - let tracked mut retired_pool = entry.1; - let tracked pool_token = pool.split(pool.frac()); - retired_pool.validate_with_frac(&pool_token); - retired_pool.combine(pool_token); - g.retired.tracked_insert(id, (entry.0, retired_pool)); - } else { - g.retired.tracked_insert(id, (Ghost(prev), pool)); - } - } - g.current = match new_perm { - Some(perm) => Some(CountResource::alloc(perm)), - None => None, - }; - assert(retired_pools_inv::

(g.retired)); + let (old_raw, Tracked(detached)) = { + proof_decl! { + let tracked tv = session.tracked_borrow_irc11_view_mut(); } + self.swap_ptr_release(raw, Tracked(perm), Tracked(tv)) }; - - if let Some(p) = NonNull::new(old_raw_ptr) { - // SAFETY: - // 1. The pointer was previously returned by `into_raw`. - // 2. The pointer is removed from the RCU slot so that no one will - // use it after the end of the current grace period. The removal - // is done atomically, so it will only be dropped once. - // unsafe { delay_drop::

(p) }; + if !old_raw.is_null() { + proof_decl! { + let tracked detached = detached.tracked_unwrap(); + let tracked root_inv = self.ptr.tracked_atomic_inv(); + } + let (callback, cert) = callback_from_detached::

( + old_raw, + Tracked(detached), + Tracked(root_inv), + ); + if let Some(monitor) = RCU_MONITOR.get() { + #[verus_spec(with Tracked(session))] + monitor.after_grace_period(callback, cert); + } } } - #[verus_spec(obj_ptr => - with - -> tracked_ref_perm: Tracked>>, + fn read<'a>(&'a self, Tracked(session): Tracked<&'a mut RunningTaskContext>) -> (res: + RcuReadGuardInner<'a, P>) + requires + self.type_inv(), + old(session).wf(), + old(session).scheduler() == self.ptr.constant().scheduler, + old(session).available_fractions() > 1, ensures - !self.is_nullable() ==> tracked_ref_perm@ is Some, - match tracked_ref_perm@ { - Some(perm) => { - &&& !obj_ptr.is_null() - &&& P::ptr_perm_match(obj_ptr, perm.resource()) - &&& perm.resource().inv() - &&& perm.frac() == 1 - }, - None => obj_ptr.is_null(), - }, - )] - fn load_read_token(&self) -> *mut

::Target { + res.type_inv(), + res.rcu.is_nullable() == self.is_nullable(), + res.proof_active, + { + let ghost context_before_disable = *session; + let inner_guard = disable_preempt_in_context(Tracked(session)); + proof { + assert(session.wf()); + session.lemma_rcu_participant_view_le(); + } + let ghost context_before_reader = *session; proof_decl! { - let tracked mut tracked_ref_perm: Option> = None; + let tracked cpu_reader = session.tracked_start_rcu_reader(); + let tracked rcu_binding = session.tracked_rcu_binding(); } proof { - use_type_invariant(self); + inner_guard.lemma_matches_context_preserved(context_before_reader, session); + session.lemma_cpu_online(); + assert(session.rcu_participant_id() == context_before_disable.rcu_participant_id()); + assert(session.rcu_generation() == context_before_disable.rcu_generation()); + assert(session.rcu_participant_view() == context_before_disable.rcu_participant_view()); + assert(context_before_reader.wf()); + assert(context_before_reader.rcu_participant_view().spec_le( + context_before_reader.irc11_view(), + )); + assert(cpu_reader.participant_view() == context_before_reader.rcu_participant_view()); + assert(session.view() == context_before_reader.view()); + assert(cpu_reader.participant_view().spec_le(session.irc11_view())); } - let obj_ptr = - atomic_with_ghost! { - self.ptr => load(); - update prev -> _next; - returning loaded; - ghost g => { - if g.current is Some { - let tracked mut perm = g.current.tracked_unwrap(); - assert(loaded == prev); - assert(!loaded.is_null()); - assert(P::ptr_perm_match(loaded, perm@)); - assert(perm@.inv()); - let ghost perm_snapshot = perm@; - - // Verification-only admission for the bounded read-token pool. - // This is not a runtime reader limit; it only reflects that - // `CountResource` uses a Rust const-generic `u64` budget rather - // than an unbounded mathematical `nat`. - assume(perm.not_empty()); - assume(1 < perm.frac()); - let tracked token = perm.split_one(); - assert(perm@ == perm_snapshot); - assert(token.frac() == 1); - tracked_ref_perm = Some(token); - g.current = Some(perm); - } else { - } - assert(retired_pools_inv::

(g.retired)); - } + let ghost reader = rcu_spec::RcuReaderContext { + scheduler: session.scheduler(), + task: session.task(), + session: session.session_id(), + cpu: session.cpu(), + generation: session.rcu_generation(), }; - proof_with! { |= Tracked(tracked_ref_perm) } - obj_ptr - } - - #[verus_spec(r => - ensures - r.type_inv(), - r.rcu.is_nullable() == self.is_nullable(), - !self.is_nullable() ==> r.tracked_ref_perm@ is Some, - )] - fn read(&self) -> RcuReadGuardInner<'_, P> { - let guard = disable_preempt(); + let ghost context_before_load = *session; proof_decl! { - let tracked mut tracked_ref_perm: Option> = None; + let tracked tv = DisabledPreemptGuard::tracked_borrow_irc11_view_mut_from_context( + session, + &inner_guard, + ); + } + let (obj_ptr, tracked_info, tracked_guard, tracked_lease) = self.load_ptr_acquire_guarded( + Ghost(reader), + Tracked(cpu_reader), + Tracked(rcu_binding), + Tracked(tv), + ); + proof { + assert(session.rcu_participant_id() == context_before_disable.rcu_participant_id()); + assert(session.rcu_generation() == context_before_disable.rcu_generation()); + assert(session.rcu_participant_view() == context_before_disable.rcu_participant_view()); + assert(tracked_guard@.participant_id() == cpu_reader.participant_id()); + assert(cpu_reader.participant_id() == context_before_reader.rcu_participant_id()); + assert(session.rcu_participant_id() == context_before_reader.rcu_participant_id()); + assert(session.wf()); + inner_guard.lemma_matches_context_preserved(context_before_load, session); + assert(inner_guard.matches_context(*session)); + assert(inner_guard.has_resource()); + assert(tracked_guard@.wf()); + assert(tracked_guard@.domain() == self.ptr.constant().domain); + assert(tracked_guard@.root() == self.ptr.id()); + assert(tracked_guard@.reader_registry() == self.ptr.constant().reader_registry); + assert(tracked_guard@.retire_observation_registry() + == self.ptr.constant().retire_observation_registry); + assert(tracked_guard@.cpu() == session.cpu()); + assert(tracked_guard@.generation() == session.rcu_generation()); + assert(cpu_reader.fraction() == context_before_reader.rcu_fraction() / 2real); + assert(context_before_load.rcu_fraction() == context_before_reader.rcu_fraction() + / 2real); + assert(session.rcu_fraction() == context_before_load.rcu_fraction()); + assert(tracked_guard@.reader_context() == (rcu_spec::RcuReaderContext { + scheduler: session.scheduler(), + task: session.task(), + session: session.session_id(), + cpu: session.cpu(), + generation: session.rcu_generation(), + })); + match (tracked_info@, tracked_lease@) { + (None, None) => { + assert(obj_ptr.is_null()); + assert(tracked_guard@.reader_fragment() == cpu_reader); + assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction()); + }, + (Some(info), Some(lease)) => { + assert(!obj_ptr.is_null()); + assert(info.wf()); + assert(info.domain() == tracked_guard@.domain()); + assert(equal(info.ptr(), obj_ptr)); + assert(!tracked_guard@.expired().contains(info.obj())); + assert(!tracked_guard@.seen_removed().removed.contains(info.obj())); + assert(tracked_guard@.protects(info.addr(), info.obj())); + assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction() + / 2real); + assert(lease.key() == info.obj()); + assert(lease.participant_id() == tracked_guard@.participant_id()); + assert(lease.reader_fraction() == tracked_guard@.reader_fragment().fraction()); + assert(lease.domain() == tracked_guard@.domain()); + assert(lease.root() == tracked_guard@.root()); + assert(lease.reader_context() == tracked_guard@.reader_context()); + assert(lease.start_view() == tracked_guard@.start_view()); + assert(lease.protected_addr() == info.addr()); + assert(RcuPointerOwnership::

::owns(obj_ptr, lease.resource())); + }, + _ => assert(false), + } } - let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] - self.load_read_token(); - RcuReadGuardInner { + let res = RcuReadGuardInner { obj_ptr, rcu: self, - _inner_guard: guard, - tracked_ref_perm: Tracked(tracked_ref_perm), + proof_active: true, + _inner_guard: inner_guard, + tracked_info, + tracked_guard: Tracked(Some(tracked_guard.get())), + tracked_lease, + tracked_session: Tracked(Some(session)), + }; + proof { + let ghost stored_context = *res.tracked_session@->Some_0; + assert(res._inner_guard.matches_context(stored_context)); + assert(res.guard_token().participant_id() == stored_context.rcu_participant_id()); + assert(res.guard_token().cpu() == stored_context.cpu()); + assert(res.guard_token().generation() == stored_context.rcu_generation()); + match res.tracked_info@ { + None => assert(res.guard_token().reader_fragment().fraction() + == stored_context.rcu_fraction()), + Some(_) => assert(res.guard_token().reader_fragment().fraction() * 2real + == stored_context.rcu_fraction()), + } + assert(res.guard_token().reader_context() == reader); + assert(res.matches_context(stored_context)); } + res + } +} + +/// Detaches the proof-only reader state while leaving the executable +/// preemption guard in place. +/// +/// The surrounding guard enters a private transitional state that still owns +/// the preemption resource. Normal completion returns the updated session +/// before the executable guard can be observed again. +fn take_reader_state<'a, T, O>( + proof_active: &mut bool, + Tracked(guard_slot): Tracked<&mut Tracked>>>, + Tracked(lease_slot): Tracked<&mut Tracked>>>, + Tracked(session_slot): Tracked<&mut Tracked>>, +) -> (res: Tracked< + ( + rcu_cpu_spec::CpuRcuReadGuardToken, + Option>, + &'a mut RunningTaskContext, + ), +>) + requires + *old(proof_active), + old(guard_slot)@ is Some, + old(session_slot)@ is Some, + ensures + !*final(proof_active), + final(guard_slot)@ is None, + final(lease_slot)@ is None, + final(session_slot)@ is None, + res@.0 == old(guard_slot)@->Some_0, + res@.1 == old(lease_slot)@, + equal(*res@.2, *old(session_slot)@->Some_0), + opens_invariants none + no_unwind +{ + proof_decl! { + let tracked guard = guard_slot.borrow_mut().tracked_take(); + let tracked mut lease = None; + vstd::modes::tracked_swap(lease_slot.borrow_mut(), &mut lease); + let tracked session = session_slot.borrow_mut().tracked_take(); + } + * proof_active = false; + Tracked((guard, lease, session)) +} + +/// Completes `Guard -> Inactive` and returns both reader fractions. +fn finish_reader_state<'a, P: NonNullPtr>( + rcu: &RcuInner

, + inner_guard: &mut DisabledPreemptGuard, + Tracked(guard): Tracked::Target>>, + Tracked(lease): Tracked::Permission>>>, + Tracked(session): Tracked<&'a mut RunningTaskContext>, +) -> (res: Tracked<&'a mut RunningTaskContext>) + requires + rcu.type_inv(), + old(session).wf(), + old(inner_guard).matches_context(*old(session)), + guard.wf(), + guard.domain() == rcu.ptr.constant().domain, + guard.root() == rcu.ptr.id(), + guard.retire_observation_registry() == rcu.ptr.constant().retire_observation_registry, + guard.participant_id() == old(session).rcu_participant_id(), + match lease { + None => guard.reader_fragment().fraction() == old(session).rcu_fraction(), + Some(lease) => { + &&& lease.active_registry() == rcu.ptr.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + &&& guard.reader_fragment().fraction() * 2real == old(session).rcu_fraction() + }, + }, + ensures + !final(inner_guard).has_resource(), + (*res@).wf(), + (*res@).task() == old(session).task(), + (*res@).scheduler() == old(session).scheduler(), + (*res@).cpu() == old(session).cpu(), + old(session).view().spec_le((*res@).view()), + (*res@).session_id() == old(session).session_id(), + (*res@).quiescent_generation() == old(session).quiescent_generation(), + (*res@).available_fractions() == old(session).available_fractions() + 1, + (*res@).preempt_depth() + 1 == old(session).preempt_depth(), + (*res@).rcu_participant_id() == old(session).rcu_participant_id(), + (*res@).rcu_generation() == old(session).rcu_generation(), + (*res@).rcu_participant_view() == old(session).rcu_participant_view(), + (*res@).rcu_fraction() == old(session).rcu_fraction() * 2real, + no_unwind +{ + let ghost context_at_entry = *session; + proof_decl! { + let tracked tv = DisabledPreemptGuard::tracked_borrow_irc11_view_mut_from_context( + session, + inner_guard, + ); + } + let Tracked(guard) = rcu.ptr.return_cpu_rcu_read_lease( + Tracked(lease), + Tracked(guard), + Tracked(tv), + ); + proof { + assert(guard.reader_fragment().fraction() == session.rcu_fraction()); + assert(context_at_entry.view().spec_le(session.view())); + } + let ghost context_before_stop = *session; + let Tracked(cpu_reader) = rcu.ptr.stop_cpu_rcu_reader(Tracked(guard)); + proof { + inner_guard.lemma_matches_context_depth(session); + session.tracked_stop_rcu_reader(cpu_reader); + assert(session.view() == context_before_stop.view()); + inner_guard.lemma_matches_context_preserved(context_before_stop, session); + inner_guard.lemma_matches_context_depth(session); + } + let ghost context_before_release = *session; + inner_guard.release_in_place_to_context(Tracked(session)); + proof { + assert(context_before_release.view() == context_before_stop.view()); + assert(session.view() == context_before_release.view()); + assert(context_at_entry.view().spec_le(session.view())); } + Tracked(session) +} + +fn restore_reader_session<'a>( + Tracked(session_slot): Tracked<&mut Tracked>>, + Tracked(session): Tracked<&'a mut RunningTaskContext>, + Ghost(restored): Ghost, +) + requires + old(session_slot)@ is None, + old(session).wf(), + *old(session) == restored, + ensures + final(session_slot)@ is Some, + (*final(session_slot)@->Some_0).wf(), + *final(session_slot)@->Some_0 == restored, + opens_invariants none + no_unwind +{ + proof_decl! { + *session_slot.borrow_mut() = Some(session); + } +} - #[verus_spec] +impl RcuInner

{ + #[inline] pub fn read_with<'a, A: InAtomicMode>( &'a self, - _guard: &'a A, // &'a dyn InAtomicMode is not well-supported in Verus. - ) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + _guard: &'a A, + Tracked(session): Tracked<&mut RunningTaskContext>, + ) -> Option<

>::Ref> where P: NonNullPtrRef<'a> + requires + self.type_inv(), + old(session).wf(), + ensures + final(session).wf(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + { proof_decl! { - let tracked mut tracked_ref_perm: Option> = None; + let tracked tv = session.tracked_borrow_irc11_view_mut(); } - let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] - self.load_read_token(); + let (obj_ptr, _tracked_info) = self.load_ptr_acquire(Tracked(tv)); if obj_ptr.is_null() { return None; } - proof_decl! { - // `read_with` returns only the reference and has no guard object to - // store the read token. For this temporary skeleton, leak the - // verification-only token so the returned ref can borrow it for - // `'a`. The final RCU proof should attach this token to the - // atomic-mode/CPU epoch state instead. - let tracked tracked_ref_perm = tracked_ref_perm.tracked_unwrap(); - let tracked tracked_ref_perm = tracked_static_ref(tracked_ref_perm); - let tracked tracked_ref_perm:

>::RefPermission = - P::borrow_perm_as_ref_perm(tracked_ref_perm.tracked_borrow()); - } // SAFETY: // 1. This pointer is not NULL. // 2. The `_guard` guarantees atomic mode for the duration of lifetime // `'a`, the pointer is valid because other writers won't release the // allocation until this task passes the quiescent state. - NonNull::new(obj_ptr).map( - |ptr| - requires - P::ptr_perm_match( - ptr.view_ptr_mut(), - P::ref_perm_view_permission(tracked_ref_perm), - ), - { - unsafe { P::raw_as_ref(ptr, Tracked(tracked_ref_perm)) } - }, - ) - } -} -/* -impl Drop for RcuInner

{ - fn drop(&mut self) { - let ptr = self.ptr.load(Acquire); - if let Some(p) = NonNull::new(ptr) { - // SAFETY: It was previously returned by `into_raw` when creating - // the RCU primitive. - let pointer = unsafe {

::from_raw(p) }; - // It is OK not to delay the drop because the RCU primitive is - // owned by nobody else. - drop(pointer); - } + NonNull::new(obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }) } } -*/ - -/// The inner implementation of both [`RcuReadGuard`] and [`RcuOptionReadGuard`]. -struct RcuReadGuardInner<'a, P: NonNullPtr> { - obj_ptr: *mut

::Target, - rcu: &'a RcuInner

, - _inner_guard: DisabledPreemptGuard, - tracked_ref_perm: Tracked>>, -} #[verus_verify] impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { #[inline] - #[verus_spec(r => + #[verus_spec(res => + requires + self.proof_active, ensures - self.tracked_ref_perm@ is Some ==> r is Some, + !self.rcu.is_nullable() ==> res is Some, )] fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> { proof { use_type_invariant(self); + reveal(RcuReadGuardInner::type_inv); + if !self.obj_ptr.is_null() { + match self.tracked_info@ { + None => assert(false), + Some(info) => { + assert(self.tracked_lease@ is Some); + assert(RcuPointerOwnership::

::owns( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + )); + assert(P::ptr_perm_match( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + )); + assert(self.tracked_lease@->Some_0.resource().inv()); + }, + } + } } - - // SAFETY: The guard ensures that `P` will not be dropped. Thus, `P` - // outlives the lifetime of `&self`. Additionally, during this period, - // it is impossible to create a mutable reference to `P`. - NonNull::new(self.obj_ptr).map( + let res = NonNull::new(self.obj_ptr).map( |ptr| requires - self.tracked_ref_perm@ is Some, - P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_ref_perm->0.resource()), + self.tracked_lease@ is Some, + P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_lease@->Some_0.resource()), { - unsafe { - P::raw_as_ref( - ptr, - Tracked( - P::borrow_perm_as_ref_perm( - self.tracked_ref_perm.tracked_borrow().tracked_borrow(), - ), - ), - ) + proof_decl! { + let tracked lease = self.tracked_lease.tracked_borrow(); + let tracked ref_perm = P::borrow_perm_as_ref_perm(lease.borrow()); } + unsafe { P::raw_as_ref(ptr, Tracked(ref_perm)) } }, - ) + ); + proof { + if !self.rcu.is_nullable() { + assert(!self.obj_ptr.is_null()); + assert(res is Some); + } + } + res } - #[verus_spec(r => + fn compare_exchange(self, new_ptr: Option

) -> (res: Result<(), Option

>) requires self.rcu.is_nullable() || new_ptr is Some, + self.type_inv(), + self.proof_active, ensures - new_ptr is Some && r is Err ==> r->Err_0 is Some, - )] - fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { - let obj_ptr = self.obj_ptr; + new_ptr is Some && res is Err ==> res->Err_0 is Some, + { + let mut this = self; proof { - use_type_invariant(&self); - use_type_invariant(self.rcu); + use_type_invariant(&this); + } + let expected = this.obj_ptr; + let rcu = this.rcu; + let tracked_state = take_reader_state::< +

::Target, +

::Permission, + >( + &mut this.proof_active, + Tracked(&mut this.tracked_guard), + Tracked(&mut this.tracked_lease), + Tracked(&mut this.tracked_session), + ); + proof_decl! { + let tracked (guard, lease, session) = tracked_state.get(); } + let ghost context_at_entry = *session; proof_decl! { - let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); let ghost new_ptr_is_some = new_ptr is Some; - let tracked mut old_perm: Option> = None; - let tracked mut err_new_perm: Option::Permission>> = None; } - let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { - //

::into_raw(new_ptr).as_ptr() - let (ptr, Tracked(perm)) =

::into_raw(new_ptr); + + let (new_raw, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { + let (ptr, Tracked(perm)) = P::into_raw(new_ptr); (ptr.as_ptr(), Tracked(Some(perm))) } else { (core::ptr::null_mut(), Tracked(None)) }; - let res = - atomic_with_ghost! { - self.rcu.ptr => compare_exchange(obj_ptr, new_ptr); - update _prev -> next; - returning res; - ghost g => { - if res is Ok { - old_perm = g.current; - if old_perm is Some { - let tracked mut pool = old_perm.tracked_unwrap(); - let ghost id = pool.id(); - if g.retired.contains_key(id) { - // use tracked_borrow_mut instead - let tracked entry = g.retired.tracked_remove(id); - let tracked mut retired_pool = entry.1; - let tracked pool_token = pool.split(pool.frac()); - retired_pool.validate_with_frac(&pool_token); - retired_pool.combine(pool_token); - g.retired.tracked_insert(id, (entry.0, retired_pool)); - } else { - g.retired.tracked_insert(id, (Ghost(_prev), pool)); - } - } - g.current = match new_perm { - Some(perm) => Some(CountResource::alloc(perm)), - None => None, - }; - } else { - err_new_perm = Some(new_perm); - } - if tracked_ref_perm is Some { - let tracked token = tracked_ref_perm.tracked_unwrap(); - let ghost id = token.id(); - if g.retired.contains_key(id) { - let tracked entry = g.retired.tracked_remove(id); - let tracked mut pool = entry.1; - pool.combine(token); - g.retired.tracked_insert(id, (entry.0, pool)); - } else if g.current is Some { - let tracked mut pool = g.current.tracked_unwrap(); - if pool.id() == id { - pool.combine(token); - g.current = Some(pool); - } else { - g.current = Some(pool); - assume(false); - } - } else { - assume(false); - } - } - assert(retired_pools_inv::

(g.retired)); - } - }; - if res.is_ok() { - if let Some(p) = NonNull::new(obj_ptr) { - // SAFETY: - // 1. The pointer was previously returned by `into_raw`. - // 2. The pointer is removed from the RCU slot so that no one will - // use it after the end of the current grace period. The removal - // is done atomically, so it will only be dropped once. - // unsafe { delay_drop::

(p) }; - } - Ok(()) - } else { - let Some(new_nonnull) = NonNull::new(new_ptr) else { - return Err(None); - }; - proof_decl! { - let tracked new_perm = err_new_perm.tracked_unwrap().tracked_unwrap(); + proof { + if !rcu.is_nullable() { + assert(new_ptr_is_some); } - // SAFETY: - // 1. It was previously returned by `into_raw`. - // 2. The `compare_exchange` fails so the pointer will not - // be used by other threads via reading the RCU primitive. - Err(Some(unsafe {

::from_raw(new_nonnull, Tracked(new_perm)) })) + assert(rcu.is_nullable() || !new_raw.is_null()); + assert(rcu.ptr.constant().nullable == rcu.is_nullable()); + assert(rcu.ptr.constant().nullable || !new_raw.is_null()); } - } - /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. - #[inline] - #[verus_spec] - fn drop(self) { - let rcu = self.rcu; - let obj_ptr = self.obj_ptr; + let cas_res = { + proof_decl! { + let tracked tv = DisabledPreemptGuard::tracked_borrow_irc11_view_mut_from_context( + session, + &this._inner_guard, + ); + } + rcu.ptr.compare_exchange_acqrel_acquire_rcu( + expected, + new_raw, + Tracked(new_perm), + Tracked(tv), + ) + }; + let ghost context_before_enqueue = *session; proof { - use_type_invariant(&self); - use_type_invariant(rcu); + this._inner_guard.lemma_matches_context_preserved(context_at_entry, session); + assert(this._inner_guard.matches_context(context_before_enqueue)); } proof_decl! { - let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + let tracked (detached, rejected_new_perm) = cas_res.2.get(); } - atomic_with_ghost! { - rcu.ptr => load(); - update prev -> _next; - returning _loaded; - ghost g => { - if tracked_ref_perm is Some { - let tracked token = tracked_ref_perm.tracked_unwrap(); - let ghost id = token.id(); - if g.current is Some { - let tracked mut pool = g.current.tracked_unwrap(); - if prev == obj_ptr && pool.id() == id { - assert(!obj_ptr.is_null()); - assert(P::ptr_perm_match(obj_ptr, token.resource())); - assert(token.resource().inv()); - assert(token.frac() == 1); - assert(P::ptr_perm_match(prev, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - pool.combine(token); - assert(P::ptr_perm_match(prev, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - g.current = Some(pool); - } else { - g.current = Some(pool); - if g.retired.contains_key(id) { - let tracked entry = g.retired.tracked_remove(id); - let tracked mut pool = entry.1; - assert(pool.id() == id); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - pool.combine(token); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - g.retired.tracked_insert(id, (entry.0, pool)); - } else { - assume(false); - } - } - } else { - if g.retired.contains_key(id) { - let tracked entry = g.retired.tracked_remove(id); - let tracked mut pool = entry.1; - assert(pool.id() == id); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - pool.combine(token); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - g.retired.tracked_insert(id, (entry.0, pool)); - } else { - assume(false); - } + + let res = match cas_res.0 { + Result::Ok(old_raw) => { + if !old_raw.is_null() { + proof_decl! { + let tracked detached = detached.tracked_unwrap(); + let tracked root_inv = rcu.ptr.tracked_atomic_inv(); + } + let (callback, cert) = callback_from_detached::

( + old_raw, + Tracked(detached), + Tracked(root_inv), + ); + if let Some(monitor) = RCU_MONITOR.get() { + #[verus_spec(with Tracked(session))] + monitor.after_grace_period(callback, cert); } } - match &g.current { - Some(pool) => { - assert(!prev.is_null()); - assert(P::ptr_perm_match(prev, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - }, - None => { - assert(rcu.ghost_nullable@); - assert(prev.is_null()); - }, + Ok(()) + }, + Result::Err(_) => { + if let Some(new_nonnull) = NonNull::new(new_raw) { + proof_decl! { + let tracked perm = rejected_new_perm.tracked_unwrap(); + } + Err(Some(unsafe { P::from_raw(new_nonnull, Tracked(perm)) })) + } else { + Err(None) } - assert(retired_pools_inv::

(g.retired)); - } + }, }; + proof { + this._inner_guard.lemma_matches_context_preserved(context_before_enqueue, session); + } + let Tracked(session) = finish_reader_state( + rcu, + &mut this._inner_guard, + Tracked(guard), + Tracked(lease), + Tracked(session), + ); + let ghost restored = *session; + restore_reader_session( + Tracked(&mut this.tracked_session), + Tracked(session), + Ghost(restored), + ); + res + } +} + +impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + fn finish(self) + no_unwind + { + let mut this = self; + proof { + use_type_invariant(&this); + } + if this.proof_active { + let tracked_state = take_reader_state::< +

::Target, +

::Permission, + >( + &mut this.proof_active, + Tracked(&mut this.tracked_guard), + Tracked(&mut this.tracked_lease), + Tracked(&mut this.tracked_session), + ); + proof_decl! { + let tracked (guard, lease, session) = tracked_state.get(); + } + let Tracked(session) = finish_reader_state( + this.rcu, + &mut this._inner_guard, + Tracked(guard), + Tracked(lease), + Tracked(session), + ); + let ghost restored = *session; + restore_reader_session( + Tracked(&mut this.tracked_session), + Tracked(session), + Ghost(restored), + ); + } + } +} + +#[verifier::external_body] +unsafe fn assume_shared_ref<'a, P: NonNullPtrRef<'a>>(ptr: NonNull) -> P::Ref { + proof_decl! { + let tracked perm: P::RefPermission = Tracked::::assume_new().get(); } + unsafe { P::raw_as_ref(ptr, Tracked(perm)) } } #[verus_verify] impl Rcu

{ - /// Creates a new RCU primitive with the given pointer `pointer`. - #[verus_spec] + /// Creates a new RCU primitive with the given pointer. + #[inline] pub fn new(pointer: P) -> Self { Self( #[verus_spec(with Ghost(false))] @@ -738,54 +1973,51 @@ impl Rcu

{ ) } - /// Replaces the current pointer with a null pointer. - /// - /// This function updates the pointer to the new pointer regardless of the - /// original pointer. The original pointer will be dropped after the grace - /// period. - /// - /// Oftentimes this function is not recommended unless you have serialized - /// writes with locks. Otherwise, you can use [`Self::read`] and then - /// [`RcuReadGuard::compare_exchange`] to update the pointer. + /// Replaces the current pointer with `new_ptr` using a release swap. #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + ensures + final(session).wf(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub fn update(&self, new_ptr: P) { - self.0.update(Some(new_ptr)); + proof { + use_type_invariant(self); + } + self.0.update(Some(new_ptr), Tracked(session)); } - /// Retrieves a read guard for the RCU primitive. - /// - /// The guard allows read access to the data protected by RCU, as well - /// as the ability to do compare-and-exchange. + /// Starts a read-side critical section and acquires the current pointer. #[inline] - pub fn read(&self) -> RcuReadGuard<'_, P> { + #[verus_spec(res => + with + Tracked(session): Tracked<&'a mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + old(session).available_fractions() > 1, + )] + pub fn read<'a>(&'a self) -> RcuReadGuard<'a, P> { proof { use_type_invariant(self); } - RcuReadGuard(self.0.read()) - } - // #[inline] - // pub fn read_with<'a, G: AsAtomicModeGuard + ?Sized>(&'a self, guard: &'a G) -> P::Ref<'a> where - // P: NonNullPtrRef<'a>, - // { - // self.0.read_with(guard.as_atomic_mode_guard()).unwrap() - // } - -} - -impl RcuOption

{ - /// Creates a new RCU primitive that contains nothing. - /// - /// This is a constant equivalence to [`RcuOption::new(None)`]. - #[inline(always)] - pub const fn new_none() -> Self { - Self(RcuInner::new_none()) + RcuReadGuard(self.0.read(Tracked(session))) } } #[verus_verify] impl RcuOption

{ - /// Creates a new RCU primitive with the given pointer. - #[verus_spec] + /// Creates a nullable RCU primitive. + #[inline] pub fn new(pointer: Option

) -> Self { if let Some(pointer) = pointer { Self( @@ -797,71 +2029,125 @@ impl RcuOption

{ } } - /// Replaces the current pointer with a null pointer. - /// - /// This function updates the pointer to the new pointer regardless of the - /// original pointer. If the original pointer is not NULL, it will be - /// dropped after the grace period. - /// - /// Oftentimes this function is not recommended unless you have - /// synchronized writes with locks. Otherwise, you can use [`Self::read`] - /// and then [`RcuOptionReadGuard::compare_exchange`] to update the pointer. + /// Creates an empty nullable RCU primitive. + #[inline(always)] + pub const fn new_none() -> Self { + Self(RcuInner::new_none()) + } + + /// Replaces the current pointer using a release swap. #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + ensures + final(session).wf(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub fn update(&self, new_ptr: Option

) { proof { use_type_invariant(self); } - self.0.update(new_ptr); + self.0.update(new_ptr, Tracked(session)); } - /// Retrieves a read guard for the RCU primitive. - /// - /// The guard allows read access to the data protected by RCU, as well - /// as the ability to do compare-and-exchange. + /// Starts a read-side critical section and acquires the current pointer. + #[inline] + #[verus_spec(res => + with + Tracked(session): Tracked<&'a mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + old(session).available_fractions() > 1, + )] + pub fn read<'a>(&'a self) -> RcuOptionReadGuard<'a, P> { + proof { + use_type_invariant(self); + } + RcuOptionReadGuard(self.0.read(Tracked(session))) + } + + /// Acquires the current pointer while an external atomic-mode guard is live. /// - /// The contained pointer can be NULL and you can only get a reference - /// (if checked non-NULL) via [`RcuOptionReadGuard::get`]. + /// Unlike the legacy [`Self::read_with`] compatibility API, this method + /// returns an RCU read guard that retains the loaded allocation's physical + /// read lease. Call [`RcuOptionReadGuard::get`] to borrow the pointer and + /// consume [`RcuOptionReadGuard::drop`] before the external guard expires. + /// The returned guard owns a nested preemption-disable scope, so its lease + /// protocol does not rely on an unverified projection from `InAtomicMode`. + #[inline] + #[verus_spec(res => + with + Tracked(session): Tracked<&'a mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + old(session).available_fractions() > 1, + )] + pub fn read_with_guard<'a, A: InAtomicMode>(&'a self, _guard: &'a A) -> RcuOptionReadGuard< + 'a, + P, + > { + proof { + use_type_invariant(self); + } + RcuOptionReadGuard(self.0.read(Tracked(session))) + } + #[inline] - pub fn read(&self) -> RcuOptionReadGuard<'_, P> { + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + ensures + final(session).wf(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] + pub fn read_with<'a, A: InAtomicMode>(&'a self, guard: &'a A) -> Option< +

>::Ref, + > where P: NonNullPtrRef<'a> { proof { use_type_invariant(self); } - RcuOptionReadGuard(self.0.read()) + self.0.read_with(guard, Tracked(session)) } } #[verus_verify] impl RcuReadGuard<'_, P> { - /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. #[inline] pub fn drop(self) { - self.0.drop(); + self.0.finish(); } - /// Gets the reference of the protected data. #[inline] pub fn get<'a>(&'a self) ->

>::Ref where P: NonNullPtrRef<'a> { proof { use_type_invariant(self); } - self.0.get().unwrap() + let res = self.0.get(); + res.unwrap() } - /// Tries to replace the already read pointer with a new pointer. - /// - /// If another thread has updated the pointer after the read, this - /// function will fail, and returns the given pointer back. Otherwise, - /// it will replace the pointer with the new one and drop the old pointer - /// after the grace period. - /// - /// If spinning on [`Rcu::read`] and this function, it is recommended - /// to relax the CPU or yield the task on failure. Otherwise contention - /// will occur. - /// - /// This API does not help to avoid - /// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem). + /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { + proof { + use_type_invariant(&self); + } self.0.compare_exchange(Some(new_ptr)).map_err( |err| requires @@ -871,23 +2157,18 @@ impl RcuReadGuard<'_, P> { } } -/* -impl AsAtomicModeGuard for RcuReadGuard<'_, P> { - fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { - self.0.inner_guard.as_atomic_mode_guard() - } -}*/ - #[verus_verify] impl RcuOptionReadGuard<'_, P> { - /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. #[inline] pub fn drop(self) { - self.0.drop(); + self.0.finish(); } #[inline] pub fn get<'a>(&'a self) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + proof { + use_type_invariant(self); + } self.0.get() } @@ -896,6 +2177,7 @@ impl RcuOptionReadGuard<'_, P> { self.0.obj_ptr.is_null() } + /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] pub fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { proof { @@ -905,60 +2187,11 @@ impl RcuOptionReadGuard<'_, P> { } } -/* -impl AsAtomicModeGuard for RcuOptionReadGuard<'_, P> { - fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { - self.0.inner_guard.as_atomic_mode_guard() - } -} -*/ - -/* -/// Delays the dropping of a [`NonNullPtr`] after the RCU grace period. -/// -/// This is internally needed for implementing [`Rcu`] and [`RcuOption`] -/// because we cannot alias a [`Box`]. Restoring `P` and use [`RcuDrop`] for it -/// can lead to multiple [`Box`]es simultaneously pointing to the same -/// content. -/// -/// # Safety -/// -/// The pointer must be previously returned by `into_raw`, will not be used -/// after the end of the current grace period, and will only be dropped once. -/// -/// [`Box`]: alloc::boxed::Box -unsafe fn delay_drop(pointer: NonNull<

::Target>) { - struct ForceSend(NonNull<

::Target>); - // SAFETY: Sending a raw pointer to another task is safe as long as - // the pointer access in another task is safe (guaranteed by the trait - // bound `P: Send`). - unsafe impl Send for ForceSend

{} - - let pointer: ForceSend

= ForceSend(pointer); - - let rcu_monitor = RCU_MONITOR.get().unwrap(); - rcu_monitor.after_grace_period(move || { - // This is necessary to make the Rust compiler to move the entire - // `ForceSend` structure into the closure. - let pointer = pointer; - - // SAFETY: - // 1. The pointer was previously returned by `into_raw`. - // 2. The pointer won't be used anymore since the grace period has - // finished and this is the only time the pointer gets dropped. - let p = unsafe {

::from_raw(pointer.0) }; - drop(p); - }); -} */ - -/// A wrapper to delay calling destructor of `T` after the RCU grace period. -/// -/// Upon dropping this structure, a callback will be registered to the global -/// RCU monitor and the destructor of `T` will be delayed until the callback. +/// A wrapper whose destructor will eventually be delayed until after an RCU +/// grace period. /// -/// [`RcuDrop`] is guaranteed to have the same layout as `T`. You can also -/// access the inner value safely via [`RcuDrop`]. -#[verifier::allow(autoderive_clone_without_spec)] +/// The delayed-drop path is deliberately not restored in this first weak-memory +/// cut; `__mod.rs` contains the old callback-monitor reference. #[repr(transparent)] #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct RcuDrop { @@ -975,150 +2208,248 @@ impl View for RcuDrop { #[verus_verify] impl RcuDrop { - /// Creates a new [`RcuDrop`] that wraps the given value. #[inline] - #[verus_spec(r => + #[verus_spec(res => ensures - r@ == value, + res@ == value, )] pub fn new(value: T) -> Self { Self { value: ManuallyDrop::new(value) } - }/* - /// Extracts the value from the `RcuDrop` container. - /// - /// # Safety - /// - /// The caller must ensure that the returned value will be dropped after - /// all the threads cannot access it anymore. Specifically, dropping it - /// after the RCU grace period is guaranteed to be safe. - /// - /// Note that panic unwinding may cause the returned value to be dropped - /// immediately, which is not sound. Therefore, the caller must forget the - /// [`PanicGuard`] after it ensures that the value will be dropped at the - /// correct time. - pub(crate) unsafe fn into_inner(slot: RcuDrop) -> (T, PanicGuard) { - let mut slot = ManuallyDrop::new(slot); - let panic_guard = PanicGuard::new(); - // SAFETY: The `slot` will not be used after this point. - let val = unsafe { ManuallyDrop::take(&mut slot.value) }; - (val, panic_guard) - } - */ - + } } #[verus_verify] impl Deref for RcuDrop { type Target = T; - #[verus_spec(r => + #[inline] + #[verus_spec(res => ensures - *r == self@, + *res == self@, )] - #[inline] fn deref(&self) -> &Self::Target { &self.value } } -/* -impl Drop for RcuDrop { - fn drop(&mut self) { - // SAFETY: The `ManuallyDrop` will not be used after this point. - let taken = unsafe { ManuallyDrop::take(&mut self.value) }; - let rcu_monitor = RCU_MONITOR.get().unwrap(); - rcu_monitor.after_grace_period(|| { - drop(taken); - }); - } -} - -/// Finishes the current grace period. -/// -/// This function is called when the current grace period on current CPU is -/// finished. If this CPU is the last CPU to finish the current grace period, -/// it takes all the current callbacks and invokes them. -/// -/// # Safety +/// Finishes a grace period on the current CPU. /// -/// The caller must ensure that this CPU is not executing in a RCU read-side -/// critical section. +#[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + old(session).is_quiescent(), + ensures + final(session).wf(), + final(session).is_quiescent(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).session_id() == old(session).session_id(), + old(session).quiescent_generation() <= final(session).quiescent_generation(), + final(session).quiescent_generation() <= old(session).quiescent_generation() + 1, + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), +)] pub unsafe fn finish_grace_period() { - let rcu_monitor = RCU_MONITOR.get().unwrap(); - // SAFETY: The caller ensures safety. - unsafe { - rcu_monitor.finish_grace_period(); + if let Some(monitor) = RCU_MONITOR.get() { + unsafe { + #[verus_spec(with Tracked(session))] + monitor.finish_grace_period(); + } } } -*/ - -exec static RCU_MONITOR: Once - ensures - RCU_MONITOR.wf(), - RCU_MONITOR.inv() == RcuMonitorPred, -{ - Once::new(Ghost(RcuMonitorPred)) -} - pub fn init() { - RCU_MONITOR.init(RcuMonitor::new_data()); + RCU_MONITOR.init(monitor::RcuMonitor::new_data()); } } // verus! +// Verus requires trait destructors to open no invariants. The verified API uses +// the consuming `Rcu(Read|OptionRead)Guard::drop` methods above; runtime builds +// retain ordinary Rust destruction, after which the embedded preemption guard +// performs the executable counter decrement. +#[cfg(not(verus_keep_ghost))] +impl<'a, P: NonNullPtr> Drop for RcuReadGuardInner<'a, P> { + fn drop(&mut self) {} +} + verus! { impl RcuInner

{ #[verifier::type_invariant] - closed spec fn type_inv(self) -> bool { + pub closed spec fn type_inv(self) -> bool { self.wf() } } -impl RcuOption

{ +impl Rcu

{ #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() - &&& self.0.is_nullable() + &&& !self.0.is_nullable() } } -impl Rcu

{ +impl RcuOption

{ #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() - &&& !self.0.is_nullable() + &&& self.0.is_nullable() } } impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { + /// Relates this guard to the task session that supplied its weak-memory + /// view. Consuming operations require the same session. + pub closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + self.0.matches_context(session) + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() &&& !self.0.rcu.is_nullable() - &&& self.0.tracked_ref_perm@ is Some + &&& self.0.is_active() } } impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { + /// Relates this guard to the task session that supplied its weak-memory + /// view. Consuming operations require the same session. + pub closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + self.0.matches_context(session) + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() &&& self.0.rcu.is_nullable() + &&& self.0.is_active() } } impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + pub closed spec fn is_active(self) -> bool { + self.proof_active + } + + pub closed spec fn has_stored_context(self) -> bool { + self.tracked_session@ is Some + } + + pub closed spec fn stored_context(self) -> RunningTaskContext + recommends + self.has_stored_context(), + { + *self.tracked_session@->Some_0 + } + + closed spec fn guard_token(self) -> rcu_cpu_spec::CpuRcuReadGuardToken< +

::Target, + > + recommends + self.tracked_guard@ is Some, + { + self.tracked_guard@->Some_0 + } + + closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + &&& self.proof_active + &&& self.tracked_guard@ is Some + &&& self._inner_guard.matches_context(session) + &&& self.guard_token().participant_id() == session.rcu_participant_id() + &&& self.guard_token().cpu() == session.cpu() + &&& self.guard_token().generation() == session.rcu_generation() + &&& self.guard_token().reader_context() == (rcu_spec::RcuReaderContext { + scheduler: session.scheduler(), + task: session.task(), + session: session.session_id(), + cpu: session.cpu(), + generation: session.rcu_generation(), + }) + } + + proof fn lemma_matches_context_preserved( + &self, + before: RunningTaskContext, + tracked after: &RunningTaskContext, + ) + requires + self.matches_context(before), + after.wf(), + after.task() == before.task(), + after.scheduler() == before.scheduler(), + after.cpu() == before.cpu(), + after.session_id() == before.session_id(), + after.quiescent_generation() == before.quiescent_generation(), + after.available_fractions() == before.available_fractions(), + after.preempt_depth() == before.preempt_depth(), + after.rcu_participant_id() == before.rcu_participant_id(), + after.rcu_generation() == before.rcu_generation(), + ensures + self.matches_context(*after), + { + self._inner_guard.lemma_matches_context_preserved(before, after); + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { - match self.tracked_ref_perm@ { - Some(perm) => { - &&& !self.obj_ptr.is_null() - &&& P::ptr_perm_match(self.obj_ptr, perm.resource()) - &&& perm.resource().inv() - &&& perm.frac() == 1 - }, - None => self.obj_ptr.is_null(), + &&& self.rcu.type_inv() + &&& !self.rcu.is_nullable() ==> !self.obj_ptr.is_null() + &&& self.proof_active == (self.tracked_guard@ is Some) + &&& self.proof_active ==> self.tracked_session@ is Some + &&& self.proof_active ==> ((self.tracked_info@ is Some) == (self.tracked_lease@ is Some)) + &&& self.tracked_session@ is Some ==> self.stored_context().wf() + &&& self.proof_active ==> { + &&& self._inner_guard.has_resource() + &&& self.stored_context().scheduler() == self.rcu.ptr.constant().scheduler + &&& self.guard_token().wf() + &&& self.guard_token().domain() == self.rcu.ptr.constant().domain + &&& self.guard_token().root() == self.rcu.ptr.id() + &&& self.guard_token().reader_registry() == self.rcu.ptr.constant().reader_registry + &&& self.guard_token().retire_observation_registry() + == self.rcu.ptr.constant().retire_observation_registry + &&& self.matches_context(self.stored_context()) + &&& match self.tracked_info@ { + None => { + &&& self.obj_ptr.is_null() + &&& self.tracked_lease@ is None + &&& self.guard_token().reader_fragment().fraction() + == self.stored_context().rcu_fraction() + }, + Some(info) => { + &&& self.tracked_lease@ is Some + &&& !self.obj_ptr.is_null() + &&& info.wf() + &&& info.domain() == self.guard_token().domain() + &&& equal(info.ptr(), self.obj_ptr) + &&& !self.guard_token().expired().contains(info.obj()) + &&& !self.guard_token().seen_removed().removed.contains(info.obj()) + &&& self.guard_token().protects(info.addr(), info.obj()) + &&& self.guard_token().reader_fragment().fraction() * 2real + == self.stored_context().rcu_fraction() + &&& self.tracked_lease@->Some_0.key() == info.obj() + &&& self.tracked_lease@->Some_0.active_registry() + == self.rcu.ptr.constant().active_lease_registry + &&& self.tracked_lease@->Some_0.participant_id() + == self.guard_token().participant_id() + &&& self.tracked_lease@->Some_0.reader_fraction() + == self.guard_token().reader_fragment().fraction() + &&& self.tracked_lease@->Some_0.domain() == self.guard_token().domain() + &&& self.tracked_lease@->Some_0.root() == self.guard_token().root() + &&& self.tracked_lease@->Some_0.reader_context() + == self.guard_token().reader_context() + &&& self.tracked_lease@->Some_0.start_view() == self.guard_token().start_view() + &&& self.tracked_lease@->Some_0.protected_addr() == info.addr() + &&& RcuPointerOwnership::

::owns( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + ) + }, + } } } } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index c44c97ab4..9d8cad538 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -1,286 +1,2328 @@ // SPDX-License-Identifier: MPL-2.0 -use vstd::{ - atomic_ghost::AtomicBool, atomic_with_ghost, predicate::Predicate as DataPredicate, prelude::*, -}; -use vstd_extra::ownership::Inv; +use alloc::collections::VecDeque; +use core::sync::atomic::Ordering; + +use vstd::{predicate::Predicate as DataPredicate, prelude::*, resource::Loc}; +use vstd_extra::raw_callback::RawCallbackWithProof; +use vstd_extra::rcu_read_pool::RcuTrackedReadPoolRegistry; -use crate::{ - specs::mm::cpu::{AtomicCpuSet, CpuSet}, - sync::{AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate}, +use crate::specs::{ + mm::cpu::{AtomicCpuSet, AtomicCpuSetToken, CpuId, CpuSet, online_cpus}, + sync::{ + rcu as rcu_spec, + rcu::{GracePeriodView, MonitorStateView}, + rcu_cpu as rcu_cpu_spec, + weak_memory::RcuMonitorWeakAtomicBool, + }, +}; +use crate::sync::{ + AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate, +}; +use crate::task::RunningTaskContext; +use vstd_extra::atomic_irc11::{ + AtomicHistory as Irc11History, ThreadView, ThreadViewOrder, ThreadViewToken, ViewSeen, }; verus! { -// This thing can be very tricky to deal with -// type Callbacks = VecDeque>; -pub(super) struct GracePeriod { - // callbacks: Callbacks, - cpu_mask: AtomicCpuSet, - is_complete: bool, +broadcast use vstd::thread_view::group_thread_view_axioms; + +pub type Callbacks = VecDeque; + +type MonitorAtomicBool = RcuMonitorWeakAtomicBool; + +/// Evidence captured at a call site where the current task is quiescent. +/// +/// This token deliberately does not claim that a complete RCU grace period has +/// elapsed. It records one CPU-local quiescent boundary and carries the +/// resource proving that the CPU's previous reader generation is closed. The +/// monitor must still collect one such resource from every online CPU. +tracked struct RcuQuiescentContext { + ghost cpu: CpuId, + ghost task: Loc, + ghost scheduler: Loc, + ghost session: Loc, + ghost participant: Loc, + ghost generation: nat, + ghost view: ThreadView, + closed: rcu_cpu_spec::CpuRcuClosedGeneration, } -pub(super) struct State { - current_gp: GracePeriod, - // next_callbacks: Callbacks, +impl RcuQuiescentContext { + closed spec fn wf(self) -> bool { + &&& self.closed.wf() + &&& self.closed.scheduler() == self.scheduler + &&& self.closed.participant_id() == self.participant + &&& self.closed.cpu() == self.cpu + &&& self.closed.closed_generation() == self.generation + &&& self.closed.view() == self.view + } + + proof fn tracked_from_running_context( + tracked context: &mut RunningTaskContext, + cpu: CpuId, + tracked retired_facts: &rcu_spec::RcuRetiredFacts, + ) -> (tracked res: Self) + requires + old(context).wf(), + old(context).is_quiescent(), + cpu == old(context).cpu(), + retired_facts.observed_by(old(context).irc11_view()), + ensures + res.cpu == cpu, + res.task == old(context).task(), + res.scheduler == old(context).scheduler(), + res.session == old(context).session_id(), + res.participant == old(context).rcu_participant_id(), + res.generation == old(context).rcu_generation(), + res.view == old(context).irc11_view(), + res.closed.wf(), + res.closed.scheduler() == res.scheduler, + res.closed.participant_id() == res.participant, + res.closed.cpu() == cpu, + res.closed.closed_generation() == res.generation, + res.closed.view() == res.view, + retired_facts.records().subset_of(res.closed.known_retired()), + res.wf(), + final(context).wf(), + final(context).is_quiescent(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation() + 1, + final(context).rcu_participant_id() == res.participant, + final(context).rcu_generation() == res.generation + 1, + final(context).rcu_participant_view() == res.view, + final(context).rcu_fraction() == 1real, + final(context).view() == old(context).view(), + final(context).irc11_view() == old(context).irc11_view(), + { + let ghost task = context.task(); + let ghost scheduler = context.scheduler(); + let ghost session = context.session_id(); + let ghost participant = context.rcu_participant_id(); + let ghost view = context.irc11_view(); + let ghost generation = context.rcu_generation(); + let tracked closed = context.tracked_report_rcu_quiescent_with(retired_facts); + let ghost _session_generation = context.tracked_record_quiescent(); + RcuQuiescentContext { cpu, task, scheduler, session, participant, generation, view, closed } + } } -/// Owner of this [`RcuMonitor`]. -pub(super) tracked struct RcuMonitorOwner {} +/// Historical record of one quiescent context bound to a monitor generation. +pub(super) ghost struct RcuCpuQuiescentReport { + cpu: CpuId, + task: Loc, + scheduler: Loc, + session: Loc, + /// Stable identity of the scheduler-owned CPU participant. + participant: Loc, + /// Last reader generation closed by this quiescent transition. + generation: nat, + view: ThreadView, + epoch: nat, +} -struct_with_invariants! { -/// A RCU monitor ensures the completion of _grace periods_ by keeping track -/// of each CPU's passing _quiescent states_. -pub(super) struct RcuMonitor { - pub(super) is_monitoring: AtomicBool<_, bool, _>, - pub(super) state: SpinLock, +impl RcuCpuQuiescentReport { + closed spec fn matches_closed(self, closed: rcu_cpu_spec::CpuRcuClosedGeneration) -> bool { + &&& closed.wf() + &&& closed.scheduler() == self.scheduler + &&& closed.participant_id() == self.participant + &&& closed.cpu() == self.cpu + &&& closed.closed_generation() == self.generation + &&& closed.view() == self.view + } } -closed spec fn wf(self) -> bool { - invariant on is_monitoring with (state) is (v: bool, g: bool) { - &&& v == g - &&& state.type_inv() +/// Copies persistent closed-generation facts for a finite CPU set without +/// removing the originals from the grace-period invariant. +proof fn duplicate_closed_generations( + tracked source: &Map, + keys: Set, +) -> (tracked duplicates: Map) + requires + keys.subset_of(source.dom()), + forall|cpu: CpuId| #[trigger] source.dom().contains(cpu) ==> source[cpu].wf(), + ensures + duplicates.dom() == keys, + forall|cpu: CpuId| #[trigger] + keys.contains(cpu) ==> { + &&& duplicates[cpu].wf() + &&& duplicates[cpu].participant_id() == source[cpu].participant_id() + &&& duplicates[cpu].cpu() == source[cpu].cpu() + &&& duplicates[cpu].closed_generation() == source[cpu].closed_generation() + &&& duplicates[cpu].view() == source[cpu].view() + &&& duplicates[cpu].known_retired() == source[cpu].known_retired() + &&& duplicates[cpu].scheduler() == source[cpu].scheduler() + }, + decreases keys.len(), +{ + if keys.is_empty() { + Map::tracked_empty() + } else { + let ghost cpu = keys.choose(); + let ghost rest = keys.remove(cpu); + let tracked mut duplicates = duplicate_closed_generations(source, rest); + let tracked closed = source.tracked_borrow(cpu); + let tracked duplicate = closed.tracked_duplicate_from_ref(); + duplicates.tracked_insert(cpu, duplicate); + assert(keys == rest.insert(cpu)); + assert(duplicates.dom() == keys); + assert forall|other: CpuId| #[trigger] keys.contains(other) implies { + &&& duplicates[other].wf() + &&& duplicates[other].participant_id() == source[other].participant_id() + &&& duplicates[other].cpu() == source[other].cpu() + &&& duplicates[other].closed_generation() == source[other].closed_generation() + &&& duplicates[other].view() == source[other].view() + &&& duplicates[other].known_retired() == source[other].known_retired() + &&& duplicates[other].scheduler() == source[other].scheduler() + } by { + if other == cpu { + } else { + assert(rest.contains(other)); + } + }; + duplicates } } + +/// RCU-specific wrapper around a type-erased executable callback. +/// +/// The executable callback is type-erased but records that invocation must +/// supply an [`RcuReclaimPermit`]. The summary records the object identity that +/// the monitor invariant uses to decide when the callback is safe to run. +#[must_use] +pub struct RcuCallback { + raw: RawCallbackWithProof, + summary: Ghost, + safety: Tracked, } -impl DataPredicate for RcuMonitorOwner { - closed spec fn predicate(&self, v: RcuMonitor) -> bool { - true +impl View for RcuCallback { + type V = rcu_spec::RcuCallbackSummary; + + closed spec fn view(&self) -> rcu_spec::RcuCallbackSummary { + self.summary@ } } -impl RcuMonitor { +impl RcuCallback { + /// Converts a raw callback into an RCU callback, given a proof that the callback is + /// safe to run after a grace period. + #[inline] + fn from_raw( + raw: RawCallbackWithProof, + Tracked(cert): Tracked, + Ghost(retire_epoch): Ghost, + Ghost(retire_view): Ghost, + Ghost(scheduler): Ghost, + ) -> (res: Self) + requires + cert.removal().observed_by(retire_view), + forall|permit: RcuReclaimPermit| + #![auto] + permit.wf() && permit.callback().domain == cert.domain() && permit.callback().obj + == cert.obj() && permit.callback().removal == cert.removal() + && permit.callback().retire_observation_registry + == cert.retire_observation_registry() && permit.callback().scheduler + == scheduler ==> raw.call_requires(permit), + ensures + res.wf(), + res@ == (rcu_spec::RcuCallbackSummary { + scheduler, + domain: cert.domain(), + obj: cert.obj(), + removal: cert.removal(), + retire_observation_registry: cert.retire_observation_registry(), + retire_epoch, + retire_view, + }), + { + let ghost summary = rcu_spec::RcuCallbackSummary { + scheduler, + domain: cert.domain(), + obj: cert.obj(), + removal: cert.removal(), + retire_observation_registry: cert.retire_observation_registry(), + retire_epoch, + retire_view, + }; + proof { + cert.lemma_matches(summary); + assert forall|permit: RcuReclaimPermit| + permit.authorizes(summary) implies raw.call_requires(permit) by { + assert(permit.callback() == summary); + assert(permit.wf()); + }; + } + Self { raw, summary: Ghost(summary), safety: Tracked(cert) } + } + + /// Runs the underlying callback once the monitor has completed the grace + /// period that contained this callback's retire summary. + /// + /// `RcuReclaimPermit` proves batch membership, weak-memory view coverage, + /// and carries a closed-generation resource for every online CPU. The + /// proof token crosses the executable type-erasure boundary and is handed + /// to the concrete callback context. Recovering the callback object's + /// physical permission still depends on the separate read-lease protocol. + #[inline] + unsafe fn call_once(self, Tracked(permit): Tracked) + requires + self.wf(), + permit.authorizes(self@), + { + proof { + use_type_invariant(&self); + use_type_invariant(&permit); + assert(self.raw.call_requires(permit)); + } + unsafe { + self.raw.call_once(Tracked(permit)); + } + } + + closed spec fn wf(self) -> bool { + &&& self.safety@.matches(self@) + &&& self@.removal.observed_by(self@.retire_view) + &&& forall|permit: RcuReclaimPermit| + permit.authorizes(self@) ==> self.raw.call_requires(permit) + } + + /// Duplicates the persistent base-retirement fact retained by this + /// type-erased callback. + proof fn tracked_retired_fact(tracked &self) -> (tracked fact: rcu_spec::RcuRetiredFact) + requires + self.wf(), + ensures + fact.wf(), + fact.record() == self@.retired_record(), + { + let tracked fact = self.safety.borrow().tracked_retired_fact(self@); + assert(fact.matches(self@)); + fact + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { self.wf() } } -pub(super) struct RcuMonitorPred; +/// Proof token produced by the monitor when a grace period finishes. +/// +/// The token is private to the monitor implementation. External code can +/// certify that a callback is safe to enqueue, but cannot manufacture the +/// completion fact needed to execute the callback. +tracked struct CompletedGracePeriod { + epoch: Ghost, + callbacks: Ghost>, + reported_cpus: Ghost>, + reports: Ghost>, + closed_generations: Map, +} -impl OncePredicate> for RcuMonitorPred { - closed spec fn inv(self, v: AtomicDataWithOwner) -> bool { - &&& v.permission@.predicate(v.data) - &&& v.data.inv() - } +/// Object-level authorization to execute one reclamation callback. +/// +/// Unlike [`CompletedGracePeriod`], this token is specific to one callback. +/// It combines the callback's traversal-retirement certificate with monitor +/// completion of the batch containing that callback. Keeping its constructor +/// private prevents executable callback code from treating batch membership +/// alone as proof that an arbitrary object was retired safely. +/// +/// `authorizes` includes one closed-generation resource for every reported CPU. +/// Those resources classify every coexisting executable guard as a later +/// reader whose start view observes the callback's removal. Physical +/// permission recovery remains the responsibility of the read-lease layer. +pub(super) tracked struct RcuReclaimPermit { + summary: Ghost, + retired: rcu_spec::RcuRetiredFact, + reports: Ghost>, + closed_generations: Map, } -impl Inv for RcuMonitor { - closed spec fn inv(self) -> bool { +impl RcuReclaimPermit { + pub closed spec fn callback(self) -> rcu_spec::RcuCallbackSummary { + self.summary@ + } + + pub closed spec fn reports(self) -> Map { + self.reports@ + } + + pub closed spec fn closed_generations(self) -> Map< + CpuId, + rcu_cpu_spec::CpuRcuClosedGeneration, + > { + self.closed_generations + } + + pub closed spec fn authorizes(self, callback: rcu_spec::RcuCallbackSummary) -> bool { + &&& self.summary@ == callback + &&& self.retired.wf() + &&& self.retired.matches(callback) + &&& self.retired.record() == callback.retired_record() + &&& self.reports@.dom() == online_cpus() + &&& self.closed_generations().dom() == self.reports@.dom() + &&& forall|cpu: CpuId| #[trigger] + self.reports@.contains_key(cpu) ==> { + &&& self.reports@[cpu].cpu == cpu + &&& self.reports@[cpu].scheduler == callback.scheduler + &&& self.reports@[cpu].epoch == callback.retire_epoch + &&& self.reports@[cpu].matches_closed(self.closed_generations()[cpu]) + &&& callback.retire_view.spec_le(self.reports@[cpu].view) + &&& callback.removal.observed_by(self.reports@[cpu].view) + &&& self.closed_generations()[cpu].known_retired().contains( + callback.retired_record(), + ) + } + } + + /// Exposes the callback-specific authorization carried by this permit. + pub(super) proof fn lemma_authorizes_callback(tracked &self) + ensures + self.authorizes(self.callback()), + self.reports().dom() == online_cpus(), + { + use_type_invariant(self); + } + + /// Classifies any still-live guard on a reported CPU as a later reader. + /// + /// The old-reader branch is excluded by resource validity: a guard from + /// the closed generation cannot coexist with the report token in this + /// permit. The surviving branch starts after the report and therefore + /// observes the callback's root-removal message. + proof fn tracked_later_guard( + tracked &self, + callback: rcu_spec::RcuCallbackSummary, + cpu: CpuId, + tracked guard: rcu_cpu_spec::CpuRcuReadGuardToken, + ) -> (tracked res: rcu_cpu_spec::CpuRcuReadGuardToken) + requires + self.authorizes(callback), + self.reports@.contains_key(cpu), + guard.wf(), + guard.cpu() == cpu, + guard.participant_id() == self.closed_generations()[cpu].participant_id(), + ensures + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.reader_fragment() == guard.reader_fragment(), + res.scheduler() == guard.scheduler(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.domain() == guard.domain(), + res.root() == guard.root(), + res.retire_observation_registry() == guard.retire_observation_registry(), + res.start_view() == guard.start_view(), + res.expired() == guard.expired(), + res.seen_removed() == guard.seen_removed(), + self.reports@[cpu].generation < res.generation(), + self.reports@[cpu].view.spec_le(res.start_view()), + callback.removal.observed_by(res.start_view()), + res.known_retired().contains(callback.retired_record()), + { + let tracked closed = self.closed_generations.tracked_borrow(cpu); + assert(self.reports@[cpu].matches_closed(*closed)); + assert(closed.known_retired().contains(callback.retired_record())); + let tracked guard = closed.lemma_later_guard(guard); + assert(closed.known_retired().subset_of(guard.known_retired())); + assert(guard.known_retired().contains(callback.retired_record())); + assert(callback.removal.observed_by(self.reports@[cpu].view)); + assert(self.reports@[cpu].view.spec_le(guard.start_view())); + self.reports@[cpu].view.lemma_spec_le_transitive(guard.start_view(), guard.start_view()); + guard + } + + /// End-to-end safety statement for one completed callback and one live + /// protected pointer on a reported CPU. + /// + /// A pre-existing reader cannot coexist with the closed-generation + /// resource. A coexisting later reader imports the callback's observed + /// retirement fact, so the reclaimed object is in its expired/removed set + /// and cannot simultaneously be protected. + proof fn tracked_excludes_protected_callback_object( + tracked &self, + callback: rcu_spec::RcuCallbackSummary, + cpu: CpuId, + tracked guard: rcu_cpu_spec::CpuRcuReadGuardToken, + tracked protected: rcu_spec::RcuProtectedPtr, + ) + requires + self.authorizes(callback), + self.reports@.contains_key(cpu), + guard.wf(), + guard.cpu() == cpu, + guard.participant_id() == self.closed_generations()[cpu].participant_id(), + callback.domain == guard.domain(), + callback.retire_observation_registry == guard.retire_observation_registry(), + callback.removal.root == guard.root(), + protected.obj() == callback.obj, + protected.protected_by(guard.paper_guard()), + ensures + false, + { + let ghost guard_domain = guard.domain(); + let ghost guard_root = guard.root(); + let ghost guard_retire_observation_registry = guard.retire_observation_registry(); + let ghost protected_guard = guard.paper_guard(); + assert(protected.protected_by(protected_guard)); + let tracked guard = self.tracked_later_guard(callback, cpu, guard); + assert(guard.paper_guard() == protected_guard); + assert(guard.domain() == guard_domain); + assert(guard.root() == guard_root); + assert(guard.retire_observation_registry() == guard_retire_observation_registry); + assert(guard.known_retired().contains(callback.retired_record())); + assert(callback.retired_record().domain == callback.domain); + assert(callback.retired_record().obj == callback.obj); + assert(callback.retired_record().removal == callback.removal); + assert(callback.retired_record().retire_observation_registry + == callback.retire_observation_registry); + guard.lemma_known_retired_expired(callback.retired_record()); + assert(guard.expired().contains(callback.obj)); + assert(protected.protected_by(guard.paper_guard())); + guard.lemma_protected_not_expired(&protected); + assert(!guard.expired().contains(protected.obj())); + assert(false); + } + + /// A completed callback excludes every active physical lease for its + /// retired allocation. + /// + /// The caller supplies the registry invariant connecting each matching + /// allocation key to the split CPU-reader fragment retained with that + /// lease. The conclusion is exactly the side condition required by + /// `RcuTrackedReadPoolRegistry::reclaim`. + pub(super) proof fn tracked_excludes_active_leases( + tracked &self, + callback: rcu_spec::RcuCallbackSummary, + tracked registry: &mut RcuTrackedReadPoolRegistry< + nat, + O, + rcu_cpu_spec::CpuRcuReadLeaseWitness, + >, + ) + requires + self.authorizes(callback), + old(registry).wf(), + forall|lease_id: nat| + #![auto] + old(registry).active_ids().contains(lease_id) && old(registry).active_record( + lease_id, + ).key() == callback.obj ==> { + let witness = old(registry).active_record(lease_id).witness(); + &&& witness.wf() + &&& witness.protected().obj() == callback.obj + &&& witness.reader().cpu() == witness.paper_guard().reader().cpu + &&& self.reports().contains_key(witness.reader().cpu()) + &&& callback.scheduler == witness.binding().registry() + &&& callback.domain == witness.paper_guard().domain() + &&& callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry() + &&& callback.removal.root == witness.paper_guard().root() + }, + ensures + *final(registry) == *old(registry), + (*final(registry)).wf(), + !(*final(registry)).has_active(callback.obj), + { + if old(registry).has_active(callback.obj) { + let ghost lease_id = choose|lease_id: nat| + #![auto] + old(registry).active_ids().contains(lease_id) && old(registry).active_record( + lease_id, + ).key() == callback.obj; + let tracked witness = registry.tracked_borrow_active_witness_mut(lease_id); + assert(witness.wf()); + let ghost cpu = witness.reader().cpu(); + let tracked closed = self.closed_generations.tracked_borrow(cpu); + assert(self.reports@[cpu].matches_closed(*closed)); + assert(closed.scheduler() == callback.scheduler); + witness.lemma_same_participant_as_closed(closed); + assert(witness.reader().participant_id() == closed.participant_id()); + assert(closed.known_retired().contains(callback.retired_record())); + closed.lemma_later_lease_witness_ref(witness); + assert(witness.reader().known_retired().contains(callback.retired_record())); + assert(witness.paper_guard().expired().contains(callback.obj)); + assert(witness.paper_guard().expired().subset_of( + witness.paper_guard().seen_removed().removed, + )); + assert(witness.paper_guard().seen_removed().removed.contains(callback.obj)); + assert(!witness.protected().seen_removed().removed.contains(witness.protected().obj())); + assert(witness.protected().seen_removed() == witness.paper_guard().seen_removed()); + assert(false); + } + } + + /// Retains the completed per-CPU generations for future stale-history loads. + pub(super) proof fn tracked_into_reclaimed_witness( + tracked self, + callback: rcu_spec::RcuCallbackSummary, + ) -> (tracked res: rcu_cpu_spec::RcuReclaimedWitness) + requires + self.authorizes(callback), + ensures + res.wf(), + res.record() == callback.retired_record(), + res.scheduler() == callback.scheduler, + { + let tracked RcuReclaimPermit { summary: _, retired, reports: _, closed_generations } = self; + assert forall|cpu: CpuId| #[trigger] closed_generations.contains_key(cpu) implies { + let closed = closed_generations[cpu]; + &&& closed.wf() + &&& closed.cpu() == cpu + &&& closed.scheduler() == callback.scheduler + &&& closed.known_retired().contains(callback.retired_record()) + } by {}; + rcu_cpu_spec::RcuReclaimedWitness::tracked_new( + callback.scheduler, + retired, + closed_generations, + ) + } + + pub closed spec fn wf(self) -> bool { + self.authorizes(self.callback()) + } + + #[verifier::type_invariant] + pub(super) closed spec fn type_inv(self) -> bool { self.wf() } } -impl Inv for State { - closed spec fn inv(self) -> bool { - self.current_gp.inv() +impl CompletedGracePeriod { + closed spec fn callbacks(self) -> Seq { + self.callbacks@ } + + closed spec fn epoch(self) -> nat { + self.epoch@ + } + + closed spec fn reported_cpus(self) -> Set { + self.reported_cpus@ + } + + closed spec fn reports(self) -> Map { + self.reports@ + } + + closed spec fn closed_generations(self) -> Map { + self.closed_generations + } + + closed spec fn reports_wf(self) -> bool { + &&& self.reports().dom() == self.reported_cpus() + &&& self.closed_generations().dom() == self.reported_cpus() + &&& forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) ==> { + &&& self.reports()[cpu].cpu == cpu + &&& self.reports()[cpu].epoch == self.epoch() + &&& self.reports()[cpu].matches_closed(self.closed_generations()[cpu]) + } + } + + closed spec fn callbacks_covered(self) -> bool { + forall|i: int, cpu: CpuId| + 0 <= i < self.callbacks().len() && #[trigger] self.reports().contains_key(cpu) ==> { + &&& self.reports()[cpu].scheduler == self.callbacks()[i].scheduler + &&& (#[trigger] self.callbacks()[i]).retire_view.spec_le(self.reports()[cpu].view) + &&& self.closed_generations()[cpu].known_retired().contains( + self.callbacks()[i].retired_record(), + ) + } + } + + closed spec fn covers(self, callback: rcu_spec::RcuCallbackSummary) -> bool { + &&& self.callbacks().contains(callback) + &&& callback.retire_epoch == self.epoch() + &&& self.reported_cpus() == online_cpus() + &&& self.reports_wf() + &&& forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) ==> { + &&& self.reports()[cpu].scheduler == callback.scheduler + &&& callback.retire_view.spec_le(self.reports()[cpu].view) + &&& self.closed_generations()[cpu].known_retired().contains( + callback.retired_record(), + ) + } + } + + /// Combines traversal retirement with monitor completion for one callback. + proof fn tracked_authorize_callback( + tracked &self, + tracked safety: &rcu_spec::RcuCallbackSafety, + callback: rcu_spec::RcuCallbackSummary, + ) -> (tracked permit: RcuReclaimPermit) + requires + safety.matches(callback), + self.covers(callback), + callback.removal.observed_by(callback.retire_view), + ensures + permit.authorizes(callback), + { + let tracked retired = safety.tracked_retired_fact(callback); + assert forall|cpu: CpuId| #[trigger] + self.closed_generations().dom().contains( + cpu, + ) implies self.closed_generations()[cpu].wf() by { + assert(self.reports().contains_key(cpu)); + assert(self.reports()[cpu].matches_closed(self.closed_generations()[cpu])); + }; + let tracked closed_generations = duplicate_closed_generations( + &self.closed_generations, + self.closed_generations().dom(), + ); + assert forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) implies callback.removal.observed_by( + self.reports()[cpu].view, + ) by { + assert(callback.retire_view.spec_le(self.reports()[cpu].view)); + }; + assert forall|cpu: CpuId| #[trigger] + self.reports().contains_key( + cpu, + ) implies self.closed_generations()[cpu].known_retired().contains( + callback.retired_record(), + ) by {}; + assert forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) implies self.reports()[cpu].matches_closed( + closed_generations[cpu], + ) by { + assert(self.closed_generations().dom().contains(cpu)); + assert(self.reports()[cpu].matches_closed(self.closed_generations()[cpu])); + }; + RcuReclaimPermit { + summary: Ghost(callback), + retired, + reports: Ghost(self.reports()), + closed_generations, + } + } +} + +pub open spec fn callback_summaries(callbacks: Callbacks) -> Seq { + Seq::new(callbacks@.len(), |i: int| callbacks@[i]@) } -impl Inv for GracePeriod { - closed spec fn inv(self) -> bool { - true +fn run_completed_callbacks( + mut callbacks: Callbacks, + Tracked(completed): Tracked, +) + requires + completed.callbacks() == callback_summaries(callbacks), + completed.reported_cpus() == online_cpus(), + completed.reports_wf(), + completed.callbacks_covered(), + forall|i: int| + 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( + callbacks, + )[i]).retire_epoch == completed.epoch(), +{ + proof { + assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( + (#[trigger] callbacks@[i])@, + ) by { + callback_summaries(callbacks).lemma_index_contains(i); + assert forall|cpu: CpuId| #[trigger] + completed.reports().contains_key(cpu) implies callbacks@[i]@.retire_view.spec_le( + completed.reports()[cpu].view, + ) by { + assert(callback_summaries(callbacks)[i] == callbacks@[i]@); + }; + } + } + + #[verus_spec( + invariant + forall|i: int| + 0 <= i < callbacks@.len() ==> completed.covers((#[trigger] callbacks@[i])@), + decreases callbacks@.len(), + )] + while callbacks.len() > 0 { + let ghost before = callbacks@; + let callback = callbacks.pop_front().unwrap(); + proof { + assert(completed.covers(callback@)); + assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( + (#[trigger] callbacks@[i])@, + ) by { + assert(callbacks@ == before.subrange(1, before.len() as int)); + assert(callbacks@[i] == before[i + 1]); + assert(0 <= i + 1 < before.len()); + } + } + unsafe { + proof { + use_type_invariant(&callback); + } + proof_decl! { + let tracked permit = completed.tracked_authorize_callback( + callback.safety.borrow(), + callback@, + ); + } + callback.call_once(Tracked(permit)); + } } } -#[verus_verify] -impl RcuMonitor { - /// Creates a new RCU monitor. - pub(super) fn new() -> Self { - let state = SpinLock::new(State::new()); +proof fn callback_summaries_empty(callbacks: Callbacks) + requires + callbacks@ == Seq::::empty(), + ensures + callback_summaries(callbacks) == Seq::::empty(), +{ + assert(callback_summaries(callbacks).len() == 0); + vstd::seq_lib::assert_seqs_equal!( + callback_summaries(callbacks) == Seq::::empty() + ); +} + +proof fn callback_summaries_len(callbacks: Callbacks) + ensures + callback_summaries(callbacks).len() == callbacks@.len(), +{ +} + +fn push_callback(callbacks: &mut Callbacks, callback: RcuCallback) + ensures + callback_summaries(*final(callbacks)) == callback_summaries(*old(callbacks)).push( + callback@, + ), +{ + let ghost before = callbacks@; + callbacks.push_back(callback); + proof { + assert(callbacks@ == before.push(callback)); + vstd::seq_lib::assert_seqs_equal!( + callback_summaries(*callbacks) + == callback_summaries(*old(callbacks)).push(callback@), + i => { + if i < before.len() { + assert(callbacks@[i] == before[i]); + } else { + assert(i == before.len()); + assert(callbacks@[i] == callback); + } + } + ); + } +} + +// The proof-facing views `GracePeriodView` and `MonitorStateView` live in +// `specs::sync::rcu` so that the monitor flag's weak-memory ghost state can +// record a state snapshot per flag message without depending on this module. +pub(super) struct GracePeriod { + callbacks: Callbacks, + cpu_mask: AtomicCpuSet, + tracked_cpu_mask: Tracked, + ghost_reports: Ghost>, + tracked_closed_generations: Tracked>, + is_complete: bool, + ghost_epoch: Ghost, +} + +impl View for GracePeriod { + type V = GracePeriodView; + + closed spec fn view(&self) -> GracePeriodView { + GracePeriodView { + epoch: self.ghost_epoch@, + callbacks: callback_summaries(self.callbacks), + is_complete: self.is_complete, + } + } +} + +impl GracePeriod { + /// Creates the initial completed grace period. A completed grace period may + /// not retain callbacks, so both executable callback storage and the proof + /// summary start empty. + pub(super) fn new() -> (res: Self) + ensures + res@ == GracePeriodView::initial(), + res.wf(), + { + let callbacks = Callbacks::new(); + let empty_cpu_set = CpuSet::new_empty(); proof { - use_type_invariant(&state); + assert(empty_cpu_set.cpus == Set::::empty()); + } + let mut cpu_mask = AtomicCpuSet::new(empty_cpu_set); + proof_decl! { + let tracked cpu_mask_token = cpu_mask.tracked_take_token(); + let tracked closed_generations = Map::tracked_empty(); } - let res = RcuMonitor { - is_monitoring: AtomicBool::new(Ghost(state), false, Tracked(false)), - state, + let res = Self { + callbacks, + cpu_mask, + tracked_cpu_mask: Tracked(cpu_mask_token), + ghost_reports: Ghost(Map::empty()), + tracked_closed_generations: Tracked(closed_generations), + is_complete: true, + ghost_epoch: Ghost(0), }; - proof { - use_type_invariant(&res.state); + callback_summaries_empty(res.callbacks); + assert(res.cpu_mask.initial_cpus() == Set::::empty()); + assert(res.ghost_reports@.dom() == Set::::empty()); + assert(res.tracked_cpu_mask@.cpus() == Set::::empty()); + assert(res.tracked_closed_generations@.dom() == Set::::empty()); } res } - /// Creates a new RCU monitor together with its tracked owner for `Once`. - #[verus_spec(r => + /// Starts a new incomplete grace period with the callbacks that should run + /// after it completes. The CPU mask is reset because all CPUs must pass a + /// fresh quiescent state for this new batch. Keep the same atomic object so + /// later weak-memory ghost state can attach stable identity to this mask. + fn restart(&mut self, callbacks: Callbacks, Ghost(epoch): Ghost) + requires + old(self).wf(), + callback_summaries(callbacks).len() > 0, + forall|i: int| + 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( + callbacks, + )[i]).scheduler == rcu_spec::rcu_scheduler(), + forall|i: int| + 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( + callbacks, + )[i]).retire_epoch == epoch, ensures - r.inv(), - r.data.inv(), - RcuMonitorPred.inv(r), - )] - pub(super) fn new_data() -> AtomicDataWithOwner { - let data = Self::new(); + final(self).callback_summaries() == callback_summaries(callbacks), + final(self)@.epoch == epoch, + !final(self).is_complete, + final(self).wf(), + no_unwind + { proof { - use_type_invariant(&data); + let tracked mut empty_closed = Map::tracked_empty(); + vstd::modes::tracked_swap( + self.tracked_closed_generations.borrow_mut(), + &mut empty_closed, + ); } - AtomicDataWithOwner { data, permission: Tracked(RcuMonitorOwner { }) } + self.is_complete = false; + self.callbacks = callbacks; + self.ghost_epoch = Ghost(epoch); + self.ghost_reports = Ghost(Map::empty()); + proof_decl! { + let tracked cpu_mask_token = self.tracked_cpu_mask.borrow_mut(); + } + #[verus_spec(with Tracked(cpu_mask_token))] + self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); } - fn is_monitoring(&self) -> bool { + /// Records one generation-bound quiescent context and updates the + /// executable CPU mask in the same invariant-preserving transition. + fn record_quiescent_state( + &mut self, + this_cpu: CpuId, + Tracked(context): Tracked, + ) -> (complete: bool) + requires + old(self).wf(), + online_cpus().contains(this_cpu), + context.cpu == this_cpu, + context.wf(), + context.scheduler == rcu_spec::rcu_scheduler(), + old(self).tracked_closed_generations@.dom() == old(self).ghost_reports@.dom(), + forall|i: int| + 0 <= i < old(self).callback_summaries().len() ==> (#[trigger] old( + self, + ).callback_summaries()[i]).retire_view.spec_le(context.view), + forall|i: int| + 0 <= i < old(self).callback_summaries().len() + ==> context.closed.known_retired().contains( + (#[trigger] old(self).callback_summaries()[i]).retired_record(), + ), + ensures + final(self).wf(), + final(self)@ == old(self)@, + complete == (final(self).tracked_cpu_mask@.cpus() == online_cpus()), + no_unwind + { + let ghost old_reports = self.ghost_reports@; + let ghost old_closed_generations = self.tracked_closed_generations@; + let ghost report = RcuCpuQuiescentReport { + cpu: this_cpu, + task: context.task, + scheduler: context.scheduler, + session: context.session, + participant: context.participant, + generation: context.generation, + view: context.view, + epoch: self@.epoch, + }; proof { - use_type_invariant(self); + let tracked closed = context.closed; + assert(report.matches_closed(closed)); + self.tracked_closed_generations.borrow_mut().tracked_insert(this_cpu, closed); + assert(old_closed_generations.dom().insert(this_cpu) == old_reports.dom().insert( + this_cpu, + )); + assert(self.tracked_closed_generations@ == old_closed_generations.insert( + this_cpu, + closed, + )); + self.ghost_reports = Ghost(self.ghost_reports@.insert(this_cpu, report)); + assert(self.tracked_closed_generations@.dom() == old_closed_generations.dom().insert( + this_cpu, + )); + assert(self.ghost_reports@.dom() == old_reports.dom().insert(this_cpu)); + assert(self.tracked_closed_generations@.dom() == self.ghost_reports@.dom()); } - self.is_monitoring.load() - } - - fn set_monitoring(&self, value: bool) { + #[verus_spec(with Tracked(self.tracked_cpu_mask.borrow_mut()))] + self.cpu_mask.add(this_cpu, Ordering::Relaxed); + let cpu_mask = #[verus_spec(with Tracked(self.tracked_cpu_mask.borrow()))] + self.cpu_mask.load(Ordering::Relaxed); + let complete = cpu_mask.is_full(); proof { - use_type_invariant(self); + assert(self.ghost_reports@ == old_reports.insert(this_cpu, report)); + assert(self.tracked_closed_generations@ == old_closed_generations.insert( + this_cpu, + self.tracked_closed_generations@[this_cpu], + )); + assert(self.tracked_closed_generations@.dom() == self.ghost_reports@.dom()); + assert(self.callback_summaries() == old(self).callback_summaries()); + assert forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) implies { + &&& self.ghost_reports@[cpu].cpu == cpu + &&& self.ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler() + &&& self.ghost_reports@[cpu].epoch == self@.epoch + &&& self.ghost_reports@[cpu].matches_closed(self.tracked_closed_generations@[cpu]) + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).scheduler == rcu_spec::rcu_scheduler() + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() + ==> self.tracked_closed_generations@[cpu].known_retired().contains( + (#[trigger] self.callback_summaries()[i]).retired_record(), + ) + } by { + assert(self.tracked_closed_generations@.contains_key(cpu)); + assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( + #[trigger] self.callback_summaries()[i]).scheduler == rcu_spec::rcu_scheduler() by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + if cpu == this_cpu { + assert(self.ghost_reports@[cpu] == report); + assert(report.scheduler == context.scheduler); + assert(report.scheduler == rcu_spec::rcu_scheduler()); + assert(self.ghost_reports@[cpu].matches_closed( + self.tracked_closed_generations@[cpu], + )); + assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + assert forall|i: int| + #![auto] + 0 <= i + < self.callback_summaries().len() implies self.tracked_closed_generations@[cpu].known_retired().contains( + self.callback_summaries()[i].retired_record()) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + } else { + assert(self.ghost_reports@[cpu] == old_reports[cpu]); + assert(old(self).ghost_reports@.contains_key(cpu)); + assert(old(self).ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler()); + assert(self.ghost_reports@[cpu] == old(self).ghost_reports@[cpu]); + assert(self.ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler()); + assert(self.tracked_closed_generations@[cpu] == old( + self, + ).tracked_closed_generations@[cpu]); + assert(self.ghost_reports@[cpu].matches_closed( + self.tracked_closed_generations@[cpu], + )); + assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + assert forall|i: int| + #![auto] + 0 <= i + < self.callback_summaries().len() implies self.tracked_closed_generations@[cpu].known_retired().contains( + self.callback_summaries()[i].retired_record()) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + } + }; + assert(self.wf()); } - atomic_with_ghost! { - self.is_monitoring => store(value); - ghost g => { - g = value; + complete + } + + closed spec fn callback_summaries(self) -> Seq { + self@.callbacks + } + + closed spec fn has_pending_work(self) -> bool { + self@.has_pending_work() + } + + /// Lock-protected invariant: a completed grace period has already had its + /// callbacks taken. Monitor methods may break this transiently inside a + /// critical section (between completing a grace period and taking its + /// callbacks), but must restore it before releasing the monitor lock. + pub(super) closed spec fn wf(self) -> bool { + &&& self@.wf() + &&& self.tracked_cpu_mask@.id() == self.cpu_mask.id() + &&& self.tracked_cpu_mask@.wf() + &&& self.ghost_reports@.dom() == self.tracked_cpu_mask@.cpus() + &&& self.tracked_closed_generations@.dom() == self.ghost_reports@.dom() + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).scheduler == rcu_spec::rcu_scheduler() + &&& forall|cpu: CpuId| #[trigger] + self.ghost_reports@.contains_key(cpu) ==> self.ghost_reports@[cpu].matches_closed( + self.tracked_closed_generations@[cpu], + ) + &&& forall|cpu: CpuId| #[trigger] + self.ghost_reports@.contains_key(cpu) ==> { + &&& self.ghost_reports@[cpu].cpu == cpu + &&& self.ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler() + &&& self.ghost_reports@[cpu].epoch == self@.epoch + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() + ==> self.tracked_closed_generations@[cpu].known_retired().contains( + (#[trigger] self.callback_summaries()[i]).retired_record(), + ) } + } +} + +pub(super) struct State { + current_gp: GracePeriod, + next_callbacks: Callbacks, + /// Monotonic registry of persistent `Retired(a, Q)` facts for callbacks + /// observed through this monitor. + tracked_retired_facts: Tracked, + /// Release view of the monitor lock. + /// + /// This proof-only token is updated before unlocking and imported after + /// locking. It gives the existing executable spin lock the release/acquire + /// semantics needed by the RCU proof without changing its runtime layout. + tracked_lock_view: Tracked, +} + +impl View for State { + type V = MonitorStateView; + + closed spec fn view(&self) -> MonitorStateView { + MonitorStateView { + current_gp: self.current_gp@, + next_callbacks: callback_summaries(self.next_callbacks), } } } impl State { - fn new() -> (res: Self) + closed spec fn lock_view(self) -> ThreadView { + self.tracked_lock_view@@ + } + + closed spec fn next_callback_epoch(self) -> nat { + self@.current_gp.epoch + 1 + } + + /// Imports the view published by the previous monitor-lock holder. + fn tracked_acquire_lock_view(&self, Tracked(thread_view): Tracked<&mut ViewSeen>) ensures - res.inv(), + self.wf(), + final(thread_view)@ == old(thread_view)@.join(self.lock_view()), + old(thread_view)@.spec_le(final(thread_view)@), + self.lock_view().spec_le(final(thread_view)@), { - Self { current_gp: GracePeriod::new() } + proof { + use_type_invariant(self); + } + proof_decl! { + let tracked published_view = self.tracked_lock_view.borrow(); + } + proof { + let ghost before = thread_view@; + let ghost lock_view = self.lock_view(); + published_view.tracked_join_into_view_seen(thread_view); + before.lemma_join_left(lock_view); + before.lemma_join_right(lock_view); + } } -} -impl GracePeriod { - fn new() -> (res: Self) + /// Publishes the current holder's observations to the next lock acquirer. + fn tracked_publish_lock_view(&mut self, Tracked(thread_view): Tracked<&ViewSeen>) + requires + old(self).wf(), ensures - res.inv(), + final(self).wf(), + final(self)@ == old(self)@, + final(self).lock_view() == old(self).lock_view().join(thread_view@), + old(self).lock_view().spec_le(final(self).lock_view()), + thread_view@.spec_le(final(self).lock_view()), { - Self { cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), is_complete: true } + proof { + let ghost old_lock_view = old(self).lock_view(); + let ghost holder_view = thread_view@; + let tracked joined = self.tracked_lock_view.borrow().tracked_joined_view_seen( + thread_view, + ); + assert(old_lock_view.spec_le(joined@)); + assert(holder_view.spec_le(joined@)); + assert forall|i: int| 0 <= i < self.current_gp.callback_summaries().len() implies ( + #[trigger] self.current_gp.callback_summaries()[i]).retire_view.spec_le(joined@) by { + self.current_gp.callback_summaries()[i].retire_view.lemma_spec_le_transitive( + old_lock_view, + joined@, + ); + }; + assert forall|i: int| 0 <= i < callback_summaries(self.next_callbacks).len() implies ( + #[trigger] callback_summaries(self.next_callbacks)[i]).retire_view.spec_le(joined@) by { + callback_summaries(self.next_callbacks)[i].retire_view.lemma_spec_le_transitive( + old_lock_view, + joined@, + ); + }; + assert(self.tracked_retired_facts@.observed_by(joined@)) by { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + self.tracked_retired_facts@.records().contains( + record, + ) implies record.removal.observed_by(joined@) by { + assert(record.removal.observed_by(old_lock_view)); + old_lock_view.lemma_spec_le_transitive(joined@, joined@); + }; + }; + self.tracked_lock_view = Tracked(joined); + old_lock_view.lemma_join_left(holder_view); + old_lock_view.lemma_join_right(holder_view); + assert forall|i: int| + 0 <= i < final(self).current_gp.callback_summaries().len() implies ( + #[trigger] final(self).current_gp.callback_summaries()[i]).retire_view.spec_le( + final(self).lock_view(), + ) by { + assert(final(self).current_gp.callback_summaries()[i] == old( + self, + ).current_gp.callback_summaries()[i]); + old(self).current_gp.callback_summaries()[i].retire_view.lemma_spec_le_transitive( + old_lock_view, + final(self).lock_view(), + ); + }; + assert forall|i: int| + 0 <= i < callback_summaries(final(self).next_callbacks).len() implies ( + #[trigger] callback_summaries(final(self).next_callbacks)[i]).retire_view.spec_le( + final(self).lock_view(), + ) by { + assert(callback_summaries(final(self).next_callbacks)[i] == callback_summaries( + old(self).next_callbacks, + )[i]); + callback_summaries( + old(self).next_callbacks, + )[i].retire_view.lemma_spec_le_transitive(old_lock_view, final(self).lock_view()); + }; + assert(self.tracked_retired_facts@.observed_by(final(self).lock_view())) by { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + self.tracked_retired_facts@.records().contains( + record, + ) implies record.removal.observed_by(final(self).lock_view()) by { + assert(record.removal.observed_by(old_lock_view)); + old_lock_view.lemma_spec_le_transitive( + final(self).lock_view(), + final(self).lock_view(), + ); + }; + }; + } } -} - -} // verus! -/*use alloc::collections::VecDeque; -use core::sync::atomic::{ - AtomicBool, - Ordering::{self, Relaxed}, -}; - -use crate::{ - cpu::{AtomicCpuSet, CpuId, CpuSet, PinCurrentCpu}, - prelude::*, - sync::SpinLock, - task::atomic_mode::AsAtomicModeGuard, -}; - + /// Creates the lock-protected initial monitor state: there is no active + /// grace period and no callbacks waiting to be attached to the next one. + pub(super) fn new() -> (res: Self) + ensures + res@ == MonitorStateView::initial(), + res.no_pending_work(), + { + let current_gp = GracePeriod::new(); + let next_callbacks = Callbacks::new(); + proof_decl! { + let tracked lock_view = ThreadViewToken::new(); + let tracked retired_facts = rcu_spec::RcuRetiredFacts::empty(); + } + let res = Self { + current_gp, + next_callbacks, + tracked_retired_facts: Tracked(retired_facts), + tracked_lock_view: Tracked(lock_view), + }; + proof { + callback_summaries_empty(res.next_callbacks); + } + res + } -impl RcuMonitor { - /// Creates a new RCU monitor. + /// Enqueues one callback and starts a grace period if the monitor was idle. /// - /// This function is used to initialize a singleton instance of `RcuMonitor`. - /// The singleton instance is globally accessible via the `RCU_MONITOR`. - pub(super) fn new() -> Self { - Self { - is_monitoring: AtomicBool::new(false), - state: SpinLock::new(State::new()), + /// This follows the upstream monitor protocol at the observable boundary: + /// an idle monitor starts a new current grace period and the caller must + /// publish `is_monitoring = true`; an already active monitor only appends + /// the callback to the next batch and does not publish another flag + /// message. The upstream implementation transiently stages the idle + /// callback in `next_callbacks` before promoting it, but our type invariant + /// keeps `next_callbacks` empty whenever the current grace period is + /// complete, so the idle case constructs the current batch directly. + fn enqueue_after_grace_period(&mut self, callback: RcuCallback) -> (started_gp: bool) + requires + callback.wf(), + callback@.scheduler == rcu_spec::rcu_scheduler(), + callback@.retire_epoch == old(self).next_callback_epoch(), + callback@.retire_view.spec_le(old(self).lock_view()), + ensures + final(self).wf(), + final(self).has_pending_work(), + started_gp ==> !final(self)@.current_gp.is_complete, + !started_gp ==> !old(self)@.current_gp.is_complete, + { + proof { + use_type_invariant(&*self); + let tracked fact = callback.tracked_retired_fact(); + self.tracked_retired_facts.borrow_mut().tracked_insert(&fact); + assert(callback@.retired_record() == fact.record()); + assert(callback@.retire_view.spec_le(self.lock_view())); + assert(callback@.removal.observed_by(self.lock_view())); + } + let ghost callback_epoch = self.next_callback_epoch(); + if self.current_gp.is_complete { + let mut callbacks = Callbacks::new(); + push_callback(&mut callbacks, callback); + proof { + assert(callback_summaries(callbacks).len() == 1); + assert(callback_summaries(self.next_callbacks).len() == 0); + } + self.current_gp.restart(callbacks, Ghost(callback_epoch)); + true + } else { + let mut next_callbacks = Callbacks::new(); + let ghost existing_summaries = callback_summaries(self.next_callbacks); + proof { + assert(self.current_gp.wf()); + assert(!self.current_gp.is_complete); + assert(self.wf()); + assert(self@.wf()); + assert(self@.next_callbacks == existing_summaries); + assert forall|i: int| 0 <= i < existing_summaries.len() implies ( + #[trigger] existing_summaries[i]).retire_epoch == self.current_gp@.epoch + 1 by {}; + } + core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); + let ghost before_summaries = callback_summaries(next_callbacks); + proof { + assert(before_summaries == existing_summaries); + } + push_callback(&mut next_callbacks, callback); + proof { + assert forall|i: int| 0 <= i < callback_summaries(next_callbacks).len() implies ( + #[trigger] callback_summaries(next_callbacks)[i]).retire_epoch + == self.current_gp@.epoch + 1 by { + if i < before_summaries.len() { + assert(callback_summaries(next_callbacks)[i] == before_summaries[i]); + } else { + assert(i == before_summaries.len()); + assert(callback_summaries(next_callbacks)[i] == callback@); + } + }; + } + self.next_callbacks = next_callbacks; + false } } - pub(super) unsafe fn finish_grace_period(&self) { - // Fast path - if !self.is_monitoring.load(Relaxed) { - return; + /// Records one CPU report while preserving the lock-protected callback + /// state. The linear mask token makes the executable `is_full` result + /// equivalent to coverage of the fixed online-CPU set. + fn record_quiescent_state( + &mut self, + this_cpu: CpuId, + Tracked(context): Tracked, + ) -> (complete: bool) + requires + old(self).wf(), + !old(self).current_gp.is_complete, + online_cpus().contains(this_cpu), + context.cpu == this_cpu, + context.wf(), + context.scheduler == rcu_spec::rcu_scheduler(), + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( + self, + ).current_gp.callback_summaries()[i]).retire_view.spec_le(context.view), + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() + ==> context.closed.known_retired().contains( + (#[trigger] old(self).current_gp.callback_summaries()[i]).retired_record(), + ), + ensures + final(self).wf(), + final(self)@ == old(self)@, + complete == (final(self).current_gp.tracked_cpu_mask@.cpus() == online_cpus()), + no_unwind + { + self.current_gp.record_quiescent_state(this_cpu, Tracked(context)) + } + + /// Bridges the lock-protected persistent retirement registry into the + /// scheduler-owned CPU participant at a quiescent boundary. + fn make_quiescent_context( + &self, + cpu: CpuId, + Tracked(context): Tracked<&mut RunningTaskContext>, + ) -> (res: Tracked) + requires + self.wf(), + old(context).wf(), + old(context).scheduler() == rcu_spec::rcu_scheduler(), + old(context).is_quiescent(), + cpu == old(context).cpu(), + self.tracked_retired_facts@.observed_by(old(context).irc11_view()), + ensures + res@.cpu == cpu, + res@.scheduler == rcu_spec::rcu_scheduler(), + res@.view == old(context).irc11_view(), + res@.wf(), + self.tracked_retired_facts@.records().subset_of(res@.closed.known_retired()), + final(context).wf(), + final(context).is_quiescent(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation() + 1, + final(context).rcu_generation() == old(context).rcu_generation() + 1, + final(context).rcu_fraction() == 1real, + final(context).view() == old(context).view(), + final(context).irc11_view() == old(context).irc11_view(), + no_unwind + { + proof_decl! { + let tracked retired_facts = self.tracked_retired_facts.borrow(); + let tracked quiescent_context = + RcuQuiescentContext::tracked_from_running_context( + context, + cpu, + retired_facts, + ); } + Tracked(quiescent_context) + } - // Check if the current GP is complete after passing the quiescent state - // on the current CPU. If GP is complete, take the callbacks of the current - // GP. - let callbacks = { - let mut state = self.state.disable_irq().lock(); - let cpu = state.as_atomic_mode_guard().current_cpu(); - if state.current_gp.is_complete() { - return; + /// Records a quiescent state for the current CPU, returns the callbacks + /// that become reclaimable if this completes the grace period, and + /// immediately starts the next grace period if callbacks accumulated while + /// the current one was running. + /// + /// This mirrors the upstream state machine: an incomplete CPU mask leaves + /// the current grace period running and returns no completed callbacks. + /// The mask's linear shadow token records the exact reported-CPU set, so a + /// completed batch also carries proof that every online CPU reported. + fn finish_grace_period( + &mut self, + this_cpu: CpuId, + Tracked(context): Tracked, + ) -> ((completed_gp, completed_callbacks, completed_token): ( + bool, + Callbacks, + Tracked, + )) + requires + online_cpus().contains(this_cpu), + context.cpu == this_cpu, + context.wf(), + context.scheduler == rcu_spec::rcu_scheduler(), + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( + self, + ).current_gp.callback_summaries()[i]).retire_view.spec_le(context.view), + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() + ==> context.closed.known_retired().contains( + (#[trigger] old(self).current_gp.callback_summaries()[i]).retired_record(), + ), + ensures + final(self).wf(), + completed_token@.callbacks() == callback_summaries(completed_callbacks), + completed_gp ==> !old(self)@.current_gp.is_complete, + completed_gp ==> completed_token@.callbacks() == old(self)@.current_gp.callbacks, + completed_gp ==> completed_token@.epoch() == old(self)@.current_gp.epoch, + completed_gp ==> completed_token@.reported_cpus() == online_cpus(), + completed_gp ==> completed_token@.reports_wf(), + completed_token@.callbacks_covered(), + !completed_gp ==> completed_token@.callbacks() == Seq::< + rcu_spec::RcuCallbackSummary, + >::empty(), + forall|i: int| + 0 <= i < callback_summaries(completed_callbacks).len() ==> ( + #[trigger] callback_summaries(completed_callbacks)[i]).retire_epoch + == completed_token@.epoch(), + (!completed_gp && !(old(self)@.current_gp.is_complete)) ==> !( + final(self)@.current_gp.is_complete), + { + proof { + use_type_invariant(&*self); + } + let ghost initially_complete = self.current_gp.is_complete; + let ghost initial_current_callbacks = self.current_gp@.callbacks; + let ghost initial_current_epoch = self.current_gp@.epoch; + let ghost initial_next_callbacks = callback_summaries(self.next_callbacks); + proof { + assert(self.wf()); + assert(self@.wf()); + assert(self@.next_callbacks == initial_next_callbacks); + assert forall|i: int| 0 <= i < initial_current_callbacks.len() implies ( + #[trigger] initial_current_callbacks[i]).retire_epoch == initial_current_epoch by { + assert(self.current_gp.wf()); + }; + assert forall|i: int| 0 <= i < initial_next_callbacks.len() implies ( + #[trigger] initial_next_callbacks[i]).retire_epoch == initial_current_epoch + 1 by { + assert(self.wf()); + }; + } + let mut completed_callbacks = Callbacks::new(); + let mut completed_gp = false; + let ghost mut completed_cpu_mask = Set::::empty(); + let ghost mut completed_reports = Map::::empty(); + let ghost mut completed_closed_generations_view = Map::< + CpuId, + rcu_cpu_spec::CpuRcuClosedGeneration, + >::empty(); + proof_decl! { + let tracked mut completed_closed_generations = Map::tracked_empty(); + } + if !self.current_gp.is_complete { + let is_complete = self.record_quiescent_state(this_cpu, Tracked(context)); + if is_complete { + completed_gp = true; + proof { + completed_cpu_mask = self.current_gp.tracked_cpu_mask@.cpus(); + completed_reports = self.current_gp.ghost_reports@; + assert forall|cpu: CpuId| #[trigger] + completed_reports.contains_key(cpu) implies completed_reports[cpu].scheduler + == rcu_spec::rcu_scheduler() by { + assert(self.current_gp.ghost_reports@.contains_key(cpu)); + }; + completed_closed_generations_view = self.current_gp.tracked_closed_generations@; + assert(completed_cpu_mask == online_cpus()); + assert(completed_closed_generations_view.dom() == completed_reports.dom()); + assert forall|cpu: CpuId| #[trigger] + completed_reports.contains_key( + cpu, + ) implies completed_reports[cpu].matches_closed( + completed_closed_generations_view[cpu], + ) by { + assert(self.current_gp.wf()); + }; + assert(self.current_gp.callback_summaries() == initial_current_callbacks); + assert forall|i: int, cpu: CpuId| + 0 <= i < initial_current_callbacks.len() + && #[trigger] completed_reports.contains_key(cpu) implies ( + #[trigger] initial_current_callbacks[i]).retire_view.spec_le( + completed_reports[cpu].view, + ) by { + assert(self.current_gp.wf()); + }; + } + core::mem::swap(&mut completed_callbacks, &mut self.current_gp.callbacks); + proof { + assert(callback_summaries(completed_callbacks) == initial_current_callbacks); + callback_summaries_empty(self.current_gp.callbacks); + assert forall|cpu: CpuId| #[trigger] + self.current_gp.tracked_closed_generations@.dom().contains( + cpu, + ) implies self.current_gp.tracked_closed_generations@[cpu].wf() by { + assert(self.current_gp.ghost_reports@.contains_key(cpu)); + assert(self.current_gp.wf()); + }; + completed_closed_generations = + duplicate_closed_generations( + self.current_gp.tracked_closed_generations.borrow(), + self.current_gp.tracked_closed_generations@.dom(), + ); + } + if self.next_callbacks.len() > 0 { + let mut next_callbacks = Callbacks::new(); + core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); + proof { + callback_summaries_empty(self.next_callbacks); + assert(callback_summaries(next_callbacks) == initial_next_callbacks); + } + self.current_gp.restart(next_callbacks, Ghost(initial_current_epoch + 1)); + } else { + self.current_gp.is_complete = true; + proof { + callback_summaries_empty(self.current_gp.callbacks); + callback_summaries_len(self.next_callbacks); + assert(self.next_callbacks@.len() == 0); + assert(callback_summaries(self.next_callbacks).len() == 0); + } + } } - - state.current_gp.finish_grace_period(cpu); - if !state.current_gp.is_complete() { - return; + } + proof_decl! { + let tracked completed = CompletedGracePeriod { + epoch: Ghost(if completed_gp { initial_current_epoch } else { 0 }), + callbacks: Ghost(callback_summaries(completed_callbacks)), + reported_cpus: Ghost(completed_cpu_mask), + reports: Ghost(completed_reports), + closed_generations: completed_closed_generations, + }; + } + proof { + callback_summaries_len(self.current_gp.callbacks); + callback_summaries_len(self.next_callbacks); + assert(completed.callbacks() == callback_summaries(completed_callbacks)); + if !completed_gp { + callback_summaries_empty(completed_callbacks); + assert(completed.callbacks_covered()); + } else { + assert(!initially_complete); + assert(completed.reported_cpus() == online_cpus()); + assert(completed.reports_wf()); + assert(callback_summaries(completed_callbacks) == initial_current_callbacks); + assert(completed.callbacks_covered()) by { + assert forall|i: int, cpu: CpuId| + 0 <= i < completed.callbacks().len() + && #[trigger] completed.reports().contains_key(cpu) implies { + &&& completed.reports()[cpu].scheduler == completed.callbacks()[i].scheduler + &&& (#[trigger] completed.callbacks()[i]).retire_view.spec_le( + completed.reports()[cpu].view, + ) + &&& completed.closed_generations()[cpu].known_retired().contains( + completed.callbacks()[i].retired_record(), + ) + } by { + assert(completed.callbacks()[i] == initial_current_callbacks[i]); + assert(completed.callbacks()[i].scheduler == rcu_spec::rcu_scheduler()); + assert(completed.reports() == completed_reports); + assert(completed.reports()[cpu].scheduler == rcu_spec::rcu_scheduler()); + assert(completed.closed_generations()[cpu].known_retired() + == completed_closed_generations_view[cpu].known_retired()); + }; + }; + assert forall|i: int| + 0 <= i < callback_summaries(completed_callbacks).len() implies ( + #[trigger] callback_summaries(completed_callbacks)[i]).retire_epoch + == completed.epoch() by { + assert(callback_summaries(completed_callbacks)[i] + == initial_current_callbacks[i]); + }; + } + if initially_complete { + assert(initial_current_callbacks.len() == 0); + vstd::seq_lib::assert_seqs_equal!( + initial_current_callbacks == Seq::::empty() + ); } + if self.current_gp.is_complete { + if initially_complete { + assert(self.wf()); + } + assert(self.current_gp@.callbacks.len() == 0); + assert(self.next_callbacks@.len() == 0); + assert(callback_summaries(self.next_callbacks).len() == 0); + } + assert(self.current_gp.wf()); + assert(self.wf()); + } + (completed_gp, completed_callbacks, Tracked(completed)) + } - // Now that the current GP is complete, take its callbacks - let current_callbacks = state.current_gp.take_callbacks(); + closed spec fn pending_summaries(self) -> Seq { + self@.pending_summaries() + } - // Check if we need to watch for a next GP - if !state.next_callbacks.is_empty() { - let callbacks = core::mem::take(&mut state.next_callbacks); - state.current_gp.restart(callbacks); - } else { - self.is_monitoring.store(false, Relaxed); - } + closed spec fn has_pending_work(self) -> bool { + self@.has_pending_work() + } - current_callbacks - }; + pub closed spec fn no_pending_work(self) -> bool { + self@.no_pending_work() + } - // Invoke the callbacks to notify the completion of GP - for f in callbacks { - (f)(); - } + /// Lock-protected invariant of the whole monitor state: the current grace + /// period is well-formed, and a complete grace period implies an empty + /// next-callback queue (the monitor either restarted the grace period with + /// the queued callbacks or stopped monitoring). Holds whenever the monitor + /// lock is free. + closed spec fn wf(self) -> bool { + &&& self@.wf() + &&& self.current_gp.wf() + &&& forall|i: int| + 0 <= i < self.current_gp.callback_summaries().len() ==> ( + #[trigger] self.current_gp.callback_summaries()[i]).retire_view.spec_le( + self.lock_view(), + ) + &&& forall|i: int| + 0 <= i < callback_summaries(self.next_callbacks).len() ==> ( + #[trigger] callback_summaries(self.next_callbacks)[i]).retire_view.spec_le( + self.lock_view(), + ) + &&& forall|i: int| + 0 <= i < callback_summaries(self.next_callbacks).len() ==> ( + #[trigger] callback_summaries(self.next_callbacks)[i]).scheduler + == rcu_spec::rcu_scheduler() + &&& self.tracked_retired_facts@.observed_by(self.lock_view()) + &&& forall|i: int| + 0 <= i < self.current_gp.callback_summaries().len() + ==> self.tracked_retired_facts@.records().contains( + (#[trigger] self.current_gp.callback_summaries()[i]).retired_record(), + ) + &&& forall|i: int| + 0 <= i < callback_summaries(self.next_callbacks).len() + ==> self.tracked_retired_facts@.records().contains( + (#[trigger] callback_summaries(self.next_callbacks)[i]).retired_record(), + ) } - pub(super) fn after_grace_period(&self, f: F) - where - F: FnOnce() + Send + 'static, - { - let mut state = self.state.disable_irq().lock(); + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} - state.next_callbacks.push_back(Box::new(f)); +/// Relationship maintained between one message of the weak `is_monitoring` +/// flag and the monitor-state snapshot recorded with it. A `true` flag may +/// over-approximate work, but a `false` flag must be precise enough to certify +/// no pending callbacks or grace period. +pub open spec fn monitor_flag_matches_state(flag: bool, state: MonitorStateView) -> bool { + !flag ==> state.no_pending_work() +} - if !state.current_gp.is_complete() { - return; - } +/// View-level form of the monitor flag write obligation. Executable monitor +/// code can call this while holding a guard by passing the protected state's +/// view, without moving the `State` value out of the lock. +proof fn monitor_flag_view_store_obligation(flag: bool, state: MonitorStateView) + requires + state.wf(), + state.has_pending_work() ==> flag, + ensures + state.wf(), + !flag ==> state.no_pending_work(), + monitor_flag_matches_state(flag, state), +{ +} - let callbacks = core::mem::take(&mut state.next_callbacks); - state.current_gp.restart(callbacks); - self.is_monitoring.store(true, Relaxed); +/// Every message of the monitor flag matches the state snapshot recorded with +/// it: the weak-memory history invariant implies the per-message relation +/// above, for stale messages as well as the latest one. +proof fn monitor_flag_message_matches_state( + history: Irc11History, + flag_ghost: rcu_spec::RcuMonitorFlagGhost, + ts: nat, +) + requires + rcu_spec::rcu_monitor_flag_history_inv(history, flag_ghost), + history.contains_timestamp(ts), + ensures + monitor_flag_matches_state(history.value(ts), flag_ghost.states[ts]), +{ + if !history.value(ts) { + rcu_spec::rcu_monitor_flag_false_has_no_pending(history, flag_ghost, ts); } } +/// The fast-path certificate at the executable `State` level: reading a +/// `false` flag message whose snapshot agrees with the lock-protected state +/// proves that this state has no queued callbacks and no incomplete grace +/// period. The agreement precondition is discharged by the writer protocol: +/// every flag store happens under the monitor lock and records the +/// lock-protected state as its snapshot. +proof fn monitor_flag_false_certifies_no_pending( + history: Irc11History, + flag_ghost: rcu_spec::RcuMonitorFlagGhost, + ts: nat, + state: State, +) + requires + rcu_spec::rcu_monitor_flag_history_inv(history, flag_ghost), + history.contains_timestamp(ts), + !history.value(ts), + flag_ghost.states[ts] == state@, + ensures + state.no_pending_work(), + state.pending_summaries() == Seq::::empty(), +{ + rcu_spec::rcu_monitor_flag_false_has_no_pending(history, flag_ghost, ts); +} -impl State { - fn new() -> Self { - Self { - current_gp: GracePeriod::new(), - next_callbacks: VecDeque::new(), - } - } +/// Bridge for the future `set_monitoring` helper: while holding the monitor +/// lock with a well-formed state, writing any flag value that over-approximates +/// the state's pending work discharges the insertion obligation of +/// [`rcu_spec::preserve_rcu_monitor_flag_inv_on_insert`]. +proof fn monitor_flag_store_obligation(flag: bool, state: State) + requires + state.wf(), + state.has_pending_work() ==> flag, + ensures + state@.wf(), + !flag ==> state@.no_pending_work(), + monitor_flag_matches_state(flag, state@), +{ + monitor_flag_view_store_obligation(flag, state@); } -impl GracePeriod { - fn new() -> Self { - Self { - callbacks: Callbacks::new(), - cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), - is_complete: true, +/// A RCU monitor ensures the completion of _grace periods_ by keeping track +/// of each CPU's passing _quiescent states_. +pub(super) struct RcuMonitor { + pub(super) is_monitoring: MonitorAtomicBool, + pub(super) state: SpinLock, +} + +impl RcuMonitor { + /// Creates the monitor with an initially false weak flag. The flag ghost + /// records the same initial lock-protected state snapshot, so every + /// possible read of the initial `false` message certifies that there is no + /// pending monitor work. + pub(super) fn new() -> (res: Self) { + let state = State::new(); + let is_monitoring = MonitorAtomicBool::new(); + let state = SpinLock::new(state); + proof { + use_type_invariant(&is_monitoring); + assert(is_monitoring.well_formed()); + use_type_invariant(&state); } + let res = Self { is_monitoring, state }; + res } - fn is_complete(&self) -> bool { - self.is_complete + /// Stores the monitor fast-path flag together with the monitor-state + /// snapshot that justifies the new flag message. Callers should hold the + /// monitor lock and pass the view of the lock-protected state. + fn set_monitoring( + &self, + value: bool, + Ghost(state): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) + requires + self.wf(), + state.wf(), + state.has_pending_work() ==> value, + ensures + old(tv)@.spec_le(final(tv)@), + { + proof { + use_type_invariant(self); + monitor_flag_view_store_obligation(value, state); + } + self.is_monitoring.store_relaxed_rcu_monitor(value, Ghost(state), Tracked(tv)); } - fn finish_grace_period(&mut self, this_cpu: CpuId) { - self.cpu_mask.add(this_cpu, Ordering::Relaxed); + /// Schedules `callback` to run after a future grace period. + /// + /// Matches the upstream protocol: callbacks are first queued for the next + /// grace period. Only an idle monitor promotes that queue into a new + /// current grace period and publishes a `true` flag; if a grace period is + /// already running, the existing monitor flag is left unchanged. + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + cert@.removal().observed_by(old(session).irc11_view()), + forall|permit: RcuReclaimPermit| #![auto] + permit.wf() + && permit.callback().domain == cert@.domain() + && permit.callback().obj == cert@.obj() + && permit.callback().removal == cert@.removal() + && permit.callback().retire_observation_registry + == cert@.retire_observation_registry() + && permit.callback().scheduler == old(session).scheduler() + ==> raw.call_requires(permit), + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + final(session).rcu_participant_id() == old(session).rcu_participant_id(), + final(session).rcu_generation() == old(session).rcu_generation(), + final(session).rcu_participant_view() == old(session).rcu_participant_view(), + final(session).rcu_fraction() == old(session).rcu_fraction(), + )] + pub(super) fn after_grace_period( + &self, + raw: RawCallbackWithProof, + cert: Tracked, + ) { + proof { + use_type_invariant(self); + } + let ghost retire_view = session.irc11_view(); + let mut state = self.state.lock(); + let ghost before_acquire = session.irc11_view(); + proof_decl! { + let tracked acquire_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_acquire_lock_view(Tracked(acquire_view)); + proof { + retire_view.lemma_spec_le_transitive(before_acquire, session.irc11_view()); + } + proof_decl! { + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); + proof { + retire_view.lemma_spec_le_transitive(session.irc11_view(), state.value().lock_view()); + } + let ghost retire_epoch = state.view()@.current_gp.epoch + 1; + proof_decl! { + let tracked cert = cert.get(); + } + let callback = RcuCallback::from_raw( + raw, + Tracked(cert), + Ghost(retire_epoch), + Ghost(retire_view), + Ghost(session.scheduler()), + ); + let started_gp = state.enqueue_after_grace_period(callback); + if started_gp { + proof { + assert(state.view().wf()); + assert(state.view().has_pending_work()); + use_type_invariant(self); + } + proof_decl! { + let tracked tv = session.tracked_borrow_irc11_view_mut(); + } + self.set_monitoring(true, Ghost(state.view()@), Tracked(tv)); + } + proof_decl! { + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); + state.drop(); + } - if self.cpu_mask.load(Ordering::Relaxed).is_full() { - self.is_complete = true; + /// Reports this CPU's quiescent state and runs a completed callback batch + /// outside the monitor lock. + /// + /// The control flow is aligned with upstream: a relaxed false flag returns + /// immediately, a stale true flag may still find a completed state under + /// the lock and return, an incomplete CPU mask keeps monitoring without + /// touching the flag, and callback bodies run outside the monitor lock. + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + old(session).is_quiescent(), + ensures + final(session).wf(), + final(session).is_quiescent(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).session_id() == old(session).session_id(), + old(session).quiescent_generation() <= final(session).quiescent_generation(), + final(session).quiescent_generation() <= old(session).quiescent_generation() + 1, + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] + pub(super) unsafe fn finish_grace_period(&self) { + proof { + use_type_invariant(self); + } + proof_decl! { + let tracked fast_tv = session.tracked_borrow_irc11_view_mut(); + } + let is_monitoring = self.is_monitoring.load_relaxed(Tracked(fast_tv)).0; + if !is_monitoring { + return; + } + let mut state = self.state.lock(); + proof_decl! { + let tracked acquire_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_acquire_lock_view(Tracked(acquire_view)); + if state.current_gp.is_complete { + proof_decl! { + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); + state.drop(); + return; } + let this_cpu = CpuId::current(Tracked(&*session)); + proof { + assert(state.value().tracked_retired_facts@.observed_by(session.irc11_view())) by { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + state.value().tracked_retired_facts@.records().contains( + record, + ) implies record.removal.observed_by(session.irc11_view()) by { + assert(record.removal.observed_by(state.value().lock_view())); + state.value().lock_view().lemma_spec_le_transitive( + session.irc11_view(), + session.irc11_view(), + ); + }; + }; + } + let Tracked(quiescent_context) = state.make_quiescent_context( + CpuId::current(Tracked(&*session)), + Tracked(session), + ); + proof { + assert forall|i: int| + 0 <= i < state.value().current_gp.callback_summaries().len() implies ( + #[trigger] state.value().current_gp.callback_summaries()[i]).retire_view.spec_le( + quiescent_context.view, + ) by { + let ghost callback = state.value().current_gp.callback_summaries()[i]; + callback.retire_view.lemma_spec_le_transitive( + state.value().lock_view(), + session.irc11_view(), + ); + }; + assert forall|i: int| + 0 <= i + < state.value().current_gp.callback_summaries().len() implies quiescent_context.closed.known_retired().contains( + (#[trigger] state.value().current_gp.callback_summaries()[i]).retired_record()) by { + assert(state.value().tracked_retired_facts@.records().contains( + state.value().current_gp.callback_summaries()[i].retired_record(), + )); + }; + } + let (completed_gp, completed_callbacks, Tracked(completed)) = state.finish_grace_period( + this_cpu, + Tracked(quiescent_context), + ); + if !completed_gp { + proof_decl! { + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); + state.drop(); + return; + } + if state.current_gp.is_complete { + proof { + assert(state.view().wf()); + rcu_spec::monitor_state_pending_iff_incomplete(state.view()@); + assert(state.view()@.no_pending_work()); + use_type_invariant(self); + } + proof_decl! { + let tracked tv = session.tracked_borrow_irc11_view_mut(); + } + self.set_monitoring(false, Ghost(state.view()@), Tracked(tv)); + } + proof_decl! { + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); + state.drop(); + run_completed_callbacks(completed_callbacks, Tracked(completed)); } - fn take_callbacks(&mut self) -> Callbacks { - core::mem::take(&mut self.callbacks) + closed spec fn wf(self) -> bool { + &&& self.is_monitoring.well_formed() + &&& self.state.type_inv() } - fn restart(&mut self, callbacks: Callbacks) { - self.is_complete = false; - self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); - self.callbacks = callbacks; + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +/// Tracked witness stored beside the global monitor in `Once`. +pub(super) tracked struct RcuMonitorOwner {} + +impl DataPredicate for RcuMonitorOwner { + closed spec fn predicate(&self, monitor: RcuMonitor) -> bool { + monitor.wf() } } -*/ + +/// Invariant used by the global `Once` cell. +pub(super) struct RcuMonitorPred; + +impl OncePredicate> for RcuMonitorPred { + closed spec fn inv(self, value: AtomicDataWithOwner) -> bool { + value.permission@.predicate(value.data) + } +} + +impl RcuMonitor { + /// Packages a fresh monitor with the owner expected by the global `Once`. + pub(super) fn new_data() -> (res: AtomicDataWithOwner) + ensures + RcuMonitorPred.inv(res), + { + let data = Self::new(); + proof_decl! { + let tracked owner = RcuMonitorOwner {}; + } + proof { + use_type_invariant(&data); + } + let res = AtomicDataWithOwner { data, permission: Tracked(owner) }; + proof { + assert(res.permission@.predicate(res.data)); + } + res + } +} + +} // verus! +// use vstd::{ +// atomic_ghost::AtomicBool, atomic_with_ghost, predicate::Predicate as DataPredicate, prelude::*, +// }; +// use vstd_extra::ownership::Inv; +// use crate::{ +// specs::mm::cpu::{AtomicCpuSet, CpuSet}, +// sync::{AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate}, +// }; +// verus! { +// // This thing can be very tricky to deal with +// // type Callbacks = VecDeque>; +// pub(super) struct GracePeriod { +// // callbacks: Callbacks, +// cpu_mask: AtomicCpuSet, +// is_complete: bool, +// } +// pub(super) struct State { +// current_gp: GracePeriod, +// // next_callbacks: Callbacks, +// } +// /// Owner of this [`RcuMonitor`]. +// pub(super) tracked struct RcuMonitorOwner {} +// impl DataPredicate for RcuMonitorOwner { +// closed spec fn predicate(&self, v: RcuMonitor) -> bool { +// true +// } +// } +// impl RcuMonitor { +// #[verifier::type_invariant] +// closed spec fn type_inv(self) -> bool { +// self.wf() +// } +// } +// pub(super) struct RcuMonitorPred; +// impl OncePredicate> for RcuMonitorPred { +// closed spec fn inv(self, v: AtomicDataWithOwner) -> bool { +// &&& v.permission@.predicate(v.data) +// &&& v.data.inv() +// } +// } +// impl Inv for RcuMonitor { +// closed spec fn inv(self) -> bool { +// self.wf() +// } +// } +// impl Inv for State { +// closed spec fn inv(self) -> bool { +// self.current_gp.inv() +// } +// } +// impl Inv for GracePeriod { +// closed spec fn inv(self) -> bool { +// true +// } +// } +// #[verus_verify] +// impl RcuMonitor { +// /// Creates a new RCU monitor. +// pub(super) fn new() -> Self { +// let state = SpinLock::new(State::new()); +// proof { +// use_type_invariant(&state); +// assert(state.type_inv()); +// } +// let res = RcuMonitor { +// is_monitoring: AtomicBool::new(Ghost(state), false, Tracked(false)), +// state, +// }; +// proof { +// use_type_invariant(&res.state); +// assert(res.state.type_inv()); +// assert(res.inv()); +// } +// res +// } +// /// Creates a new RCU monitor together with its tracked owner for `Once`. +// #[verus_spec(r => +// ensures +// r.inv(), +// r.data.inv(), +// RcuMonitorPred.inv(r), +// )] +// pub(super) fn new_data() -> AtomicDataWithOwner { +// let data = Self::new(); +// proof { +// use_type_invariant(&data); +// assert(data.inv()); +// } +// AtomicDataWithOwner { data, permission: Tracked(RcuMonitorOwner { }) } +// } +// fn is_monitoring(&self) -> bool { +// proof { +// use_type_invariant(self); +// } +// self.is_monitoring.load() +// } +// fn set_monitoring(&self, value: bool) { +// proof { +// use_type_invariant(self); +// } +// atomic_with_ghost! { +// self.is_monitoring => store(value); +// ghost g => { +// g = value; +// } +// } +// } +// } +// impl State { +// fn new() -> (res: Self) +// ensures +// res.inv(), +// { +// Self { current_gp: GracePeriod::new() } +// } +// } +// impl GracePeriod { +// fn new() -> (res: Self) +// ensures +// res.inv(), +// { +// Self { cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), is_complete: true } +// } +// } +// } // verus! +// /*use alloc::collections::VecDeque; +// use core::sync::atomic::{ +// AtomicBool, +// Ordering::{self, Relaxed}, +// }; +// use crate::{ +// cpu::{AtomicCpuSet, CpuId, CpuSet, PinCurrentCpu}, +// prelude::*, +// sync::SpinLock, +// task::atomic_mode::AsAtomicModeGuard, +// }; +// impl RcuMonitor { +// /// Creates a new RCU monitor. +// /// +// /// This function is used to initialize a singleton instance of `RcuMonitor`. +// /// The singleton instance is globally accessible via the `RCU_MONITOR`. +// pub(super) fn new() -> Self { +// Self { +// is_monitoring: AtomicBool::new(false), +// state: SpinLock::new(State::new()), +// } +// } +// pub(super) unsafe fn finish_grace_period(&self) { +// // Fast path +// if !self.is_monitoring.load(Relaxed) { +// return; +// } +// // Check if the current GP is complete after passing the quiescent state +// // on the current CPU. If GP is complete, take the callbacks of the current +// // GP. +// let callbacks = { +// let mut state = self.state.disable_irq().lock(); +// let cpu = state.as_atomic_mode_guard().current_cpu(); +// if state.current_gp.is_complete() { +// return; +// } +// state.current_gp.finish_grace_period(cpu); +// if !state.current_gp.is_complete() { +// return; +// } +// // Now that the current GP is complete, take its callbacks +// let current_callbacks = state.current_gp.take_callbacks(); +// // Check if we need to watch for a next GP +// if !state.next_callbacks.is_empty() { +// let callbacks = core::mem::take(&mut state.next_callbacks); +// state.current_gp.restart(callbacks); +// } else { +// self.is_monitoring.store(false, Relaxed); +// } +// current_callbacks +// }; +// // Invoke the callbacks to notify the completion of GP +// for f in callbacks { +// (f)(); +// } +// } +// pub(super) fn after_grace_period(&self, f: F) +// where +// F: FnOnce() + Send + 'static, +// { +// let mut state = self.state.disable_irq().lock(); +// state.next_callbacks.push_back(Box::new(f)); +// if !state.current_gp.is_complete() { +// return; +// } +// let callbacks = core::mem::take(&mut state.next_callbacks); +// state.current_gp.restart(callbacks); +// self.is_monitoring.store(true, Relaxed); +// } +// } +// impl State { +// fn new() -> Self { +// Self { +// current_gp: GracePeriod::new(), +// next_callbacks: VecDeque::new(), +// } +// } +// } +// impl GracePeriod { +// fn new() -> Self { +// Self { +// callbacks: Callbacks::new(), +// cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), +// is_complete: true, +// } +// } +// fn is_complete(&self) -> bool { +// self.is_complete +// } +// fn finish_grace_period(&mut self, this_cpu: CpuId) { +// self.cpu_mask.add(this_cpu, Ordering::Relaxed); +// if self.cpu_mask.load(Ordering::Relaxed).is_full() { +// self.is_complete = true; +// } +// } +// fn take_callbacks(&mut self) -> Callbacks { +// core::mem::take(&mut self.callbacks) +// } +// fn restart(&mut self, callbacks: Callbacks) { +// self.is_complete = false; +// self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); +// self.callbacks = callbacks; +// } +// } +// */ diff --git a/ostd/src/sync/rcu/non_null/mod.rs b/ostd/src/sync/rcu/non_null/mod.rs index 25ca3c959..096324a90 100644 --- a/ostd/src/sync/rcu/non_null/mod.rs +++ b/ostd/src/sync/rcu/non_null/mod.rs @@ -4,6 +4,7 @@ use alloc::{boxed::Box, sync::Arc}; use vstd::prelude::*; use vstd::raw_ptr::*; +use vstd::thread_view::Objective; use vstd_extra::prelude::*; mod either; @@ -41,7 +42,7 @@ pub unsafe trait NonNullPtr: Sized + 'static { where Self: 'a;*/ /// A verification-only permission type that represents the ownership of the memory managed by the pointer. - type Permission: Inv; + type Permission: Inv + Objective; /// The power of two of the pointer alignment. const ALIGN_BITS: u32; diff --git a/ostd/src/sync/rwlock.rs b/ostd/src/sync/rwlock.rs index 1a3415266..1a0442561 100644 --- a/ostd/src/sync/rwlock.rs +++ b/ostd/src/sync/rwlock.rs @@ -3,6 +3,7 @@ use vstd::atomic_ghost::*; use vstd::cell::{self, CellId, pcell::*}; use vstd::prelude::*; use vstd::resource::Loc; +use vstd::thread_view::Objective; use vstd_extra::resource::ghost_resource::{count_auth::*, count_ghost::*, csum::*, excl::*}; use vstd_extra::sum::*; use vstd_extra::{prelude::*, resource}; @@ -56,6 +57,10 @@ tracked struct RwPerms { read_guard_token: CountResource, MAX_READER>, } +unsafe impl Objective for RwPerms { + +} + ghost struct RwId { core_token_id: Loc, frac_id: Loc, diff --git a/ostd/src/sync/rwmutex.rs b/ostd/src/sync/rwmutex.rs index 4a8c5fa21..7bdb7a31d 100644 --- a/ostd/src/sync/rwmutex.rs +++ b/ostd/src/sync/rwmutex.rs @@ -3,6 +3,7 @@ use vstd::atomic_ghost::*; use vstd::cell::{self, CellId, pcell::*}; use vstd::prelude::*; use vstd::resource::Loc; +use vstd::thread_view::Objective; use vstd_extra::resource::ghost_resource::{count_auth::*, count_ghost::*, csum::*, excl::*}; use vstd_extra::sum::*; @@ -33,6 +34,10 @@ tracked struct RwPerms { read_guard_token: CountResource, MAX_READER>, } +unsafe impl Objective for RwPerms { + +} + ghost struct RwId { core_token_id: Loc, frac_id: Loc, diff --git a/ostd/src/task/mod.rs b/ostd/src/task/mod.rs index 1bf2f5a5e..dbc8c1047 100644 --- a/ostd/src/task/mod.rs +++ b/ostd/src/task/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 //! Tasks are the unit of code execution. use vstd::prelude::*; +use vstd::resource::Loc; /* pub mod atomic_mode; mod kernel_stack; */ @@ -22,8 +23,11 @@ use kernel_stack::KernelStack; use processor::current_task;*/ use spin::Once; // use utils::ForceSync; +pub(crate) use self::preempt::disable_preempt_in_context; pub use self::{ - preempt::{DisabledPreemptGuard, disable_preempt}, + preempt::{ + DisabledPreemptGuard, PreemptThreadViewSession, RunningTaskContext, disable_preempt, + }, /* scheduler::info::{AtomicCpuId, TaskScheduleInfo}, */ }; /* @@ -44,6 +48,8 @@ pub fn inject_post_schedule_handler(handler: fn()) { POST_SCHEDULE_HANDLER.call_once(|| handler); }*/ +verus! { + /// A task that executes a function to the end. /// /// Each task is associated with per-task data and an optional user space. @@ -71,17 +77,46 @@ pub struct Task { schedule_info: TaskScheduleInfo,*/ } -/* + +#[verus_verify] impl Task { + /// Returns the unique identifier of the task. + pub uninterp spec fn id(&self) -> Loc; + /// Gets the current task. /// /// It returns `None` if the function is called in the bootstrap context. pub fn current() -> Option { - let current_task = current_task()?; + Some(CurrentTask { }) + // let current_task = current_task()?; + // // SAFETY: `current_task` is the current task. + // Some(unsafe { CurrentTask::new(current_task) }) - // SAFETY: `current_task` is the current task. - Some(unsafe { CurrentTask::new(current_task) }) } +} + +/// The current task. +/// +/// This type is not `Send`, so it cannot outlive the current task. +/// +/// This type is also not `Sync`, so it can provide access to the local data of the current task. +pub struct CurrentTask; + +#[verifier::external] +impl !Send for Task { + +} + +#[verifier::external] +impl !Sync for Task { + +} + +} // verus! +/* +impl Task { + + pub(super) fn ctx(&self) -> &SyncUnsafeCell { &self.ctx @@ -267,13 +302,7 @@ impl TaskOptions { } } -/// The current task. -/// -/// This type is not `Send`, so it cannot outlive the current task. -/// -/// This type is also not `Sync`, so it can provide access to the local data of the current task. -#[derive(Debug)] -pub struct CurrentTask(NonNull); + // The intern `NonNull` contained by `CurrentTask` implies that `CurrentTask` is `!Send` and // `!Sync`. But it is still good to do this explicitly because these properties are key for diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 8e44ce12b..a58e4f19d 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -1,16 +1,1187 @@ // SPDX-License-Identifier: MPL-2.0 -use vstd::prelude::*; +use vstd::thread_view::{ThreadView as Irc11ThreadView, ViewSeen}; +use vstd::{prelude::*, resource::Loc}; +use vstd_extra::atomic_irc11::ThreadViewOrder; +use vstd_extra::resource::ghost_resource::count_ghost::{CountGhost, CountGhostResource}; -use crate::{sync::GuardTransfer /*, task::atomic_mode::InAtomicMode*/}; +use crate::{ + specs::sync::{ + rcu::RcuRetiredFacts, + rcu_cpu::{ + CpuRcuClosedGeneration, CpuRcuCoreBinding, CpuRcuParticipant, CpuRcuReaderFragment, + }, + }, + specs::task::cpu_core::{CpuCoreOwner, CpuCoreOwnerHandle, CpuCoreRegistration}, + sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ + task::scheduler::{SchedulerView, TaskThreadView}, +}; + +verus! { + +broadcast use vstd::thread_view::group_thread_view_axioms; + +pub const PREEMPT_SESSION_FRACTIONS: usize = 1 << 31; + +/// Proof token carried by a nested preemption-disable guard. +/// +/// Nested guards deliberately do not carry a `TaskThreadView`; the current +/// task's weak-memory view has already been checked out by the outermost +/// guard. This token only records that the guard was created while preemption +/// was already disabled. +pub tracked struct NestedPreemptToken { + depth_before: Ghost, +} + +impl NestedPreemptToken { + pub proof fn new(depth_before: nat) -> (tracked res: Self) + requires + depth_before > 0, + ensures + res.depth_before() == depth_before, + res.wf(), + { + NestedPreemptToken { depth_before: Ghost(depth_before) } + } + + pub closed spec fn depth_before(self) -> nat { + self.depth_before@ + } + + pub closed spec fn wf(self) -> bool { + self.depth_before() > 0 + } +} + +/// A shareable proof token tying a guard to the active preemption session. +/// +/// The token is a fractional resource-algebra fragment. Its generation changes +/// only while the session owns the full fraction, which is exactly the +/// quiescent state in which no preemption-disabled reader can remain live. +pub ghost struct PreemptSessionState { + task: Loc, + quiescent_generation: nat, +} + +pub tracked struct PreemptSessionToken { + token: CountGhost, +} + +impl PreemptSessionToken { + proof fn new_placeholder() -> (tracked res: Self) + ensures + res.wf(), + { + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); + assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); + let tracked mut tokens = CountGhostResource::< + PreemptSessionState, + PREEMPT_SESSION_FRACTIONS, + >::alloc(arbitrary()); + let tracked token = tokens.split_one(); + assert(token.frac() == 1); + let tracked res = PreemptSessionToken { token }; + assert(res.wf()); + res + } + + pub closed spec fn id(self) -> Loc { + self.token.id() + } + + pub closed spec fn task(self) -> Loc { + self.token@.task + } + + pub closed spec fn quiescent_generation(self) -> nat { + self.token@.quiescent_generation + } + + pub closed spec fn frac(self) -> int { + self.token.frac() + } + + pub closed spec fn wf(self) -> bool { + &&& self.frac() == 1 + &&& 0 < self.frac() <= PREEMPT_SESSION_FRACTIONS + } + + pub proof fn agree(tracked &self, tracked other: &Self) + requires + self.id() == other.id(), + ensures + self.task() == other.task(), + self.quiescent_generation() == other.quiescent_generation(), + { + self.token.agree(&other.token); + } +} + +/// The active preemption-disable session that owns the task-local view. +/// +/// Nested preemption-disable guards do not own this resource. They can only +/// use it by borrowing the session that was established by the outermost +/// preemption-disable scope. This keeps the model to one linear +/// `ThreadView` per running task while still allowing nested RCU code to +/// perform weak atomic operations. +pub tracked struct PreemptThreadViewSession { + task_view: TaskThreadView, + tokens: CountGhostResource, +} + +impl PreemptThreadViewSession { + pub proof fn new(tracked task_view: TaskThreadView, sched_view: SchedulerView) -> (tracked res: + Self) + requires + task_view.wf(sched_view), + ensures + res.scheduler() == task_view.scheduler(), + res.task() == task_view.task(), + res.view() == task_view.view(), + res.irc11_view() == task_view.irc11_view(), + res.session_task() == task_view.task(), + res.quiescent_generation() == 0, + res.available_fractions() == PREEMPT_SESSION_FRACTIONS, + res.wf_session_resource(), + res.wf(sched_view), + { + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); + assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); + let task = task_view.task(); + let ghost state = PreemptSessionState { task, quiescent_generation: 0 }; + let tracked tokens = CountGhostResource::< + PreemptSessionState, + PREEMPT_SESSION_FRACTIONS, + >::alloc(state); + assert(tokens.is_full()); + tokens.validate(); + assert(tokens.frac() == PREEMPT_SESSION_FRACTIONS); + let tracked res = PreemptThreadViewSession { task_view, tokens }; + assert(res.available_fractions() == PREEMPT_SESSION_FRACTIONS); + assert(res.wf_session_resource()); + assert(res.wf(sched_view)); + res + } + + pub closed spec fn task(self) -> Loc { + self.task_view.task() + } + + pub closed spec fn scheduler(self) -> Loc { + self.task_view.scheduler() + } + + pub closed spec fn view(self) -> Irc11ThreadView { + self.task_view.view() + } + + pub open spec fn irc11_view(self) -> Irc11ThreadView { + self.view() + } + + pub closed spec fn session_id(self) -> Loc { + self.tokens.id() + } + + pub closed spec fn session_task(self) -> Loc { + self.tokens@.task + } + + pub closed spec fn quiescent_generation(self) -> nat { + self.tokens@.quiescent_generation + } + + pub closed spec fn available_fractions(self) -> int { + self.tokens.frac() + } + + pub closed spec fn has_full_authority(self) -> bool { + self.tokens.is_full() + } + + pub closed spec fn wf_session_resource(self) -> bool { + &&& self.tokens.wf() + &&& self.session_task() == self.task() + &&& 0 < self.available_fractions() <= PREEMPT_SESSION_FRACTIONS + } + + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + &&& self.task_view.wf(sched_view) + &&& self.wf_session_resource() + } + + pub closed spec fn token_matches(self, token: PreemptSessionToken) -> bool { + &&& token.wf() + &&& token.id() == self.session_id() + &&& token.task() == self.session_task() + &&& token.quiescent_generation() == self.quiescent_generation() + } + + /// Splits one guard fragment from the active session. + /// + /// The session keeps at least one fraction after the split so future + /// agreement checks can still relate guard fragments back to the session. + pub proof fn tracked_split_guard_token(tracked &mut self) -> (tracked token: + PreemptSessionToken) + requires + old(self).wf_session_resource(), + old(self).available_fractions() > 1, + ensures + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() + 1 == old(self).available_fractions(), + final(self).wf_session_resource(), + token.wf(), + final(self).token_matches(token), + { + let tracked token = self.tokens.split_one(); + PreemptSessionToken { token } + } + + /// Returns a guard fragment when a preemption-disable guard is dropped. + pub proof fn tracked_return_guard_token(tracked &mut self, tracked token: PreemptSessionToken) + requires + old(self).wf_session_resource(), + old(self).token_matches(token), + ensures + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions() + token.frac(), + final(self).wf_session_resource(), + { + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); + let ghost old_frac = self.tokens.frac(); + let tracked PreemptSessionToken { token } = token; + let ghost returned_frac = token.frac(); + assert(returned_frac == 1); + self.tokens.combine(token); + assert(old_frac + returned_frac > PREEMPT_SESSION_FRACTIONS ==> false); + assert(old_frac + returned_frac <= PREEMPT_SESSION_FRACTIONS); + self.tokens.validate(); + assert(self.tokens.frac() == old_frac + returned_frac); + assert(0 < self.tokens.frac() <= PREEMPT_SESSION_FRACTIONS); + assert(self.tokens@.task == self.task_view.task()); + assert(self.wf_session_resource()); + } + + /// Borrows the single task-local `ThreadView` for weak atomic operations. + /// + /// After the borrow mutates the view, the caller must update the scheduler + /// snapshot with `SchedulerView::update_checked_out_task_view` before + /// relying on `wf` again. + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ViewSeen) + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).wf_session_resource() == old(self).wf_session_resource(), + final(self).view() == (*final(tv))@, + final(self).irc11_view() == (*final(tv))@, + { + let tracked token = self.task_view.tracked_borrow_thread_view_mut(); + token.tracked_borrow_mut() + } + + /// Borrows the task view while preserving an existing lower bound whenever + /// the atomic operation grows the native view. + proof fn tracked_borrow_thread_view_mut_above( + tracked &mut self, + lower: Irc11ThreadView, + ) -> (tracked tv: &mut ViewSeen) + requires + lower.spec_le(old(self).view()), + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).wf_session_resource() == old(self).wf_session_resource(), + final(self).view() == (*final(tv))@, + final(self).irc11_view() == (*final(tv))@, + old(self).view().spec_le((*final(tv))@) ==> lower.spec_le((*final(tv))@), + { + let ghost old_view = self.view(); + let tracked token = self.task_view.tracked_borrow_thread_view_mut(); + let tracked tv = token.tracked_borrow_mut(); + if old_view.spec_le((*final(tv))@) { + lower.lemma_spec_le_transitive(old_view, (*final(tv))@); + } + tv + } + + /// Borrows the native subjective view for an IRC11 atomic operation. + pub proof fn tracked_borrow_irc11_view_mut(tracked &mut self) -> (tracked view: &mut ViewSeen) + ensures + (*view)@ == old(self).irc11_view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == (*final(view))@, + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).wf_session_resource() == old(self).wf_session_resource(), + final(self).irc11_view() == (*final(view))@, + { + let tracked token = self.task_view.tracked_borrow_irc11_view_mut(); + token.tracked_borrow_mut() + } + + /// Advances the session's quiescent boundary. + /// + /// Updating the fractional resource requires full ownership. Therefore no + /// `PreemptSessionToken` from the previous generation can coexist with + /// this transition. Tokens split afterwards carry the new generation. + proof fn tracked_advance_quiescent_generation(tracked &mut self) -> (generation: nat) + requires + old(self).wf_session_resource(), + old(self).available_fractions() == PREEMPT_SESSION_FRACTIONS, + old(self).has_full_authority(), + ensures + generation == old(self).quiescent_generation(), + final(self).quiescent_generation() == generation + 1, + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).wf_session_resource(), + { + let ghost generation = self.quiescent_generation(); + let ghost state = PreemptSessionState { + task: self.task(), + quiescent_generation: generation + 1, + }; + self.tokens.update(state); + self.tokens.validate(); + assert(self.tokens.frac() == PREEMPT_SESSION_FRACTIONS); + assert(self.tokens@.task == self.task_view.task()); + generation + } + + /// Returns the checked-out view to the caller for scheduler check-in. + /// + /// This is the proof-side counterpart of dropping the outermost + /// preemption-disable scope: the session stops owning the task view, and + /// the caller can write it back with + /// `SchedulerGhostState::tracked_schedule_out`. + pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) + requires + self.wf_session_resource(), + self.available_fractions() == PREEMPT_SESSION_FRACTIONS, + ensures + res.scheduler() == self.scheduler(), + res.task() == self.task(), + res.view() == self.view(), + res.irc11_view() == self.irc11_view(), + { + self.task_view + } + + /// Returns the checked-out view while preserving its scheduler relation. + pub proof fn tracked_into_task_view_for_scheduler( + tracked self, + sched_view: SchedulerView, + ) -> (tracked res: TaskThreadView) + requires + self.wf(sched_view), + self.available_fractions() == PREEMPT_SESSION_FRACTIONS, + ensures + res.scheduler() == self.scheduler(), + res.task() == self.task(), + res.view() == self.view(), + res.irc11_view() == self.irc11_view(), + res.wf(sched_view), + { + self.task_view + } +} + +/// The proof-owned state for one task while it is running. +/// +/// The scheduler creates this context after checking out the task's +/// `TaskThreadView`. Every preemption-disable guard consumes one fractional +/// session token and increments `preempt_depth`; releasing the guard performs +/// the inverse transition. Consequently the context can only be returned to +/// the scheduler when no guard remains live. +pub tracked struct RunningTaskContext { + session: PreemptThreadViewSession, + core_handle: CpuCoreOwnerHandle, + rcu_participant: CpuRcuParticipant, + rcu_binding: CpuRcuCoreBinding, + preempt_depth: Ghost, + cpu: Ghost, +} + +impl RunningTaskContext { + /// Starts a running interval for a checked-out task view. + pub proof fn new( + tracked task_view: TaskThreadView, + tracked core_handle: CpuCoreOwnerHandle, + tracked rcu_participant: CpuRcuParticipant, + tracked rcu_binding: CpuRcuCoreBinding, + sched_view: SchedulerView, + cpu: crate::specs::mm::cpu::CpuId, + ) -> (tracked res: Self) + requires + task_view.wf(sched_view), + core_handle.wf(), + core_handle.cpu() == cpu, + core_handle.current_task() == Some(task_view.task()), + core_handle.expected_locals_key() == seq![rcu_participant.id()], + rcu_participant.wf(), + rcu_participant.cpu() == cpu, + rcu_participant.fraction() == 1real, + rcu_participant.view().spec_le(task_view.irc11_view()), + rcu_binding.registry() == task_view.scheduler(), + rcu_binding.cpu() == cpu, + rcu_binding.owner_id() == core_handle.id(), + rcu_binding.locals_key() == core_handle.expected_locals_key(), + rcu_binding.single_local_id() == rcu_participant.id(), + sched_view.cpu_has_rcu_participant(cpu), + sched_view.cpu_rcu_participant_id(cpu) == rcu_participant.id(), + sched_view.cpu_core_registration(cpu).owner_id == core_handle.id(), + sched_view.cpu_core_registration(cpu).locals_key == core_handle.expected_locals_key(), + !sched_view.cpu_rcu_participant_is_stored(cpu), + sched_view.current.contains_key(cpu), + sched_view.current[cpu] == Some(task_view.task()), + crate::specs::mm::cpu::online_cpus().contains(cpu), + ensures + res.scheduler() == task_view.scheduler(), + res.task() == task_view.task(), + res.view() == task_view.view(), + res.irc11_view() == task_view.irc11_view(), + res.cpu() == cpu, + res.core_owner_id() == core_handle.id(), + res.preempt_depth() == 0, + res.quiescent_generation() == 0, + res.available_fractions() == PREEMPT_SESSION_FRACTIONS, + res.rcu_participant_id() == rcu_participant.id(), + res.rcu_generation() == rcu_participant.generation(), + res.rcu_participant_view() == rcu_participant.view(), + res.rcu_fraction() == 1real, + res.rcu_binding().registry() == res.scheduler(), + res.rcu_binding().cpu() == cpu, + res.rcu_binding().single_local_id() == res.rcu_participant_id(), + res.wf(), + res.is_quiescent(), + res.wf_scheduler(sched_view), + { + let tracked session = PreemptThreadViewSession::new(task_view, sched_view); + let tracked res = RunningTaskContext { + session, + core_handle, + rcu_participant, + rcu_binding, + preempt_depth: Ghost(0), + cpu: Ghost(cpu), + }; + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); + assert(res.wf()); + assert(res.session.wf(sched_view)); + assert(res.wf_scheduler(sched_view)); + res + } + + pub closed spec fn task(self) -> Loc { + self.session.task() + } + + pub closed spec fn scheduler(self) -> Loc { + self.session.scheduler() + } + + pub closed spec fn view(self) -> Irc11ThreadView { + self.session.view() + } + + pub open spec fn irc11_view(self) -> Irc11ThreadView { + self.view() + } + + pub closed spec fn cpu(self) -> crate::specs::mm::cpu::CpuId { + self.cpu@ + } + + pub closed spec fn core_owner_id(self) -> Loc { + self.core_handle.id() + } + + pub closed spec fn session_id(self) -> Loc { + self.session.session_id() + } + + pub closed spec fn quiescent_generation(self) -> nat { + self.session.quiescent_generation() + } + + pub closed spec fn available_fractions(self) -> int { + self.session.available_fractions() + } + + pub closed spec fn has_full_authority(self) -> bool { + self.session.has_full_authority() + } + + pub closed spec fn preempt_depth(self) -> nat { + self.preempt_depth@ + } + + pub closed spec fn rcu_participant_id(self) -> Loc { + self.rcu_participant.id() + } + + pub closed spec fn rcu_generation(self) -> nat { + self.rcu_participant.generation() + } + + pub closed spec fn rcu_participant_view(self) -> Irc11ThreadView { + self.rcu_participant.view() + } + + pub closed spec fn rcu_fraction(self) -> real { + self.rcu_participant.fraction() + } + + pub closed spec fn rcu_binding(self) -> CpuRcuCoreBinding { + self.rcu_binding + } + + pub closed spec fn wf(self) -> bool { + &&& self.session.wf_session_resource() + &&& self.view() == self.irc11_view() + &&& self.available_fractions() + self.preempt_depth() == PREEMPT_SESSION_FRACTIONS + &&& self.core_handle.wf() + &&& self.core_handle.cpu() == self.cpu() + &&& self.core_handle.current_task() == Some(self.task()) + &&& self.core_handle.expected_locals_key() == seq![self.rcu_participant_id()] + &&& self.rcu_participant.wf() + &&& self.rcu_binding().registry() == self.scheduler() + &&& self.rcu_binding().cpu() == self.cpu() + &&& self.rcu_binding().owner_id() == self.core_owner_id() + &&& self.rcu_binding().locals_key() == self.core_handle.expected_locals_key() + &&& self.rcu_binding().single_local_id() == self.rcu_participant_id() + &&& self.rcu_participant.cpu() == self.cpu() + &&& crate::specs::mm::cpu::online_cpus().contains(self.cpu()) + &&& self.rcu_participant_view().spec_le(self.irc11_view()) + } + + /// The checked-out task view includes the persistent view of this CPU's + /// RCU participant. + pub proof fn lemma_rcu_participant_view_le(tracked &self) + requires + self.wf(), + ensures + self.rcu_participant_view().spec_le(self.irc11_view()), + { + } + + /// A running task is checked out on a CPU in the scheduler's online set. + pub proof fn lemma_cpu_online(tracked &self) + requires + self.wf(), + ensures + crate::specs::mm::cpu::online_cpus().contains(self.cpu()), + { + } + + /// Relates this running context to the scheduler snapshot from which its + /// task view was checked out. + pub closed spec fn wf_scheduler(self, sched_view: SchedulerView) -> bool { + &&& self.wf() + &&& self.session.wf(sched_view) + &&& sched_view.current.contains_key(self.cpu()) + &&& sched_view.current[self.cpu()] == Some(self.task()) + &&& sched_view.cpu_has_rcu_participant(self.cpu()) + &&& sched_view.cpu_rcu_participant_id(self.cpu()) == self.rcu_participant_id() + &&& sched_view.cpu_core_registration(self.cpu()).owner_id == self.core_owner_id() + &&& sched_view.cpu_core_registration(self.cpu()).locals_key == seq![ + self.rcu_participant_id(), + ] + &&& !sched_view.cpu_rcu_participant_is_stored(self.cpu()) + } + + /// Re-establishes the scheduler relation after the checked-out task view + /// has been updated to this context's current weak-memory view. + pub proof fn lemma_wf_scheduler(tracked &self, sched_view: SchedulerView) + requires + self.wf(), + sched_view.wf(), + sched_view.id == self.scheduler(), + sched_view.task_view_is_checked_out(self.task()), + sched_view.checked_out_views[self.task()] == self.view(), + sched_view.task_views.contains_key(self.task()), + sched_view.task_views[self.task()] == self.view(), + sched_view.current.contains_key(self.cpu()), + sched_view.current[self.cpu()] == Some(self.task()), + sched_view.cpu_has_rcu_participant(self.cpu()), + sched_view.cpu_rcu_participant_id(self.cpu()) == self.rcu_participant_id(), + sched_view.cpu_core_registration(self.cpu()).owner_id == self.core_owner_id(), + sched_view.cpu_core_registration(self.cpu()).locals_key == seq![ + self.rcu_participant_id(), + ], + !sched_view.cpu_rcu_participant_is_stored(self.cpu()), + ensures + self.wf_scheduler(sched_view), + { + self.session.task_view.lemma_wf(sched_view); + assert(self.session.wf(sched_view)); + } + + pub open spec fn is_quiescent(self) -> bool { + &&& self.preempt_depth() == 0 + &&& self.available_fractions() == PREEMPT_SESSION_FRACTIONS + &&& self.has_full_authority() + &&& self.rcu_fraction() == 1real + } + + /// Borrows the running task's persistent weak-memory view. + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ViewSeen) + requires + old(self).wf(), + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), + final(self).view() == (*final(tv))@, + old(self).view().spec_le((*final(tv))@) ==> final(self).wf(), + { + let ghost participant_view = self.rcu_participant_view(); + assert(self.irc11_view() == self.view()); + assert(participant_view.spec_le(self.view())); + self.session.tracked_borrow_thread_view_mut_above(participant_view) + } + + /// Borrows the running task's native IRC11 view. + pub proof fn tracked_borrow_irc11_view_mut(tracked &mut self) -> (tracked view: &mut ViewSeen) + requires + old(self).wf(), + ensures + (*view)@ == old(self).irc11_view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == (*final(view))@, + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), + final(self).irc11_view() == (*final(view))@, + old(self).irc11_view().spec_le((*final(view))@) ==> final(self).wf(), + { + self.session.tracked_borrow_irc11_view_mut() + } + + /// Starts one RCU reader from this CPU's persistent participant. + /// + /// Preemption must already be disabled. The returned fragment names the + /// participant's current CPU generation and remains live until the reader + /// guard is destroyed. + pub proof fn tracked_start_rcu_reader(tracked &mut self) -> (tracked reader: + CpuRcuReaderFragment) + requires + old(self).wf(), + old(self).preempt_depth() > 0, + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction() / 2real, + reader.wf(), + reader.participant_id() == old(self).rcu_participant_id(), + reader.cpu() == old(self).cpu(), + reader.generation() == old(self).rcu_generation(), + reader.participant_view() == old(self).rcu_participant_view(), + reader.fraction() == old(self).rcu_fraction() / 2real, + { + self.rcu_participant.tracked_start_reader_in_place(self.irc11_view()) + } + + /// Copies the persistent scheduler binding for a guard or quiescent report. + pub proof fn tracked_rcu_binding(tracked &self) -> (tracked binding: CpuRcuCoreBinding) + requires + self.wf(), + ensures + binding.registry() == self.scheduler(), + binding.cpu() == self.cpu(), + binding.owner_id() == self.core_owner_id(), + binding.locals_key() == seq![self.rcu_participant_id()], + binding.locals_key().len() == 1, + binding.single_local_id() == self.rcu_participant_id(), + { + self.rcu_binding.tracked_duplicate() + } + + /// Returns a completed reader to this CPU's persistent participant. + pub proof fn tracked_stop_rcu_reader(tracked &mut self, tracked reader: CpuRcuReaderFragment) + requires + old(self).wf(), + old(self).preempt_depth() > 0, + reader.wf(), + reader.participant_id() == old(self).rcu_participant_id(), + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction() + reader.fraction(), + { + self.rcu_participant.tracked_stop_reader_in_place(reader); + } + + /// Closes the current CPU participation generation at a quiescent point. + /// + /// Unlike [`Self::tracked_record_quiescent`], this transition is backed by + /// the persistent CPU participant PCM and returns an unforgeable token + /// that conflicts with every reader fragment from the closed generation. + pub proof fn tracked_report_rcu_quiescent(tracked &mut self) -> (tracked closed: + CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).is_quiescent(), + ensures + closed.wf(), + closed.scheduler() == old(self).scheduler(), + closed.participant_id() == old(self).rcu_participant_id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).rcu_generation(), + closed.view() == old(self).irc11_view(), + final(self).wf(), + final(self).is_quiescent(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation() + 1, + final(self).rcu_participant_view() == old(self).irc11_view(), + final(self).rcu_fraction() == 1real, + { + let tracked binding = self.rcu_binding.tracked_duplicate(); + self.rcu_participant.tracked_report_quiescent_in_place(binding, self.irc11_view()) + } + + /// Closes the current CPU generation while publishing retirement facts + /// whose detachment observations are covered by this task's current view. + pub proof fn tracked_report_rcu_quiescent_with( + tracked &mut self, + tracked learned: &RcuRetiredFacts, + ) -> (tracked closed: CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).is_quiescent(), + learned.observed_by(old(self).irc11_view()), + ensures + closed.wf(), + closed.scheduler() == old(self).scheduler(), + closed.participant_id() == old(self).rcu_participant_id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).rcu_generation(), + closed.view() == old(self).irc11_view(), + learned.records().subset_of(closed.known_retired()), + final(self).wf(), + final(self).is_quiescent(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation() + 1, + final(self).rcu_participant_view() == old(self).irc11_view(), + final(self).rcu_fraction() == 1real, + { + let tracked binding = self.rcu_binding.tracked_duplicate(); + self.rcu_participant.tracked_report_quiescent_with_in_place( + binding, + self.irc11_view(), + learned, + ) + } + + /// Records one quiescent boundary for this running session. + /// + /// The returned generation names the interval that has just ended. The + /// context advances to the next generation before another RCU reader can + /// split a preemption-session fragment. + pub proof fn tracked_record_quiescent(tracked &mut self) -> (generation: nat) + requires + old(self).wf(), + old(self).is_quiescent(), + ensures + generation == old(self).quiescent_generation(), + final(self).quiescent_generation() == generation + 1, + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), + final(self).wf(), + final(self).is_quiescent(), + { + self.session.tracked_advance_quiescent_generation() + } + + /// Ends a running interval and returns the updated task view to scheduler + /// ownership. The full-fraction requirement rules out live preempt guards. + pub proof fn tracked_into_task_view(tracked self) -> (tracked res: ( + TaskThreadView, + CpuCoreOwner, + )) + requires + self.wf(), + self.preempt_depth() == 0, + self.rcu_fraction() == 1real, + ensures + res.0.scheduler() == self.scheduler(), + res.0.task() == self.task(), + res.0.view() == self.view(), + res.0.irc11_view() == self.irc11_view(), + res.1.id() == self.core_owner_id(), + res.1.cpu() == self.cpu(), + res.1.current_task() == Some(self.task()), + res.1.locals_key() == seq![self.rcu_participant_id()], + res.1.registration() == (CpuCoreRegistration { + owner_id: self.core_owner_id(), + locals_key: seq![self.rcu_participant_id()], + }), + res.1.locals().id() == self.rcu_participant_id(), + res.1.locals().generation() == self.rcu_generation(), + res.1.locals().view() == self.rcu_participant_view(), + res.1.locals().fraction() == 1real, + res.1.locals().view().spec_le(res.0.irc11_view()), + res.1.wf(), + { + assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); + let ghost cpu = self.cpu(); + let ghost task = self.task(); + let ghost core_owner_id = self.core_owner_id(); + let ghost rcu_participant_id = self.rcu_participant_id(); + let ghost core_view = self.core_handle@; + assert(self.core_handle.cpu() == cpu); + assert(self.core_handle.current_task() == Some(task)); + let tracked core = self.core_handle.tracked_restore(self.rcu_participant); + assert(core@ == core_view); + assert(core.id() == core_owner_id); + assert(core.cpu() == cpu); + assert(core.current_task() == Some(task)); + assert(core.locals_key() == seq![rcu_participant_id]); + assert(core.registration() == (CpuCoreRegistration { + owner_id: core_owner_id, + locals_key: seq![rcu_participant_id], + })); + (self.session.tracked_into_task_view(), core) + } + + /// Scheduler-facing form of `tracked_into_task_view` that preserves the + /// checked-out token's relation to the supplied scheduler snapshot. + pub proof fn tracked_into_task_view_for_scheduler( + tracked self, + sched_view: SchedulerView, + ) -> (tracked res: (TaskThreadView, CpuCoreOwner)) + requires + self.wf_scheduler(sched_view), + self.is_quiescent(), + ensures + res.0.scheduler() == self.scheduler(), + res.0.task() == self.task(), + res.0.view() == self.view(), + res.0.irc11_view() == self.irc11_view(), + res.0.wf(sched_view), + res.1.id() == self.core_owner_id(), + res.1.cpu() == self.cpu(), + res.1.current_task() == Some(self.task()), + res.1.locals_key() == seq![self.rcu_participant_id()], + res.1.registration() == (CpuCoreRegistration { + owner_id: self.core_owner_id(), + locals_key: seq![self.rcu_participant_id()], + }), + res.1.locals().id() == self.rcu_participant_id(), + res.1.locals().generation() == self.rcu_generation(), + res.1.locals().view() == self.rcu_participant_view(), + res.1.locals().fraction() == 1real, + res.1.locals().view().spec_le(res.0.irc11_view()), + res.1.wf(), + { + assert(self.preempt_depth() == 0); + assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); + assert(self.session.wf(sched_view)); + let ghost cpu = self.cpu(); + let ghost task = self.task(); + let ghost core_owner_id = self.core_owner_id(); + let ghost rcu_participant_id = self.rcu_participant_id(); + let ghost core_view = self.core_handle@; + assert(self.core_handle.cpu() == cpu); + assert(self.core_handle.current_task() == Some(task)); + let tracked task_view = self.session.tracked_into_task_view_for_scheduler(sched_view); + let tracked core = self.core_handle.tracked_restore(self.rcu_participant); + assert(core@ == core_view); + assert(core.id() == core_owner_id); + assert(core.cpu() == cpu); + assert(core.current_task() == Some(task)); + assert(core.locals_key() == seq![rcu_participant_id]); + assert(core.registration() == (CpuCoreRegistration { + owner_id: core_owner_id, + locals_key: seq![rcu_participant_id], + })); + (task_view, core) + } +} + +/// Proof resource carried by a `DisabledPreemptGuard`. +/// +/// The guard records whether this preemption-disable scope is the outermost +/// one or a nested one. It deliberately does not own the checked-out +/// `TaskThreadView`; that linear resource lives in `PreemptThreadViewSession`. +/// This keeps nested guards from minting another `ThreadView` while allowing +/// them to borrow the active session. +pub tracked enum PreemptGuardResource { + Outermost(PreemptSessionToken), + Nested { tracked session: PreemptSessionToken, tracked nested: NestedPreemptToken }, +} + +impl PreemptGuardResource { + proof fn new_placeholder() -> (tracked res: Self) + ensures + res.wf(arbitrary()), + { + let tracked token = PreemptSessionToken::new_placeholder(); + PreemptGuardResource::Outermost(token) + } + + pub closed spec fn is_outermost(self) -> bool { + self is Outermost + } + + pub closed spec fn is_nested(self) -> bool { + self is Nested + } + + pub closed spec fn session_token(self) -> PreemptSessionToken + recommends + self is Outermost || self is Nested, + { + match self { + PreemptGuardResource::Outermost(token) => token, + PreemptGuardResource::Nested { session, nested: _ } => session, + } + } + + pub closed spec fn session_id(self) -> Loc { + self.session_token().id() + } + + pub closed spec fn task(self) -> Loc { + self.session_token().task() + } + + pub closed spec fn quiescent_generation(self) -> nat { + self.session_token().quiescent_generation() + } + + pub closed spec fn wf(self, _sched_view: SchedulerView) -> bool { + match self { + PreemptGuardResource::Outermost(token) => token.wf(), + PreemptGuardResource::Nested { session, nested } => { + &&& session.wf() + &&& nested.wf() + }, + } + } + + pub closed spec fn matches_session(self, session: PreemptThreadViewSession) -> bool { + &&& self.wf(arbitrary()) + &&& session.token_matches(self.session_token()) + } + + pub closed spec fn matches_context(self, context: RunningTaskContext) -> bool { + &&& context.wf() + &&& context.preempt_depth() > 0 + &&& self.matches_session(context.session) + } + + /// Returns this guard's session fragment to its owning session. + pub proof fn tracked_return_to_session( + tracked self, + tracked session: &mut PreemptThreadViewSession, + ) + requires + old(session).wf_session_resource(), + self.matches_session(*old(session)), + ensures + final(session).wf_session_resource(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).view() == old(session).view(), + final(session).irc11_view() == old(session).irc11_view(), + final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions() + 1, + { + match self { + PreemptGuardResource::Outermost(token) => { + session.tracked_return_guard_token(token); + }, + PreemptGuardResource::Nested { session: token, nested: _ } => { + session.tracked_return_guard_token(token); + }, + } + } +} + +impl RunningTaskContext { + /// Performs the proof transition corresponding to incrementing the + /// executable preemption counter. + pub proof fn tracked_disable_preempt(tracked &mut self) -> (tracked resource: + PreemptGuardResource) + requires + old(self).wf(), + old(self).available_fractions() > 1, + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() + 1 == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth() + 1, + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), + resource.matches_context(*final(self)), + resource.is_outermost() <==> old(self).preempt_depth() == 0, + resource.is_nested() <==> old(self).preempt_depth() > 0, + { + let ghost depth_before = self.preempt_depth@; + let tracked token = self.session.tracked_split_guard_token(); + let tracked resource = if depth_before == 0 { + PreemptGuardResource::Outermost(token) + } else { + let tracked nested = NestedPreemptToken::new(depth_before); + PreemptGuardResource::Nested { session: token, nested } + }; + self.preempt_depth = Ghost(depth_before + 1); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); + assert(self.wf()); + resource + } + + /// Performs the inverse transition when a preemption-disable guard is + /// consumed. + pub proof fn tracked_enable_preempt(tracked &mut self, tracked resource: PreemptGuardResource) + requires + old(self).wf(), + old(self).preempt_depth() > 0, + resource.matches_context(*old(self)), + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions() + 1, + final(self).preempt_depth() + 1 == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), + { + let ghost old_depth = self.preempt_depth@; + resource.tracked_return_to_session(&mut self.session); + self.preempt_depth = Ghost((old_depth - 1) as nat); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); + assert(self.wf()); + } +} /// A guard for disable preempt. -#[verus_verify] #[clippy::has_significant_drop] #[must_use] #[derive(Debug)] pub struct DisabledPreemptGuard { // This private field prevents user from constructing values of this type directly. _private: (), + // Proof-only guard resource. + // + // The guard only records whether this scope is outermost or nested. The + // checked-out `TaskThreadView` is owned by `PreemptThreadViewSession`. + tracked_resource: Tracked>, } /* impl !Send for DisabledPreemptGuard {} @@ -18,14 +1189,255 @@ pub struct DisabledPreemptGuard { // SAFETY: The guard disables preemptions, which meets the second // sufficient condition for atomic mode. unsafe impl InAtomicMode for DisabledPreemptGuard {} +*/ impl DisabledPreemptGuard { - fn new() -> Self { + fn new(Tracked(tracked_resource): Tracked) -> (res: DisabledPreemptGuard) + requires + tracked_resource.wf(arbitrary()), + ensures + res.wf(arbitrary()), + res.tracked_resource@ == Some(tracked_resource), + { + // The CPU-local backend is outside the current Verus dependency + // closure, but executable builds must still perform the real + // preemption-disable transition. + #[cfg(not(verus_keep_ghost))] super::cpu_local::inc_guard_count(); - Self { _private: () } + Self { _private: (), tracked_resource: Tracked(Some(tracked_resource)) } } } -*/ + +impl DisabledPreemptGuard { + pub(crate) closed spec fn has_resource(&self) -> bool { + self.tracked_resource@ is Some + } + + closed spec fn resource(&self) -> PreemptGuardResource + recommends + self.tracked_resource@ is Some, + { + self.tracked_resource@->Some_0 + } + + pub closed spec fn is_outermost(&self) -> bool { + self.resource().is_outermost() + } + + pub closed spec fn is_nested(&self) -> bool { + self.resource().is_nested() + } + + pub closed spec fn wf(&self, sched_view: SchedulerView) -> bool { + &&& self.tracked_resource@ is Some + &&& self.resource().wf(sched_view) + } + + pub closed spec fn matches_session(&self, session: PreemptThreadViewSession) -> bool { + &&& self.tracked_resource@ is Some + &&& self.resource().matches_session(session) + } + + pub closed spec fn matches_context(&self, context: RunningTaskContext) -> bool { + &&& self.tracked_resource@ is Some + &&& self.resource().matches_context(context) + } + + pub closed spec fn quiescent_generation(&self) -> nat { + self.resource().quiescent_generation() + } + + /// Extracts the positive preemption depth witnessed by this guard. + pub proof fn lemma_matches_context_depth(&self, tracked context: &RunningTaskContext) + requires + self.matches_context(*context), + ensures + context.preempt_depth() > 0, + self.quiescent_generation() == context.quiescent_generation(), + { + } + + /// A live preemption guard rules out a quiescent report from the same + /// running context. Returning every session fraction is therefore a + /// necessary proof step before the monitor can close this generation. + pub proof fn lemma_blocks_quiescent_report(&self, tracked context: &RunningTaskContext) + requires + self.matches_context(*context), + ensures + !context.is_quiescent(), + { + self.lemma_matches_context_depth(context); + } + + /// Changing only the task's weak-memory view preserves this guard's + /// relation to the running context. + pub proof fn lemma_matches_context_preserved( + &self, + before: RunningTaskContext, + tracked after: &RunningTaskContext, + ) + requires + self.matches_context(before), + after.wf(), + after.task() == before.task(), + after.scheduler() == before.scheduler(), + after.session_id() == before.session_id(), + after.quiescent_generation() == before.quiescent_generation(), + after.available_fractions() == before.available_fractions(), + after.preempt_depth() == before.preempt_depth(), + ensures + self.matches_context(*after), + { + assert(before.session.session_task() == before.task()); + assert(after.session.session_task() == after.task()); + assert(self.resource().session_token().task() == before.task()); + assert(self.resource().session_token().task() == after.task()); + assert(self.resource().session_token().quiescent_generation() + == before.quiescent_generation()); + assert(self.resource().session_token().quiescent_generation() + == after.quiescent_generation()); + assert(after.session.token_matches(self.resource().session_token())); + } + + /// Borrows the running task's view while this guard witnesses that + /// preemption is disabled. Both outermost and nested guards use the same + /// context-owned view. + pub proof fn tracked_borrow_thread_view_mut_from_context<'context>( + tracked context: &'context mut RunningTaskContext, + guard: &DisabledPreemptGuard, + ) -> (tracked tv: &'context mut ViewSeen) + requires + old(context).wf(), + guard.matches_context(*old(context)), + ensures + (*tv)@ == old(context).view(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() == old(context).available_fractions(), + final(context).has_full_authority() == old(context).has_full_authority(), + final(context).preempt_depth() == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + final(context).view() == (*final(tv))@, + old(context).view().spec_le((*final(tv))@) ==> final(context).wf(), + old(context).view().spec_le((*final(tv))@) ==> guard.matches_context(*final(context)), + { + context.tracked_borrow_thread_view_mut() + } + + /// Borrows the current task's native IRC11 view while preemption is disabled. + pub proof fn tracked_borrow_irc11_view_mut_from_context<'context>( + tracked context: &'context mut RunningTaskContext, + guard: &DisabledPreemptGuard, + ) -> (tracked view: &'context mut ViewSeen) + requires + old(context).wf(), + guard.matches_context(*old(context)), + ensures + (*view)@ == old(context).irc11_view(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).view() == (*final(view))@, + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() == old(context).available_fractions(), + final(context).has_full_authority() == old(context).has_full_authority(), + final(context).preempt_depth() == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + final(context).irc11_view() == (*final(view))@, + old(context).irc11_view().spec_le((*final(view))@) ==> final(context).wf(), + old(context).irc11_view().spec_le((*final(view))@) ==> guard.matches_context( + *final(context), + ), + { + context.tracked_borrow_irc11_view_mut() + } + + /// Returns this guard's fractional witness and decrements the modeled + /// preemption depth. + /// + /// The proof resource is stored in an `Option` so a containing guard's + /// standard `Drop::drop(&mut self)` can consume it exactly once. The + /// executable preemption counter is still decremented by this guard's Rust + /// destructor. + pub(crate) fn release_in_place_to_context( + &mut self, + Tracked(context): Tracked<&mut RunningTaskContext>, + ) + requires + old(context).wf(), + old(context).preempt_depth() > 0, + self.matches_context(*old(context)), + ensures + final(context).wf(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).view() == old(context).view(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() == old(context).available_fractions() + 1, + final(context).preempt_depth() + 1 == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + !final(self).has_resource(), + opens_invariants none + no_unwind + { + proof_decl! { + let tracked resource = self.tracked_resource.borrow_mut().tracked_take(); + } + proof { + context.tracked_enable_preempt(resource); + } + } + + /// Consuming compatibility wrapper for callers that do not need standard + /// destructor integration. + pub(crate) fn release_to_context(self, Tracked(context): Tracked<&mut RunningTaskContext>) + requires + old(context).wf(), + old(context).preempt_depth() > 0, + self.matches_context(*old(context)), + ensures + final(context).wf(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).view() == old(context).view(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() == old(context).available_fractions() + 1, + final(context).preempt_depth() + 1 == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + { + let mut this = self; + this.release_in_place_to_context(Tracked(context)); + } +} + +} // verus! +#[cfg(not(verus_keep_ghost))] +impl Drop for DisabledPreemptGuard { + fn drop(&mut self) { + super::cpu_local::dec_guard_count(); + } +} + #[verus_verify] impl GuardTransfer for DisabledPreemptGuard { #[verifier::external_body] @@ -34,16 +1446,55 @@ impl GuardTransfer for DisabledPreemptGuard { } } -/* -impl Drop for DisabledPreemptGuard { - fn drop(&mut self) { - super::cpu_local::dec_guard_count(); - } -} */ +verus! { /// Disables preemption. -#[verifier::external_body] -pub fn disable_preempt() -> DisabledPreemptGuard { - // DisabledPreemptGuard::new() - unimplemented!() +/// +/// TODO: This API is still unsound. +pub fn disable_preempt() -> (res: DisabledPreemptGuard) + ensures + res.wf(arbitrary()), +{ + proof_decl! { + let tracked tracked_resource = PreemptGuardResource::new_placeholder(); + } + DisabledPreemptGuard::new(Tracked(tracked_resource)) } + +/// Disables preemption inside the current running-task context. +/// +/// This has the same executable effect as [`disable_preempt`]. Its additional +/// tracked argument updates the modeled preemption depth and ties the guard to +/// the task view checked out by the scheduler. +pub(crate) fn disable_preempt_in_context( + Tracked(context): Tracked<&mut RunningTaskContext>, +) -> (res: DisabledPreemptGuard) + requires + old(context).wf(), + old(context).available_fractions() > 1, + ensures + final(context).wf(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).view() == old(context).view(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() + 1 == old(context).available_fractions(), + final(context).preempt_depth() == old(context).preempt_depth() + 1, + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + res.has_resource(), + res.is_outermost() <==> old(context).preempt_depth() == 0, + res.is_nested() <==> old(context).preempt_depth() > 0, + res.matches_context(*final(context)), +{ + proof_decl! { + let tracked resource = context.tracked_disable_preempt(); + } + DisabledPreemptGuard::new(Tracked(resource)) +} + +} // verus! diff --git a/ostd/src/task/preempt/mod.rs b/ostd/src/task/preempt/mod.rs index 101aac5c3..561d6e5e0 100644 --- a/ostd/src/task/preempt/mod.rs +++ b/ostd/src/task/preempt/mod.rs @@ -1,8 +1,13 @@ // SPDX-License-Identifier: MPL-2.0 -// pub(super) mod cpu_local; +#[cfg(not(verus_keep_ghost))] +pub(super) mod cpu_local; mod guard; -pub use self::guard::{DisabledPreemptGuard, disable_preempt}; +pub(crate) use self::guard::disable_preempt_in_context; +pub use self::guard::{ + DisabledPreemptGuard, PreemptSessionToken, PreemptThreadViewSession, RunningTaskContext, + disable_preempt, +}; /* /// Halts the CPU until interrupts if no preemption is required. /// diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 0f31b0af1..f6e3085ab 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -63,6 +63,1439 @@ //! Violating this invariant—e.g., running the same task on two CPUs concurrently— //! can have catastrophic consequences, //! as the task's stack and internal state may be corrupted by concurrent modifications. +use vstd::resource::map::GhostMapAuth; +use vstd::{map::Map, prelude::*, resource::Loc}; +use vstd_extra::atomic_irc11::{ThreadView as Irc11ThreadView, ThreadViewOrder, ThreadViewToken}; + +use super::{Task, preempt::RunningTaskContext}; +use crate::{ + specs::{ + mm::cpu::CpuId, + sync::rcu_cpu::CpuRcuParticipant, + task::cpu_core::{ + CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreOwnerHandle, CpuCoreRegistration, + }, + }, + sync::{OnceImpl, RoArc, TrivialPred}, +}; + +verus! { + +broadcast use vstd::thread_view::group_thread_view_axioms; + +/// A task-like object that can be identified in scheduler ghost state. +pub trait Schedulable { + spec fn sched_id(&self) -> Loc; +} + +impl Schedulable for Task { + open spec fn sched_id(&self) -> Loc { + self.id() + } +} + +#[derive(PartialEq, Eq, Clone, Copy)] +pub ghost enum TaskSchedState { + New, + Runnable, + Blocked, + Running, + Exited, +} + +/// High-level model for scheduler-owned weak-memory views. +/// +/// The scheduler proof state has two layers. `SchedulerView` is the copyable +/// ghost snapshot used in specifications: it records scheduling state and the +/// current weak-memory view for each live task. `SchedulerGhostState` is the +/// authoritative tracked root that stores both that snapshot and the actual +/// linear `ThreadView` resources. +/// +/// When a task is scheduled in, its `ThreadView` is checked out of +/// `SchedulerGhostState` and moved into a `RunningTaskContext`. While it is +/// checked out, +/// `SchedulerView::task_views` remains the logical source of truth, but the +/// ownership partition records that the view is in `checked_out_views` rather +/// than `stored_views`. Weak-memory operations mutate the token's +/// `ThreadView`; the scheduler view must be updated with +/// `update_checked_out_task_view` to keep the logical snapshot synchronized. +/// A quiescent schedule-out checks the updated token back into `stored_views`. +/// It also joins the outgoing task view into the persistent view of that CPU. +/// The next task scheduled on the CPU joins that CPU view into its checked-out +/// task view. Thus observations survive both task migration and context +/// switches without minting a fresh `ThreadView`. +/// +/// In short, the resource flow is: +/// `stored task view + CPU view -> RunningTaskContext -> stored task view + CPU view`. +/// Scheduler-policy transitions may change runqueues, current tasks, and task +/// states only through `same_thread_view_ownership`, which frames all three +/// weak-memory ownership maps. +/// +/// Abstract proof view of the scheduler state. +/// +/// `runqueues` contains runnable-but-not-current tasks, `current` contains +/// the task currently running on each CPU, and `state` records the scheduler +/// state for every known task. `task_views` is the logical per-task +/// weak-memory view. `stored_views` records views still owned by the scheduler +/// resource; `checked_out_views` records views temporarily held by guards. +/// `cpu_views` persists observations across context switches on each CPU. +/// `cpu_core_registrations` permanently binds each registered CPU to one core +/// owner and its stable CPU-local aggregate. `stored_cpu_cores` records which +/// complete owners are currently held by the scheduler. +pub ghost struct SchedulerView { + pub id: Loc, + pub runqueues: Map>, + pub current: Map>, + pub state: Map, + pub task_views: Map, + pub stored_views: Map, + pub checked_out_views: Map, + pub cpu_views: Map, + pub cpu_core_registrations: Map, + pub stored_cpu_cores: Set, +} + +impl SchedulerView { + /// Initial scheduler state before any task has been registered. + pub open spec fn empty(id: Loc) -> Self { + SchedulerView { + id, + runqueues: Map::empty(), + current: Map::empty(), + state: Map::empty(), + task_views: Map::empty(), + stored_views: Map::empty(), + checked_out_views: Map::empty(), + cpu_views: Map::empty(), + cpu_core_registrations: Map::empty(), + stored_cpu_cores: Set::empty(), + } + } + + pub proof fn lemma_empty_wf(id: Loc) + ensures + Self::empty(id).wf(), + { + } + + pub open spec fn task_is_known(self, task: Loc) -> bool { + self.state.contains_key(task) + } + + pub open spec fn task_has_thread_view(self, task: Loc) -> bool { + self.task_views.contains_key(task) + } + + pub open spec fn task_thread_view(self, task: Loc) -> Irc11ThreadView + recommends + self.task_has_thread_view(task), + { + self.task_views[task] + } + + pub open spec fn task_view_is_stored(self, task: Loc) -> bool { + self.stored_views.contains_key(task) + } + + pub open spec fn task_view_is_checked_out(self, task: Loc) -> bool { + self.checked_out_views.contains_key(task) + } + + pub open spec fn cpu_has_thread_view(self, cpu: CpuId) -> bool { + self.cpu_views.contains_key(cpu) + } + + pub open spec fn cpu_thread_view(self, cpu: CpuId) -> Irc11ThreadView + recommends + self.cpu_has_thread_view(cpu), + { + self.cpu_views[cpu] + } + + pub open spec fn cpu_has_rcu_participant(self, cpu: CpuId) -> bool { + self.cpu_core_registrations.contains_key(cpu) + && self.cpu_core_registrations[cpu].locals_key.len() == 1 + } + + pub open spec fn cpu_rcu_participant_id(self, cpu: CpuId) -> Loc + recommends + self.cpu_has_rcu_participant(cpu), + { + self.cpu_core_registrations[cpu].locals_key[0] + } + + pub open spec fn cpu_rcu_participant_is_stored(self, cpu: CpuId) -> bool { + self.stored_cpu_cores.contains(cpu) + } + + pub open spec fn cpu_core_registration(self, cpu: CpuId) -> CpuCoreRegistration + recommends + self.cpu_core_registrations.contains_key(cpu), + { + self.cpu_core_registrations[cpu] + } + + /// The scheduling policy changed no weak-memory ownership state. + /// + /// This relation deliberately ignores runqueues, current tasks, and task + /// scheduling states. A concrete scheduler may update those fields while + /// choosing where a task runs, but it must leave all three view maps to the + /// authoritative weak-memory transitions below. + pub open spec fn same_thread_view_ownership(self, other: Self) -> bool { + &&& self.id == other.id + &&& self.task_views == other.task_views + &&& self.stored_views == other.stored_views + &&& self.checked_out_views == other.checked_out_views + &&& self.cpu_views == other.cpu_views + &&& self.cpu_core_registrations == other.cpu_core_registrations + &&& self.stored_cpu_cores == other.stored_cpu_cores + } + + pub open spec fn task_in_runqueue(self, task: Loc) -> bool { + exists|cpu: CpuId, idx: int| + self.runqueues.contains_key(cpu) && 0 <= idx && idx < self.runqueues[cpu].len() && ( + #[trigger] self.runqueues[cpu][idx]) == task + } + + pub open spec fn task_is_current(self, task: Loc) -> bool { + exists|cpu: CpuId| #[trigger] + self.current.contains_key(cpu) && self.current[cpu] is Some && self.current[cpu]->0 + == task + } + + pub open spec fn task_is_runnable(self, task: Loc) -> bool { + &&& self.state.contains_key(task) + &&& (self.state[task] is Runnable || self.state[task] is Running) + } + + pub open spec fn task_is_live(self, task: Loc) -> bool { + &&& self.state.contains_key(task) + &&& !(self.state[task] is Exited) + } + + /// States the scheduler's main safety invariant for one task id. + /// + /// A task may occur at most once across all runqueues, at most once in the + /// `current` map, and never in both places at the same time. + pub open spec fn no_duplicate_task(self, task: Loc) -> bool { + &&& forall|cpu1: CpuId, cpu2: CpuId, idx1: int, idx2: int| + #![trigger self.runqueues[cpu1][idx1], self.runqueues[cpu2][idx2]] + self.runqueues.contains_key(cpu1) && self.runqueues.contains_key(cpu2) && 0 <= idx1 + && idx1 < self.runqueues[cpu1].len() && 0 <= idx2 && idx2 + < self.runqueues[cpu2].len() && self.runqueues[cpu1][idx1] == task + && self.runqueues[cpu2][idx2] == task ==> cpu1 == cpu2 && idx1 == idx2 + &&& forall|cpu1: CpuId, cpu2: CpuId| + #![trigger self.current.contains_key(cpu1), self.current.contains_key(cpu2)] + self.current.contains_key(cpu1) && self.current.contains_key(cpu2) + && self.current[cpu1] is Some && self.current[cpu2] is Some && self.current[cpu1]->0 + == task && self.current[cpu2]->0 == task ==> cpu1 == cpu2 + &&& !(self.task_in_runqueue(task) && self.task_is_current(task)) + } + + /// Registers a task and installs its initial weak-memory view. + /// + /// Registration is the only transition that introduces a task view. A new + /// task starts with the empty view and remains in `New` until the scheduler + /// enqueues it. In particular, a caller cannot transfer observations from + /// another task into a newly-created task. + pub open spec fn register_task(self, task: Loc) -> SchedulerView + recommends + !self.state.contains_key(task), + { + SchedulerView { + state: self.state.insert(task, TaskSchedState::New), + task_views: self.task_views.insert(task, Irc11ThreadView::empty()), + stored_views: self.stored_views.insert(task, Irc11ThreadView::empty()), + ..self + } + } + + /// Registers the persistent weak-memory view for one CPU. + /// + /// Scheduler policy may populate that CPU's runqueue/current slot only + /// after this transition. The initial empty view carries no observations. + pub open spec fn register_cpu( + self, + cpu: CpuId, + registration: CpuCoreRegistration, + ) -> SchedulerView + recommends + !self.cpu_views.contains_key(cpu), + valid_cpu(cpu), + registration.locals_key.len() == 1, + { + SchedulerView { + cpu_views: self.cpu_views.insert(cpu, Irc11ThreadView::empty()), + cpu_core_registrations: self.cpu_core_registrations.insert(cpu, registration), + stored_cpu_cores: self.stored_cpu_cores.insert(cpu), + ..self + } + } + + pub proof fn lemma_register_cpu_preserves_wf( + self, + cpu: CpuId, + registration: CpuCoreRegistration, + ) + requires + self.wf(), + !self.cpu_views.contains_key(cpu), + valid_cpu(cpu), + registration.locals_key.len() == 1, + ensures + self.register_cpu(cpu, registration).wf(), + self.register_cpu(cpu, registration).cpu_has_thread_view(cpu), + self.register_cpu(cpu, registration).cpu_thread_view(cpu) == Irc11ThreadView::empty(), + self.register_cpu(cpu, registration).cpu_core_registration(cpu) == registration, + self.register_cpu(cpu, registration).cpu_rcu_participant_id(cpu) + == registration.locals_key[0], + self.register_cpu(cpu, registration).cpu_rcu_participant_is_stored(cpu), + { + } + + /// Task registration preserves the scheduler ownership partition. + pub proof fn lemma_register_task_preserves_wf(self, task: Loc) + requires + self.wf(), + !self.state.contains_key(task), + ensures + self.register_task(task).wf(), + self.register_task(task).state[task] is New, + self.register_task(task).task_thread_view(task) == Irc11ThreadView::empty(), + self.register_task(task).task_view_is_stored(task), + !self.register_task(task).task_view_is_checked_out(task), + { + } + + /// Moves the current task's weak-memory view out of scheduler storage. + /// + /// This is the logical transition for the outermost preemption-disable + /// guard. The total logical snapshot `task_views` is unchanged; only the + /// ownership partition changes from `stored_views` to `checked_out_views`. + pub open spec fn checkout_task_view(self, cpu: CpuId) -> SchedulerView + recommends + self.current.contains_key(cpu), + self.current[cpu] is Some, + self.cpu_has_thread_view(cpu), + self.task_view_is_stored(self.current[cpu]->0), + !self.task_view_is_checked_out(self.current[cpu]->0), + { + let task = self.current[cpu]->0; + let joined = self.stored_views[task].join(self.cpu_views[cpu]); + SchedulerView { + task_views: self.task_views.insert(task, joined), + stored_views: self.stored_views.remove(task), + checked_out_views: self.checked_out_views.insert(task, joined), + stored_cpu_cores: self.stored_cpu_cores.remove(cpu), + ..self + } + } + + /// Records a mutation to a checked-out `ThreadView`. + /// + /// Weak-memory operations mutate the linear `ThreadView` carried by the + /// guard. This transition keeps the logical snapshot and checked-out + /// partition synchronized with that updated view. + pub open spec fn update_checked_out_task_view( + self, + task: Loc, + view: Irc11ThreadView, + ) -> SchedulerView + recommends + self.task_view_is_checked_out(task), + { + SchedulerView { + task_views: self.task_views.insert(task, view), + checked_out_views: self.checked_out_views.insert(task, view), + ..self + } + } + + /// Updating the view of the currently checked-out task preserves the + /// scheduler ownership partition and all scheduling invariants. + pub proof fn lemma_update_checked_out_task_view_preserves_wf( + self, + task: Loc, + view: Irc11ThreadView, + ) + requires + self.wf(), + self.task_view_is_checked_out(task), + ensures + self.update_checked_out_task_view(task, view).wf(), + self.update_checked_out_task_view(task, view).task_view_is_checked_out(task), + self.update_checked_out_task_view(task, view).task_views.contains_key(task), + self.update_checked_out_task_view(task, view).task_views[task] == view, + self.update_checked_out_task_view(task, view).checked_out_views[task] == view, + !self.update_checked_out_task_view(task, view).stored_views.contains_key(task), + { + } + + /// Writes a checked-out task view back to scheduler storage. + /// + /// The caller must provide the same view that is recorded as checked out; + /// this prevents check-in from overwriting the task with an unrelated view. + pub open spec fn checkin_task_view( + self, + cpu: CpuId, + task: Loc, + view: Irc11ThreadView, + ) -> SchedulerView + recommends + self.task_view_is_checked_out(task), + !self.task_view_is_stored(task), + self.checked_out_views[task] == view, + self.current.contains_key(cpu), + self.current[cpu] == Some(task), + !self.cpu_rcu_participant_is_stored(cpu), + { + SchedulerView { + task_views: self.task_views.insert(task, view), + stored_views: self.stored_views.insert(task, view), + checked_out_views: self.checked_out_views.remove(task), + stored_cpu_cores: self.stored_cpu_cores.insert(cpu), + ..self + } + } + + pub proof fn lemma_checkin_task_view_preserves_wf( + self, + cpu: CpuId, + task: Loc, + view: Irc11ThreadView, + ) + requires + self.wf(), + self.task_view_is_checked_out(task), + !self.task_view_is_stored(task), + self.checked_out_views[task] == view, + self.current.contains_key(cpu), + self.current[cpu] == Some(task), + !self.cpu_rcu_participant_is_stored(cpu), + ensures + self.checkin_task_view(cpu, task, view).wf(), + { + } + + /// Publishes the outgoing task's observations into the persistent CPU + /// view. A subsequent task scheduled on this CPU imports the result in + /// `checkout_task_view`. + pub open spec fn publish_cpu_view(self, cpu: CpuId, view: Irc11ThreadView) -> SchedulerView + recommends + self.cpu_has_thread_view(cpu), + { + SchedulerView { + cpu_views: self.cpu_views.insert(cpu, self.cpu_views[cpu].join(view)), + ..self + } + } + + pub proof fn lemma_publish_cpu_view_preserves_wf(self, cpu: CpuId, view: Irc11ThreadView) + requires + self.wf(), + self.cpu_has_thread_view(cpu), + ensures + self.publish_cpu_view(cpu, view).wf(), + self.publish_cpu_view(cpu, view).cpu_thread_view(cpu) == self.cpu_thread_view(cpu).join( + view, + ), + self.cpu_thread_view(cpu).spec_le( + self.publish_cpu_view(cpu, view).cpu_thread_view(cpu), + ), + view.spec_le(self.publish_cpu_view(cpu, view).cpu_thread_view(cpu)), + { + self.cpu_thread_view(cpu).lemma_join_left(view); + self.cpu_thread_view(cpu).lemma_join_right(view); + assert(self.publish_cpu_view(cpu, view).cpu_views.dom() == self.cpu_views.dom()); + } + + pub open spec fn wf(self) -> bool { + // CPU-indexed maps may only mention valid CPUs. + &&& forall|cpu: CpuId| #[trigger] self.runqueues.contains_key(cpu) ==> valid_cpu(cpu) + &&& forall|cpu: CpuId| #[trigger] + self.current.contains_key(cpu) ==> valid_cpu(cpu) && self.cpu_views.contains_key(cpu) + &&& forall|cpu: CpuId| #[trigger] + self.runqueues.contains_key(cpu) ==> self.cpu_views.contains_key(cpu) + &&& forall|cpu: CpuId| #[trigger] + self.cpu_views.contains_key(cpu) ==> valid_cpu( + cpu, + ) + // The complete RCU participant follows the same checkout boundary as + // the current task view, while its identity remains CPU-stable. + &&& self.cpu_core_registrations.dom() == self.cpu_views.dom() + &&& forall|cpu: CpuId| #[trigger] + self.cpu_core_registrations.contains_key(cpu) + ==> self.cpu_core_registrations[cpu].locals_key.len() == 1 + &&& self.stored_cpu_cores.subset_of(self.cpu_views.dom()) + &&& forall|cpu: CpuId| #[trigger] + self.cpu_views.contains_key(cpu) ==> (self.stored_cpu_cores.contains(cpu) <==> !( + self.current.contains_key(cpu) && self.current[cpu] is Some + && self.checked_out_views.contains_key( + self.current[cpu]->0, + ))) + // Runqueues contain exactly runnable tasks; current slots contain + // running tasks. + &&& forall|cpu: CpuId, idx: int| + #![trigger self.runqueues[cpu][idx]] + self.runqueues.contains_key(cpu) && 0 <= idx && idx < self.runqueues[cpu].len() + ==> self.state.contains_key(self.runqueues[cpu][idx]) + && self.state[self.runqueues[cpu][idx]] is Runnable + &&& forall|cpu: CpuId| #[trigger] + self.current.contains_key(cpu) && self.current[cpu] is Some ==> self.state.contains_key( + self.current[cpu]->0, + ) + && self.state[self.current[cpu]->0] is Running + // No known task may be duplicated across scheduler positions. + &&& forall|task: Loc| #[trigger] + self.state.contains_key(task) ==> self.no_duplicate_task( + task, + ) + // Live tasks have a weak-memory view, while exited tasks do not have + // to keep one around. + &&& forall|task: Loc| #[trigger] + self.state.contains_key(task) && !(self.state[task] is Exited) + ==> self.task_views.contains_key(task) + &&& forall|task: Loc| #[trigger] + self.task_views.contains_key(task) ==> self.state.contains_key(task) && !( + self.state[task] is Exited) + // `stored_views` is the part of `task_views` still owned by the + // scheduler resource. + &&& forall|task: Loc| #[trigger] + self.stored_views.contains_key(task) ==> self.task_views.contains_key(task) + && self.stored_views[task] == self.task_views[task] + && !self.checked_out_views.contains_key(task) && self.task_is_live( + task, + ) + // `checked_out_views` is the part temporarily held by guards. For the + // preemption-disable path, only the current running task may be checked + // out. + &&& forall|task: Loc| #[trigger] + self.checked_out_views.contains_key(task) ==> self.task_views.contains_key(task) + && self.checked_out_views[task] == self.task_views[task] + && !self.stored_views.contains_key(task) && self.task_is_live(task) + && self.task_is_current( + task, + ) + // Together, the stored and checked-out partitions cover all logical + // task views. + &&& forall|task: Loc| #[trigger] + self.task_views.contains_key(task) ==> (self.stored_views.contains_key(task) + || self.checked_out_views.contains_key(task)) + } +} + +/// Tracked owner of per-task weak-memory views. +/// +/// This is the internal resource backing `SchedulerGhostState`: +/// a guard borrows or moves out the current task's `ThreadView`, atomic +/// operations update it, and schedule-out writes it back to the same task +/// entry after all preemption guards have been released. +tracked struct SchedulerThreadViews { + scheduler: Ghost, + views: Map, + cpu_views: Map, + cpu_cores: Map>, + core_bindings: Map>, +} + +/// A checked-out per-task `ThreadView`. +/// +/// The `task` field records where the linear `ThreadView` must be written +/// back. This prevents proofs from taking one task's view and re-inserting it +/// under another task id. +pub tracked struct TaskThreadView { + scheduler: Ghost, + task: Ghost, + thread_view: ThreadViewToken, +} + +impl TaskThreadView { + proof fn new(scheduler: Loc, task: Loc, tracked thread_view: ThreadViewToken) -> (tracked res: + Self) + ensures + res.scheduler() == scheduler, + res.task() == task, + res.view() == thread_view@, + res.irc11_view() == thread_view@, + { + TaskThreadView { scheduler: Ghost(scheduler), task: Ghost(task), thread_view } + } + + pub closed spec fn scheduler(self) -> Loc { + self.scheduler@ + } + + pub closed spec fn task(self) -> Loc { + self.task@ + } + + pub closed spec fn view(self) -> Irc11ThreadView { + self.thread_view@ + } + + /// Transitional alias for callers already written against the native view. + pub open spec fn irc11_view(self) -> Irc11ThreadView { + self.view() + } + + /// Connects the checked-out token to the scheduler view that owns it. + /// + /// The token's linear `ThreadView` must agree with both the checked-out + /// partition and the total logical `task_views` snapshot. + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + &&& sched_view.wf() + &&& sched_view.id == self.scheduler() + &&& sched_view.checked_out_views.contains_key(self.task()) + &&& sched_view.checked_out_views[self.task()] == self.view() + &&& sched_view.task_views.contains_key(self.task()) + &&& sched_view.task_views[self.task()] == self.view() + } + + /// Packages the public scheduler facts needed to establish ownership of a + /// checked-out task view. + pub proof fn lemma_wf(tracked &self, sched_view: SchedulerView) + requires + sched_view.wf(), + sched_view.id == self.scheduler(), + sched_view.task_view_is_checked_out(self.task()), + sched_view.checked_out_views[self.task()] == self.view(), + sched_view.task_views.contains_key(self.task()), + sched_view.task_views[self.task()] == self.view(), + ensures + self.wf(sched_view), + { + } + + /// Borrows the linear `ThreadView` for weak-memory operations. + /// + /// After the borrow mutates the view, the caller must use + /// `update_checked_out_task_view` on the scheduler view before relying on + /// `TaskThreadView::wf` again. + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: + &mut ThreadViewToken) + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == (*final(tv))@, + final(self).irc11_view() == (*final(tv))@, + { + &mut self.thread_view + } + + /// Borrows the native IRC11 view carried by this task. + pub proof fn tracked_borrow_irc11_view_mut(tracked &mut self) -> (tracked view: + &mut ThreadViewToken) + ensures + (*view)@ == old(self).irc11_view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == (*final(view))@, + final(self).irc11_view() == (*final(view))@, + { + &mut self.thread_view + } +} + +impl SchedulerThreadViews { + proof fn empty(scheduler: Loc) -> (tracked res: Self) + ensures + res.scheduler() == scheduler, + res.view() == Map::::empty(), + res.cpu_view_map() == Map::::empty(), + res.core_registration_map() == Map::::empty(), + res.binding_registration_map() == Map::::empty(), + { + let tracked views = Map::::tracked_empty(); + let tracked cpu_views = Map::::tracked_empty(); + let tracked cpu_cores = Map::>::tracked_empty(); + let tracked core_bindings = Map::< + CpuId, + CpuCoreOwnerBinding, + >::tracked_empty(); + SchedulerThreadViews { + scheduler: Ghost(scheduler), + views, + cpu_views, + cpu_cores, + core_bindings, + } + } + + closed spec fn scheduler(self) -> Loc { + self.scheduler@ + } + + pub closed spec fn view(self) -> Map { + Map::new(self.views.dom(), |task: Loc| self.views[task]@) + } + + pub closed spec fn cpu_view_map(self) -> Map { + Map::new(self.cpu_views.dom(), |cpu: CpuId| self.cpu_views[cpu]@) + } + + pub closed spec fn core_registration_map(self) -> Map { + Map::new(self.cpu_cores.dom(), |cpu: CpuId| self.cpu_cores[cpu].registration()) + } + + pub closed spec fn binding_registration_map(self) -> Map { + Map::new( + self.core_bindings.dom(), + |cpu: CpuId| + CpuCoreRegistration { + owner_id: self.core_bindings[cpu].owner_id(), + locals_key: self.core_bindings[cpu].locals_key(), + }, + ) + } + + pub closed spec fn contains(self, task: Loc) -> bool { + self.views.contains_key(task) + } + + pub closed spec fn thread_view(self, task: Loc) -> Irc11ThreadView + recommends + self.contains(task), + { + self.views[task]@ + } + + pub closed spec fn irc11_thread_view(self, task: Loc) -> Irc11ThreadView + recommends + self.contains(task), + { + self.thread_view(task) + } + + pub closed spec fn contains_cpu(self, cpu: CpuId) -> bool { + self.cpu_views.contains_key(cpu) + } + + pub closed spec fn cpu_thread_view(self, cpu: CpuId) -> Irc11ThreadView + recommends + self.contains_cpu(cpu), + { + self.cpu_views[cpu]@ + } + + pub closed spec fn irc11_cpu_thread_view(self, cpu: CpuId) -> Irc11ThreadView + recommends + self.contains_cpu(cpu), + { + self.cpu_thread_view(cpu) + } + + closed spec fn cpu_cores_wf(self, sched_view: SchedulerView) -> bool { + &&& self.cpu_cores.dom() == sched_view.stored_cpu_cores + &&& forall|cpu: CpuId| #[trigger] + self.cpu_cores.contains_key(cpu) ==> { + &&& self.cpu_cores[cpu].wf() + &&& self.cpu_cores[cpu].cpu() == cpu + &&& self.cpu_cores[cpu].is_idle() + &&& self.cpu_cores[cpu].id() == sched_view.cpu_core_registrations[cpu].owner_id + &&& self.cpu_cores[cpu].locals_key() + == sched_view.cpu_core_registrations[cpu].locals_key + &&& self.cpu_cores[cpu].registration() == sched_view.cpu_core_registrations[cpu] + &&& self.cpu_cores[cpu].locals().fraction() == 1real + &&& self.cpu_cores[cpu].locals().view().spec_le(self.cpu_views[cpu]@) + } + } + + closed spec fn core_bindings_wf(self, sched_view: SchedulerView) -> bool { + &&& self.core_bindings.dom() == sched_view.cpu_core_registrations.dom() + &&& forall|cpu: CpuId| #[trigger] + self.core_bindings.contains_key(cpu) ==> { + &&& self.core_bindings[cpu].registry() == sched_view.id + &&& self.core_bindings[cpu].cpu() == cpu + &&& self.core_bindings[cpu].owner_id() + == sched_view.cpu_core_registrations[cpu].owner_id + &&& self.core_bindings[cpu].locals_key() + == sched_view.cpu_core_registrations[cpu].locals_key + } + } + + proof fn lemma_cpu_cores_frame(tracked &self, before: SchedulerView, after: SchedulerView) + requires + self.cpu_cores_wf(before), + before.stored_cpu_cores == after.stored_cpu_cores, + before.cpu_core_registrations == after.cpu_core_registrations, + before.cpu_views == after.cpu_views, + ensures + self.cpu_cores_wf(after), + { + } + + /// The tracked owner contains exactly the views still stored in scheduler + /// state. Checked-out views are represented by `TaskThreadView` tokens + /// instead, so they are intentionally absent here. + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + &&& self.scheduler() == sched_view.id + &&& self.view() == sched_view.stored_views + &&& self.cpu_view_map() == sched_view.cpu_views + &&& self.cpu_cores_wf(sched_view) + &&& self.core_bindings_wf(sched_view) + } + + proof fn tracked_register_cpu( + tracked &mut self, + tracked identity: &mut GhostMapAuth, + sched_view: SchedulerView, + cpu: CpuId, + ) -> (registration: CpuCoreRegistration) + requires + old(self).wf(sched_view), + sched_view.wf(), + !sched_view.cpu_has_thread_view(cpu), + valid_cpu(cpu), + old(identity).id() == sched_view.id, + old(identity)@ == sched_view.cpu_core_registrations, + ensures + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view(), + registration.locals_key.len() == 1, + final(self).cpu_view_map() == sched_view.register_cpu(cpu, registration).cpu_views, + final(self).wf(sched_view.register_cpu(cpu, registration)), + final(self).contains_cpu(cpu), + final(self).cpu_thread_view(cpu) == Irc11ThreadView::empty(), + final(self).core_registration_map().contains_key(cpu), + final(self).core_registration_map()[cpu] == registration, + final(self).binding_registration_map() == sched_view.register_cpu( + cpu, + registration, + ).cpu_core_registrations, + final(identity).id() == old(identity).id(), + final(identity)@ == sched_view.register_cpu(cpu, registration).cpu_core_registrations, + { + let tracked cpu_view = ThreadViewToken::new(); + let tracked participant = CpuRcuParticipant::new(cpu, Irc11ThreadView::empty()); + participant.lemma_cpu_core_local_state(); + let ghost participant_id = participant.id(); + let tracked core = CpuCoreOwner::new(cpu, participant); + let ghost registration = core.registration(); + assert(core.locals_key() == seq![participant_id]); + assert(registration.locals_key.len() == 1); + let tracked entry = identity.insert(cpu, registration); + let tracked binding = CpuCoreOwnerBinding::tracked_new(entry, &core); + sched_view.lemma_register_cpu_preserves_wf(cpu, registration); + self.cpu_views.tracked_insert(cpu, cpu_view); + self.cpu_cores.tracked_insert(cpu, core); + self.core_bindings.tracked_insert(cpu, binding); + assert(final(self).cpu_view_map() == sched_view.register_cpu(cpu, registration).cpu_views); + assert(final(self).wf(sched_view.register_cpu(cpu, registration))); + registration + } + + /// Inserts a task view created during task registration. + /// + /// This is separate from check-in: initial insertion creates a new stored + /// entry, while check-in returns an existing checked-out view. + proof fn tracked_insert_initial_thread_view( + tracked &mut self, + tracked token: TaskThreadView, + rcu_view: SchedulerView, + ) + requires + !old(self).contains(token.task()), + token.scheduler() == old(self).scheduler(), + old(self).cpu_cores_wf(rcu_view), + old(self).core_bindings_wf(rcu_view), + ensures + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view().insert(token.task(), token.view()), + final(self).cpu_view_map() == old(self).cpu_view_map(), + final(self).cpu_cores == old(self).cpu_cores, + final(self).cpu_cores_wf(rcu_view), + final(self).core_bindings == old(self).core_bindings, + final(self).core_bindings_wf(rcu_view), + { + let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; + self.views.tracked_insert(task, thread_view); + } + + /// Registers a task with its unique initial weak-memory view. + /// + /// This is the linear counterpart of [`SchedulerView::register_task`]. It + /// mints the empty `ThreadView` internally, so callers cannot initialize a + /// task using a view detached from another scheduler entry. + proof fn tracked_register_task(tracked &mut self, sched_view: SchedulerView, task: Loc) + requires + old(self).wf(sched_view), + sched_view.wf(), + !sched_view.state.contains_key(task), + ensures + final(self).scheduler() == old(self).scheduler(), + final(self).view() == sched_view.register_task(task).stored_views, + final(self).cpu_view_map() == old(self).cpu_view_map(), + final(self).wf(sched_view.register_task(task)), + final(self).contains(task), + final(self).thread_view(task) == Irc11ThreadView::empty(), + { + sched_view.lemma_register_task_preserves_wf(task); + let tracked thread_view = ThreadViewToken::new(); + let tracked token = TaskThreadView::new(self.scheduler(), task, thread_view); + self.lemma_cpu_cores_frame(sched_view, sched_view.register_task(task)); + self.tracked_insert_initial_thread_view(token, sched_view.register_task(task)); + assert(final(self).view() == sched_view.register_task(task).stored_views); + assert(final(self).wf(sched_view.register_task(task))); + } + + /// Checks out the current CPU's task view from the tracked owner. + /// + /// The proof-side owner loses this task entry and returns the linear token + /// that a guard will carry until check-in. + proof fn tracked_take_current_thread_view( + tracked &mut self, + sched_view: SchedulerView, + cpu: CpuId, + ) -> (tracked res: ( + TaskThreadView, + CpuCoreOwnerHandle, + CpuRcuParticipant, + CpuCoreOwnerBinding, + )) + requires + old(self).wf(sched_view), + sched_view.wf(), + sched_view.current.contains_key(cpu), + sched_view.current[cpu] is Some, + sched_view.cpu_has_thread_view(cpu), + sched_view.task_view_is_stored(sched_view.current[cpu]->0), + old(self).contains(sched_view.current[cpu]->0), + old(self).contains_cpu(cpu), + ensures + res.0.task() == sched_view.current[cpu]->0, + res.0.scheduler() == sched_view.id, + res.0.view() == old(self).thread_view(sched_view.current[cpu]->0).join( + old(self).cpu_thread_view(cpu), + ), + res.1.wf(), + res.1.cpu() == cpu, + res.1.current_task() == Some(res.0.task()), + res.1.id() == sched_view.cpu_core_registration(cpu).owner_id, + res.1.expected_locals_key() == seq![res.2.id()], + res.1.expected_locals_key() == sched_view.cpu_core_registration(cpu).locals_key, + res.2.id() == sched_view.cpu_rcu_participant_id(cpu), + res.2.cpu() == cpu, + res.2.fraction() == 1real, + res.2.view().spec_le(res.0.irc11_view()), + res.2.wf(), + res.3.registry() == sched_view.id, + res.3.cpu() == cpu, + res.3.owner_id() == res.1.id(), + res.3.locals_key() == seq![res.2.id()], + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), + final(self).cpu_view_map() == old(self).cpu_view_map(), + final(self).view() == sched_view.checkout_task_view(cpu).stored_views, + final(self).wf(sched_view.checkout_task_view(cpu)), + res.0.wf(sched_view.checkout_task_view(cpu)), + { + let task = sched_view.current[cpu]->0; + let tracked mut thread_view = self.views.tracked_remove(task); + let tracked cpu_view = self.cpu_views.tracked_borrow(cpu); + let tracked mut core = self.cpu_cores.tracked_remove(cpu); + assert(core.is_idle()); + assert(core.id() == sched_view.cpu_core_registration(cpu).owner_id); + assert(core.locals_key() == sched_view.cpu_core_registration(cpu).locals_key); + assert(core.registration() == sched_view.cpu_core_registration(cpu)); + core.tracked_schedule_in(task); + assert(core.registration() == sched_view.cpu_core_registration(cpu)); + let tracked (core_handle, participant) = core.tracked_open(); + let ghost participant_view = participant.view(); + let ghost task_irc11_view = thread_view@; + let ghost cpu_irc11_view = cpu_view@; + let tracked binding = self.core_bindings.tracked_borrow(cpu).tracked_duplicate(); + thread_view.tracked_join(cpu_view); + assert(participant_view.spec_le(cpu_irc11_view)); + task_irc11_view.lemma_join_right(cpu_irc11_view); + participant_view.lemma_spec_le_transitive(cpu_irc11_view, thread_view@); + let tracked token = TaskThreadView { + scheduler: Ghost(self.scheduler()), + task: Ghost(task), + thread_view, + }; + let next = sched_view.checkout_task_view(cpu); + assert(core_handle.cpu() == cpu); + assert(core_handle.current_task() == Some(task)); + assert(core_handle.id() == sched_view.cpu_core_registration(cpu).owner_id); + assert(participant.cpu() == cpu); + assert(core_handle.expected_locals_key() == seq![participant.id()]); + assert(sched_view.cpu_core_registration(cpu).locals_key == seq![participant.id()]); + assert(participant.id() == sched_view.cpu_rcu_participant_id(cpu)); + assert(final(self).view() == next.stored_views); + assert(final(self).wf(next)); + assert(token.wf(next)); + (token, core_handle, participant, binding) + } + + /// Checks out the current task's weak-memory view and starts its running + /// context. + proof fn tracked_take_current_running_context( + tracked &mut self, + sched_view: SchedulerView, + cpu: CpuId, + ) -> (tracked context: RunningTaskContext) + requires + old(self).wf(sched_view), + sched_view.wf(), + sched_view.current.contains_key(cpu), + sched_view.current[cpu] is Some, + sched_view.cpu_has_thread_view(cpu), + sched_view.task_view_is_stored(sched_view.current[cpu]->0), + old(self).contains(sched_view.current[cpu]->0), + old(self).contains_cpu(cpu), + ensures + context.task() == sched_view.current[cpu]->0, + context.scheduler() == sched_view.id, + context.view() == old(self).thread_view(sched_view.current[cpu]->0).join( + old(self).cpu_thread_view(cpu), + ), + context.cpu() == cpu, + context.is_quiescent(), + context.wf_scheduler(sched_view.checkout_task_view(cpu)), + final(self).view() == sched_view.checkout_task_view(cpu).stored_views, + final(self).scheduler() == old(self).scheduler(), + final(self).wf(sched_view.checkout_task_view(cpu)), + { + let ghost next = sched_view.checkout_task_view(cpu); + let tracked (task_view, core_handle, participant, binding) = + self.tracked_take_current_thread_view(sched_view, cpu); + let tracked context = RunningTaskContext::new( + task_view, + core_handle, + participant, + binding, + next, + cpu, + ); + context + } + + /// Ends a quiescent running interval and checks its updated task view back + /// into scheduler ownership. + proof fn tracked_put_running_context( + tracked &mut self, + sched_view: SchedulerView, + tracked context: RunningTaskContext, + ) + requires + old(self).wf(sched_view), + sched_view.wf(), + sched_view.task_view_is_checked_out(context.task()), + context.scheduler() == sched_view.id, + sched_view.current.contains_key(context.cpu()), + sched_view.current[context.cpu()] == Some(context.task()), + sched_view.cpu_has_thread_view(context.cpu()), + sched_view.cpu_has_rcu_participant(context.cpu()), + sched_view.cpu_rcu_participant_id(context.cpu()) == context.rcu_participant_id(), + context.wf(), + context.is_quiescent(), + !old(self).contains(context.task()), + old(self).contains_cpu(context.cpu()), + ensures + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view().insert(context.task(), context.view()), + final(self).cpu_view_map() == old(self).cpu_view_map().insert( + context.cpu(), + old(self).cpu_thread_view(context.cpu()).join(context.view()), + ), + final(self).wf( + sched_view.update_checked_out_task_view( + context.task(), + context.view(), + ).checkin_task_view(context.cpu(), context.task(), context.view()).publish_cpu_view( + context.cpu(), + context.view(), + ), + ), + { + let ghost task = context.task(); + let ghost view = context.view(); + let ghost irc11_context_view = context.irc11_view(); + let ghost cpu = context.cpu(); + let ghost expected_registration = CpuCoreRegistration { + owner_id: context.core_owner_id(), + locals_key: seq![context.rcu_participant_id()], + }; + let tracked context_binding = context.tracked_rcu_binding(); + let tracked canonical_binding = self.core_bindings.tracked_borrow(cpu); + canonical_binding.lemma_same_cpu_agree(&context_binding); + assert(canonical_binding.owner_id() == sched_view.cpu_core_registration(cpu).owner_id); + assert(canonical_binding.locals_key() == sched_view.cpu_core_registration(cpu).locals_key); + assert(context_binding.owner_id() == context.core_owner_id()); + assert(context_binding.locals_key() == seq![context.rcu_participant_id()]); + assert(expected_registration.owner_id == sched_view.cpu_core_registration(cpu).owner_id); + assert(expected_registration.locals_key == sched_view.cpu_core_registration( + cpu, + ).locals_key); + assert(expected_registration == sched_view.cpu_core_registration(cpu)); + sched_view.lemma_update_checked_out_task_view_preserves_wf(task, view); + let ghost updated = sched_view.update_checked_out_task_view(task, view); + context.lemma_wf_scheduler(updated); + let tracked (task_view, mut core) = context.tracked_into_task_view_for_scheduler(updated); + let ghost participant_view = core.locals().view(); + let ghost participant_id = core.locals().id(); + let ghost core_registration = core.registration(); + assert(core_registration == expected_registration); + assert(core.id() == core_registration.owner_id); + assert(core.locals_key() == core_registration.locals_key); + let ghost scheduled_task = core.tracked_schedule_out(); + assert(core.registration() == core_registration); + assert(core.id() == core_registration.owner_id); + assert(core.locals_key() == core_registration.locals_key); + assert(scheduled_task == task); + let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = task_view; + let tracked cpu_view = self.cpu_views.tracked_borrow_mut(cpu); + cpu_view.tracked_join(&thread_view); + self.views.tracked_insert(task, thread_view); + self.cpu_cores.tracked_insert(cpu, core); + updated.lemma_checkin_task_view_preserves_wf(cpu, task, view); + let ghost checked = updated.checkin_task_view(cpu, task, view); + assert(checked.wf()); + checked.lemma_publish_cpu_view_preserves_wf(cpu, view); + let ghost next = checked.publish_cpu_view(cpu, view); + assert(next.wf()); + assert(participant_view.spec_le(irc11_context_view)); + old(self).cpu_thread_view(cpu).lemma_join_right(irc11_context_view); + participant_view.lemma_spec_le_transitive(irc11_context_view, self.cpu_views[cpu]@); + assert(self.cpu_cores.dom() == next.stored_cpu_cores); + assert forall|stored_cpu: CpuId| #[trigger] + self.cpu_cores.contains_key(stored_cpu) implies { + &&& self.cpu_cores[stored_cpu].wf() + &&& self.cpu_cores[stored_cpu].cpu() == stored_cpu + &&& self.cpu_cores[stored_cpu].is_idle() + &&& self.cpu_cores[stored_cpu].id() == next.cpu_core_registrations[stored_cpu].owner_id + &&& self.cpu_cores[stored_cpu].locals_key() + == next.cpu_core_registrations[stored_cpu].locals_key + &&& self.cpu_cores[stored_cpu].registration() == next.cpu_core_registrations[stored_cpu] + &&& self.cpu_cores[stored_cpu].locals().fraction() == 1real + &&& self.cpu_cores[stored_cpu].locals().view().spec_le(self.cpu_views[stored_cpu]@) + } by { + assert(next.cpu_core_registrations.contains_key(stored_cpu)); + if stored_cpu == cpu { + assert(self.cpu_cores[stored_cpu].registration() == core_registration); + assert(core_registration == next.cpu_core_registrations[stored_cpu]); + assert(self.cpu_cores[stored_cpu].wf()); + assert(self.cpu_cores[stored_cpu].cpu() == stored_cpu); + assert(self.cpu_cores[stored_cpu].is_idle()); + assert(self.cpu_cores[stored_cpu].id() == core_registration.owner_id); + assert(self.cpu_cores[stored_cpu].locals_key() == core_registration.locals_key); + assert(self.cpu_cores[stored_cpu].locals().id() == participant_id); + } else { + assert(old(self).cpu_cores.contains_key(stored_cpu)); + assert(self.cpu_cores[stored_cpu] == old(self).cpu_cores[stored_cpu]); + assert(next.cpu_core_registrations[stored_cpu] + == sched_view.cpu_core_registrations[stored_cpu]); + assert(old(self).cpu_cores[stored_cpu].wf()); + assert(old(self).cpu_cores[stored_cpu].cpu() == stored_cpu); + assert(old(self).cpu_cores[stored_cpu].is_idle()); + assert(old(self).cpu_cores[stored_cpu].id() + == sched_view.cpu_core_registrations[stored_cpu].owner_id); + assert(old(self).cpu_cores[stored_cpu].locals_key() + == sched_view.cpu_core_registrations[stored_cpu].locals_key); + assert(old(self).cpu_cores[stored_cpu].registration() + == sched_view.cpu_core_registrations[stored_cpu]); + } + }; + assert(final(self).view() == next.stored_views); + assert(final(self).cpu_view_map() == next.cpu_views); + assert(final(self).wf(next)); + } +} + +/// Authoritative scheduler proof state. +/// +/// This linear root keeps the copyable scheduler snapshot and the owned +/// per-task `ThreadView` resources in one object. Its transitions update both +/// layers together, preventing a proof from changing `SchedulerView` without +/// moving the corresponding linear token (or vice versa). +pub tracked struct SchedulerGhostState { + identity: GhostMapAuth, + view: Ghost, + thread_views: SchedulerThreadViews, +} + +impl SchedulerGhostState { + /// Creates an empty authority for one scheduler instance. + pub proof fn new() -> (tracked res: Self) + ensures + res.wf(), + res.view() == SchedulerView::empty(res.id()), + { + let tracked (identity, _entries) = GhostMapAuth::new( + Map::::empty(), + ); + let ghost id = identity.id(); + SchedulerView::lemma_empty_wf(id); + let tracked thread_views = SchedulerThreadViews::empty(id); + let tracked res = SchedulerGhostState { + identity, + view: Ghost(SchedulerView::empty(id)), + thread_views, + }; + assert(res.wf()); + res + } + + pub closed spec fn view(self) -> SchedulerView { + self.view@ + } + + pub closed spec fn id(self) -> Loc { + self.identity.id() + } + + pub closed spec fn wf(self) -> bool { + &&& self.view().wf() + &&& self.view().id == self.id() + &&& self.identity@ == self.view().cpu_core_registrations + &&& self.thread_views.wf(self.view()) + } + + /// Applies a scheduler-policy transition without changing thread views. + /// + /// The caller proves the concrete runqueue/current/state update is a valid + /// `SchedulerView`. Equality of the ownership maps prevents this glue step + /// from minting a view, discarding observations, or moving a checked-out + /// token behind the linear root's back. + pub proof fn tracked_apply_scheduling_transition(tracked &mut self, next: SchedulerView) + requires + old(self).wf(), + next.wf(), + old(self).view().same_thread_view_ownership(next), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == next, + { + self.view = Ghost(next); + assert(self.thread_views.wf(self.view())); + assert(self.wf()); + } + + /// Registers one persistent CPU view before scheduler policy installs a + /// runqueue or current-task slot for that CPU. + pub proof fn tracked_register_cpu(tracked &mut self, cpu: CpuId) + requires + old(self).wf(), + !old(self).view().cpu_has_thread_view(cpu), + valid_cpu(cpu), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().register_cpu( + cpu, + final(self).view().cpu_core_registration(cpu), + ), + final(self).view().cpu_has_thread_view(cpu), + final(self).view().cpu_thread_view(cpu) == Irc11ThreadView::empty(), + final(self).view().cpu_rcu_participant_is_stored(cpu), + { + let ghost old_view = self.view@; + let ghost registration = self.thread_views.tracked_register_cpu( + &mut self.identity, + old_view, + cpu, + ); + self.view = Ghost(old_view.register_cpu(cpu, registration)); + assert(self.wf()); + } + + /// Registers a new task with one empty weak-memory view. + pub proof fn tracked_register_task(tracked &mut self, task: Loc) + requires + old(self).wf(), + !old(self).view().state.contains_key(task), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().register_task(task), + final(self).view().state[task] is New, + final(self).view().task_view_is_stored(task), + final(self).view().task_thread_view(task) == Irc11ThreadView::empty(), + { + let ghost old_view = self.view@; + self.thread_views.tracked_register_task(old_view, task); + self.view = Ghost(old_view.register_task(task)); + assert(self.wf()); + } + + /// Checks the current task's view out for one running interval. + pub proof fn tracked_schedule_in(tracked &mut self, cpu: CpuId) -> (tracked context: + RunningTaskContext) + requires + old(self).wf(), + old(self).view().current.contains_key(cpu), + old(self).view().current[cpu] is Some, + old(self).view().cpu_has_thread_view(cpu), + old(self).view().task_view_is_stored(old(self).view().current[cpu]->0), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().checkout_task_view(cpu), + context.task() == old(self).view().current[cpu]->0, + context.scheduler() == old(self).id(), + context.cpu() == cpu, + context.view() == old(self).view().task_thread_view(context.task()).join( + old(self).view().cpu_thread_view(cpu), + ), + old(self).view().task_thread_view(context.task()).spec_le(context.view()), + old(self).view().cpu_thread_view(cpu).spec_le(context.view()), + context.is_quiescent(), + context.wf_scheduler(final(self).view()), + { + let ghost old_view = self.view@; + let tracked context = self.thread_views.tracked_take_current_running_context(old_view, cpu); + old_view.task_thread_view(context.task()).lemma_join_left(old_view.cpu_thread_view(cpu)); + old_view.task_thread_view(context.task()).lemma_join_right(old_view.cpu_thread_view(cpu)); + self.view = Ghost(old_view.checkout_task_view(cpu)); + assert(self.wf()); + context + } + + /// Ends a quiescent running interval and stores its updated view. + pub proof fn tracked_schedule_out(tracked &mut self, tracked context: RunningTaskContext) + requires + old(self).wf(), + old(self).view().task_view_is_checked_out(context.task()), + context.scheduler() == old(self).id(), + old(self).view().current.contains_key(context.cpu()), + old(self).view().current[context.cpu()] == Some(context.task()), + old(self).view().cpu_has_thread_view(context.cpu()), + old(self).view().cpu_has_rcu_participant(context.cpu()), + old(self).view().cpu_rcu_participant_id(context.cpu()) == context.rcu_participant_id(), + context.wf(), + context.is_quiescent(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().update_checked_out_task_view( + context.task(), + context.view(), + ).checkin_task_view(context.cpu(), context.task(), context.view()).publish_cpu_view( + context.cpu(), + context.view(), + ), + final(self).view().task_view_is_stored(context.task()), + final(self).view().task_thread_view(context.task()) == context.view(), + !final(self).view().task_view_is_checked_out(context.task()), + context.view().spec_le(final(self).view().cpu_thread_view(context.cpu())), + { + let ghost old_view = self.view@; + let ghost cpu = context.cpu(); + let ghost task = context.task(); + let ghost context_view = context.view(); + let ghost next = old_view.update_checked_out_task_view( + task, + context_view, + ).checkin_task_view(cpu, task, context_view).publish_cpu_view(cpu, context_view); + self.thread_views.tracked_put_running_context(old_view, context); + old_view.lemma_update_checked_out_task_view_preserves_wf(task, context_view); + let ghost updated = old_view.update_checked_out_task_view(task, context_view); + updated.lemma_checkin_task_view_preserves_wf(cpu, task, context_view); + let ghost checked = updated.checkin_task_view(cpu, task, context_view); + checked.lemma_publish_cpu_view_preserves_wf(cpu, context_view); + assert(next.wf()); + self.view = Ghost(next); + old_view.cpu_thread_view(cpu).lemma_join_right(context_view); + assert(self.thread_views.wf(next)); + assert(next.id == self.id()); + assert(self.wf()); + } +} + +/// Logical identity of a runnable task handle. +/// +/// `RoArc` does not yet expose a proof-level view of its pointee. Keep this +/// abstract at the scheduler boundary and connect it to `RoArc`'s internals +/// later when the task registry is introduced. +pub uninterp spec fn runnable_id(runnable: &RoArc) -> Loc; + +pub open spec fn valid_cpu(cpu: CpuId) -> bool { + crate::specs::mm::cpu::online_cpus().contains(cpu) +} + +pub open spec fn can_enqueue(view: SchedulerView, task: Loc, flags: EnqueueFlags) -> bool { + match flags { + EnqueueFlags::Spawn => { + &&& view.state.contains_key(task) + &&& view.state[task] is New + &&& view.task_view_is_stored(task) + &&& view.task_thread_view(task) == Irc11ThreadView::empty() + }, + EnqueueFlags::Wake => view.state.contains_key(task) && !(view.state[task] is Exited), + } +} + +/// An SMP-aware task scheduler. +pub trait Scheduler: Sync + Send { + spec fn view(&self) -> SchedulerView; + + spec fn wf(&self) -> bool; + + /// Enqueues a runnable task. + /// + /// The scheduler implementer can perform load-balancing or some time accounting work here. + /// + /// The newly-enqueued task may have a higher priority than the currently running one on a CPU + /// and thus should preempt the latter. + /// In this case, this method returns the ID of that CPU. + fn enqueue(&self, runnable: RoArc, flags: EnqueueFlags) -> (r: Option) + requires + self.wf(), + self.view().wf(), + can_enqueue(self.view(), runnable_id(&runnable), flags), + ensures + self.wf(), + self.view().wf(), + self.view().task_is_runnable(runnable_id(&runnable)), + self.view().no_duplicate_task(runnable_id(&runnable)), + r matches Some(cpu) ==> valid_cpu(cpu), + ; +} + +exec static SCHEDULER: OnceImpl<&'static dyn Scheduler, TrivialPred> + ensures + SCHEDULER.wf(), + SCHEDULER.inv() == TrivialPred, +{ + OnceImpl::new(Ghost(TrivialPred)) +} + +/// Possible actions of a rescheduling. +enum ReschedAction { + /// Keep running current task and do nothing. + DoNothing, + /// Loop until finding a task to swap out the current. + Retry, + /// Switch to target task. + SwitchTo(RoArc), +} + +/// Possible triggers of an `enqueue` action. +#[derive(PartialEq, Copy, Clone)] +pub enum EnqueueFlags { + /// Spawn a new task. + Spawn, + /// Wake a sleeping task. + Wake, +} + +/// Possible triggers of an `update_current` action. +#[derive(PartialEq, Copy, Clone)] +pub enum UpdateFlags { + /// Timer interrupt. + Tick, + /// Task waiting. + Wait, + /// Task yielding. + Yield, + /// Task exiting. + Exit, +} + +} // verus! +/* // mod fifo_scheduler; // pub mod info; use alloc::sync::Arc; @@ -95,18 +1528,9 @@ pub fn inject_scheduler(scheduler: &'static dyn Scheduler) { }); */ } -static SCHEDULER: Once<&'static dyn Scheduler> = Once::new(); /// A SMP-aware task scheduler. pub trait Scheduler: Sync + Send { - /// Enqueues a runnable task. - /// - /// The scheduler implementer can perform load-balancing or some time accounting work here. - /// - /// The newly-enqueued task may have a higher priority than the currently running one on a CPU - /// and thus should preempt the latter. - /// In this case, this method returns the ID of that CPU. - fn enqueue(&self, runnable: Arc, flags: EnqueueFlags) -> Option; /// Gets an immutable access to the local runqueue of the current CPU. fn local_rq_with(&self, f: &mut dyn FnMut(&dyn LocalRunQueue)); @@ -357,28 +1781,6 @@ pub trait LocalRunQueue { fn dequeue_current(&mut self) -> Option>; } -/// Possible triggers of an `enqueue` action. -#[derive(PartialEq, Copy, Clone)] -pub enum EnqueueFlags { - /// Spawn a new task. - Spawn, - /// Wake a sleeping task. - Wake, -} - -/// Possible triggers of an `update_current` action. -#[derive(PartialEq, Copy, Clone)] -pub enum UpdateFlags { - /// Timer interrupt. - Tick, - /// Task waiting. - Wait, - /// Task yielding. - Yield, - /// Task exiting. - Exit, -} - /// Preempts the current task. #[track_caller] pub(crate) fn might_preempt() { @@ -579,12 +1981,5 @@ where // processor::switch_to_task(next_task); } -/// Possible actions of a rescheduling. -enum ReschedAction { - /// Keep running current task and do nothing. - DoNothing, - /// Loop until finding a task to swap out the current. - Retry, - /// Switch to target task. - SwitchTo(Arc), -} + +*/ diff --git a/tools/bootstrap-verus-irc11.sh b/tools/bootstrap-verus-irc11.sh new file mode 100644 index 000000000..be7a7cd17 --- /dev/null +++ b/tools/bootstrap-verus-irc11.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +verus_dir="$repo_root/tools/verus" +source_dir="$verus_dir/source" +vargo_manifest="$verus_dir/tools/vargo/Cargo.toml" +vargo_bin="$verus_dir/tools/vargo/target/release/vargo" + +if [[ ! -f "$source_dir/vstd/atomic_weak.rs" ]]; then + echo "The pinned Verus checkout does not contain IRC11 support" >&2 + exit 1 +fi + +if [[ ! -x "$source_dir/z3" ]]; then + ( + cd "$source_dir" + ./tools/get-z3.sh + ) +fi + +cargo build --release --manifest-path "$vargo_manifest" + +# The IRC11 patch predates weak-memory in Vargo's build fingerprint, so force +# vstd to rebuild whenever this bootstrap path is invoked. +rm -f "$source_dir/target-verus/release/.vstd-fingerprint" + +( + cd "$source_dir" + "$vargo_bin" build --release --features singular --vstd-weak-memory + "$vargo_bin" build --release -p verusdoc +) + +# This Verus revision places standalone package builds in Cargo's default +# target directory, while the current dv searches beside the Verus binary. +cp "$source_dir/target/release/verusdoc" "$source_dir/target-verus/release/verusdoc" + +test -x "$source_dir/target-verus/release/verus" +test -x "$source_dir/target-verus/release/verusdoc" +test -f "$source_dir/target-verus/release/verus-root" +test -f "$source_dir/target-verus/release/vstd.vir" diff --git a/tools/patches/verus-irc11-vstd.patch b/tools/patches/verus-irc11-vstd.patch new file mode 100644 index 000000000..f002105b8 --- /dev/null +++ b/tools/patches/verus-irc11-vstd.patch @@ -0,0 +1,199 @@ +diff --git a/source/vstd/Cargo.toml b/source/vstd/Cargo.toml +index 1c106c81..ef05ea7e 100644 +--- a/source/vstd/Cargo.toml ++++ b/source/vstd/Cargo.toml +@@ -33,6 +33,7 @@ allocator = [] + strict_provenance_atomic_ptr = [] + allow_panic = [] # code is allowed to panic. + nonzero_internals = [] ++weak-memory = [] + + + [package.metadata.verus] +diff --git a/source/vstd/atomic.rs b/source/vstd/atomic.rs +--- a/source/vstd/atomic.rs ++++ b/source/vstd/atomic.rs +@@ -5,10 +5,8 @@ use super::prelude::*; + use super::raw_ptr::PointsTo; + use super::view::*; + +-#[cfg(not(feature = "weak-memory"))] + pub use sc_atomic_types::*; + +-#[cfg(not(feature = "weak-memory"))] + mod sc_atomic_types { + + use core::sync::atomic::{ +@@ -22,7 +20,8 @@ mod sc_atomic_types { + use super::super::modes::*; + use super::super::pervasive::*; + use super::super::prelude::*; + use super::super::raw_ptr::PointsTo; ++ use super::super::thread_view::Objective; + use super::super::view::*; + use super::super::wrapping::*; + +@@ -73,6 +73,9 @@ mod sc_atomic_types { + unused: $value_ty, + } + ++ #[cfg(verus_keep_ghost)] ++ unsafe impl Objective for $p_ident {} ++ + pub ghost struct $p_data_ident { + pub patomic: int, + pub value: $value_ty, +@@ -122,6 +125,9 @@ mod sc_atomic_types { + unusued: $value_ty, + } + ++ #[cfg(verus_keep_ghost)] ++ unsafe impl Objective for $p_ident {} ++ + #[verifier::accept_recursive_types(T)] + pub ghost struct $p_data_ident { + pub patomic: int, +diff --git a/source/vstd/atomic_ghost.rs b/source/vstd/atomic_ghost.rs +index 0d6a3b8a..4b456bfb 100644 +--- a/source/vstd/atomic_ghost.rs ++++ b/source/vstd/atomic_ghost.rs +@@ -2,16 +2,15 @@ + //! See the [`atomic_with_ghost!`] documentation for more information. + #![allow(unused_imports)] + +-#[cfg(not(feature = "weak-memory"))] + pub use atomic_ghost::*; + +-#[cfg(not(feature = "weak-memory"))] + mod atomic_ghost { + + use super::super::atomic::*; + use super::super::invariant::*; + use super::super::modes::*; + use super::super::prelude::*; ++ use super::super::thread_view::Objective; + + verus! { + +@@ -26,7 +25,7 @@ pub trait AtomicInvariantPredicate { + + pub struct $atomic_pred_ty { p: Pred } + +- impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty ++ impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty + where Pred: AtomicInvariantPredicate + { + open spec fn inv(k_loc: (K, int), perm_g: ($perm_ty, G)) -> bool { +@@ -46,7 +45,7 @@ pub trait AtomicInvariantPredicate { + /// + /// See the [`atomic_with_ghost!`] documentation for usage information. + +- pub struct $at_ident ++ pub struct $at_ident + //where Pred: AtomicInvariantPredicate + { + #[doc(hidden)] +@@ -56,7 +55,7 @@ pub trait AtomicInvariantPredicate { + pub atomic_inv: Tracked>>, + } + +- impl $at_ident { ++ impl $at_ident { + pub open spec fn constant(&self) -> K { + self.atomic_inv@.constant().0 + } +@@ -66,7 +65,7 @@ pub trait AtomicInvariantPredicate { + } + } + +- impl $at_ident ++ impl $at_ident + where Pred: AtomicInvariantPredicate + { + #[inline(always)] +@@ -117,7 +116,7 @@ pub trait AtomicInvariantPredicate { + + pub struct $atomic_pred_ty { t: T, p: Pred } + +- impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty ++ impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty + where Pred: AtomicInvariantPredicate + { + open spec fn inv(k_loc: (K, int), perm_g: ($perm_ty, G)) -> bool { +@@ -137,7 +136,7 @@ pub trait AtomicInvariantPredicate { + /// + /// See the [`atomic_with_ghost!`] documentation for usage information. + +- pub struct $at_ident ++ pub struct $at_ident + //where Pred: AtomicInvariantPredicate + { + #[doc(hidden)] +@@ -147,7 +146,7 @@ pub trait AtomicInvariantPredicate { + pub atomic_inv: Tracked>>, + } + +- impl $at_ident ++ impl $at_ident + where Pred: AtomicInvariantPredicate + { + pub open spec fn well_formed(&self) -> bool { +diff --git a/source/vstd/thread_view.rs b/source/vstd/thread_view.rs +index bfb1c293..2a2ad51b 100644 +--- a/source/vstd/thread_view.rs ++++ b/source/vstd/thread_view.rs +@@ -223,6 +223,34 @@ unsafe impl Objective for algebra::Resource { + + } + ++// Memory ownership permissions are objective independently of the type stored ++// at the location. Their views describe global memory, not one thread's ++// subjective weak-memory observations. ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::cell::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::cell::pcell::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::cell::pcell_maybe_uninit::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::raw_ptr::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::simple_pptr::PointsTo { ++ ++} ++ + // primitive types are objective because they do not hold permissions + macro_rules! declare_primitive_is_objective { + ($($a:ty),*) => { +@@ -237,6 +265,19 @@ macro_rules! declare_primitive_is_objective { + + declare_primitive_is_objective!(bool, char, (), u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, int, nat, str); + ++// Verus does not currently derive the auto-trait bound for a tuple when one of ++// its elements is a type parameter. State the binary case explicitly since ++// invariants commonly package a permission and user ghost state as a pair. ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for (A, B) { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for Option { ++ ++} ++ + // note: the fact that tuples are Objective (above) suffices for OBJMOD-SEP + // OBJ with wand update + #[cfg(verus_keep_ghost)] diff --git a/tools/patches/verus-irc11.patch b/tools/patches/verus-irc11.patch new file mode 100644 index 000000000..e22bfedf5 --- /dev/null +++ b/tools/patches/verus-irc11.patch @@ -0,0 +1,3053 @@ +diff --git a/source/rust_verify_test/tests/core_special_setup.rs b/source/rust_verify_test/tests/core_special_setup.rs +index d9f26d56..7dcb1fbf 100644 +--- a/source/rust_verify_test/tests/core_special_setup.rs ++++ b/source/rust_verify_test/tests/core_special_setup.rs +@@ -28,6 +28,7 @@ test_verify_one_file_with_options! { + verus_keep_ghost, + feature(fn_traits), + )] ++ #![cfg_attr(verus_keep_ghost, feature(auto_traits))] + #![cfg_attr(verus_keep_ghost, verifier::exec_allows_no_decreases_clause)] + + #[verifier::external] +diff --git a/source/vstd/atomic.rs b/source/vstd/atomic.rs +index bcc34e13..8bb30533 100644 +--- a/source/vstd/atomic.rs ++++ b/source/vstd/atomic.rs +@@ -1,56 +1,67 @@ + #![allow(unused_imports)] + +-use core::sync::atomic::{ +- AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, +- AtomicU32, AtomicUsize, Ordering, +-}; +- +-#[cfg(target_has_atomic = "64")] +-use core::sync::atomic::{AtomicI64, AtomicU64}; +- +-use super::modes::*; + use super::pervasive::*; + use super::prelude::*; + use super::raw_ptr::PointsTo; + use super::view::*; +-use super::wrapping::*; +- +-macro_rules! make_unsigned_integer_atomic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { +- atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); +- #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] +- impl $at_ident { +- atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); +- atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); +- } +- }; +-} + +-macro_rules! make_signed_integer_atomic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { +- atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); +- #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] +- impl $at_ident { +- atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); +- atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); +- } +- }; +-} ++#[cfg(not(feature = "weak-memory"))] ++pub use sc_atomic_types::*; + +-macro_rules! make_bool_atomic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); +- #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] +- impl $at_ident { +- atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); +- atomic_bool_methods!($at_ident, $p_ident, $rust_ty, $value_ty); +- } ++#[cfg(not(feature = "weak-memory"))] ++mod sc_atomic_types { ++ ++ use core::sync::atomic::{ ++ AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, ++ AtomicU32, AtomicUsize, Ordering, + }; +-} + +-macro_rules! atomic_types { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- verus! { ++ #[cfg(target_has_atomic = "64")] ++ use core::sync::atomic::{AtomicI64, AtomicU64}; ++ ++ use super::super::modes::*; ++ use super::super::pervasive::*; ++ use super::super::prelude::*; ++ use super::super::raw_ptr::PointsTo; ++ use super::super::view::*; ++ use super::super::wrapping::*; ++ ++ macro_rules! make_unsigned_integer_atomic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++ } ++ ++ macro_rules! make_signed_integer_atomic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++ } ++ ++ macro_rules! make_bool_atomic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); ++ atomic_bool_methods!($at_ident, $p_ident, $rust_ty, $value_ty); ++ } ++ }; ++ } ++ ++ macro_rules! atomic_types { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus! { + + #[verifier::external_body] /* vattr */ + pub struct $at_ident { +@@ -92,12 +102,12 @@ macro_rules! atomic_types { + } + + } +- }; +-} ++ }; ++ } + +-macro_rules! atomic_types_generic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- verus! { ++ macro_rules! atomic_types_generic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus! { + + #[verifier::accept_recursive_types(T)] + #[verifier::external_body] /* vattr */ +@@ -142,12 +152,12 @@ macro_rules! atomic_types_generic { + } + + } +- }; +-} ++ }; ++ } + +-pub type AtomicCellId = int; ++ pub type AtomicCellId = int; + +-macro_rules! atomic_common_methods { ++ macro_rules! atomic_common_methods { + ($at_ident: ty, $p_ident: ty, $p_data_ident: ty, $rust_ty: ty, $value_ty: ty, [ $($addr:tt)* ]) => { + verus_impl!{ + +@@ -268,7 +278,7 @@ macro_rules! atomic_common_methods { + }; + } + +-macro_rules! atomic_integer_methods { ++ macro_rules! atomic_integer_methods { + ($at_ident:ident, $p_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { + verus_impl!{ + +@@ -436,9 +446,9 @@ macro_rules! atomic_integer_methods { + }; + } + +-macro_rules! atomic_bool_methods { +- ($at_ident:ident, $p_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- verus!{ ++ macro_rules! atomic_bool_methods { ++ ($at_ident:ident, $p_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus!{ + + #[inline(always)] + #[verifier::external_body] /* vattr */ +@@ -505,8 +515,8 @@ macro_rules! atomic_bool_methods { + } + + } +- }; +-} ++ }; ++ } + + macro_rules! ptr_atomic_methods { + ($at_ty: ty, $rust_ty: ty, $value_ty: ty) => { +@@ -593,90 +603,163 @@ ptr_atomic_methods!(PAtomicIsize, AtomicIsize, isize); + + make_bool_atomic!(PAtomicBool, PermissionBool, PermissionDataBool, AtomicBool, bool); + +-make_unsigned_integer_atomic!(PAtomicU8, PermissionU8, PermissionDataU8, AtomicU8, u8, u8_specs); +-make_unsigned_integer_atomic!( +- PAtomicU16, +- PermissionU16, +- PermissionDataU16, +- AtomicU16, +- u16, +- u16_specs +-); +-make_unsigned_integer_atomic!( +- PAtomicU32, +- PermissionU32, +- PermissionDataU32, +- AtomicU32, +- u32, +- u32_specs +-); ++ make_unsigned_integer_atomic!( ++ PAtomicU8, ++ PermissionU8, ++ PermissionDataU8, ++ AtomicU8, ++ u8, ++ u8_specs ++ ); ++ make_unsigned_integer_atomic!( ++ PAtomicU16, ++ PermissionU16, ++ PermissionDataU16, ++ AtomicU16, ++ u16, ++ u16_specs ++ ); ++ make_unsigned_integer_atomic!( ++ PAtomicU32, ++ PermissionU32, ++ PermissionDataU32, ++ AtomicU32, ++ u32, ++ u32_specs ++ ); + +-#[cfg(target_has_atomic = "64")] +-make_unsigned_integer_atomic!( +- PAtomicU64, +- PermissionU64, +- PermissionDataU64, +- AtomicU64, +- u64, +- u64_specs +-); +-make_unsigned_integer_atomic!( +- PAtomicUsize, +- PermissionUsize, +- PermissionDataUsize, +- AtomicUsize, +- usize, +- usize_specs +-); +- +-make_signed_integer_atomic!(PAtomicI8, PermissionI8, PermissionDataI8, AtomicI8, i8, i8_specs); +-make_signed_integer_atomic!( +- PAtomicI16, +- PermissionI16, +- PermissionDataI16, +- AtomicI16, +- i16, +- i16_specs +-); +-make_signed_integer_atomic!( +- PAtomicI32, +- PermissionI32, +- PermissionDataI32, +- AtomicI32, +- i32, +- i32_specs +-); ++ #[cfg(target_has_atomic = "64")] ++ make_unsigned_integer_atomic!( ++ PAtomicU64, ++ PermissionU64, ++ PermissionDataU64, ++ AtomicU64, ++ u64, ++ u64_specs ++ ); ++ make_unsigned_integer_atomic!( ++ PAtomicUsize, ++ PermissionUsize, ++ PermissionDataUsize, ++ AtomicUsize, ++ usize, ++ usize_specs ++ ); + +-#[cfg(target_has_atomic = "64")] +-make_signed_integer_atomic!( +- PAtomicI64, +- PermissionI64, +- PermissionDataI64, +- AtomicI64, +- i64, +- i64_specs +-); +-make_signed_integer_atomic!( +- PAtomicIsize, +- PermissionIsize, +- PermissionDataIsize, +- AtomicIsize, +- isize, +- isize_specs +-); +- +-atomic_types_generic!(PAtomicPtr, PermissionPtr, PermissionDataPtr, AtomicPtr, *mut T); +- +-#[cfg_attr(verus_keep_ghost, verifier::verus_macro)] +-impl PAtomicPtr { +- atomic_common_methods!( +- PAtomicPtr::, +- PermissionPtr::, +- PermissionDataPtr::, +- AtomicPtr::, +- *mut T, +- [ .view().addr ] ++ make_signed_integer_atomic!(PAtomicI8, PermissionI8, PermissionDataI8, AtomicI8, i8, i8_specs); ++ make_signed_integer_atomic!( ++ PAtomicI16, ++ PermissionI16, ++ PermissionDataI16, ++ AtomicI16, ++ i16, ++ i16_specs ++ ); ++ make_signed_integer_atomic!( ++ PAtomicI32, ++ PermissionI32, ++ PermissionDataI32, ++ AtomicI32, ++ i32, ++ i32_specs ++ ); ++ ++ #[cfg(target_has_atomic = "64")] ++ make_signed_integer_atomic!( ++ PAtomicI64, ++ PermissionI64, ++ PermissionDataI64, ++ AtomicI64, ++ i64, ++ i64_specs ++ ); ++ make_signed_integer_atomic!( ++ PAtomicIsize, ++ PermissionIsize, ++ PermissionDataIsize, ++ AtomicIsize, ++ isize, ++ isize_specs + ); ++ ++ atomic_types_generic!(PAtomicPtr, PermissionPtr, PermissionDataPtr, AtomicPtr, *mut T); ++ ++ #[cfg_attr(verus_keep_ghost, verifier::verus_macro)] ++ impl PAtomicPtr { ++ atomic_common_methods!( ++ PAtomicPtr::, ++ PermissionPtr::, ++ PermissionDataPtr::, ++ AtomicPtr::, ++ *mut T, ++ [ .view().addr ] ++ ); ++ } ++ ++ impl PAtomicPtr { ++ verus_impl! { ++ ++ #[inline(always)] ++ #[verifier::external_body] /* vattr */ ++ #[verifier::atomic] /* vattr */ ++ #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] ++ pub fn fetch_and(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: ++ *mut T) ++ requires ++ equal(self.id(), old(perm).view().patomic), ++ ensures ++ equal(old(perm).view().value, ret), ++ final(perm).view().patomic == old(perm).view().patomic, ++ final(perm).view().value@.addr == (old(perm).view().value@.addr & n), ++ final(perm).view().value@.provenance == old(perm).view().value@.provenance, ++ final(perm).view().value@.metadata == old(perm).view().value@.metadata, ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.fetch_and(n, Ordering::SeqCst) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] /* vattr */ ++ #[verifier::atomic] /* vattr */ ++ #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] ++ pub fn fetch_xor(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: ++ *mut T) ++ requires ++ equal(self.id(), old(perm).view().patomic), ++ ensures ++ equal(old(perm).view().value, ret), ++ final(perm).view().patomic == old(perm).view().patomic, ++ final(perm).view().value@.addr == (old(perm).view().value@.addr ^ n), ++ final(perm).view().value@.provenance == old(perm).view().value@.provenance, ++ final(perm).view().value@.metadata == old(perm).view().value@.metadata, ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.fetch_xor(n, Ordering::SeqCst) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] /* vattr */ ++ #[verifier::atomic] /* vattr */ ++ #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] ++ pub fn fetch_or(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: *mut T) ++ requires ++ equal(self.id(), old(perm).view().patomic), ++ ensures ++ equal(old(perm).view().value, ret), ++ final(perm).view().patomic == old(perm).view().patomic, ++ final(perm).view().value@.addr == (old(perm).view().value@.addr | n), ++ final(perm).view().value@.provenance == old(perm).view().value@.provenance, ++ final(perm).view().value@.metadata == old(perm).view().value@.metadata, ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.fetch_or(n, Ordering::SeqCst) ++ } ++ ++ } ++ } + } + + impl core::fmt::Debug for AtomicUpdate { +@@ -1125,65 +1208,4 @@ pub use { + try_open_atomic_update_in_proof, + }; + +-impl PAtomicPtr { +- #[inline(always)] +- #[verifier::external_body] /* vattr */ +- #[verifier::atomic] /* vattr */ +- #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] +- pub fn fetch_and(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: +- *mut T) +- requires +- equal(self.id(), old(perm).view().patomic), +- ensures +- equal(old(perm).view().value, ret), +- final(perm).view().patomic == old(perm).view().patomic, +- final(perm).view().value@.addr == (old(perm).view().value@.addr & n), +- final(perm).view().value@.provenance == old(perm).view().value@.provenance, +- final(perm).view().value@.metadata == old(perm).view().value@.metadata, +- opens_invariants none +- no_unwind +- { +- self.ato.fetch_and(n, Ordering::SeqCst) +- } +- +- #[inline(always)] +- #[verifier::external_body] /* vattr */ +- #[verifier::atomic] /* vattr */ +- #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] +- pub fn fetch_xor(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: +- *mut T) +- requires +- equal(self.id(), old(perm).view().patomic), +- ensures +- equal(old(perm).view().value, ret), +- final(perm).view().patomic == old(perm).view().patomic, +- final(perm).view().value@.addr == (old(perm).view().value@.addr ^ n), +- final(perm).view().value@.provenance == old(perm).view().value@.provenance, +- final(perm).view().value@.metadata == old(perm).view().value@.metadata, +- opens_invariants none +- no_unwind +- { +- self.ato.fetch_xor(n, Ordering::SeqCst) +- } +- +- #[inline(always)] +- #[verifier::external_body] /* vattr */ +- #[verifier::atomic] /* vattr */ +- #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] +- pub fn fetch_or(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: *mut T) +- requires +- equal(self.id(), old(perm).view().patomic), +- ensures +- equal(old(perm).view().value, ret), +- final(perm).view().patomic == old(perm).view().patomic, +- final(perm).view().value@.addr == (old(perm).view().value@.addr | n), +- final(perm).view().value@.provenance == old(perm).view().value@.provenance, +- final(perm).view().value@.metadata == old(perm).view().value@.metadata, +- opens_invariants none +- no_unwind +- { +- self.ato.fetch_or(n, Ordering::SeqCst) +- } +-} +- + } // verus! +diff --git a/source/vstd/atomic_ghost.rs b/source/vstd/atomic_ghost.rs +index 317e8c52..0d6a3b8a 100644 +--- a/source/vstd/atomic_ghost.rs ++++ b/source/vstd/atomic_ghost.rs +@@ -2,21 +2,27 @@ + //! See the [`atomic_with_ghost!`] documentation for more information. + #![allow(unused_imports)] + +-use super::atomic::*; +-use super::invariant::*; +-use super::modes::*; +-use super::prelude::*; ++#[cfg(not(feature = "weak-memory"))] ++pub use atomic_ghost::*; + +-verus! { ++#[cfg(not(feature = "weak-memory"))] ++mod atomic_ghost { ++ ++ use super::super::atomic::*; ++ use super::super::invariant::*; ++ use super::super::modes::*; ++ use super::super::prelude::*; ++ ++ verus! { + + pub trait AtomicInvariantPredicate { + spec fn atomic_inv(k: K, v: V, g: G) -> bool; + } + + } // verus! +-macro_rules! declare_atomic_type { +- ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { +- verus!{ ++ macro_rules! declare_atomic_type { ++ ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { ++ verus!{ + + pub struct $atomic_pred_ty { p: Pred } + +@@ -103,11 +109,11 @@ macro_rules! declare_atomic_type { + } + + } +- }; +-} +-macro_rules! declare_atomic_type_generic { +- ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { +- verus!{ ++ }; ++ } ++ macro_rules! declare_atomic_type_generic { ++ ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { ++ verus!{ + + pub struct $atomic_pred_ty { t: T, p: Pred } + +@@ -190,142 +196,142 @@ macro_rules! declare_atomic_type_generic { + } + + } +- }; +-} ++ }; ++ } + +-#[cfg(target_has_atomic = "64")] +-declare_atomic_type!(AtomicU64, PAtomicU64, PermissionU64, u64, AtomicPredU64); +- +-declare_atomic_type!(AtomicU32, PAtomicU32, PermissionU32, u32, AtomicPredU32); +-declare_atomic_type!(AtomicU16, PAtomicU16, PermissionU16, u16, AtomicPredU16); +-declare_atomic_type!(AtomicU8, PAtomicU8, PermissionU8, u8, AtomicPredU8); +-declare_atomic_type!(AtomicUsize, PAtomicUsize, PermissionUsize, usize, AtomicPredUsize); +- +-#[cfg(target_has_atomic = "64")] +-declare_atomic_type!(AtomicI64, PAtomicI64, PermissionI64, i64, AtomicPredI64); +- +-declare_atomic_type!(AtomicI32, PAtomicI32, PermissionI32, i32, AtomicPredI32); +-declare_atomic_type!(AtomicI16, PAtomicI16, PermissionI16, i16, AtomicPredI16); +-declare_atomic_type!(AtomicI8, PAtomicI8, PermissionI8, i8, AtomicPredI8); +-declare_atomic_type!(AtomicIsize, PAtomicIsize, PermissionIsize, isize, AtomicPredIsize); +- +-declare_atomic_type!(AtomicBool, PAtomicBool, PermissionBool, bool, AtomicPredBool); +- +-declare_atomic_type_generic!(AtomicPtr, PAtomicPtr, PermissionPtr, *mut T, AtomicPredPtr); +- +-/// Performs a given atomic operation on a given atomic +-/// while providing access to its ghost state. +-/// +-/// `atomic_with_ghost!` supports the types +-/// [`AtomicU64`] [`AtomicU32`], [`AtomicU16`], [`AtomicU8`], +-/// [`AtomicI64`], [`AtomicI32`], [`AtomicI16`], [`AtomicI8`], and [`AtomicBool`]. +-/// +-/// For each type, it supports all applicable atomic operations among +-/// `load`, `store`, `swap`, `compare_exchange`, `compare_exchange_weak`, +-/// `fetch_add`, `fetch_add_wrapping`, `fetch_sub`, `fetch_sub_wrapping`, +-/// `fetch_or`, `fetch_and`, `fetch_xor`, `fetch_nand`, `fetch_max`, and `fetch_min`. +-/// +-/// Naturally, `AtomicBool` does not support the arithmetic-specific operations. +-/// +-/// In general, the syntax is: +-/// +-/// let result = atomic_with_ghost!( +-/// $atomic => $operation_name($operands...); +-/// update $prev -> $next; // `update` line is optional +-/// returning $ret; // `returning` line is optional +-/// ghost $g => { +-/// /* Proof code with access to `tracked` variable `g: G` */ +-/// } +-/// ); +-/// +-/// Here, the `$operation_name` is one of `load`, `store`, etc. Meanwhile, +-/// `$prev`, `$next`, and `$ret` are all identifiers which +-/// will be available as spec variable inside the block to describe the +-/// atomic action which is performed. +-/// +-/// For example, suppose the user performs `fetch_add(1)`. The atomic +-/// operation might load the value 5, add 1, store the value 6, +-/// and return the original value, 5. In that case, we would have +-/// `prev == 5`, `next == 6`, and `ret == 5`. +-/// +-/// The specification for a given operation is given as a relation between +-/// `prev`, `next`, and `ret`; that is, at the beginning of the proof block, +-/// the user may assume the given specification holds: +-/// +-/// | operation | specification | +-/// |-------------------------------|----------------------------------------------------------------------------------------------------------------------------| +-/// | `load()` | `next == prev && rev == prev` | +-/// | `store(x)` | `next == x && ret == ()` | +-/// | `swap(x)` | `next == x && ret == prev` | +-/// | `compare_exchange(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`prev != x && next == prev && ret == Err(prev)` ("failure") | +-/// | `compare_exchange_weak(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`next == prev && ret == Err(prev)` ("failure") | +-/// | `fetch_add(x)` (*) | `next == prev + x && ret == prev` | +-/// | `fetch_add_wrapping(x)` | `next == wrapping_add(prev, x) && ret == prev` | +-/// | `fetch_sub(x)` (*) | `next == prev - x && ret == prev` | +-/// | `fetch_sub_wrapping(x)` | `next == wrapping_sub(prev, x) && ret == prev` | +-/// | `fetch_or(x)` | next == prev \| x && ret == prev | +-/// | `fetch_and(x)` | `next == prev & x && ret == prev` | +-/// | `fetch_xor(x)` | `next == prev ^ x && ret == prev` | +-/// | `fetch_nand(x)` | `next == !(prev & x) && ret == prev` | +-/// | `fetch_max(x)` | `next == max(prev, x) && ret == prev` | +-/// | `fetch_min(x)` | `next == max(prev, x) && ret == prev` | +-/// | `no_op()` (**) | `next == prev && ret == ()` | +-/// +-/// (*) Note that `fetch_add` and `fetch_sub` do not specify +-/// wrapping-on-overflow; instead, they require the user to +-/// prove that overflow _does not occur_, i.e., the user must show +-/// that `next` is in bounds for the integer type in question. +-/// Furthermore, for `fetch_add` and `fetch_sub`, the spec values of +-/// `prev`, `next`, and `ret` are all given with type `int`, so the +-/// user may reason about boundedness within the proof block. +-/// +-/// (As executable code, `fetch_add` is equivalent to `fetch_add_wrapping`, +-/// and likewise for `fetch_sub` and `fetch_sub_wrapping`. +-/// We have both because it's frequently the case that the user needs to verify +-/// lack-of-overflow _anyway_, and having it as an explicit precondition by default +-/// then makes verification errors easier to diagnose. Furthermore, when overflow is +-/// intended, the wrapping operations document that intent.) +-/// +-/// (**) `no_op` is entirely a ghost operation and doesn't emit any actual instruction. +-/// This allows the user to access the ghost state and the stored value (as `spec` data) +-/// without actually performing a load. +-/// +-/// --- +-/// +-/// At the beginning of the proof block, the user may assume, in addition +-/// to the specified relation between `prev`, `next`, and `ret`, that +-/// `atomic.inv(prev, g)` holds. The user is required to update `g` such that +-/// `atomic.inv(next, g)` holds at the end of the block. +-/// In other words, the ghost block has the implicit pre- and post-conditions: +-/// +-/// let result = atomic_with_ghost!( +-/// $atomic => $operation_name($operands...); +-/// update $prev -> $next; +-/// returning $ret; +-/// ghost $g => { +-/// assume(specified relation on (prev, next, ret)); +-/// assume(atomic.inv(prev, g)); +-/// +-/// // User code here; may update variable `g` with full +-/// // access to variables in the outer context. +-/// +-/// assert(atomic.inv(next, g)); +-/// } +-/// ); +-/// +-/// Note that the necessary action on ghost state might depend +-/// on the result of the operation; for example, if the user performs a +-/// compare-and-swap, then the ghost action that they then need to do +-/// will probably depend on whether the operation succeeded or not. +-/// +-/// The value returned by the `atomic_with_ghost!(...)` expression will be equal +-/// to `ret`, although the return value is an `exec` value (the actual result of +-/// the operation) while `ret` is a `spec` value. +-/// +-/// ### Example (TODO) +- +-#[macro_export] +-macro_rules! atomic_with_ghost { ++ #[cfg(target_has_atomic = "64")] ++ declare_atomic_type!(AtomicU64, PAtomicU64, PermissionU64, u64, AtomicPredU64); ++ ++ declare_atomic_type!(AtomicU32, PAtomicU32, PermissionU32, u32, AtomicPredU32); ++ declare_atomic_type!(AtomicU16, PAtomicU16, PermissionU16, u16, AtomicPredU16); ++ declare_atomic_type!(AtomicU8, PAtomicU8, PermissionU8, u8, AtomicPredU8); ++ declare_atomic_type!(AtomicUsize, PAtomicUsize, PermissionUsize, usize, AtomicPredUsize); ++ ++ #[cfg(target_has_atomic = "64")] ++ declare_atomic_type!(AtomicI64, PAtomicI64, PermissionI64, i64, AtomicPredI64); ++ ++ declare_atomic_type!(AtomicI32, PAtomicI32, PermissionI32, i32, AtomicPredI32); ++ declare_atomic_type!(AtomicI16, PAtomicI16, PermissionI16, i16, AtomicPredI16); ++ declare_atomic_type!(AtomicI8, PAtomicI8, PermissionI8, i8, AtomicPredI8); ++ declare_atomic_type!(AtomicIsize, PAtomicIsize, PermissionIsize, isize, AtomicPredIsize); ++ ++ declare_atomic_type!(AtomicBool, PAtomicBool, PermissionBool, bool, AtomicPredBool); ++ ++ declare_atomic_type_generic!(AtomicPtr, PAtomicPtr, PermissionPtr, *mut T, AtomicPredPtr); ++ ++ /// Performs a given atomic operation on a given atomic ++ /// while providing access to its ghost state. ++ /// ++ /// `atomic_with_ghost!` supports the types ++ /// [`AtomicU64`] [`AtomicU32`], [`AtomicU16`], [`AtomicU8`], ++ /// [`AtomicI64`], [`AtomicI32`], [`AtomicI16`], [`AtomicI8`], and [`AtomicBool`]. ++ /// ++ /// For each type, it supports all applicable atomic operations among ++ /// `load`, `store`, `swap`, `compare_exchange`, `compare_exchange_weak`, ++ /// `fetch_add`, `fetch_add_wrapping`, `fetch_sub`, `fetch_sub_wrapping`, ++ /// `fetch_or`, `fetch_and`, `fetch_xor`, `fetch_nand`, `fetch_max`, and `fetch_min`. ++ /// ++ /// Naturally, `AtomicBool` does not support the arithmetic-specific operations. ++ /// ++ /// In general, the syntax is: ++ /// ++ /// let result = atomic_with_ghost!( ++ /// $atomic => $operation_name($operands...); ++ /// update $prev -> $next; // `update` line is optional ++ /// returning $ret; // `returning` line is optional ++ /// ghost $g => { ++ /// /* Proof code with access to `tracked` variable `g: G` */ ++ /// } ++ /// ); ++ /// ++ /// Here, the `$operation_name` is one of `load`, `store`, etc. Meanwhile, ++ /// `$prev`, `$next`, and `$ret` are all identifiers which ++ /// will be available as spec variable inside the block to describe the ++ /// atomic action which is performed. ++ /// ++ /// For example, suppose the user performs `fetch_add(1)`. The atomic ++ /// operation might load the value 5, add 1, store the value 6, ++ /// and return the original value, 5. In that case, we would have ++ /// `prev == 5`, `next == 6`, and `ret == 5`. ++ /// ++ /// The specification for a given operation is given as a relation between ++ /// `prev`, `next`, and `ret`; that is, at the beginning of the proof block, ++ /// the user may assume the given specification holds: ++ /// ++ /// | operation | specification | ++ /// |-------------------------------|----------------------------------------------------------------------------------------------------------------------------| ++ /// | `load()` | `next == prev && rev == prev` | ++ /// | `store(x)` | `next == x && ret == ()` | ++ /// | `swap(x)` | `next == x && ret == prev` | ++ /// | `compare_exchange(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`prev != x && next == prev && ret == Err(prev)` ("failure") | ++ /// | `compare_exchange_weak(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`next == prev && ret == Err(prev)` ("failure") | ++ /// | `fetch_add(x)` (*) | `next == prev + x && ret == prev` | ++ /// | `fetch_add_wrapping(x)` | `next == wrapping_add(prev, x) && ret == prev` | ++ /// | `fetch_sub(x)` (*) | `next == prev - x && ret == prev` | ++ /// | `fetch_sub_wrapping(x)` | `next == wrapping_sub(prev, x) && ret == prev` | ++ /// | `fetch_or(x)` | next == prev \| x && ret == prev | ++ /// | `fetch_and(x)` | `next == prev & x && ret == prev` | ++ /// | `fetch_xor(x)` | `next == prev ^ x && ret == prev` | ++ /// | `fetch_nand(x)` | `next == !(prev & x) && ret == prev` | ++ /// | `fetch_max(x)` | `next == max(prev, x) && ret == prev` | ++ /// | `fetch_min(x)` | `next == max(prev, x) && ret == prev` | ++ /// | `no_op()` (**) | `next == prev && ret == ()` | ++ /// ++ /// (*) Note that `fetch_add` and `fetch_sub` do not specify ++ /// wrapping-on-overflow; instead, they require the user to ++ /// prove that overflow _does not occur_, i.e., the user must show ++ /// that `next` is in bounds for the integer type in question. ++ /// Furthermore, for `fetch_add` and `fetch_sub`, the spec values of ++ /// `prev`, `next`, and `ret` are all given with type `int`, so the ++ /// user may reason about boundedness within the proof block. ++ /// ++ /// (As executable code, `fetch_add` is equivalent to `fetch_add_wrapping`, ++ /// and likewise for `fetch_sub` and `fetch_sub_wrapping`. ++ /// We have both because it's frequently the case that the user needs to verify ++ /// lack-of-overflow _anyway_, and having it as an explicit precondition by default ++ /// then makes verification errors easier to diagnose. Furthermore, when overflow is ++ /// intended, the wrapping operations document that intent.) ++ /// ++ /// (**) `no_op` is entirely a ghost operation and doesn't emit any actual instruction. ++ /// This allows the user to access the ghost state and the stored value (as `spec` data) ++ /// without actually performing a load. ++ /// ++ /// --- ++ /// ++ /// At the beginning of the proof block, the user may assume, in addition ++ /// to the specified relation between `prev`, `next`, and `ret`, that ++ /// `atomic.inv(prev, g)` holds. The user is required to update `g` such that ++ /// `atomic.inv(next, g)` holds at the end of the block. ++ /// In other words, the ghost block has the implicit pre- and post-conditions: ++ /// ++ /// let result = atomic_with_ghost!( ++ /// $atomic => $operation_name($operands...); ++ /// update $prev -> $next; ++ /// returning $ret; ++ /// ghost $g => { ++ /// assume(specified relation on (prev, next, ret)); ++ /// assume(atomic.inv(prev, g)); ++ /// ++ /// // User code here; may update variable `g` with full ++ /// // access to variables in the outer context. ++ /// ++ /// assert(atomic.inv(next, g)); ++ /// } ++ /// ); ++ /// ++ /// Note that the necessary action on ghost state might depend ++ /// on the result of the operation; for example, if the user performs a ++ /// compare-and-swap, then the ghost action that they then need to do ++ /// will probably depend on whether the operation succeeded or not. ++ /// ++ /// The value returned by the `atomic_with_ghost!(...)` expression will be equal ++ /// to `ret`, although the return value is an `exec` value (the actual result of ++ /// the operation) while `ret` is a `spec` value. ++ /// ++ /// ### Example (TODO) ++ ++ #[macro_export] ++ macro_rules! atomic_with_ghost { + ($($tokens:tt)*) => { + // The helper is used to parse things using Verus syntax + // The helper then calls atomic_with_ghost_inner, below: +@@ -335,127 +341,127 @@ macro_rules! atomic_with_ghost { + } + } + +-pub use atomic_with_ghost; +- +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_inner { +- (load, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_load!($e, $prev, $next, $ret, $g, $b) +- }; +- (store, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_store!( +- $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (swap, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- swap, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- +- (fetch_or, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_or, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_and, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_and, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_xor, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_xor, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_nand, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_nand, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_max, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_max, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_min, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_min, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_add_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_add_wrapping, +- $e, +- $operand, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- (fetch_sub_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_sub_wrapping, +- $e, +- $operand, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- +- (fetch_add, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_add!( +- $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_sub, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_sub!( +- $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- +- (compare_exchange, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( +- compare_exchange, +- $e, +- $operand1, +- $operand2, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- (compare_exchange_weak, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( +- compare_exchange_weak, +- $e, +- $operand1, +- $operand2, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- (no_op, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_no_op!($e, $prev, $next, $ret, $g, $b) +- }; +-} ++ pub use atomic_with_ghost; ++ ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_inner { ++ (load, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_load!($e, $prev, $next, $ret, $g, $b) ++ }; ++ (store, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_store!( ++ $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (swap, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ swap, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ ++ (fetch_or, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_or, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_and, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_and, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_xor, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_xor, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_nand, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_nand, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_max, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_max, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_min, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_min, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_add_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_add_wrapping, ++ $e, ++ $operand, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ (fetch_sub_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_sub_wrapping, ++ $e, ++ $operand, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ ++ (fetch_add, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_add!( ++ $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_sub, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_sub!( ++ $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ ++ (compare_exchange, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( ++ compare_exchange, ++ $e, ++ $operand1, ++ $operand2, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ (compare_exchange_weak, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( ++ compare_exchange_weak, ++ $e, ++ $operand1, ++ $operand2, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ (no_op, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_no_op!($e, $prev, $next, $ret, $g, $b) ++ }; ++ } + +-pub use atomic_with_ghost_inner; ++ pub use atomic_with_ghost_inner; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_store { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_store { + ($e:expr, $operand:expr, $prev:pat, $next:pat, $res:pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let atomic = &($e); +@@ -474,11 +480,11 @@ macro_rules! atomic_with_ghost_store { + } } + }; + } +-pub use atomic_with_ghost_store; ++ pub use atomic_with_ghost_store; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_load { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_load { + ($e:expr, $prev:pat, $next: pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -500,11 +506,11 @@ macro_rules! atomic_with_ghost_load { + }; + } + +-pub use atomic_with_ghost_load; ++ pub use atomic_with_ghost_load; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_no_op { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_no_op { + ($e:expr, $prev:pat, $next: pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let atomic = &($e); +@@ -524,11 +530,11 @@ macro_rules! atomic_with_ghost_no_op { + }; + } + +-pub use atomic_with_ghost_no_op; ++ pub use atomic_with_ghost_no_op; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_with_1_operand { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_with_1_operand { + ($name:ident, $e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -551,11 +557,11 @@ macro_rules! atomic_with_ghost_update_with_1_operand { + }; + } + +-pub use atomic_with_ghost_update_with_1_operand; ++ pub use atomic_with_ghost_update_with_1_operand; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_with_2_operand { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_with_2_operand { + ($name:ident, $e:expr, $operand1:expr, $operand2:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -579,11 +585,11 @@ macro_rules! atomic_with_ghost_update_with_2_operand { + }; + } + +-pub use atomic_with_ghost_update_with_2_operand; ++ pub use atomic_with_ghost_update_with_2_operand; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_fetch_add { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_fetch_add { + ($e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + ($crate::vstd::prelude::verus_exec_expr!( { + let result; +@@ -610,11 +616,11 @@ macro_rules! atomic_with_ghost_update_fetch_add { + } + } + +-pub use atomic_with_ghost_update_fetch_add; ++ pub use atomic_with_ghost_update_fetch_add; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_fetch_sub { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_fetch_sub { + ($e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -641,4 +647,5 @@ macro_rules! atomic_with_ghost_update_fetch_sub { + }; + } + +-pub use atomic_with_ghost_update_fetch_sub; ++ pub use atomic_with_ghost_update_fetch_sub; ++} +diff --git a/source/vstd/atomic_weak.rs b/source/vstd/atomic_weak.rs +new file mode 100644 +index 00000000..644ac089 +--- /dev/null ++++ b/source/vstd/atomic_weak.rs +@@ -0,0 +1,743 @@ ++#[cfg(feature = "weak-memory")] ++pub use weak_atomic_types::*; ++ ++#[cfg(feature = "weak-memory")] ++mod weak_atomic_types { ++ ++ use core::sync::atomic::{ ++ AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, ++ AtomicU32, AtomicUsize, Ordering, ++ }; ++ ++ #[cfg(target_has_atomic = "64")] ++ use core::sync::atomic::{AtomicI64, AtomicU64}; ++ ++ use super::super::cell::CellId; ++ use super::super::prelude::*; ++ use super::super::thread_view::*; ++ use super::super::wrapping::*; ++ ++ verus! { ++ ++broadcast use crate::group_vstd_default; ++ ++#[verifier::external_body] ++pub fn fence_release(Tracked(vs): Tracked) -> (rel_vs: Tracked) ++ ensures ++ vs.view() == rel_vs@.view(), ++ opens_invariants none ++ no_unwind ++{ ++ core::sync::atomic::fence(Ordering::Release); ++ Tracked::assume_new() ++} ++ ++#[verifier::external_body] ++pub fn fence_acquire(Tracked(acq_vs): Tracked) -> (vs: Tracked) ++ ensures ++ acq_vs.view() == vs@.view(), ++ opens_invariants none ++ no_unwind ++{ ++ core::sync::atomic::fence(Ordering::Acquire); ++ Tracked::assume_new() ++} ++ ++pub ghost struct AtomicHistory(pub Map); ++ ++impl AtomicHistory { ++ pub open spec fn dom(&self) -> Set { ++ self.0.dom() ++ } ++ ++ pub open spec fn contains_timestamp(&self, timestamp: nat) -> bool { ++ self.0.dom().contains(timestamp) ++ } ++ ++ pub open spec fn index(&self, timestamp: nat) -> (T, ThreadView) ++ recommends ++ self.contains_timestamp(timestamp), ++ { ++ self.0.index(timestamp) ++ } ++ ++ pub open spec fn value(&self, timestamp: nat) -> T ++ recommends ++ self.contains_timestamp(timestamp), ++ { ++ self.0.index(timestamp).0 ++ } ++ ++ pub open spec fn thread_view(&self, timestamp: nat) -> ThreadView ++ recommends ++ self.contains_timestamp(timestamp), ++ { ++ self.0.index(timestamp).1 ++ } ++ ++ pub open spec fn get(&self, timestamp: nat) -> Option<(T, ThreadView)> { ++ self.0.get(timestamp) ++ } ++ ++ pub open spec fn get_value(&self, timestamp: nat) -> Option { ++ match self.get(timestamp) { ++ Some((val, _)) => Some(val), ++ None => None, ++ } ++ } ++ ++ pub open spec fn get_thread_view(&self, timestamp: nat) -> Option { ++ match self.get(timestamp) { ++ Some((_, view)) => Some(view), ++ None => None, ++ } ++ } ++ ++ pub open spec fn insert(&self, timestamp: nat, val: T, view: ThreadView) -> Self ++ recommends ++ !self.contains_timestamp(timestamp), ++ { ++ AtomicHistory(self.0.insert(timestamp, (val, view))) ++ } ++ ++ pub open spec fn remove(&self, timestamp: nat) -> Self { ++ AtomicHistory(self.0.remove(timestamp)) ++ } ++ ++ pub open spec fn is_singleton(&self, timestamp: nat, val: (T, ThreadView)) -> bool { ++ &&& self.contains_timestamp(timestamp) ++ &&& forall|ts| #[trigger] ++ self.contains_timestamp(ts) ==> ts == timestamp && self.get(ts) == Some(val) ++ } ++ ++ pub open spec fn is_max_timestamp(&self, timestamp: nat) -> bool { ++ &&& self.contains_timestamp(timestamp) ++ &&& forall|ts| #[trigger] self.contains_timestamp(ts) ==> ts <= timestamp ++ } ++} ++ ++pub broadcast proof fn history_insert_contains_timestamp_cases( ++ h: AtomicHistory, ++ t: nat, ++ v: T, ++ o: ThreadView, ++ t2: nat, ++) ++ requires ++ #[trigger] h.insert(t, v, o).contains_timestamp(t2), ++ ensures ++ t == t2 || h.contains_timestamp(t2), ++{ ++} ++ ++pub broadcast proof fn history_insert_contains_inserted_timestamp( ++ h: AtomicHistory, ++ t: nat, ++ v: T, ++ o: ThreadView, ++) ++ ensures ++ (#[trigger] h.insert(t, v, o)).contains_timestamp(t), ++{ ++} ++ ++pub broadcast proof fn history_get_contains_timestamp(h: AtomicHistory, t: nat) ++ requires ++ (#[trigger] h.get(t)).is_some(), ++ ensures ++ h.contains_timestamp(t), ++{ ++} ++ ++pub broadcast proof fn history_singleton_dom_singleton( ++ h: AtomicHistory, ++ ts: nat, ++ val: (T, ThreadView), ++) ++ requires ++ #[trigger] h.is_singleton(ts, val), ++ ensures ++ h.0.dom().is_singleton(), ++{ ++ assert(forall|ts1| #[trigger] h.0.dom().contains(ts1) ==> h.contains_timestamp(ts1)); ++ assert(forall|ts1| #[trigger] h.0.dom().contains(ts1) ==> ts1 == ts); ++} ++ ++pub broadcast group group_view_history { ++ group_thread_view_axioms, ++ history_insert_contains_inserted_timestamp, ++ history_insert_contains_timestamp_cases, ++ history_get_contains_timestamp, ++} ++ ++#[verifier::external_body] ++#[verifier::accept_recursive_types(T)] ++pub tracked struct AtomicPointsTo { ++ no_copy: NoCopy, ++ unused: T, ++} ++ ++unsafe impl Objective for AtomicPointsTo { ++ ++} ++ ++impl AtomicPointsTo { ++ pub uninterp spec fn loc(&self) -> CellId; ++ ++ pub uninterp spec fn hist(&self) -> AtomicHistory; ++ ++ pub uninterp spec fn get_timestamp(&self, view: ThreadView) -> Option; ++ ++ pub axiom fn get_timestamp_monotonic(tracked &self, v1: ThreadView, v2: ThreadView) ++ requires ++ v1.contains(v2), ++ ensures ++ self.get_timestamp(v2).is_some() ==> { ++ &&& self.get_timestamp(v1).is_some() ++ &&& self.get_timestamp(v2).unwrap() <= self.get_timestamp(v1).unwrap() ++ }, ++ ; ++ ++ pub axiom fn disjoint(tracked &mut self, tracked other: &Self) ++ ensures ++ final(self).loc() != other.loc(), ++ ; ++} ++ ++/// On a load, the thread must read a timestamp no smaller than that in its old view. ++/// After a load, the thread's new view will contain the timestamp that was read. ++pub open spec fn load_timestamp_in_view( ++ pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ timestamp: nat, ++) -> bool { ++ &&& pt.get_timestamp(old_view).is_none() || pt.get_timestamp(old_view).unwrap() <= timestamp ++ &&& pt.get_timestamp(new_view) == Some(timestamp) ++} ++ ++/// On a load, the location's AtomicHistory must have included [timestamp -> (val, message_view)]. ++pub open spec fn load_reads_from_history( ++ hist: AtomicHistory, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ hist.get(timestamp) == Some((val, message_view)) ++} ++ ++/// After a load, the thread's new view will contain the old view. ++pub open spec fn load_view_nondecreasing(old_view: ThreadView, new_view: ThreadView) -> bool { ++ new_view.contains(old_view) ++} ++ ++pub open spec fn load_acquire( ++ pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& load_timestamp_in_view(pt, old_view, new_view, timestamp) ++ &&& load_reads_from_history(pt.hist(), val, timestamp, message_view) ++ &&& load_view_nondecreasing( ++ old_view, ++ new_view, ++ ) ++ // because this is an acquire load, the message view is joined to the thread's current view ++ &&& new_view.contains(message_view) ++} ++ ++pub open spec fn load_relaxed( ++ pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ acquire_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& load_timestamp_in_view(pt, old_view, new_view, timestamp) ++ &&& load_reads_from_history(pt.hist(), val, timestamp, message_view) ++ &&& load_view_nondecreasing( ++ old_view, ++ new_view, ++ ) ++ // because this is a relaxed load, the message view is joined to the thread's acquire view ++ &&& acquire_view.contains(message_view) ++} ++ ++/// On a store, the store's timestamp must be greater than that in the thread's old view. ++/// After a store, the thread's new view will contain the timestamp of the store. ++/// The message view for the store will also contain the timestamp of the store. ++pub open spec fn store_timestamp_in_view( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ message_view: ThreadView, ++ timestamp: nat, ++) -> bool { ++ &&& old_pt.get_timestamp(old_view).is_none() || old_pt.get_timestamp(old_view).unwrap() ++ < timestamp ++ &&& new_pt.get_timestamp(new_view) == Some(timestamp) ++ &&& new_pt.get_timestamp(message_view) == Some(timestamp) ++} ++ ++/// After a store, the thread's new view will strictly contain its old view. ++/// This is a strict containment because the new view will contain the timestamp of the store. ++pub open spec fn store_view_increasing(old_view: ThreadView, new_view: ThreadView) -> bool { ++ &&& new_view.contains_strict(old_view) ++} ++ ++/// After a store, the locations's AtomicHistory is updated to contain the store. ++/// The timestamp of the store must not have previously been an entry in the location's AtomicHistory. ++pub open spec fn store_insert_history( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& !old_pt.hist().contains_timestamp(timestamp) ++ &&& new_pt.loc() == old_pt.loc() ++ &&& new_pt.hist() == old_pt.hist().insert(timestamp, val, message_view) ++} ++ ++pub open spec fn store_release( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_insert_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a release store, the message view is the thread's current view ++ &&& message_view == new_view ++} ++ ++pub open spec fn store_relaxed( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ release_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_insert_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a relaxed store, the message view contains the release view ++ &&& message_view.contains( ++ release_view, ++ ) ++ // and the thread's current view will now contain the message view ++ &&& new_view.contains(message_view) ++} ++ ++/// After a store_mut, the locations's AtomicHistory is updated to be a singleton containing only the new store. ++/// The timestamp of the store must not have previously been an entry in the location's AtomicHistory. ++pub open spec fn store_mut_truncate_history( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& !old_pt.hist().contains_timestamp(timestamp) ++ &&& new_pt.loc() == old_pt.loc() ++ &&& new_pt.hist().is_singleton(timestamp, (val, message_view)) ++} ++ ++pub open spec fn store_mut_release( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_mut_truncate_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a release store, the message view is the thread's current view ++ &&& message_view == new_view ++} ++ ++pub open spec fn store_mut_relaxed( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ release_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_mut_truncate_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a relaxed store, the message view contains the release view ++ &&& message_view.contains( ++ release_view, ++ ) ++ // and the thread's current view will now contain the message view ++ &&& new_view.contains(message_view) ++} ++ ++pub ghost struct LoadData { ++ pub timestamp: nat, ++ pub message_view: ThreadView, ++} ++ ++pub ghost struct StoreData { ++ pub timestamp: nat, ++ pub message_view: ThreadView, ++} ++ ++pub ghost struct UpdateData { ++ pub load_timestamp: nat, ++ pub load_message_view: ThreadView, ++ pub store_message_view: ThreadView, ++ pub intermediate_thread_view: ThreadView, ++} ++ ++macro_rules! make_unsigned_integer_atomic { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++} ++ ++macro_rules! make_signed_integer_atomic { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++} ++ ++macro_rules! make_bool_atomic { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ atomic_types!($at_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $rust_ty, $value_ty, []); ++ atomic_bool_methods!($at_ident, $rust_ty, $value_ty); ++ } ++ }; ++} ++ ++macro_rules! atomic_types { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus! { ++ ++ #[verifier::external_body] ++ pub struct $at_ident { ++ ato: $rust_ty, ++ } ++ ++ } ++ }; ++} ++ ++macro_rules! atomic_common_methods { ++ ($at_ident: ty, $rust_ty: ty, $value_ty: ty, [ $($addr:tt)* ]) => { ++ verus_impl!{ ++ ++ pub uninterp spec fn loc(&self) -> CellId; ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ pub const fn new(i: $value_ty) -> ((ato, pt, vs, ts): ( ++ Self, ++ Tracked>, ++ Tracked, ++ Ghost, ++ )) ++ ensures ++ ato.loc() == pt@.loc(), ++ pt@.hist().is_singleton(ts@, (i, vs@@)), ++ pt@.get_timestamp(vs@@) == Some(ts@) ++ { ++ let p = $at_ident { ato: $rust_ty::new(i) }; ++ (p, Tracked::assume_new(), Tracked::assume_new(), Ghost::assume_new()) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ pub const fn new_incl(i: $value_ty, Tracked(vs0) : Tracked) -> ((ato, pt, vs, ts): ( ++ Self, ++ Tracked>, ++ Tracked, ++ Ghost, ++ )) ++ ensures ++ ato.loc() == pt@.loc(), ++ pt@.hist().is_singleton(ts@, (i, vs@@)), ++ pt@.get_timestamp(vs@@) == Some(ts@), ++ vs@@.contains(vs0@) ++ { ++ let p = $at_ident { ato: $rust_ty::new(i) }; ++ (p, Tracked::assume_new(), Tracked::assume_new(), Ghost::assume_new()) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn load( ++ &self, ++ order: Ordering, ++ Tracked(vs): Tracked<&mut ViewSeen>, ++ Tracked(pt): Tracked<&AtomicPointsTo<$value_ty>>, ++ ) -> ((val, acq_vs, ld): ($value_ty, Tracked, Ghost)) ++ requires ++ self.loc() == pt.loc(), ++ order matches Ordering::Acquire || order matches Ordering::Relaxed ++ ensures ++ match order { ++ Ordering::Acquire => load_acquire(*pt, old(vs)@, final(vs)@, val, ld@.timestamp, ld@.message_view), ++ Ordering::Relaxed => load_relaxed(*pt, old(vs)@, final(vs)@, acq_vs@@, val, ld@.timestamp, ld@.message_view) ++ } ++ opens_invariants none ++ no_unwind ++ { ++ return (self.ato.load(order), Tracked::assume_new(), Ghost::assume_new()); ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn store( ++ &self, ++ v: $value_ty, ++ order: Ordering, ++ Tracked(vs): Tracked<&mut ViewSeen>, ++ Tracked(rel_vs): Tracked, ++ Tracked(pt): Tracked<&mut AtomicPointsTo<$value_ty>>, ++ ) -> (st: (Ghost)) ++ requires ++ self.loc() == old(pt).loc(), ++ order matches Ordering::Release || order matches Ordering::Relaxed ++ ensures ++ match order { ++ Ordering::Release => store_release(*old(pt), *final(pt), old(vs)@, final(vs)@, v, st@.timestamp, st@.message_view), ++ Ordering::Relaxed => store_relaxed(*old(pt), *final(pt), old(vs)@, final(vs)@, rel_vs@, v, st@.timestamp, st@.message_view) ++ } ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.store(v, order); ++ (Ghost::assume_new()) ++ } ++ ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn store_mut( ++ &mut self, ++ v: $value_ty, ++ order: Ordering, ++ Tracked(v_sn): Tracked<&mut ViewSeen>, ++ Tracked(rel_v_sn): Tracked, ++ Tracked(pt): Tracked<&mut AtomicPointsTo<$value_ty>>, ++ ) -> (st: (Ghost)) ++ requires ++ old(self).loc() == old(pt).loc(), ++ order matches Ordering::Release || order matches Ordering::Relaxed ++ ensures ++ match order { ++ Ordering::Release => store_mut_release(*old(pt), *final(pt), old(v_sn)@, final(v_sn)@, v, st@.timestamp, st@.message_view), ++ Ordering::Relaxed => store_mut_relaxed(*old(pt), *final(pt), old(v_sn)@, final(v_sn)@, rel_v_sn@, v, st@.timestamp, st@.message_view) ++ }, ++ final(self).loc() == old(self).loc() ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.store(v, order); ++ (Ghost::assume_new()) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn compare_exchange( ++ &self, ++ current: $value_ty, ++ new: $value_ty, ++ success: Ordering, ++ failure: Ordering, ++ Tracked(vs): Tracked<&mut ViewSeen>, ++ Tracked(rel_vs): Tracked, ++ Tracked(pt): Tracked<&mut AtomicPointsTo<$value_ty>>, ++ ) -> ((res, acq_vs, up): (Result<$value_ty, $value_ty>, Tracked, Ghost)) ++ requires ++ self.loc() == old(pt).loc(), ++ success matches Ordering::AcqRel || success matches Ordering::Acquire || success matches Ordering::Release || success matches Ordering::Relaxed, ++ failure matches Ordering::Acquire || failure matches Ordering::Relaxed ++ ensures ++ match res { ++ Ok(v) => { ++ &&& current == v ++ &&& up@.store_message_view.contains_strict(up@.load_message_view) ++ &&& match success { ++ Ordering::AcqRel => { ++ &&& load_acquire(*old(pt), old(vs)@, up@.intermediate_thread_view, current, up@.load_timestamp, up@.load_message_view) ++ &&& store_release(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, new, up@.load_timestamp + 1, up@.store_message_view) ++ }, ++ Ordering::Acquire => { ++ &&& load_acquire(*old(pt), old(vs)@, up@.intermediate_thread_view, current, up@.load_timestamp, up@.load_message_view) ++ &&& store_relaxed(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, rel_vs@, new, up@.load_timestamp + 1, up@.store_message_view) ++ }, ++ Ordering::Release => { ++ &&& load_relaxed(*old(pt), old(vs)@, up@.intermediate_thread_view, acq_vs@@, v, up@.load_timestamp, up@.load_message_view) ++ &&& store_release(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, new, up@.load_timestamp + 1, up@.store_message_view) ++ }, ++ Ordering::Relaxed => { ++ &&& load_relaxed(*old(pt), old(vs)@, up@.intermediate_thread_view, acq_vs@@, v, up@.load_timestamp, up@.load_message_view) ++ &&& store_relaxed(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, rel_vs@, new, up@.load_timestamp + 1, up@.store_message_view) ++ } ++ } ++ }, ++ Err(v) => { ++ &&& current != v ++ &&& *final(pt) == *old(pt) ++ &&& match failure { ++ Ordering::Acquire => load_acquire(*old(pt), old(vs)@, final(vs)@, v, up@.load_timestamp, up@.load_message_view), ++ Ordering::Relaxed => load_relaxed(*old(pt), old(vs)@, final(vs)@, acq_vs@@, v, up@.load_timestamp, up@.load_message_view) ++ } ++ } ++ } ++ opens_invariants none ++ no_unwind ++ { ++ return (self.ato.compare_exchange(current, new, success, failure), Tracked::assume_new(), Ghost::assume_new()); ++ } ++ ++ // TODO - compare_exchange_weak, swap ++ ++ #[inline(always)] ++ pub axiom fn truncate_history(tracked &mut self, tracked pt: &mut AtomicPointsTo<$value_ty>, tracked vs: &mut ViewSeen) -> (ts: nat) ++ requires ++ old(self).loc() == old(pt).loc() ++ ensures ++ *final(self) == *old(self), ++ final(pt).loc() == old(pt).loc(), ++ old(pt).hist().is_max_timestamp(ts), ++ final(pt).hist().is_singleton(ts, old(pt).hist().get(ts).unwrap()), ++ final(vs)@.contains(old(vs)@), ++ final(pt).get_timestamp(final(vs)@) == Some(ts), ++ forall |t| #[trigger] old(pt).hist().contains_timestamp(t) ==> final(vs)@.contains(old(pt).hist().thread_view(t)) ++ ++ opens_invariants none ++ ; ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ pub const fn into_inner(self, Tracked(pt): Tracked>) -> ((val, vs, ts): ($value_ty, Tracked, Ghost)) ++ requires ++ self.loc() == pt.loc(), ++ ensures ++ pt.hist().is_max_timestamp(ts@), ++ val == pt.hist().value(ts@), ++ pt.get_timestamp(vs@@) == Some(ts@), ++ forall |t| #[trigger] pt.hist().contains_timestamp(t) ==> vs@@.contains(pt.hist().thread_view(t)) ++ opens_invariants none ++ no_unwind ++ { ++ (self.ato.into_inner(), Tracked::assume_new(), Ghost::assume_new()) ++ } ++ ++ } ++ }; ++} ++ ++macro_rules! atomic_integer_methods { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ verus_impl!{ ++ ++ // this macro is currently a stub for the functions we plan to implement: ++ // TODO - fetch_add_wrapping, fetch_sub_wrapping, fetch_add, fetch_sub, fetch_and, fetch_or, fetch_xor, fetch_nand, fetch_max, fetch_min ++ ++ } ++ }; ++} ++ ++macro_rules! atomic_bool_methods { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus!{ ++ ++ // this macro is currently a stub for the functions we plan to implement: ++ // TODO - fetch_and, fetch_or, fetch_xor, fetch_nand ++ ++ } ++ }; ++} ++ ++make_bool_atomic!(PAtomicWeakBool, AtomicBool, bool); ++ ++make_unsigned_integer_atomic!(PAtomicWeakU8, AtomicU8, u8, u8_specs); ++ ++make_unsigned_integer_atomic!(PAtomicWeakU16, AtomicU16, u16, u16_specs); ++ ++make_unsigned_integer_atomic!(PAtomicWeakU32, AtomicU32, u32, u32_specs); ++ ++#[cfg(target_has_atomic = "64")] ++make_unsigned_integer_atomic!(PAtomicWeakU64, AtomicU64, u64, u64_specs); ++ ++make_unsigned_integer_atomic!(PAtomicWeakUsize, AtomicUsize, usize, usize_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakI8, AtomicI8, i8, i8_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakI16, AtomicI16, i16, i16_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakI32, AtomicI32, i32, i32_specs); ++ ++#[cfg(target_has_atomic = "64")] ++make_signed_integer_atomic!(PAtomicWeakI64, AtomicI64, i64,i64_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakIsize, AtomicIsize, isize, isize_specs); ++ ++// TODO - AtomicPtr ++} // verus! ++} +diff --git a/source/vstd/invariant.rs b/source/vstd/invariant.rs +index 927650bf..b5f0f058 100644 +--- a/source/vstd/invariant.rs ++++ b/source/vstd/invariant.rs +@@ -1,7 +1,8 @@ + #[allow(unused_imports)] + use super::pervasive::*; +-#[allow(unused_imports)] + use super::prelude::*; ++#[cfg(verus_keep_ghost)] ++use super::thread_view::Objective; + + // TODO: + // * utility for conveniently creating unique namespaces +@@ -114,6 +115,7 @@ pub trait InvariantPredicate { + /// + /// **Note:** Rather than using `AtomicInvariant` directly, we generally recommend + /// using the [`atomic_ghost` APIs](crate::atomic_ghost). ++#[cfg(not(feature = "weak-memory"))] + #[cfg_attr(verus_keep_ghost, verifier::proof)] + #[cfg_attr(verus_keep_ghost, verifier::external_body)] /* vattr */ + #[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(K))] +@@ -124,6 +126,20 @@ pub struct AtomicInvariant { + dummy1: super::prelude::AlwaysSyncSend<(K, Pred, *mut V)>, + } + ++// TODO - document ++// TODO - can we fold this into a single AtomicInvariant definition without the Objective trait bound? ++// Creusot only implements Sync on AtomicInvariant when T: Send + Objective ++#[cfg(feature = "weak-memory")] ++#[cfg_attr(verus_keep_ghost, verifier::proof)] ++#[cfg_attr(verus_keep_ghost, verifier::external_body)] /* vattr */ ++#[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(K))] ++#[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(V))] ++#[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(Pred))] ++pub struct AtomicInvariant { ++ dummy: super::prelude::SyncSendIfSend, ++ dummy1: super::prelude::AlwaysSyncSend<(K, Pred, *mut V)>, ++} ++ + /// A `LocalInvariant` is a ghost object that provides "interior mutability" + /// for ghost objects, specifically, for `tracked` ghost objects. + /// A reference `&LocalInvariant` may be shared between clients. +@@ -267,7 +283,109 @@ macro_rules! declare_invariant_impl { + }; + } + ++#[cfg(feature = "weak-memory")] ++macro_rules! declare_invariant_impl_objective { ++ ($invariant:ident => $selfid:ident => $($into_inner_clause:tt)*) => { ++ // note the path names of `inv` and `namespace` are harcoded into the VIR crate. ++ ++ verus!{ ++ ++ impl $invariant { ++ /// The constant specified upon the initialization of this ` ++ #[doc = stringify!($invariant)] ++ ///`. ++ pub uninterp spec fn constant(&self) -> K; ++ ++ /// Namespace the invariant was declared in. ++ #[rustc_diagnostic_item = concat!("verus::vstd::invariant::", stringify!($invariant), "::namespace")] ++ pub uninterp spec fn namespace(&self) -> int; ++ } ++ ++ impl> $invariant { ++ /// Returns `true` if it is possible to store the value `v` into the ` ++ #[doc = stringify!($invariant)] ++ ///`. ++ /// ++ /// This is equivalent to `Pred::inv(self.constant(), v)`. ++ ++ #[rustc_diagnostic_item = concat!("verus::vstd::invariant::", stringify!($invariant), "::inv")] ++ pub open spec fn inv(&self, v: V) -> bool { ++ Pred::inv(self.constant(), v) ++ } ++ ++ /// Initialize a new ` ++ #[doc = stringify!($invariant)] ++ ///` with constant `k`. initial stored (tracked) value `v`, ++ /// and in the namespace `ns`. ++ ++ pub axiom fn new(k: K, tracked v: V, ns: int) -> (tracked i: $invariant) ++ requires ++ Pred::inv(k, v), ++ ensures ++ i.constant() == k, ++ i.namespace() == ns; ++ ++ // Q. Why does AtomicInvariant::into_inner have an opens_invariant clause ++ // while LocalInvariant::into_inner doesn't? ++ // ++ // A. It has to do with the way we prevent double-opening via into_inner, ++ // i.e., how we prevent the user from calling into_inner on an already-open ++ // invariant: ++ // ++ // open_{atomic|local}_invariant!(&inv => i => { ++ // inv.into_inner(); // this should error ++ // } ++ // ++ // There are two broad approaches: ++ // 1. Use the mask-checking, treating into_inner the same as an invariant-open ++ // 2. Use lifetimes, ensuring the borrow used to open the block extends through ++ // the entire block. ++ // ++ // Approach (1) is easier to use, but (1) is not sound for LocalInvariants. ++ // Thus, we use approach (1) for AtomicInvariants and (2) for LocalInvariants. ++ // ++ // Q. Why is approach (1) easier for AtomicInvariants? ++ // ++ // A. This makes it easier to perform an atomic operation that relinquishes ++ // the permission to access the same atomic. ++ // ++ // Q. Why is approach (1) unsound for LocalInvariants? ++ // ++ // A. Because into_inner is not the only problem. We want to implement ++ // the bound `impl => LocalInvariant: Send`. ++ // Furthermore, moving a local invariant to another thread basically means moving ++ // the invariant from one thread's invariant pool to another, which basically ++ // needs the same restrictions as `into_inner`. ++ // ++ // open_{atomic|local}_invariant!(&inv => i => { ++ // send_to_another_thread(inv); // this must be disallowed, too ++ // } ++ // ++ // However, we cannot (easily) put a mask bound on the send operation. ++ // ++ // Q. How are the lifetime restrictions implemented for LocalInvariant? ++ // The short answer is that we use the lifetime argument of the InvariantBlockGuard ++ // and force it to remain alive for the duration of the open_local_invariant! block. ++ // However, this requires special support from Verus in the lifetime-erasure system. ++ // See rustc_mir_build_additional_files/verus_builder.rs for more information. ++ ++ /// Destroys the ` ++ #[doc = stringify!($invariant)] ++ ///`, returning the tracked value contained within. ++ ++ pub axiom fn into_inner(tracked $selfid) -> (tracked v: V) ++ ensures $selfid.inv(v), ++ $($into_inner_clause)* ; ++ } ++ ++ } ++ }; ++} ++ ++#[cfg(not(feature = "weak-memory"))] + declare_invariant_impl!(AtomicInvariant => self => opens_invariants [ self.namespace() ] ); ++#[cfg(feature = "weak-memory")] ++declare_invariant_impl_objective!(AtomicInvariant => self => opens_invariants [ self.namespace() ] ); + declare_invariant_impl!(LocalInvariant => self => ); + + #[doc(hidden)] +@@ -365,7 +483,7 @@ pub fn spend_open_invariant_credit( + // Why does this use the 'static type param for the InvariantBlockGuard? This is because + // AtomicInvariant doesn't need the lifetime-checking like LocalInvariant does. See the explanation + // over `into_inner`. +-#[cfg(verus_keep_ghost)] ++#[cfg(all(verus_keep_ghost, not(feature = "weak-memory")))] + #[rustc_diagnostic_item = "verus::vstd::invariant::open_atomic_invariant_begin"] + #[doc(hidden)] + #[verifier::external] /* vattr */ +@@ -375,6 +493,16 @@ pub fn open_atomic_invariant_begin<'a, K, V, Pred: InvariantPredicate>( + unimplemented!(); + } + ++#[cfg(all(verus_keep_ghost, feature = "weak-memory"))] ++#[rustc_diagnostic_item = "verus::vstd::invariant::open_atomic_invariant_begin"] ++#[doc(hidden)] ++#[verifier::external] /* vattr */ ++pub fn open_atomic_invariant_begin<'a, K, V: Objective, Pred: InvariantPredicate>( ++ _inv: &'a AtomicInvariant, ++) -> (InvariantBlockGuard<'static>, V) { ++ unimplemented!(); ++} ++ + #[cfg(verus_keep_ghost)] + #[rustc_diagnostic_item = "verus::vstd::invariant::open_local_invariant_begin"] + #[doc(hidden)] +diff --git a/source/vstd/rwlock.rs b/source/vstd/rwlock.rs +index ba7ed260..8b7ce233 100644 +--- a/source/vstd/rwlock.rs ++++ b/source/vstd/rwlock.rs +@@ -2,239 +2,245 @@ + #![allow(unused_imports)] + #![allow(non_shorthand_field_patterns)] + +-use super::atomic_ghost::*; +-use super::cell::CellId; +-use super::cell::pcell_maybe_uninit as un; +-use super::invariant::InvariantPredicate; +-use super::modes::*; +-use super::multiset::*; +-use super::prelude::*; +-use super::set::*; +-use core::marker::PhantomData; +-use verus_state_machines_macros::tokenized_state_machine_vstd; +- +-tokenized_state_machine_vstd!( +-RwLockToks> { +- fields { +- #[sharding(constant)] +- pub k: K, +- +- #[sharding(constant)] +- pub pred: PhantomData, +- +- #[sharding(variable)] +- pub flag_exc: bool, +- +- #[sharding(variable)] +- pub flag_rc: nat, +- +- #[sharding(storage_option)] +- pub storage: Option, +- +- #[sharding(option)] +- pub pending_writer: Option<()>, +- +- #[sharding(option)] +- pub writer: Option<()>, +- +- #[sharding(multiset)] +- pub pending_reader: Multiset<()>, +- +- #[sharding(multiset)] +- pub reader: Multiset, +- } ++#[cfg(not(feature = "weak-memory"))] ++pub use rwlock::*; ++ ++#[cfg(not(feature = "weak-memory"))] ++mod rwlock { ++ ++ use super::super::atomic_ghost::*; ++ use super::super::cell::CellId; ++ use super::super::cell::pcell_maybe_uninit as un; ++ use super::super::invariant::InvariantPredicate; ++ use super::super::modes::*; ++ use super::super::multiset::*; ++ use super::super::prelude::*; ++ use super::super::set::*; ++ use core::marker::PhantomData; ++ use verus_state_machines_macros::tokenized_state_machine_vstd; ++ ++ tokenized_state_machine_vstd!( ++ RwLockToks> { ++ fields { ++ #[sharding(constant)] ++ pub k: K, ++ ++ #[sharding(constant)] ++ pub pred: PhantomData, ++ ++ #[sharding(variable)] ++ pub flag_exc: bool, ++ ++ #[sharding(variable)] ++ pub flag_rc: nat, ++ ++ #[sharding(storage_option)] ++ pub storage: Option, + +- init!{ +- initialize_full(k: K, t: V) { +- require Pred::inv(k, t); +- init k = k; +- init pred = PhantomData; +- init flag_exc = false; +- init flag_rc = 0; +- init storage = Option::Some(t); +- init pending_writer = Option::None; +- init writer = Option::None; +- init pending_reader = Multiset::empty(); +- init reader = Multiset::empty(); ++ #[sharding(option)] ++ pub pending_writer: Option<()>, ++ ++ #[sharding(option)] ++ pub writer: Option<()>, ++ ++ #[sharding(multiset)] ++ pub pending_reader: Multiset<()>, ++ ++ #[sharding(multiset)] ++ pub reader: Multiset, + } +- } + +- #[inductive(initialize_full)] +- fn initialize_full_inductive(post: Self, k: K, t: V) { +- broadcast use group_multiset_axioms; +- } ++ init!{ ++ initialize_full(k: K, t: V) { ++ require Pred::inv(k, t); ++ init k = k; ++ init pred = PhantomData; ++ init flag_exc = false; ++ init flag_rc = 0; ++ init storage = Option::Some(t); ++ init pending_writer = Option::None; ++ init writer = Option::None; ++ init pending_reader = Multiset::empty(); ++ init reader = Multiset::empty(); ++ } ++ } + +- /// Increment the 'rc' counter, obtain a pending_reader +- transition!{ +- acquire_read_start() { +- update flag_rc = pre.flag_rc + 1; +- add pending_reader += {()}; ++ #[inductive(initialize_full)] ++ fn initialize_full_inductive(post: Self, k: K, t: V) { ++ broadcast use group_multiset_axioms; ++ } ++ ++ /// Increment the 'rc' counter, obtain a pending_reader ++ transition!{ ++ acquire_read_start() { ++ update flag_rc = pre.flag_rc + 1; ++ add pending_reader += {()}; ++ } + } +- } + +- /// Exchange the pending_reader for a reader by checking +- /// that the 'exc' bit is 0 +- transition!{ +- acquire_read_end() { +- require(pre.flag_exc == false); ++ /// Exchange the pending_reader for a reader by checking ++ /// that the 'exc' bit is 0 ++ transition!{ ++ acquire_read_end() { ++ require(pre.flag_exc == false); + +- remove pending_reader -= {()}; ++ remove pending_reader -= {()}; + +- birds_eye let x: V = pre.storage->0; +- add reader += {x}; ++ birds_eye let x: V = pre.storage->0; ++ add reader += {x}; + +- assert Pred::inv(pre.k, x); ++ assert Pred::inv(pre.k, x); ++ } + } +- } + +- /// Decrement the 'rc' counter, abandon the attempt to gain +- /// the 'read' lock. +- transition!{ +- acquire_read_abandon() { +- remove pending_reader -= {()}; +- assert(pre.flag_rc >= 1); +- update flag_rc = (pre.flag_rc - 1) as nat; ++ /// Decrement the 'rc' counter, abandon the attempt to gain ++ /// the 'read' lock. ++ transition!{ ++ acquire_read_abandon() { ++ remove pending_reader -= {()}; ++ assert(pre.flag_rc >= 1); ++ update flag_rc = (pre.flag_rc - 1) as nat; ++ } + } +- } + +- /// Atomically set 'exc' bit from 'false' to 'true' +- /// Obtain a pending_writer +- transition!{ +- acquire_exc_start() { +- require(pre.flag_exc == false); +- update flag_exc = true; +- add pending_writer += Some(()); ++ /// Atomically set 'exc' bit from 'false' to 'true' ++ /// Obtain a pending_writer ++ transition!{ ++ acquire_exc_start() { ++ require(pre.flag_exc == false); ++ update flag_exc = true; ++ add pending_writer += Some(()); ++ } + } +- } + +- /// Finish obtaining the write lock by checking that 'rc' is 0. +- /// Exchange the pending_writer for a writer and withdraw the +- /// stored object. +- transition!{ +- acquire_exc_end() { +- require(pre.flag_rc == 0); ++ /// Finish obtaining the write lock by checking that 'rc' is 0. ++ /// Exchange the pending_writer for a writer and withdraw the ++ /// stored object. ++ transition!{ ++ acquire_exc_end() { ++ require(pre.flag_rc == 0); + +- remove pending_writer -= Some(()); ++ remove pending_writer -= Some(()); + +- add writer += Some(()); ++ add writer += Some(()); + +- birds_eye let x = pre.storage->0; +- withdraw storage -= Some(x); ++ birds_eye let x = pre.storage->0; ++ withdraw storage -= Some(x); + +- assert Pred::inv(pre.k, x); ++ assert Pred::inv(pre.k, x); ++ } + } +- } + +- /// Release the write-lock. Update the 'exc' bit back to 'false'. +- /// Return the 'writer' and also deposit an object back into storage. +- transition!{ +- release_exc(x: V) { +- require Pred::inv(pre.k, x); +- remove writer -= Some(()); ++ /// Release the write-lock. Update the 'exc' bit back to 'false'. ++ /// Return the 'writer' and also deposit an object back into storage. ++ transition!{ ++ release_exc(x: V) { ++ require Pred::inv(pre.k, x); ++ remove writer -= Some(()); + +- update flag_exc = false; ++ update flag_exc = false; + +- deposit storage += Some(x); ++ deposit storage += Some(x); ++ } + } +- } + +- /// Check that the 'reader' is actually a guard for the given object. +- property!{ +- read_guard(x: V) { +- have reader >= {x}; +- guard storage >= Some(x); ++ /// Check that the 'reader' is actually a guard for the given object. ++ property!{ ++ read_guard(x: V) { ++ have reader >= {x}; ++ guard storage >= Some(x); ++ } + } +- } + +- property!{ +- read_match(x: V, y: V) { +- have reader >= {x}; +- have reader >= {y}; +- assert(equal(x, y)); ++ property!{ ++ read_match(x: V, y: V) { ++ have reader >= {x}; ++ have reader >= {y}; ++ assert(equal(x, y)); ++ } + } +- } + +- /// Release the reader-lock. Decrement 'rc' and return the 'reader' object. +- #[transition] +- transition!{ +- release_shared(x: V) { +- remove reader -= {x}; ++ /// Release the reader-lock. Decrement 'rc' and return the 'reader' object. ++ #[transition] ++ transition!{ ++ release_shared(x: V) { ++ remove reader -= {x}; + +- assert(pre.flag_rc >= 1) by { +- //assert(pre.reader.count(x) >= 1); +- assert(equal(pre.storage, Option::Some(x))); +- //assert(equal(x, pre.storage->0)); +- }; +- update flag_rc = (pre.flag_rc - 1) as nat; ++ assert(pre.flag_rc >= 1) by { ++ //assert(pre.reader.count(x) >= 1); ++ assert(equal(pre.storage, Option::Some(x))); ++ //assert(equal(x, pre.storage->0)); ++ }; ++ update flag_rc = (pre.flag_rc - 1) as nat; ++ } + } +- } + +- #[invariant] +- pub fn exc_bit_matches(&self) -> bool { +- (if self.flag_exc { 1 } else { 0 as int }) == +- (if self.pending_writer is Some { 1 } else { 0 as int }) as int +- + (if self.writer is Some { 1 } else { 0 as int }) as int +- } ++ #[invariant] ++ pub fn exc_bit_matches(&self) -> bool { ++ (if self.flag_exc { 1 } else { 0 as int }) == ++ (if self.pending_writer is Some { 1 } else { 0 as int }) as int ++ + (if self.writer is Some { 1 } else { 0 as int }) as int ++ } + +- #[invariant] +- pub fn count_matches(&self) -> bool { +- self.flag_rc == self.pending_reader.count(()) +- + self.reader.count(self.storage->0) +- } ++ #[invariant] ++ pub fn count_matches(&self) -> bool { ++ self.flag_rc == self.pending_reader.count(()) ++ + self.reader.count(self.storage->0) ++ } + +- #[invariant] +- pub fn reader_agrees_storage(&self) -> bool { +- forall |t: V| imply(#[trigger] self.reader.count(t) > 0, +- equal(self.storage, Option::Some(t))) +- } ++ #[invariant] ++ pub fn reader_agrees_storage(&self) -> bool { ++ forall |t: V| imply(#[trigger] self.reader.count(t) > 0, ++ equal(self.storage, Option::Some(t))) ++ } + +- #[invariant] +- pub fn writer_agrees_storage(&self) -> bool { +- imply(self.writer is Some, self.storage is None) +- } ++ #[invariant] ++ pub fn writer_agrees_storage(&self) -> bool { ++ imply(self.writer is Some, self.storage is None) ++ } + +- #[invariant] +- pub fn writer_agrees_storage_rev(&self) -> bool { +- imply(self.storage is None, self.writer is Some) +- } ++ #[invariant] ++ pub fn writer_agrees_storage_rev(&self) -> bool { ++ imply(self.storage is None, self.writer is Some) ++ } + +- #[invariant] +- pub fn sto_user_inv(&self) -> bool { +- self.storage.is_some() ==> Pred::inv(self.k, self.storage.unwrap()) +- } ++ #[invariant] ++ pub fn sto_user_inv(&self) -> bool { ++ self.storage.is_some() ==> Pred::inv(self.k, self.storage.unwrap()) ++ } + +- #[inductive(acquire_read_start)] +- fn acquire_read_start_inductive(pre: Self, post: Self) { +- broadcast use group_multiset_axioms; +- } ++ #[inductive(acquire_read_start)] ++ fn acquire_read_start_inductive(pre: Self, post: Self) { ++ broadcast use group_multiset_axioms; ++ } + +- #[inductive(acquire_read_end)] +- fn acquire_read_end_inductive(pre: Self, post: Self) { +- broadcast use group_multiset_axioms; +- } ++ #[inductive(acquire_read_end)] ++ fn acquire_read_end_inductive(pre: Self, post: Self) { ++ broadcast use group_multiset_axioms; ++ } + +- #[inductive(acquire_read_abandon)] +- fn acquire_read_abandon_inductive(pre: Self, post: Self) { +- broadcast use group_multiset_axioms; +- } ++ #[inductive(acquire_read_abandon)] ++ fn acquire_read_abandon_inductive(pre: Self, post: Self) { ++ broadcast use group_multiset_axioms; ++ } + +- #[inductive(acquire_exc_start)] +- fn acquire_exc_start_inductive(pre: Self, post: Self) { } ++ #[inductive(acquire_exc_start)] ++ fn acquire_exc_start_inductive(pre: Self, post: Self) { } + +- #[inductive(acquire_exc_end)] +- fn acquire_exc_end_inductive(pre: Self, post: Self) { } ++ #[inductive(acquire_exc_end)] ++ fn acquire_exc_end_inductive(pre: Self, post: Self) { } + +- #[inductive(release_exc)] +- fn release_exc_inductive(pre: Self, post: Self, x: V) { } ++ #[inductive(release_exc)] ++ fn release_exc_inductive(pre: Self, post: Self, x: V) { } + +- #[inductive(release_shared)] +- fn release_shared_inductive(pre: Self, post: Self, x: V) { +- broadcast use group_multiset_axioms; +- assert(equal(pre.storage, Option::Some(x))); +- } +-}); ++ #[inductive(release_shared)] ++ fn release_shared_inductive(pre: Self, post: Self, x: V) { ++ broadcast use group_multiset_axioms; ++ assert(equal(pre.storage, Option::Some(x))); ++ } ++ }); + +-verus! { ++ verus! { + + pub trait RwLockPredicate: Sized { + spec fn inv(self, v: V) -> bool; +@@ -709,3 +715,4 @@ impl> RwLock { + } + + } // verus! ++} +diff --git a/source/vstd/thread_view.rs b/source/vstd/thread_view.rs +new file mode 100644 +index 00000000..bfb1c293 +--- /dev/null ++++ b/source/vstd/thread_view.rs +@@ -0,0 +1,396 @@ ++use super::cell::CellId; ++use super::prelude::*; ++use super::resource::algebra; ++use super::resource::pcm; ++use super::view::*; ++ ++/// A type that implements `Objective` cannot carry permissions that depend on a thread's subjective view of memory, ++/// as determined by Rust's weak memory model. This trait is only relevant in the weak memory setting. ++// todo - add tests to ensure the (non-)implementations of this marker trait are as expected ++#[cfg(verus_keep_ghost)] ++pub unsafe auto trait Objective {} ++ ++verus! { ++ ++#[cfg(verus_keep_ghost)] ++#[verifier::external_trait_specification] ++pub trait ExObjective: core::marker::PointeeSized { ++ type ExternalTraitSpecificationFor: Objective; ++} ++ ++/// Represents a thread's subjective view of memory. ++/// For non-atomic memory locations, the relationship between two views is modeled abstractly (i.e., no timestamps are compared). ++/// For atomic memory locations, we can reason about the per-location timestamp contained in a view. ++// This type will be defined with uninterp spec fns, so it should be treated abstractly by the verifier. ++#[verifier::external_body] ++pub ghost struct ThreadView; ++ ++impl ThreadView { ++ /// The empty view. ++ pub uninterp spec fn empty() -> Self; ++ ++ /// True when `other` is contained in `self`. ++ pub uninterp spec fn contains(self, other: Self) -> bool; ++ ++ /// True when `other` is contained in `self` and `other` is not equal to `self`. ++ pub uninterp spec fn contains_strict(self, other: Self) -> bool; ++ ++ /// Returns the union of `self` and `other`. ++ pub uninterp spec fn join(self, other: Self) -> Self; ++ ++ /// View containment is reflexive. ++ pub broadcast axiom fn contains_refl(v: Self) ++ ensures ++ #[trigger] v.contains(v), ++ ; ++ ++ /// View containment is anti-symmetric. ++ pub broadcast axiom fn contains_anti_sym(v1: Self, v2: Self) ++ requires ++ #[trigger] v1.contains(v2), ++ v1 != v2, ++ ensures ++ !(#[trigger] v2.contains(v1)), ++ ; ++ ++ /// View containment is transitive. ++ pub broadcast axiom fn contains_trans(v1: Self, v2: Self, v3: Self) ++ requires ++ #[trigger] v1.contains(v2), ++ #[trigger] v2.contains(v3), ++ ensures ++ #[trigger] v1.contains(v3), ++ ; ++ ++ pub broadcast axiom fn contains_strict_contains(v1: Self, v2: Self) ++ requires ++ #[trigger] v1.contains_strict(v2), ++ ensures ++ v1.contains(v2), ++ ; ++ ++ /// Joining of views is associative. ++ pub broadcast axiom fn join_assoc(v1: Self, v2: Self, v3: Self) ++ ensures ++ #[trigger] v1.join(v2.join(v3)) =~= #[trigger] v1.join(v2).join(v3), ++ ; ++ ++ /// Joining of views is commutative. ++ pub broadcast axiom fn join_comm(v1: Self, v2: Self) ++ ensures ++ #[trigger] v1.join(v2) =~= v2.join(v1), ++ ; ++ ++ /// Joining a view with itself results in the same view. ++ pub broadcast axiom fn join_identity(v: Self) ++ ensures ++ #[trigger] v.join(v) =~= v, ++ ; ++ ++ /// The result of joining a view with another view contains the original view. ++ pub broadcast axiom fn join_contains(v1: Self, v2: Self) ++ ensures ++ #[trigger] v1.join(v2).contains(v1), ++ ; ++} ++ ++pub broadcast group group_thread_view_axioms { ++ ThreadView::contains_refl, ++ ThreadView::contains_anti_sym, ++ ThreadView::contains_trans, ++ ThreadView::contains_strict_contains, ++ ThreadView::join_assoc, ++ ThreadView::join_comm, ++ ThreadView::join_identity, ++ ThreadView::join_contains, ++} ++ ++/// Resource representing a thread's subjective view of memory. ++/// Owning a `ViewSeen` provides a lower-bound on the thread's current view. ++#[derive(Clone, Copy)] ++#[verifier::external_body] ++pub tracked struct ViewSeen; ++ ++impl View for ViewSeen { ++ type V = ThreadView; ++ ++ open spec fn view(&self) -> ThreadView { ++ self.thread_view() ++ } ++} ++ ++impl ViewSeen { ++ /// The view that this permission represents. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// Creates a [`ViewSeen`] permission corresponding to the empty view. ++ pub axiom fn new() -> (tracked out: ViewSeen) ++ ensures ++ out@ == ThreadView::empty(), ++ ; ++ ++ /// Joins this [`ViewSeen`] permission with another [`ViewSeen`] to create a new [`ViewSeen`], ++ /// representing the join of the two views. ++ pub axiom fn join(tracked self, tracked other: Self) -> (tracked out: Self) ++ ensures ++ out@ == self@.join(other@), ++ ; ++ ++ /// Creates a new [`ViewSeen`] representing a view which is contained in the view corresponding to the original [`ViewSeen`]. ++ pub axiom fn weaken(tracked self, v: ThreadView) -> (tracked out: Self) ++ requires ++ self@.contains(v), ++ ensures ++ out@ == v, ++ ; ++} ++ ++/// Resource representing the ``release view" in a thread's subjective view of memory, according to Rust's weak memory model. ++/// If a thread holds a [`ReleaseViewSeen`], then that view that was held by a thread at some point that it performed a release fence in the past. ++#[derive(Clone, Copy)] ++#[verifier::external_body] ++pub tracked struct ReleaseViewSeen; ++ ++impl View for ReleaseViewSeen { ++ type V = ThreadView; ++ ++ open spec fn view(&self) -> ThreadView { ++ self.thread_view() ++ } ++} ++ ++impl ReleaseViewSeen { ++ /// The view that this permission represents. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// Creates a new permission corresponding to the empty view. ++ pub axiom fn new() -> (tracked out: Self) ++ ensures ++ out@ == ThreadView::empty(), ++ ; ++} ++ ++/// Resource representing the ``acquire view" in a thread's subjective view of memory, according to Rust's weak memory model. ++/// If a thread holds an [`AcquireViewSeen`], then that permission represents a view that would be held by a thread ++/// if it were to perform an acquire fence in the future. ++#[derive(Clone, Copy)] ++#[verifier::external_body] ++pub tracked struct AcquireViewSeen; ++ ++impl View for AcquireViewSeen { ++ type V = ThreadView; ++ ++ open spec fn view(&self) -> ThreadView { ++ self.thread_view() ++ } ++} ++ ++impl AcquireViewSeen { ++ /// The view that this permission represents. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// Creates a new permission corresponding to the empty view. ++ pub axiom fn new() -> (tracked out: Self) ++ ensures ++ out@ == ThreadView::empty(), ++ ; ++} ++ ++// ViewSeen permissions are not objective as they represent a thread's subjective view of memory. ++#[cfg(verus_keep_ghost)] ++impl !Objective for ViewSeen { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++impl !Objective for AcquireViewSeen { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++impl !Objective for ReleaseViewSeen { ++ ++} ++ ++// PCMs and RAs are objective ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for pcm::Resource

{ ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for algebra::Resource { ++ ++} ++ ++// primitive types are objective because they do not hold permissions ++macro_rules! declare_primitive_is_objective { ++ ($($a:ty),*) => { ++ verus! { ++ $( ++ #[cfg(verus_keep_ghost)] ++ unsafe impl Objective for $a {} ++ )* ++ } ++ } ++} ++ ++declare_primitive_is_objective!(bool, char, (), u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, int, nat, str); ++ ++// note: the fact that tuples are Objective (above) suffices for OBJMOD-SEP ++// OBJ with wand update ++#[cfg(verus_keep_ghost)] ++unsafe impl<'a, P: Objective, Q: Objective, F: ProofFnOnce> Objective for proof_fn<'a, F>( ++ tracked p: P, ++) -> tracked Q { ++ ++} ++ ++/// Represents a permission of type `T` which is safe for a thread to own, provided that this thread has ++/// seen a particular view. ++#[derive(Copy)] ++#[verifier::external_body] ++#[verifier::accept_recursive_types(T)] ++pub tracked struct ViewAt { ++ _dummy: core::marker::PhantomData, ++} ++ ++impl Clone for ViewAt { ++ #[verifier::external_body] ++ fn clone(&self) -> Self { ++ unimplemented!() ++ } ++} ++ ++// ViewAt is objective, because it does not give direct access to memory permissions themselves ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for ViewAt { ++ ++} ++ ++impl ViewAt { ++ /// View that a thread must synchronize with in order to safely start using the inner permission. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// The inner permission represented by this [`ViewAt`]. ++ pub uninterp spec fn value(&self) -> T; ++ ++ /// Creates a new [`ViewAt`] from the given permission. ++ /// This permission will be safe to start using at an arbitrary view, ++ /// represented by the [`ViewSeen`] returned by this operation. ++ pub axiom fn new(tracked t: T) -> (tracked (va, vs): (Self, ViewSeen)) ++ ensures ++ va.value() == t, ++ va.thread_view() == vs@, ++ ; ++ ++ /// Creates a new [`ViewAt`] from the given permission and lower bound on the synchronizing view. ++ /// This permission will be safe to start using at some view that is larger than the given view `sn`, ++ /// represented by the [`ViewSeen`] returned by this operation. ++ pub axiom fn new_incl(tracked t: T, tracked vs_0: ViewSeen) -> (tracked (va, vs): ( ++ Self, ++ ViewSeen, ++ )) ++ ensures ++ va.value() == t, ++ va.thread_view() == vs@, ++ va.thread_view().contains(vs_0@), ++ ; ++ ++ // Weaker version of `join_tup`. ++ axiom fn join_tup_inner(tracked v0: ViewAt, tracked v1: ViewAt) -> (tracked out: ++ ViewAt<(T, U)>) ++ requires ++ v0.thread_view() == v1.thread_view(), ++ ensures ++ out.thread_view() == v0.thread_view(), ++ out.value().0 == v0.value(), ++ out.value().1 == v1.value(), ++ ; ++ ++ /// Given two [`ViewAt`] permissions, they can be joined into a single [`ViewAt`] permission, ++ /// whose inner permission a tuple of the original inner permissions, ++ /// and whose synchronizing view is the join of the original synchronizing views. ++ pub proof fn join_tup(tracked v0: ViewAt, tracked v1: ViewAt) -> (tracked out: ViewAt< ++ (T, U), ++ >) ++ ensures ++ out.thread_view() == v0.thread_view().join(v1.thread_view()), ++ out.value().0 == v0.value(), ++ out.value().1 == v1.value(), ++ { ++ let view0 = v0.thread_view(); ++ let view1 = v1.thread_view(); ++ let view_join = view0.join(view1); ++ assert(view_join.contains(view0)) by { ++ ThreadView::join_contains(view0, view1); ++ } ++ assert(view_join.contains(view1)) by { ++ ThreadView::join_comm(view0, view1); ++ ThreadView::join_contains(view1, view0); ++ } ++ let tracked v0 = v0.weaken(view_join); ++ let tracked v1 = v1.weaken(view_join); ++ ViewAt::join_tup_inner(v0, v1) ++ } ++ ++ /// Given a [`ViewAt`] permission, its synchronizing view can be weakened to a larger view. ++ pub axiom fn weaken(tracked self, v: ThreadView) -> (tracked out: Self) ++ requires ++ v.contains(self.thread_view()), ++ ensures ++ out.thread_view() == v, ++ out.value() == self.value(), ++ ; ++ ++ /// Returns the inner permission, provided that the calling thread has obtained the synchronizing view `self.thread_view()`. ++ pub axiom fn into_inner(tracked self, tracked sn: ViewSeen) -> (tracked out: T) ++ requires ++ sn@.contains(self.thread_view()), ++ ensures ++ out == self.value(), ++ ; ++ ++ /// Weaker version of `apply_fn`. ++ axiom fn apply_fn_inner( ++ tracked self, ++ tracked f: ViewAt tracked U>, ++ ) -> (tracked out: ViewAt) ++ requires ++ f.value().requires((self.value(),)), ++ f.thread_view() == self.thread_view(), ++ ensures ++ f.value().ensures((self.value(),), out.value()), ++ out.thread_view() == self.thread_view(), ++ ; ++ ++ /// Given a proof closure `f`, it can be applied to a resource `self` which is ``under" a [`ViewAt`]. ++ /// The resulting resource will be returned under a [`ViewAt`] at some larger view than the original resource. ++ pub proof fn apply_fn( ++ tracked self, ++ tracked f: proof_fn[Once](tracked v1: T) -> tracked U, ++ ) -> (tracked out: ViewAt) ++ requires ++ f.requires((self.value(),)), ++ ensures ++ f.ensures((self.value(),), out.value()), ++ out.thread_view().contains(self.thread_view()), ++ { ++ let tracked va_f = ViewAt::new(f).0; ++ let view1 = va_f.thread_view(); ++ let view2 = self.thread_view(); ++ let view_join = view1.join(view2); ++ assert(view_join.contains(view1)) by { ++ ThreadView::join_contains(view1, view2); ++ } ++ assert(view_join.contains(view2)) by { ++ ThreadView::join_comm(view1, view2); ++ ThreadView::join_contains(view2, view1); ++ } ++ let tracked va_f = va_f.weaken(view_join); ++ let tracked va_t = self.weaken(view_join); ++ va_t.apply_fn_inner(va_f) ++ } ++} ++ ++} // verus! +diff --git a/source/vstd/vstd.rs b/source/vstd/vstd.rs +index 24ef737c..900e9a6c 100644 +--- a/source/vstd/vstd.rs ++++ b/source/vstd/vstd.rs +@@ -22,6 +22,7 @@ + #![cfg_attr(verus_keep_ghost, feature(slice_index_methods))] + #![cfg_attr(all(feature = "alloc", verus_keep_ghost), feature(liballoc_internals))] + #![cfg_attr(verus_keep_ghost, feature(nonzero_internals))] ++#![cfg_attr(verus_keep_ghost, feature(auto_traits))] + + #[cfg(feature = "alloc")] + extern crate alloc; +@@ -30,6 +31,7 @@ pub mod arithmetic; + pub mod array; + pub mod atomic; + pub mod atomic_ghost; ++pub mod atomic_weak; + pub mod bits; + pub mod bytes; + pub mod calc_macro; +@@ -81,6 +83,7 @@ pub mod state_machine_internal; + pub mod string; + #[cfg(feature = "std")] + pub mod thread; ++pub mod thread_view; + pub mod tokens; + pub mod utf8; + pub mod view; +diff --git a/source/vstd_build/src/main.rs b/source/vstd_build/src/main.rs +index 6b849bf2..2250d553 100644 +--- a/source/vstd_build/src/main.rs ++++ b/source/vstd_build/src/main.rs +@@ -37,6 +37,7 @@ fn main() { + let mut no_lifetime = false; + let mut expand_errors = false; + let mut no_solver_version_check = false; ++ let mut weak_memory = false; + for arg in args { + if arg == "--release" { + release = true; +@@ -58,6 +59,8 @@ fn main() { + expand_errors = true; + } else if arg == "--no-solver-version-check" { + no_solver_version_check = true; ++ } else if arg == "--weak-memory" { ++ weak_memory = true; + } else { + panic!("unexpected argument: {:}", arg) + } +@@ -142,6 +145,10 @@ fn main() { + child_args.push("--cfg".to_string()); + child_args.push("feature=\"alloc\"".to_string()); + } ++ if weak_memory { ++ child_args.push("--cfg".to_string()); ++ child_args.push("feature=\"weak-memory\"".to_string()); ++ } + child_args.push("--cfg".to_string()); + child_args.push("feature=\"nonzero_internals\"".to_string()); + child_args.push(VSTD_RS_PATH.to_string()); +diff --git a/tools/vargo/src/cli.rs b/tools/vargo/src/cli.rs +index 47afffb8..f4f2883c 100644 +--- a/tools/vargo/src/cli.rs ++++ b/tools/vargo/src/cli.rs +@@ -66,6 +66,10 @@ pub struct BuildOptions { + /// Turn expand errors on when building vstd + #[arg(long)] + pub vstd_expand_errors: bool, ++ ++ /// Enable use of weak memory ++ #[arg(long)] ++ pub vstd_weak_memory: bool, + } + + #[derive(Clone, Debug, Args, PartialEq, Eq)] +diff --git a/tools/vargo/src/commands/build.rs b/tools/vargo/src/commands/build.rs +index 5afacfe5..affd0a32 100644 +--- a/tools/vargo/src/commands/build.rs ++++ b/tools/vargo/src/commands/build.rs +@@ -490,6 +490,9 @@ fn rebuild_vstd( + if vargo_cmd.build_options.vstd_expand_errors { + vstd_build.arg("--expand-errors"); + } ++ if vargo_cmd.build_options.vstd_weak_memory { ++ vstd_build.arg("--weak-memory"); ++ } + if options.vargo_verbose { + vstd_build.arg("--verbose"); + } diff --git a/verified_libs/vstd_extra/src/atomic_irc11.rs b/verified_libs/vstd_extra/src/atomic_irc11.rs new file mode 100644 index 000000000..582c640c3 --- /dev/null +++ b/verified_libs/vstd_extra/src/atomic_irc11.rs @@ -0,0 +1,688 @@ +//! Adapters for Verus' native IRC11 weak-memory model. +//! +//! This module is intentionally thin. It exposes the native subjective thread +//! views, per-location histories, and points-to resources without translating +//! them into the older `atomic_weak` model. In particular, native histories are +//! finite maps with abstract natural-number timestamps rather than contiguous +//! sequences. +//! +//! Verus does not yet provide a weak-memory `AtomicPtr`. This module therefore +//! supplies only that executable wrapper, specified directly in terms of +//! [`AtomicPointsTo`] and the native load/store/update relations. +use core::sync::atomic::{AtomicPtr, Ordering}; + +pub use vstd::atomic_weak::{ + AtomicHistory, AtomicPointsTo, LoadData, PAtomicWeakBool, PAtomicWeakI8, PAtomicWeakI16, + PAtomicWeakI32, PAtomicWeakIsize, PAtomicWeakU8, PAtomicWeakU16, PAtomicWeakU32, + PAtomicWeakUsize, StoreData, UpdateData, fence_acquire, fence_release, + history_get_contains_timestamp, load_acquire, load_reads_from_history, load_relaxed, + load_timestamp_in_view, load_view_nondecreasing, store_insert_history, store_relaxed, + store_release, store_timestamp_in_view, store_view_increasing, +}; +#[cfg(target_has_atomic = "64")] +pub use vstd::atomic_weak::{PAtomicWeakI64, PAtomicWeakU64}; +pub use vstd::cell::CellId as AtomicId; +use vstd::prelude::*; +pub use vstd::thread_view::{ + AcquireViewSeen, Objective, ReleaseViewSeen, ThreadView, ViewAt, ViewSeen, +}; + +verus! { + +/// Logical timestamp used by one native atomic history. +pub type Timestamp = nat; + +/// Compatibility vocabulary for ordering native subjective views. +/// +/// `old.spec_le(new)` is only notation for the native relation +/// `new.contains(old)`; it introduces no second view model. +pub trait ThreadViewOrder { + spec fn spec_le(self, newer: Self) -> bool; + + spec fn view_join(self, other: Self) -> Self; + + proof fn lemma_spec_le_transitive(self, middle: Self, newer: Self) + requires + self.spec_le(middle), + middle.spec_le(newer), + ensures + self.spec_le(newer), + ; + + proof fn lemma_join_left(self, other: Self) + ensures + self.spec_le(self.view_join(other)), + ; + + proof fn lemma_join_right(self, other: Self) + ensures + other.spec_le(self.view_join(other)), + ; +} + +impl ThreadViewOrder for ThreadView { + open spec fn spec_le(self, newer: Self) -> bool { + newer.contains(self) + } + + open spec fn view_join(self, other: Self) -> Self { + self.join(other) + } + + proof fn lemma_spec_le_transitive(self, middle: Self, newer: Self) { + ThreadView::contains_trans(newer, middle, self); + } + + proof fn lemma_join_left(self, other: Self) { + ThreadView::join_contains(self, other); + } + + proof fn lemma_join_right(self, other: Self) { + ThreadView::join_comm(self, other); + ThreadView::join_contains(other, self); + } +} + +/// Scheduler-owned native subjective view. +/// +/// OSTD should create one token when it registers an execution participant, +/// move that token through schedule-in/schedule-out, and borrow it for native +/// weak atomic operations. The wrapper deliberately exposes no operation that +/// can manufacture an arbitrary non-empty view. +pub tracked struct ThreadViewToken { + view_seen: ViewSeen, +} + +impl View for ThreadViewToken { + type V = ThreadView; + + closed spec fn view(&self) -> ThreadView { + self.view_seen@ + } +} + +impl ThreadViewToken { + /// Creates the empty view used when registering a task or CPU. + pub proof fn new() -> (tracked res: Self) + ensures + res@ == ThreadView::empty(), + { + let tracked view_seen = ViewSeen::new(); + ThreadViewToken { view_seen } + } + + /// Wraps a native view returned by atomic or synchronization setup. + pub proof fn from_view_seen(tracked view_seen: ViewSeen) -> (tracked res: Self) + ensures + res@ == view_seen@, + { + ThreadViewToken { view_seen } + } + + /// Removes the scheduler wrapper without changing the represented view. + pub proof fn into_view_seen(tracked self) -> (tracked res: ViewSeen) + ensures + res@ == self@, + { + self.view_seen + } + + /// Borrows the native token for one atomic operation. + pub proof fn tracked_borrow_mut(tracked &mut self) -> (tracked res: &mut ViewSeen) + ensures + (*res)@ == old(self)@, + final(self)@ == (*final(res))@, + { + &mut self.view_seen + } + + /// Imports observations held by another execution participant. + /// + /// `ViewSeen` is persistent knowledge, so Verus' native model permits + /// copying it before the join. The source token consequently remains + /// available to its CPU or task owner. + pub proof fn tracked_join(tracked &mut self, tracked other: &Self) + ensures + final(self)@ == old(self)@.join(other@), + { + let tracked other_view = other.view_seen; + let tracked old_view = self.view_seen; + self.view_seen = old_view.join(other_view); + } + + /// Imports this stored lower bound into an executing thread's native view. + pub proof fn tracked_join_into_view_seen(tracked &self, tracked target: &mut ViewSeen) + ensures + final(target)@ == old(target)@.join(self@), + { + let tracked source = self.view_seen; + let tracked old_target = *target; + *target = old_target.join(source); + } + + /// Publishes an executing thread's current native view into this token. + pub proof fn tracked_join_view_seen(tracked &mut self, tracked source: &ViewSeen) + ensures + final(self)@ == old(self)@.join(source@), + { + let tracked source = *source; + let tracked old_view = self.view_seen; + self.view_seen = old_view.join(source); + } + + /// Returns a joined token without mutating the stored source token. + pub proof fn tracked_joined_view_seen(tracked &self, tracked source: &ViewSeen) -> (tracked res: + Self) + ensures + res@ == self@.join(source@), + { + let tracked source = *source; + let tracked stored = self.view_seen; + let tracked view_seen = stored.join(source); + ThreadViewToken { view_seen } + } +} + +/// IRC11 wrapper around Rust's sized-pointer atomic. +/// +/// The permission describes only the atomic location's modification history; +/// ownership of the pointee remains in the client invariant. +#[repr(transparent)] +#[verifier::accept_recursive_types(T)] +#[verifier::external_body] +pub struct PAtomicWeakPtr { + value: AtomicPtr, +} + +impl PAtomicWeakPtr { + pub uninterp spec fn loc(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(value: *mut T) -> ((atomic, points_to, view, timestamp): ( + Self, + Tracked>, + Tracked, + Ghost, + )) + ensures + atomic.loc() == points_to@.loc(), + points_to@.hist().is_singleton(timestamp@, (value, view@@)), + points_to@.get_timestamp(view@@) == Some(timestamp@), + { + ( + Self { value: AtomicPtr::new(value) }, + Tracked::assume_new(), + Tracked::assume_new(), + Ghost::assume_new(), + ) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load( + &self, + order: Ordering, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(points_to): Tracked<&AtomicPointsTo<*mut T>>, + ) -> ((value, acquire_view, load): (*mut T, Tracked, Ghost)) + requires + self.loc() == points_to.loc(), + order == Ordering::Acquire || order == Ordering::Relaxed, + ensures + match order { + Ordering::Acquire => load_acquire( + *points_to, + old(view)@, + final(view)@, + value, + load@.timestamp, + load@.message_view, + ), + Ordering::Relaxed => load_relaxed( + *points_to, + old(view)@, + final(view)@, + acquire_view@@, + value, + load@.timestamp, + load@.message_view, + ), + }, + opens_invariants none + no_unwind + { + (self.value.load(order), Tracked::assume_new(), Ghost::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store( + &self, + value: *mut T, + order: Ordering, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(release_view): Tracked, + Tracked(points_to): Tracked<&mut AtomicPointsTo<*mut T>>, + ) -> (store: Ghost) + requires + self.loc() == old(points_to).loc(), + order == Ordering::Release || order == Ordering::Relaxed, + ensures + forall|observed_view: ThreadView| #[trigger] + old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( + observed_view, + ), + match order { + Ordering::Release => store_release( + *old(points_to), + *final(points_to), + old(view)@, + final(view)@, + value, + store@.timestamp, + store@.message_view, + ), + Ordering::Relaxed => store_relaxed( + *old(points_to), + *final(points_to), + old(view)@, + final(view)@, + release_view@, + value, + store@.timestamp, + store@.message_view, + ), + }, + opens_invariants none + no_unwind + { + self.value.store(value, order); + Ghost::assume_new() + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange( + &self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(release_view): Tracked, + Tracked(points_to): Tracked<&mut AtomicPointsTo<*mut T>>, + ) -> ((result, acquire_view, update): ( + Result<*mut T, *mut T>, + Tracked, + Ghost, + )) + requires + self.loc() == old(points_to).loc(), + success == Ordering::AcqRel || success == Ordering::Acquire || success + == Ordering::Release || success == Ordering::Relaxed, + failure == Ordering::Acquire || failure == Ordering::Relaxed, + ensures + result is Ok ==> old(points_to).hist().is_max_timestamp(update@.load_timestamp), + forall|observed_view: ThreadView| #[trigger] + old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( + observed_view, + ), + match result { + Ok(value) => { + &&& current.addr() == value.addr() + &&& update@.store_message_view.contains_strict(update@.load_message_view) + &&& match success { + Ordering::AcqRel => { + &&& load_acquire( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_release( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + Ordering::Acquire => { + &&& load_acquire( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_relaxed( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + release_view@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + Ordering::Release => { + &&& load_relaxed( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + acquire_view@@, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_release( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + Ordering::Relaxed => { + &&& load_relaxed( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + acquire_view@@, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_relaxed( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + release_view@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + } + }, + Err(value) => { + &&& current.addr() != value.addr() + &&& *final(points_to) == *old(points_to) + &&& match failure { + Ordering::Acquire => load_acquire( + *old(points_to), + old(view)@, + final(view)@, + value, + update@.load_timestamp, + update@.load_message_view, + ), + Ordering::Relaxed => load_relaxed( + *old(points_to), + old(view)@, + final(view)@, + acquire_view@@, + value, + update@.load_timestamp, + update@.load_message_view, + ), + } + }, + }, + opens_invariants none + no_unwind + { + ( + self.value.compare_exchange(current, new, success, failure), + Tracked::assume_new(), + Ghost::assume_new(), + ) + } + + /// Release swap reads the latest modification and immediately appends its + /// replacement in the same modification order. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn swap_release( + &self, + value: *mut T, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(points_to): Tracked<&mut AtomicPointsTo<*mut T>>, + ) -> ((old_value, acquire_view, swap): (*mut T, Tracked, Ghost)) + requires + self.loc() == old(points_to).loc(), + ensures + old(points_to).hist().is_max_timestamp(swap@.load_timestamp), + forall|observed_view: ThreadView| #[trigger] + old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( + observed_view, + ), + load_relaxed( + *old(points_to), + old(view)@, + swap@.intermediate_thread_view, + acquire_view@@, + old_value, + swap@.load_timestamp, + swap@.load_message_view, + ), + store_release( + *old(points_to), + *final(points_to), + swap@.intermediate_thread_view, + final(view)@, + value, + swap@.load_timestamp + 1, + swap@.store_message_view, + ), + opens_invariants none + no_unwind + { + (self.value.swap(value, Ordering::Release), Tracked::assume_new(), Ghost::assume_new()) + } +} + +/// The partial order used for weak-memory views, written from old to new. +pub open spec fn view_le(old_view: ThreadView, new_view: ThreadView) -> bool { + new_view.contains(old_view) +} + +/// View joins monotonically include both operands. +pub proof fn lemma_join_upper_bound(left: ThreadView, right: ThreadView) + ensures + view_le(left, left.join(right)), + view_le(right, left.join(right)), +{ + ThreadView::join_contains(left, right); + ThreadView::join_comm(left, right); + ThreadView::join_contains(right, left); +} + +// These executable examples are part of verification. They ensure that the +// native tokens can be threaded through the operation shapes needed by OSTD. +fn test_native_acquire_load() { + let (atomic, Tracked(pt), Tracked(mut view), Ghost(initial_ts)) = PAtomicWeakUsize::new(7); + let ghost old_view = view@; + assert(pt.hist().is_singleton(initial_ts, (7, old_view))); + let (value, Tracked(_acquire_view), Ghost(load)) = atomic.load( + Ordering::Acquire, + Tracked(&mut view), + Tracked(&pt), + ); + + proof { + assert(pt.hist().get(load.timestamp) == Some((value, load.message_view))); + history_get_contains_timestamp(pt.hist(), load.timestamp); + assert(pt.hist().contains_timestamp(load.timestamp)); + assert(load.timestamp == initial_ts); + assert(pt.hist().get(load.timestamp) == Some((7, old_view))); + assert(value == 7); + assert(view@.contains(old_view)); + assert(pt.get_timestamp(view@) == Some(load.timestamp)); + } +} + +fn test_native_release_store() { + let (atomic, Tracked(mut pt), Tracked(mut view), Ghost(_initial_ts)) = PAtomicWeakBool::new( + false, + ); + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let ghost old_history = pt.hist(); + let Ghost(store) = atomic.store( + true, + Ordering::Release, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut pt), + ); + + proof { + assert(pt.hist() == old_history.insert(store.timestamp, true, store.message_view)); + assert(pt.get_timestamp(view@) == Some(store.timestamp)); + assert(store.message_view == view@); + } +} + +fn test_native_compare_exchange() { + let (atomic, Tracked(mut pt), Tracked(mut view), Ghost(_initial_ts)) = PAtomicWeakUsize::new(0); + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let (result, Tracked(_acquire_view), Ghost(update)) = atomic.compare_exchange( + 0, + 1, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut pt), + ); + + proof { + match result { + Ok(value) => { + assert(value == 0); + assert(pt.hist().get_value(update.load_timestamp + 1) == Some(1)); + }, + Err(value) => { + assert(value != 0); + }, + } + } +} + +fn test_native_pointer_operations() { + let first = core::ptr::null_mut::(); + let second = core::ptr::null_mut::(); + let (atomic, Tracked(mut points_to), Tracked(mut view), Ghost(_initial_ts)) = + PAtomicWeakPtr::new(first); + + let (_loaded, Tracked(_acquire_view), Ghost(_load)) = atomic.load( + Ordering::Acquire, + Tracked(&mut view), + Tracked(&points_to), + ); + + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let Ghost(_store) = atomic.store( + second, + Ordering::Release, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut points_to), + ); + + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let (_result, Tracked(_acquire_view), Ghost(_update)) = atomic.compare_exchange( + second, + first, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut points_to), + ); + + let (_old, Tracked(_acquire_view), Ghost(swap)) = atomic.swap_release( + second, + Tracked(&mut view), + Tracked(&mut points_to), + ); + proof { + assert(points_to.get_timestamp(view@) == Some(swap.load_timestamp + 1)); + } +} + +fn test_scheduler_owned_view_token() { + let (atomic, Tracked(mut points_to), Tracked(view_seen), Ghost(initial_ts)) = + PAtomicWeakUsize::new(0); + proof_decl! { + let tracked mut token = ThreadViewToken::from_view_seen(view_seen); + } + let ghost initial_view = token@; + proof { + assert(points_to.hist().is_singleton(initial_ts, (0, initial_view))); + } + + let (value, Tracked(_acquire_view), Ghost(load)) = atomic.load( + Ordering::Acquire, + Tracked(token.tracked_borrow_mut()), + Tracked(&points_to), + ); + proof { + assert(points_to.get_timestamp(token@) == Some(load.timestamp)); + assert(points_to.hist().get(load.timestamp) == Some((value, load.message_view))); + history_get_contains_timestamp(points_to.hist(), load.timestamp); + assert(load.timestamp == initial_ts); + assert(points_to.hist().get(load.timestamp) == Some((0, initial_view))); + assert(value == 0); + } + + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let Ghost(store) = atomic.store( + 1, + Ordering::Release, + Tracked(token.tracked_borrow_mut()), + Tracked(release_view), + Tracked(&mut points_to), + ); + proof { + assert(points_to.get_timestamp(token@) == Some(store.timestamp)); + assert(points_to.hist().get_value(store.timestamp) == Some(1)); + } + + proof_decl! { + let tracked cpu_token = ThreadViewToken::new(); + } + proof { + let ghost before = token@; + token.tracked_join(&cpu_token); + assert(token@ == before.join(cpu_token@)); + ThreadView::join_contains(before, cpu_token@); + assert(token@.contains(before)); + } +} + +} // verus! diff --git a/verified_libs/vstd_extra/src/atomic_weak.rs b/verified_libs/vstd_extra/src/atomic_weak.rs new file mode 100644 index 000000000..cd625e167 --- /dev/null +++ b/verified_libs/vstd_extra/src/atomic_weak.rs @@ -0,0 +1,12 @@ +//! Compatibility exports for the native IRC11 weak-memory model. +//! +//! The original version of this module implemented a second weak-memory +//! history, thread-view, and atomic-wrapper semantics. Verus now provides +//! those primitives in `vstd::atomic_weak`, and [`crate::atomic_irc11`] adds +//! only the adapters that are still missing upstream, such as weak +//! `AtomicPtr` support. +//! +//! New code should import [`crate::atomic_irc11`] directly. This module remains +//! as a source-compatible path and deliberately contains no independent +//! weak-memory semantics. +pub use crate::atomic_irc11::*; diff --git a/verified_libs/vstd_extra/src/external/smart_ptr.rs b/verified_libs/vstd_extra/src/external/smart_ptr.rs index 711c90527..dfd359876 100644 --- a/verified_libs/vstd_extra/src/external/smart_ptr.rs +++ b/verified_libs/vstd_extra/src/external/smart_ptr.rs @@ -5,6 +5,7 @@ use alloc::sync::Arc; use vstd::layout::valid_layout; use vstd::prelude::*; use vstd::raw_ptr::*; +use vstd::thread_view::Objective; // A unified interface for the raw ptr permission returned by `into_raw` methods of smart pointers like `Box` and `Arc`. verus! { @@ -63,6 +64,16 @@ pub tracked struct ArcPointsTo { pub perm: &'static PointsTo, } +// Raw-memory ownership is global state and does not depend on a thread's +// subjective weak-memory view. +unsafe impl Objective for BoxPointsTo { + +} + +unsafe impl Objective for ArcPointsTo { + +} + impl BoxPointsTo { pub open spec fn perm(self) -> PointsTowithDealloc { self.perm diff --git a/verified_libs/vstd_extra/src/lib.rs b/verified_libs/vstd_extra/src/lib.rs index 68cf8e5ae..b49c1b8a5 100644 --- a/verified_libs/vstd_extra/src/lib.rs +++ b/verified_libs/vstd_extra/src/lib.rs @@ -17,6 +17,8 @@ extern crate alloc; pub mod arithmetic; pub mod array_ptr; +pub mod atomic_irc11; +pub mod atomic_weak; pub mod auxiliary; pub mod cast_ptr; pub mod drop_tracking; @@ -32,7 +34,9 @@ pub mod trans_macros; pub mod map_extra; pub mod prelude; +pub mod raw_callback; pub mod raw_ptr_extra; +pub mod rcu_read_pool; pub mod seq_extra; pub mod set_extra; pub mod spec_operators; diff --git a/verified_libs/vstd_extra/src/raw_callback.rs b/verified_libs/vstd_extra/src/raw_callback.rs new file mode 100644 index 000000000..2fcf5c39e --- /dev/null +++ b/verified_libs/vstd_extra/src/raw_callback.rs @@ -0,0 +1,211 @@ +//! Type-erased one-shot callback wrapper for Verus code. +//! +//! Verus does not support function pointer types in public API signatures. This +//! module therefore exposes callbacks as sized capture objects implementing +//! [`RawCallbackContext`], while the executable runner erasure is kept behind +//! trusted `external_body` wrappers. +use alloc::boxed::Box; +use core::marker::PhantomData; + +use vstd::prelude::*; + +verus! { + +/// A one-shot, type-erased callback. +/// +/// `data` is a thin pointer to a heap payload, and `run` stores the address of +/// the monomorphized shim that knows how to consume that payload. The pair is a +/// TCB boundary: proofs should reason about the resource represented by the +/// callback, not about arbitrary closure bodies. +#[must_use] +pub struct RawCallback { + data: *mut (), + run: usize, +} + +/// Captured context for a [`RawCallback`]. +/// +/// Implement this trait on a sized capture object instead of passing +/// `unsafe fn(C)` or `dyn FnOnce()`, both of which are outside Verus' supported +/// executable API surface. +pub trait RawCallbackContext: Send + 'static { + fn run(self); +} + +/// A one-shot callback that transports a tracked proof value to its runner. +/// +/// This is the proof-carrying counterpart of [`RawCallback`]. The executable +/// representation remains a thin data pointer plus a monomorphized runner, +/// while `S` records the tracked value that must be supplied at invocation. +#[must_use] +pub struct RawCallbackWithProof { + data: *mut (), + run: usize, + _proof: PhantomData, +} + +/// Captured context for a [`RawCallbackWithProof`]. +pub trait RawCallbackContextWithProof: Send + 'static { + spec fn call_requires(&self, proof: S) -> bool; + + fn run(self, proof: Tracked) + requires + self.call_requires(proof@), + ; +} + +struct RawDropContext { + value: T, +} + +// SAFETY: `RawCallback::new` only accepts `C: Send + 'static` payloads, and +// `call_once` consumes the payload through the matching monomorphized runner. +#[verifier::external] +unsafe impl Send for RawCallback { + +} + +// SAFETY: `RawCallbackWithProof::new` only accepts `C: Send + 'static` +// payloads, and `call_once` consumes the payload through the matching +// monomorphized runner. `S` is proof-only and has no runtime representation. +#[verifier::external] +unsafe impl Send for RawCallbackWithProof { + +} + +impl RawCallbackContext for RawDropContext { + #[inline] + #[verifier::external_body] + fn run(self) { + let RawDropContext { value } = self; + drop(value); + } +} + +impl RawCallback { + /// Builds a callback from an explicit captured context. + #[inline] + #[verifier::external_body] + pub fn new(context: C) -> Self { + let payload = Box::new(RawCallbackPayload { context }); + Self { data: Box::into_raw(payload).cast::<()>(), run: raw_callback_runner::() } + } + + /// Builds the common "drop this value later" callback. + #[inline] + #[verifier::external_body] + pub fn defer_drop(value: T) -> Self { + Self::new(RawDropContext { value }) + } + + /// Runs the callback exactly once. + /// + /// # Safety + /// + /// The caller must ensure this callback has not already been run and will + /// not be run again. + #[inline] + #[verifier::external_body] + pub unsafe fn call_once(self) { + unsafe { + call_raw_callback(self.data, self.run); + } + } +} + +impl RawCallbackWithProof { + /// Proof obligation retained across executable callback type erasure. + pub uninterp spec fn call_requires(&self, proof: S) -> bool; + + /// Builds a proof-carrying callback from an explicit captured context. + #[inline] + #[verifier::external_body] + pub fn new>(context: C) -> (res: Self) + ensures + forall|proof: S| #![auto] res.call_requires(proof) == context.call_requires(proof), + { + let payload = Box::new(RawCallbackPayloadWithProof { context, _proof: PhantomData }); + Self { + data: Box::into_raw(payload).cast::<()>(), + run: raw_callback_with_proof_runner::(), + _proof: PhantomData, + } + } + + /// Runs the callback exactly once and transfers `proof` to its context. + /// + /// # Safety + /// + /// The caller must ensure this callback has not already been run and will + /// not be run again. + #[inline] + #[verifier::external_body] + pub unsafe fn call_once(self, proof: Tracked) + requires + self.call_requires(proof@), + { + unsafe { + call_raw_callback_with_proof(self.data, self.run, proof); + } + } +} + +struct RawCallbackPayload { + context: C, +} + +struct RawCallbackPayloadWithProof, S> { + context: C, + _proof: PhantomData, +} + +#[inline] +#[verifier::external_body] +fn raw_callback_runner() -> usize { + run_raw_callback:: as *const () as usize +} + +#[inline] +#[verifier::external_body] +unsafe fn call_raw_callback(data: *mut (), run: usize) { + let run: unsafe fn (*mut ()) = unsafe { core::mem::transmute(run) }; + unsafe { + run(data); + } +} + +#[inline] +#[verifier::external_body] +unsafe fn run_raw_callback(data: *mut ()) { + let payload = unsafe { Box::from_raw(data.cast::>()) }; + let RawCallbackPayload { context } = *payload; + context.run(); +} + +#[inline] +#[verifier::external_body] +fn raw_callback_with_proof_runner, S>() -> usize { + run_raw_callback_with_proof:: as *const () as usize +} + +#[inline] +#[verifier::external_body] +unsafe fn call_raw_callback_with_proof(data: *mut (), run: usize, proof: Tracked) { + let run: unsafe fn (*mut (), Tracked) = unsafe { core::mem::transmute(run) }; + unsafe { + run(data, proof); + } +} + +#[inline] +#[verifier::external_body] +unsafe fn run_raw_callback_with_proof, S>( + data: *mut (), + proof: Tracked, +) { + let payload = unsafe { Box::from_raw(data.cast::>()) }; + let RawCallbackPayloadWithProof { context, _proof } = *payload; + context.run(proof); +} + +} // verus! diff --git a/verified_libs/vstd_extra/src/rcu_read_pool.rs b/verified_libs/vstd_extra/src/rcu_read_pool.rs new file mode 100644 index 000000000..0d068cfb9 --- /dev/null +++ b/verified_libs/vstd_extra/src/rcu_read_pool.rs @@ -0,0 +1,1023 @@ +//! Unbounded fractional read leases for delayed reclamation. +//! +//! The pool stores one linear resource in Verus' Leaf-style storage protocol. +//! Each lease receives half of the pool's current rational fraction, so the +//! number of outstanding leases has no fixed integer bound. A lease guards a +//! shared borrow of the stored resource. Reclamation can recover the resource +//! only after all leases have been returned and the pool fraction is whole. +use vstd::{ + prelude::*, + resource::{Loc, frac_opt::Frac}, +}; + +verus! { + +/// Owner-side accumulator for one delayed-reclamation resource. +pub tracked struct RcuReadPool { + frac: Frac, +} + +/// One read-side fraction split from an [`RcuReadPool`]. +pub tracked struct RcuReadLease { + frac: Frac, +} + +/// Allocation-indexed ownership pools retained across publication changes. +/// +/// An RCU root needs this indirection because a weak load may select an older +/// publication after a newer pointer has already been installed. +pub tracked struct RcuReadPoolRegistry { + pools: Map>, +} + +/// One allocation-indexed lease issued by a tracked registry. +/// +/// The private `lease_id` names the matching active record in the registry. +/// Returning a lease must consume that exact record, so a lease cannot be +/// returned to another allocation that happens to store an equal resource. +pub tracked struct RcuTrackedReadLease { + ghost lease_id: nat, + ghost key: K, + lease: RcuReadLease, +} + +/// Registry-side accounting for one outstanding read lease. +/// +/// `W` is a client-provided linear witness. RCU uses it to retain enough of the +/// reader's CPU-generation authority for a completed grace period to rule out +/// this record before reclamation. +pub tracked struct RcuReadLeaseRecord { + ghost key: K, + ghost pool_id: Loc, + ghost fraction: real, + witness: W, +} + +/// Allocation-indexed pools with explicit accounting for every issued lease. +/// +/// Unlike [`RcuReadPoolRegistry`], this registry records every split and only +/// removes the record when the matching lease is returned. Its invariant says +/// that each pool's owner fraction plus all of that allocation's active lease +/// fractions is exactly one. Consequently, proving that an allocation has no +/// active records is sufficient to recover its stored resource. +pub tracked struct RcuTrackedReadPoolRegistry { + pools: Map>, + active: Map>, + ghost next_lease: nat, +} + +impl RcuReadLeaseRecord { + pub closed spec fn key(self) -> K { + self.key + } + + pub closed spec fn pool_id(self) -> Loc { + self.pool_id + } + + pub closed spec fn fraction(self) -> real { + self.fraction + } + + pub closed spec fn witness(self) -> W { + self.witness + } +} + +impl RcuTrackedReadLease { + pub closed spec fn lease_id(self) -> nat { + self.lease_id + } + + pub closed spec fn key(self) -> K { + self.key + } + + pub closed spec fn pool_id(self) -> Loc { + self.lease.id() + } + + pub closed spec fn resource(self) -> T { + self.lease.resource() + } + + pub closed spec fn fraction(self) -> real { + self.lease.fraction() + } + + pub proof fn borrow(tracked &self) -> (tracked resource: &T) + ensures + *resource == self.resource(), + { + self.lease.borrow() + } +} + +/// Sum of active lease fractions for `key` among record IDs below `upto`. +pub open spec fn active_lease_fraction( + active: Map>, + key: K, + upto: nat, +) -> real + decreases upto, +{ + if upto == 0 { + 0real + } else { + let id = (upto - 1) as nat; + active_lease_fraction(active, key, id) + if active.contains_key(id) && active[id].key() + == key { + active[id].fraction() + } else { + 0real + } + } +} + +proof fn lemma_active_fraction_insert_above( + active: Map>, + inserted: nat, + record: RcuReadLeaseRecord, + key: K, + upto: nat, +) + requires + upto <= inserted, + ensures + active_lease_fraction(active.insert(inserted, record), key, upto) == active_lease_fraction( + active, + key, + upto, + ), + decreases upto, +{ + if upto > 0 { + let id = (upto - 1) as nat; + lemma_active_fraction_insert_above(active, inserted, record, key, id); + assert(id < inserted); + assert(active.insert(inserted, record).contains_key(id) == active.contains_key(id)); + if active.contains_key(id) { + assert(active.insert(inserted, record)[id] == active[id]); + } + } +} + +proof fn lemma_active_fraction_insert_next( + active: Map>, + next: nat, + record: RcuReadLeaseRecord, + key: K, +) + ensures + active_lease_fraction(active.insert(next, record), key, next + 1) == active_lease_fraction( + active, + key, + next, + ) + if record.key() == key { + record.fraction() + } else { + 0real + }, +{ + lemma_active_fraction_insert_above(active, next, record, key, next); +} + +proof fn lemma_active_fraction_remove( + active: Map>, + removed: nat, + key: K, + upto: nat, +) + requires + removed < upto, + active.contains_key(removed), + ensures + active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active, + key, + upto, + ) - if active[removed].key() == key { + active[removed].fraction() + } else { + 0real + }, + decreases upto, +{ + let id = (upto - 1) as nat; + if removed == id { + lemma_active_fraction_remove_above(active, removed, key, id); + assert(!active.remove(removed).contains_key(id)); + assert(active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active.remove(removed), + key, + id, + )); + assert(active_lease_fraction(active, key, upto) == active_lease_fraction(active, key, id) + + if active[removed].key() == key { + active[removed].fraction() + } else { + 0real + }); + } else { + assert(removed < id); + lemma_active_fraction_remove(active, removed, key, id); + assert(active.remove(removed).contains_key(id) == active.contains_key(id)); + if active.contains_key(id) { + assert(active.remove(removed)[id] == active[id]); + } + assert(active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active.remove(removed), + key, + id, + ) + if active.contains_key(id) && active[id].key() == key { + active[id].fraction() + } else { + 0real + }); + assert(active_lease_fraction(active, key, upto) == active_lease_fraction(active, key, id) + + if active.contains_key(id) && active[id].key() == key { + active[id].fraction() + } else { + 0real + }); + } +} + +proof fn lemma_active_fraction_remove_above( + active: Map>, + removed: nat, + key: K, + upto: nat, +) + requires + upto <= removed, + ensures + active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active, + key, + upto, + ), + decreases upto, +{ + if upto > 0 { + let id = (upto - 1) as nat; + lemma_active_fraction_remove_above(active, removed, key, id); + assert(id < removed); + assert(active.remove(removed).contains_key(id) == active.contains_key(id)); + if active.contains_key(id) { + assert(active.remove(removed)[id] == active[id]); + } + } +} + +proof fn lemma_active_fraction_zero( + active: Map>, + key: K, + upto: nat, +) + requires + forall|id: nat| #![auto] id < upto && active.contains_key(id) ==> active[id].key() != key, + ensures + active_lease_fraction(active, key, upto) == 0real, + decreases upto, +{ + if upto > 0 { + let id = (upto - 1) as nat; + lemma_active_fraction_zero(active, key, id); + } +} + +impl RcuReadPool { + /// Stores `resource` and creates a whole read pool. + pub proof fn new(tracked resource: T) -> (tracked res: Self) + ensures + res.resource() == resource, + res.fraction() == 1real, + { + let tracked frac = Frac::new(resource); + RcuReadPool { frac } + } + + /// Storage-protocol identity shared by this pool and all of its leases. + pub closed spec fn id(self) -> Loc { + self.frac.id() + } + + /// The resource retained in storage while read leases exist. + pub closed spec fn resource(self) -> T { + self.frac.resource() + } + + /// Rational fraction currently accumulated by the owner. + pub closed spec fn fraction(self) -> real { + self.frac.frac() + } + + /// Splits a fresh lease without imposing a fixed reader capacity. + pub proof fn split_lease(tracked &mut self) -> (tracked lease: RcuReadLease) + ensures + final(self).id() == old(self).id(), + final(self).resource() == old(self).resource(), + lease.id() == old(self).id(), + lease.resource() == old(self).resource(), + final(self).fraction() == old(self).fraction() / 2real, + lease.fraction() == old(self).fraction() / 2real, + { + let tracked frac = self.frac.split(); + RcuReadLease { frac } + } + + /// Returns one lease to its originating pool. + pub proof fn return_lease(tracked &mut self, tracked lease: RcuReadLease) + requires + old(self).id() == lease.id(), + ensures + final(self).id() == old(self).id(), + final(self).resource() == old(self).resource(), + final(self).resource() == lease.resource(), + final(self).fraction() == old(self).fraction() + lease.fraction(), + { + self.frac.combine(lease.frac); + } + + /// Recovers the stored resource after every lease has returned. + pub proof fn reclaim(tracked self) -> (tracked resource: T) + requires + self.fraction() == 1real, + ensures + resource == self.resource(), + { + let tracked (resource, _empty) = self.frac.take_resource(); + resource + } + + /// Establishes the valid range of the accumulated rational fraction. + pub proof fn lemma_fraction_bounded(tracked &self) + ensures + 0real < self.fraction() <= 1real, + { + self.frac.bounded(); + } +} + +impl RcuReadLease { + /// Storage-protocol identity of the originating pool. + pub closed spec fn id(self) -> Loc { + self.frac.id() + } + + /// The resource protected by this lease. + pub closed spec fn resource(self) -> T { + self.frac.resource() + } + + /// Rational fraction carried by this lease. + pub closed spec fn fraction(self) -> real { + self.frac.frac() + } + + /// Borrows the protected resource for the lifetime of this lease borrow. + pub proof fn borrow(tracked &self) -> (tracked resource: &T) + ensures + *resource == self.resource(), + { + self.frac.borrow() + } + + /// Establishes that every lease carries a positive rational fraction. + pub proof fn lemma_fraction_bounded(tracked &self) + ensures + 0real < self.fraction() <= 1real, + { + self.frac.bounded(); + } +} + +impl RcuReadPoolRegistry { + /// Creates an empty pool registry. + pub proof fn empty() -> (tracked res: Self) + ensures + res.keys() == Set::::empty(), + { + RcuReadPoolRegistry { pools: Map::tracked_empty() } + } + + /// Registered allocation identities. + pub closed spec fn keys(self) -> Set { + self.pools.dom() + } + + pub closed spec fn contains(self, key: K) -> bool { + self.pools.contains_key(key) + } + + /// Relates keyed lookup to membership in the registry's key set. + pub proof fn lemma_contains_iff_key(tracked &self, key: K) + ensures + self.contains(key) <==> self.keys().contains(key), + { + } + + /// Relates registry membership to the complete key set for all keys. + pub proof fn lemma_all_contains_iff_keys(tracked &self) + ensures + forall|key: K| #[trigger] self.contains(key) <==> self.keys().contains(key), + { + } + + pub closed spec fn pool(self, key: K) -> RcuReadPool + recommends + self.contains(key), + { + self.pools[key] + } + + /// Registers a fresh allocation and stores its linear permission. + pub proof fn insert(tracked &mut self, key: K, tracked resource: T) + requires + !old(self).contains(key), + ensures + final(self).keys() == old(self).keys().insert(key), + final(self).contains(key), + final(self).pool(key).resource() == resource, + final(self).pool(key).fraction() == 1real, + forall|other: K| + old(self).contains(other) ==> final(self).pool(other) == old(self).pool(other), + { + let tracked pool = RcuReadPool::new(resource); + self.pools.tracked_insert(key, pool); + } + + /// Splits a lease from the allocation selected by `key`. + pub proof fn split_lease(tracked &mut self, key: K) -> (tracked lease: RcuReadLease) + requires + old(self).contains(key), + ensures + final(self).keys() == old(self).keys(), + final(self).contains(key), + final(self).pool(key).id() == old(self).pool(key).id(), + final(self).pool(key).resource() == old(self).pool(key).resource(), + lease.id() == old(self).pool(key).id(), + lease.resource() == old(self).pool(key).resource(), + final(self).pool(key).fraction() == old(self).pool(key).fraction() / 2real, + lease.fraction() == old(self).pool(key).fraction() / 2real, + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + let tracked pool = self.pools.tracked_borrow_mut(key); + pool.split_lease() + } + + /// Returns a lease to the pool identified by `key`. + pub proof fn return_lease(tracked &mut self, key: K, tracked lease: RcuReadLease) + requires + old(self).contains(key), + old(self).pool(key).id() == lease.id(), + ensures + final(self).keys() == old(self).keys(), + final(self).contains(key), + final(self).pool(key).id() == old(self).pool(key).id(), + final(self).pool(key).resource() == old(self).pool(key).resource(), + final(self).pool(key).resource() == lease.resource(), + final(self).pool(key).fraction() == old(self).pool(key).fraction() + lease.fraction(), + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + let tracked pool = self.pools.tracked_borrow_mut(key); + pool.return_lease(lease); + } + + /// Removes a whole pool and recovers its stored ownership resource. + pub proof fn reclaim(tracked &mut self, key: K) -> (tracked resource: T) + requires + old(self).contains(key), + old(self).pool(key).fraction() == 1real, + ensures + final(self).keys() == old(self).keys().remove(key), + !final(self).contains(key), + resource == old(self).pool(key).resource(), + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + let tracked pool = self.pools.tracked_remove(key); + pool.reclaim() + } +} + +impl RcuTrackedReadPoolRegistry { + /// Creates an empty tracked registry. + pub proof fn empty() -> (tracked res: Self) + ensures + res.wf(), + res.keys() == Set::::empty(), + res.active_ids() == Set::::empty(), + res.next_lease() == 0, + { + RcuTrackedReadPoolRegistry { + pools: Map::tracked_empty(), + active: Map::tracked_empty(), + next_lease: 0, + } + } + + pub closed spec fn keys(self) -> Set { + self.pools.dom() + } + + pub closed spec fn contains(self, key: K) -> bool { + self.pools.contains_key(key) + } + + /// Relates keyed lookup to membership in the registry's key set. + pub proof fn lemma_contains_iff_key(tracked &self, key: K) + ensures + self.contains(key) <==> self.keys().contains(key), + { + } + + /// Relates registry membership to the complete key set for all keys. + pub proof fn lemma_all_contains_iff_keys(tracked &self) + ensures + forall|key: K| #[trigger] self.contains(key) <==> self.keys().contains(key), + { + } + + pub closed spec fn pool(self, key: K) -> RcuReadPool + recommends + self.contains(key), + { + self.pools[key] + } + + pub closed spec fn active_ids(self) -> Set { + self.active.dom() + } + + /// Ghost snapshot used to state the per-allocation accounting invariant. + pub closed spec fn active_records(self) -> Map> { + self.active + } + + pub closed spec fn next_lease(self) -> nat { + self.next_lease + } + + pub closed spec fn active_record(self, lease_id: nat) -> RcuReadLeaseRecord + recommends + self.active_ids().contains(lease_id), + { + self.active[lease_id] + } + + /// Borrows the client witness associated with one active lease. + /// + /// The witness remains owned by the registry until the matching lease is + /// returned. Reclamation proofs use this borrow to show that an allegedly + /// active lease is incompatible with a completed grace period. + pub proof fn tracked_borrow_active_witness(tracked &self, lease_id: nat) -> (tracked witness: + &W) + requires + self.active_ids().contains(lease_id), + ensures + *witness == self.active_record(lease_id).witness(), + { + let tracked record = self.active.tracked_borrow(lease_id); + &record.witness + } + + /// Mutably borrows an active witness while preserving the registry. + /// + /// Resource-algebra validation may require a mutable receiver even when + /// its postcondition leaves the witness unchanged. + pub proof fn tracked_borrow_active_witness_mut( + tracked &mut self, + lease_id: nat, + ) -> (tracked witness: &mut W) + requires + old(self).active_ids().contains(lease_id), + ensures + *witness == old(self).active_record(lease_id).witness(), + final(self).keys() == old(self).keys(), + final(self).active_ids() == old(self).active_ids(), + final(self).next_lease() == old(self).next_lease(), + final(self).active_record(lease_id).key() == old(self).active_record(lease_id).key(), + final(self).active_record(lease_id).pool_id() == old(self).active_record( + lease_id, + ).pool_id(), + final(self).active_record(lease_id).fraction() == old(self).active_record( + lease_id, + ).fraction(), + final(self).active_record(lease_id).witness() == *final(witness), + forall|other: nat| + #![auto] + other != lease_id && old(self).active_ids().contains(other) + ==> final(self).active_record(other) == old(self).active_record(other), + { + let tracked record = self.active.tracked_borrow_mut(lease_id); + &mut record.witness + } + + pub open spec fn has_active(self, key: K) -> bool { + exists|lease_id: nat| + #![auto] + self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == key + } + + pub open spec fn wf(self) -> bool { + &&& forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> { + let record = self.active_record(lease_id); + &&& lease_id < self.next_lease() + &&& self.contains(record.key()) + &&& record.pool_id() == self.pool(record.key()).id() + &&& record.fraction() > 0real + } + &&& forall|key: K| #[trigger] + self.contains(key) ==> self.pool(key).fraction() + active_lease_fraction( + self.active_records(), + key, + self.next_lease(), + ) == 1real + } + + /// Registers one allocation and stores its complete ownership resource. + pub proof fn insert(tracked &mut self, key: K, tracked resource: T) + requires + old(self).wf(), + !old(self).contains(key), + ensures + final(self).wf(), + final(self).keys() == old(self).keys().insert(key), + final(self).active_ids() == old(self).active_ids(), + final(self).next_lease() == old(self).next_lease(), + forall|lease_id: nat| + #![auto] + old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) + == old(self).active_record(lease_id), + final(self).contains(key), + final(self).pool(key).resource() == resource, + final(self).pool(key).fraction() == 1real, + forall|other: K| + old(self).contains(other) ==> final(self).pool(other) == old(self).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|lease_id: nat| #[trigger] old(self).active_ids().contains(lease_id) implies { + let record = old(self).active_record(lease_id); + &&& lease_id < old(self).next_lease() + &&& old(self).contains(record.key()) + &&& record.pool_id() == old(self).pool(record.key()).id() + &&& record.fraction() > 0real + } by {}; + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + let tracked pool = RcuReadPool::new(resource); + self.pools.tracked_insert(key, pool); + assert forall|lease_id: nat| #![auto] self.active_ids().contains(lease_id) implies { + &&& lease_id < self.next_lease() + &&& self.contains(self.active_record(lease_id).key()) + &&& self.active_record(lease_id).pool_id() == self.pool( + self.active_record(lease_id).key(), + ).id() + &&& self.active_record(lease_id).fraction() > 0real + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).active_record(lease_id).key() != key); + }; + assert forall|lease_id: nat| + #![auto] + lease_id < self.next_lease() && self.active_records().contains_key( + lease_id, + ) implies self.active_records()[lease_id].key() != key by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).contains(old(self).active_record(lease_id).key())); + }; + assert(active_lease_fraction(self.active_records(), key, self.next_lease()) == 0real) by { + lemma_active_fraction_zero(self.active_records(), key, self.next_lease()); + }; + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + if other == key { + assert(self.pool(key).fraction() == 1real); + } else { + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + } + }; + } + + /// Splits a lease and installs its client witness in the active registry. + pub proof fn split_lease(tracked &mut self, key: K, tracked witness: W) -> (tracked lease: + RcuTrackedReadLease) + requires + old(self).wf(), + old(self).contains(key), + ensures + final(self).wf(), + final(self).keys() == old(self).keys(), + forall|candidate: K| #[trigger] + final(self).contains(candidate) == old(self).contains(candidate), + final(self).next_lease() == old(self).next_lease() + 1, + lease.lease_id() == old(self).next_lease(), + lease.key() == key, + final(self).active_ids() == old(self).active_ids().insert(lease.lease_id()), + final(self).active_record(lease.lease_id()).key() == key, + final(self).active_record(lease.lease_id()).pool_id() == lease.pool_id(), + final(self).active_record(lease.lease_id()).fraction() == lease.fraction(), + final(self).active_record(lease.lease_id()).witness() == witness, + forall|lease_id: nat| + #![auto] + old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) + == old(self).active_record(lease_id), + lease.pool_id() == old(self).pool(key).id(), + lease.resource() == old(self).pool(key).resource(), + lease.fraction() == old(self).pool(key).fraction() / 2real, + final(self).pool(key).id() == old(self).pool(key).id(), + final(self).pool(key).resource() == old(self).pool(key).resource(), + final(self).pool(key).fraction() == old(self).pool(key).fraction() / 2real, + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + let ghost lease_id = self.next_lease; + let tracked pool = self.pools.tracked_borrow_mut(key); + let tracked lease = pool.split_lease(); + lease.lemma_fraction_bounded(); + let ghost pool_id = lease.id(); + let ghost fraction = lease.fraction(); + let tracked record = RcuReadLeaseRecord { key, pool_id, fraction, witness }; + self.active.tracked_insert(lease_id, record); + self.next_lease = lease_id + 1; + + assert forall|active_id: nat| #![auto] self.active_ids().contains(active_id) implies { + &&& active_id < self.next_lease() + &&& self.contains(self.active_record(active_id).key()) + &&& self.active_record(active_id).pool_id() == self.pool( + self.active_record(active_id).key(), + ).id() + &&& self.active_record(active_id).fraction() > 0real + } by { + if active_id == lease_id { + assert(self.active_record(active_id).fraction() == fraction); + } else { + assert(old(self).active_ids().contains(active_id)); + assert(self.active_record(active_id) == old(self).active_record(active_id)); + } + }; + + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + lemma_active_fraction_insert_next( + old(self).active_records(), + lease_id, + self.active_record(lease_id), + other, + ); + if other == key { + assert(old(self).pool(key).fraction() + active_lease_fraction( + old(self).active_records(), + key, + lease_id, + ) == 1real); + } else { + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + assert(old(self).pool(other).fraction() + active_lease_fraction( + old(self).active_records(), + other, + lease_id, + ) == 1real); + } + }; + RcuTrackedReadLease { lease_id, key, lease } + } + + /// Returns one lease and removes exactly its matching active record. + pub proof fn return_lease( + tracked &mut self, + tracked lease: RcuTrackedReadLease, + ) -> (tracked witness: W) + requires + old(self).wf(), + old(self).active_ids().contains(lease.lease_id()), + old(self).active_record(lease.lease_id()).key() == lease.key(), + old(self).active_record(lease.lease_id()).pool_id() == lease.pool_id(), + old(self).active_record(lease.lease_id()).fraction() == lease.fraction(), + ensures + final(self).wf(), + final(self).keys() == old(self).keys(), + forall|candidate: K| #[trigger] + final(self).contains(candidate) == old(self).contains(candidate), + final(self).next_lease() == old(self).next_lease(), + final(self).active_ids() == old(self).active_ids().remove(lease.lease_id()), + witness == old(self).active_record(lease.lease_id()).witness(), + forall|lease_id: nat| + #![auto] + lease_id != lease.lease_id() && old(self).active_ids().contains(lease_id) + ==> final(self).active_record(lease_id) == old(self).active_record(lease_id), + final(self).pool(lease.key()).id() == old(self).pool(lease.key()).id(), + final(self).pool(lease.key()).resource() == old(self).pool(lease.key()).resource(), + final(self).pool(lease.key()).fraction() == old(self).pool(lease.key()).fraction() + + lease.fraction(), + forall|other: K| + other != lease.key() && old(self).contains(other) ==> final(self).pool(other) + == old(self).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + let ghost lease_id = lease.lease_id; + let ghost key = lease.key; + let tracked record = self.active.tracked_remove(lease_id); + let tracked pool = self.pools.tracked_borrow_mut(key); + pool.return_lease(lease.lease); + + assert forall|active_id: nat| #![auto] self.active_ids().contains(active_id) implies { + &&& active_id < self.next_lease() + &&& self.contains(self.active_record(active_id).key()) + &&& self.active_record(active_id).pool_id() == self.pool( + self.active_record(active_id).key(), + ).id() + &&& self.active_record(active_id).fraction() > 0real + } by { + assert(old(self).active_ids().contains(active_id)); + assert(active_id != lease_id); + assert(self.active_record(active_id) == old(self).active_record(active_id)); + }; + + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + lemma_active_fraction_remove( + old(self).active_records(), + lease_id, + other, + self.next_lease(), + ); + if other == key { + assert(old(self).pool(key).fraction() + active_lease_fraction( + old(self).active_records(), + key, + self.next_lease(), + ) == 1real); + } else { + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + assert(old(self).pool(other).fraction() + active_lease_fraction( + old(self).active_records(), + other, + self.next_lease(), + ) == 1real); + } + }; + record.witness + } + + /// Recovers one allocation after a client proof rules out all active leases. + pub proof fn reclaim(tracked &mut self, key: K) -> (tracked resource: T) + requires + old(self).wf(), + old(self).contains(key), + !old(self).has_active(key), + ensures + final(self).wf(), + final(self).keys() == old(self).keys().remove(key), + final(self).active_ids() == old(self).active_ids(), + final(self).active_records() == old(self).active_records(), + final(self).next_lease() == old(self).next_lease(), + forall|lease_id: nat| + #![auto] + old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) + == old(self).active_record(lease_id), + !final(self).contains(key), + resource == old(self).pool(key).resource(), + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + assert forall|lease_id: nat| + #![auto] + lease_id < self.next_lease() && self.active_records().contains_key( + lease_id, + ) implies self.active_records()[lease_id].key() != key by { + if self.active_records()[lease_id].key() == key { + assert(self.active_ids().contains(lease_id)); + assert(exists|candidate: nat| + #![auto] + self.active_ids().contains(candidate) && self.active_record(candidate).key() + == key) by { + assert(self.active_record(lease_id).key() == key); + }; + assert(self.has_active(key)); + } + }; + lemma_active_fraction_zero(self.active_records(), key, self.next_lease()); + assert(self.pool(key).fraction() == 1real); + let tracked pool = self.pools.tracked_remove(key); + let tracked resource = pool.reclaim(); + assert forall|lease_id: nat| #![auto] self.active_ids().contains(lease_id) implies { + &&& lease_id < self.next_lease() + &&& self.contains(self.active_record(lease_id).key()) + &&& self.active_record(lease_id).pool_id() == self.pool( + self.active_record(lease_id).key(), + ).id() + &&& self.active_record(lease_id).fraction() > 0real + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).active_record(lease_id).key() != key); + }; + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + assert(other != key); + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + assert(self.active_records() == old(self).active_records()); + assert(old(self).pool(other).fraction() + active_lease_fraction( + old(self).active_records(), + other, + old(self).next_lease(), + ) == 1real); + }; + resource + } +} + +/// Regression proof for the complete indexed split/return/reclaim lifecycle. +proof fn tracked_registry_reclaims_after_returns( + key: K, + tracked resource: T, + tracked first_witness: W, + tracked second_witness: W, +) -> (tracked res: T) + ensures + res == resource, +{ + let tracked mut registry = RcuTrackedReadPoolRegistry::empty(); + registry.insert(key, resource); + let tracked first = registry.split_lease(key, first_witness); + let tracked second = registry.split_lease(key, second_witness); + let tracked _first_witness = registry.return_lease(first); + let tracked _second_witness = registry.return_lease(second); + assert(!registry.has_active(key)); + assert(registry.pool(key).resource() == resource); + let tracked res = registry.reclaim(key); + assert(res == resource); + res +} + +/// Regression proof: recursively splitting leases does not require a capacity +/// assumption, and returning them restores the whole resource. +pub proof fn split_return_reclaims(tracked resource: T) -> (tracked res: T) + ensures + res == resource, +{ + let tracked mut pool = RcuReadPool::new(resource); + let tracked first = pool.split_lease(); + let tracked second = pool.split_lease(); + pool.return_lease(first); + pool.return_lease(second); + assert(pool.fraction() == 1real); + pool.reclaim() +} + +} // verus! diff --git a/verified_libs/vstd_extra/src/resource/ghost_resource/count_auth.rs b/verified_libs/vstd_extra/src/resource/ghost_resource/count_auth.rs index 9b522f088..00786d9b9 100644 --- a/verified_libs/vstd_extra/src/resource/ghost_resource/count_auth.rs +++ b/verified_libs/vstd_extra/src/resource/ghost_resource/count_auth.rs @@ -4,6 +4,7 @@ use vstd::modes::tracked_swap; use vstd::prelude::*; use vstd::resource::Loc; use vstd::resource::storage_protocol::*; +use vstd::thread_view::Objective; verus! { @@ -72,6 +73,14 @@ pub tracked struct EmptyCount { r: StorageResource<(), T, FractionalCarrierOpt>, } +unsafe impl Objective for Count { + +} + +unsafe impl Objective for EmptyCount { + +} + impl Count { #[verifier::type_invariant] spec fn inv(self) -> bool { diff --git a/verified_libs/vstd_extra/src/sum.rs b/verified_libs/vstd_extra/src/sum.rs index bb3096b8a..fcbe83e96 100644 --- a/verified_libs/vstd_extra/src/sum.rs +++ b/verified_libs/vstd_extra/src/sum.rs @@ -1,6 +1,7 @@ use crate::ownership::Inv; use vstd::modes::tracked_swap; use vstd::prelude::*; +use vstd::thread_view::Objective; verus! { @@ -10,6 +11,10 @@ pub tracked enum Sum { Right(R), } +unsafe impl Objective for Sum { + +} + impl Sum { pub open spec fn left(self) -> L { self->Left_0