From cac5ef8ea43b4e618de9d1d01a86de0a22e8cc99 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 1 Jun 2026 04:34:07 -0400 Subject: [PATCH 01/47] basic weak memory model --- ostd/specs/sync/weak_memory.rs | 414 +++++++++++++++++++++++++++++++++ 1 file changed, 414 insertions(+) create mode 100644 ostd/specs/sync/weak_memory.rs diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs new file mode 100644 index 000000000..ff65120d8 --- /dev/null +++ b/ostd/specs/sync/weak_memory.rs @@ -0,0 +1,414 @@ +//! Weak-memory atomic wrappers used by the verification layer. +//! +//! This module is a TCB boundary: executable atomic operations are connected to +//! Rust atomics with `external_body`, while proofs rely only on the ghost specs +//! below. The first concrete wrapper is `AtomicUsizeW`; pointer atomics and CAS +//! should be layered on top after the view/history model stabilizes. +use core::sync::atomic::{AtomicUsize, Ordering}; + +use vstd::prelude::*; +use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; +use vstd::resource::Loc; +use vstd::seq::Seq; + +verus! { + +pub type AtomicId = Loc; + +/// Logical timestamp into one atomic object's message history. +/// Timestamp 0 is always the initial message installed by `new`. +pub type Timestamp = nat; + +/// A thread-local weak-memory view. +/// +/// `seen[id] = ts` means this thread has advanced past all messages for `id` +/// older than `ts`; future reads from that atomic must not go backwards. +pub ghost struct WmView { + pub seen: Map, +} + +impl WmView { + pub open spec fn empty() -> Self { + WmView { seen: Map::empty() } + } + + pub open spec fn seen_at(self, id: AtomicId) -> Timestamp { + if self.seen.contains_key(id) { + self.seen[id] + } else { + // Missing entries are equivalent to only seeing the initial write. + 0nat + } + } + + /// Monotonically advance the view for one atomic object. + pub open spec fn observe(self, id: AtomicId, ts: Timestamp) -> Self { + WmView { + seen: self.seen.insert( + id, + if self.seen_at(id) <= ts { + ts + } else { + self.seen_at(id) + }, + ), + } + } + + /// Pointwise maximum of two views. + /// + /// This is the ghost effect of an acquire read: the reader imports the + /// release view carried by the message it read. + pub open spec fn join(self, other: Self) -> Self { + WmView { + seen: Map::new( + |id: AtomicId| self.seen.contains_key(id) || other.seen.contains_key(id), + |id: AtomicId| + if self.seen_at(id) <= other.seen_at(id) { + other.seen_at(id) + } else { + self.seen_at(id) + }, + ), + } + } + + pub open spec fn le(self, other: Self) -> bool { + forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) + } +} + +/// One message in an atomic object's modification history. +/// +/// `view` is the release view published with this value. Relaxed stores publish +/// only their own timestamp; release stores publish the writer's current view. +pub ghost struct Msg { + pub value: V, + pub view: WmView, +} + +pub type History = Seq>; + +/// Authoritative ghost state for one atomic object's history. +/// +/// This is intentionally a thin wrapper around vstd's map resource algebra: +/// [`GhostMapAuth`] owns the authoritative timestamp-to-message map, while `len` +/// records that the domain is the contiguous range `0..len`. +/// +/// In the next layer this token should live inside an invariant next to the +/// executable atomic. This file intentionally keeps that glue out of scope. +pub tracked struct HistAuth { + auth: GhostMapAuth>, + pub ghost len: nat, +} + +impl HistAuth { + pub closed spec fn id(self) -> AtomicId { + self.auth.id() + } + + pub closed spec fn map(self) -> Map> { + self.auth@ + } + + pub closed spec fn len(self) -> nat { + self.len + } + + pub closed spec fn history(self) -> History { + Seq::new(self.len(), |i: int| self.map()[i as nat]) + } + + pub open spec fn wf(self) -> bool { + &&& self.len() > 0 + &&& self.map().dom() =~= Set::new(|ts: Timestamp| ts < self.len()) + } + + pub open spec fn valid_ts(self, ts: Timestamp) -> bool { + ts < self.len() + } + + pub open spec fn msg_at(self, ts: Timestamp) -> Msg + recommends + self.valid_ts(ts), + { + self.map()[ts] + } + + /// RC11-style relaxed readability: a read may choose any message that is + /// not older than the thread's current view for this location. + pub open spec fn readable(self, view: WmView, ts: Timestamp) -> bool { + &&& self.valid_ts(ts) + &&& view.seen_at(self.id()) <= ts + } + + /// Append one message to the authoritative history and return a persistent + /// snapshot for the newly allocated timestamp. + pub proof fn append_msg(tracked &mut self, msg: Msg) -> (tracked snap: MsgSnap) + requires + old(self).wf(), + ensures + final(self).id() == old(self).id(), + final(self).history() == old(self).history().push(msg), + final(self).wf(), + snap.id() == final(self).id(), + snap.ts() == old(self).history().len(), + snap.msg() == msg, + snap.agrees_with(*final(self)), + { + let ghost ts = self.len(); + + let tracked pt = self.auth.insert(ts, msg); + self.len = self.len + 1; + + let tracked psnap = pt.persist(); + MsgSnap { snap: psnap } + } +} + +/// A stable proof handle for one message; a snapshot of the message. +/// +/// The underlying vstd token is persistent/duplicable, so a message snapshot +/// can be copied through proofs without granting permission to mutate history. +/// +/// Stores return snapshots so higher layers can connect a concrete write to +/// later ownership-transfer predicates without exposing the whole history. +pub tracked struct MsgSnap { + snap: GhostPersistentPointsTo>, +} + +impl MsgSnap { + pub closed spec fn id(self) -> AtomicId { + self.snap.id() + } + + /// Fetch the timestamp of this specific message. + pub closed spec fn ts(self) -> Timestamp { + self.snap.key() + } + + /// Fetch the ghost message value of this specific message. + pub closed spec fn msg(self) -> Msg { + self.snap.value() + } + + pub open spec fn agrees_with(self, auth: HistAuth) -> bool { + &&& self.id() == auth.id() + &&& auth.valid_ts(self.ts()) + &&& self.msg() == auth.msg_at(self.ts()) + } + + pub proof fn duplicate(tracked &self) -> (tracked snap: MsgSnap) + ensures + snap.id() == self.id(), + snap.ts() == self.ts(), + snap.msg() == self.msg(), + { + let tracked psnap = self.snap.duplicate(); + MsgSnap { snap: psnap } + } + + pub proof fn agree(tracked &self, tracked auth: &HistAuth) + requires + self.id() == auth.id(), + auth.wf(), + ensures + self.agrees_with(*auth), + { + self.snap.agree(&auth.auth); + assert(auth.map().contains_pair(self.ts(), self.msg())); + } +} + +/// Explicit per-thread view token. +/// +/// Passing this token through atomic operations makes the weak-memory effects +/// visible in specs instead of hiding them in global or thread-local state. +pub tracked struct ThreadView { + pub ghost view: WmView, +} + +impl View for ThreadView { + type V = WmView; + + open spec fn view(&self) -> WmView { + self.view + } +} + +impl ThreadView { + pub proof fn new() -> (tracked res: Self) + ensures + res@ == WmView::empty(), + { + ThreadView { view: WmView::empty() } + } + + pub proof fn observe(tracked &mut self, id: AtomicId, ts: Timestamp) + ensures + final(self)@ == old(self)@.observe(id, ts), + { + self.view = self.view.observe(id, ts); + } + + pub proof fn join(tracked &mut self, view: WmView) + ensures + final(self)@ == old(self)@.join(view), + { + self.view = self.view.join(view); + } +} + +#[repr(transparent)] +#[verifier::external_body] +/// TCB wrapper around Rust's `AtomicUsize`. +/// +/// The executable field is the real atomic object. The proof layer sees only +/// the specs below, plus the uninterpreted logical identity `id`. +pub struct AtomicUsizeW { + value: AtomicUsize, +} + +impl AtomicUsizeW { + /// Logical identity of this atomic object. + /// + /// `id` has no runtime representation; it indexes ghost histories and + /// thread views. The `new` spec ties the fresh history to this identity. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: usize) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = AtomicUsizeW { value: AtomicUsize::new(init) }; + (atomic, Tracked::assume_new()) + } + + /// Relaxed load: choose a readable message and advance only this location's + /// timestamp in the caller's thread view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (usize, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + /// Acquire load: same readable message choice as relaxed, plus import the + /// release view carried by the selected message. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (usize, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + /// Relaxed store: append a new message whose published view contains only + /// this store's own timestamp. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: usize, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + /// Release store: append a new message carrying the writer's current view, + /// then advance the writer's view for this location. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: usize, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } +} + +} // verus! From 553eba183c067f19bd01954d9d6e382aa8d0bf9e Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 1 Jun 2026 05:04:30 -0400 Subject: [PATCH 02/47] test snapshots and history for usize --- ostd/specs/sync/weak_memory.rs | 218 ++++++++++++++++++++++++++++++++- 1 file changed, 216 insertions(+), 2 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index ff65120d8..7b6b3f3fd 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -6,6 +6,7 @@ //! should be layered on top after the view/history model stabilizes. use core::sync::atomic::{AtomicUsize, Ordering}; +use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::prelude::*; use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; use vstd::resource::Loc; @@ -89,14 +90,22 @@ pub ghost struct Msg { pub type History = Seq>; +/// User-supplied invariant predicate for a weak-memory atomic. +/// +/// This mirrors `vstd::atomic_ghost::AtomicInvariantPredicate`, except the +/// predicate is over the whole message history rather than one current value. +pub trait WeakAtomicInvariantPredicate { + spec fn atomic_inv(k: K, history: History, g: G) -> bool; +} + /// Authoritative ghost state for one atomic object's history. /// /// This is intentionally a thin wrapper around vstd's map resource algebra: /// [`GhostMapAuth`] owns the authoritative timestamp-to-message map, while `len` /// records that the domain is the contiguous range `0..len`. /// -/// In the next layer this token should live inside an invariant next to the -/// executable atomic. This file intentionally keeps that glue out of scope. +/// The proof-facing atomic wrapper below stores this token inside an +/// `AtomicInvariant` next to the executable atomic. pub tracked struct HistAuth { auth: GhostMapAuth>, pub ghost len: nat, @@ -220,6 +229,185 @@ impl MsgSnap { } } +/// Predicate adapter stored inside `AtomicInvariant`. +/// +/// The invariant contains the authoritative history and user ghost state. The +/// constant pairs the user key `K` with the logical atomic id. +pub struct WeakAtomicPredUsize { + p: Pred, +} + +impl InvariantPredicate<(K, AtomicId), (HistAuth, G)> for WeakAtomicPredUsize< + Pred, +> where Pred: WeakAtomicInvariantPredicate { + open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth, G)) -> bool { + let (k, id) = k_id; + let (hist, g) = hist_g; + &&& hist.id() == id + &&& hist.wf() + &&& Pred::atomic_inv(k, hist.history(), g) + } +} + +/// A weak-memory atomic with an `atomic_ghost`-style invariant. +/// +/// `AtomicUsizeW` remains the TCB executable wrapper. This type is the proof +/// facing wrapper: it stores the authoritative history in an `AtomicInvariant` +/// and exposes `well_formed`/`type_inv` predicates tying that history to the +/// executable atomic id. As in `vstd::atomic_ghost`, outer data structures put +/// this predicate in their own `#[verifier::type_invariant]`. +pub struct WeakAtomicUsize { + #[doc(hidden)] + pub atomic: AtomicUsizeW, + #[doc(hidden)] + pub atomic_inv: Tracked< + AtomicInvariant<(K, AtomicId), (HistAuth, G), WeakAtomicPredUsize>, + >, +} + +impl WeakAtomicUsize { + pub closed spec fn constant(&self) -> K { + self.atomic_inv@.constant().0 + } + + pub closed spec fn well_formed(&self) -> bool { + self.atomic_inv@.constant().1 == self.atomic.id() + } + + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } +} + +impl WeakAtomicUsize where Pred: WeakAtomicInvariantPredicate { + #[inline(always)] + pub const fn new(Ghost(k): Ghost, init: usize, Tracked(g): Tracked) -> (res: Self) + requires + Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), + ensures + res.well_formed(), + res.constant() == k, + { + let (atomic, Tracked(hist)) = AtomicUsizeW::new(init); + let tracked pair = (hist, g); + assert(WeakAtomicPredUsize::::inv((k, atomic.id()), pair)); + let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); + WeakAtomicUsize { atomic, atomic_inv: Tracked(atomic_inv) } + } + + #[inline(always)] + pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + usize, + Ghost, + )) + requires + self.well_formed(), + { + let result; + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } + + #[inline(always)] + pub fn load_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + usize, + Ghost, + )) + requires + self.well_formed(), + { + let result; + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } +} + +pub struct TrueWeakAtomicInv; + +impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { + open spec fn atomic_inv(k: K, history: History, g: G) -> bool { + true + } +} + +/// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, +/// but for weak-memory atomics with per-thread view tokens and message histories. +/// +/// The macro opens the atomic invariant, performs the specified operation, and +/// provides the previous history, new history, and operation snapshot to the user- +/// provided proof block. The user can then write proofs about the effects of the +/// operation on the history and thread view, using the snapshot to connect to the +/// authoritative history. +#[macro_export] +macro_rules! weak_atomic_with_ghost { + ( + $atomic:expr => store_release($value:expr, $tv:expr); + update $prev:ident -> $next:ident; + snapshot $snap:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let atomic = &($atomic); + let value = $value; + ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { + #[allow(unused_mut)] + let tracked (mut hist, mut $g) = pair; + let ghost $prev = hist.history(); + let snap_tracked = atomic.atomic.store_release(Tracked(&mut hist), $tv, value); + let ghost $next = hist.history(); + + proof { + let tracked $snap = snap_tracked.get(); + $b + } + + proof { + pair = (hist, $g); + } + }); + }} + }; + ( + $atomic:expr => store_relaxed($value:expr, $tv:expr); + update $prev:ident -> $next:ident; + snapshot $snap:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let atomic = &($atomic); + let value = $value; + ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { + #[allow(unused_mut)] + let tracked (mut hist, mut $g) = pair; + let ghost $prev = hist.history(); + let snap_tracked = atomic.atomic.store_relaxed(Tracked(&mut hist), $tv, value); + let ghost $next = hist.history(); + + proof { + let tracked $snap = snap_tracked.get(); + $b + } + + proof { + pair = (hist, $g); + } + }); + }} + }; +} + /// Explicit per-thread view token. /// /// Passing this token through atomic operations makes the weak-memory effects @@ -411,4 +599,30 @@ impl AtomicUsizeW { } } +#[cfg(verus_keep_ghost)] +fn smoke_test_weak_atomic_with_ghost() { + let atomic = WeakAtomicUsize::<(), (), TrueWeakAtomicInv>::new(Ghost(()), 0, Tracked(())); + let tracked mut tv = ThreadView::new(); + weak_atomic_with_ghost! { + atomic => store_release(1, Tracked(&mut tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(snap.msg().value == 1); + } + } + weak_atomic_with_ghost! { + atomic => store_relaxed(2, Tracked(&mut tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(snap.msg().value == 2); + } + } +} + } // verus! From 2399723f884e03eeb4b892580139786f500c9596 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 1 Jun 2026 05:46:59 -0400 Subject: [PATCH 03/47] load semantics --- ostd/specs/sync/weak_memory.rs | 95 +++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 7b6b3f3fd..52a5d75e4 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -4,6 +4,12 @@ //! Rust atomics with `external_body`, while proofs rely only on the ghost specs //! below. The first concrete wrapper is `AtomicUsizeW`; pointer atomics and CAS //! should be layered on top after the view/history model stabilizes. +//! +//! We focus on the repaired C11/RC11-style memory model, where relaxed behavior +//! is modeled as reading from previously written messages in a location’s modi- +//! fication history, subject to coherence. In particular, relaxed reads may ob- +//! serve stale writes, but a thread’s view prevents it from going backwards, +//! and reads do not observe future writes that have not been added to the history. use core::sync::atomic::{AtomicUsize, Ordering}; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; @@ -124,7 +130,10 @@ impl HistAuth { self.len } - pub closed spec fn history(self) -> History { + pub closed spec fn history(self) -> History + recommends + self.wf(), + { Seq::new(self.len(), |i: int| self.map()[i as nat]) } @@ -344,7 +353,7 @@ impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { /// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, /// but for weak-memory atomics with per-thread view tokens and message histories. -/// +/// /// The macro opens the atomic invariant, performs the specified operation, and /// provides the previous history, new history, and operation snapshot to the user- /// provided proof block. The user can then write proofs about the effects of the @@ -352,6 +361,64 @@ impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { /// authoritative history. #[macro_export] macro_rules! weak_atomic_with_ghost { + ( + $atomic:expr => load_acquire($tv:expr); + returning $ret:ident; + timestamp $ts:ident; + message $msg:ident; + history $history:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let result; + let atomic = &($atomic); + ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { + #[allow(unused_mut)] + let tracked (hist, mut $g) = pair; + let ghost $history = hist.history(); + result = atomic.atomic.load_acquire(Tracked(&hist), $tv); + let ghost $ret = result.0; + let ghost $ts = result.1@; + let ghost $msg = hist.msg_at($ts); + + proof { $b } + + proof { + pair = (hist, $g); + } + }); + result + }} + }; + ( + $atomic:expr => load_relaxed($tv:expr); + returning $ret:ident; + timestamp $ts:ident; + message $msg:ident; + history $history:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let result; + let atomic = &($atomic); + ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { + #[allow(unused_mut)] + let tracked (hist, mut $g) = pair; + let ghost $history = hist.history(); + result = atomic.atomic.load_relaxed(Tracked(&hist), $tv); + let ghost $ret = result.0; + let ghost $ts = result.1@; + let ghost $msg = hist.msg_at($ts); + + proof { $b } + + proof { + pair = (hist, $g); + } + }); + result + }} + }; ( $atomic:expr => store_release($value:expr, $tv:expr); update $prev:ident -> $next:ident; @@ -623,6 +690,30 @@ fn smoke_test_weak_atomic_with_ghost() { assert(snap.msg().value == 2); } } + let _ = + weak_atomic_with_ghost! { + atomic => load_acquire(Tracked(&mut tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(ret == msg.value); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + atomic => load_relaxed(Tracked(&mut tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(ret == msg.value); + assert(ts < history.len()); + } + }; } } // verus! From 1e3a29c8581ce5d85ce4d05449ce7992cdb3b9df Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 2 Jun 2026 05:15:56 -0400 Subject: [PATCH 04/47] more atomics for the weak memory model --- dv | 2 +- ostd/specs/sync/mod.rs | 1 + ostd/specs/sync/weak_memory.rs | 1345 +++++++++++++++++++++++++++++++- 3 files changed, 1320 insertions(+), 28 deletions(-) diff --git a/dv b/dv index bdf3997cc..561c8a2af 160000 --- a/dv +++ b/dv @@ -1 +1 @@ -Subproject commit bdf3997cc866fd9a2eac0f73af230e14b84f2add +Subproject commit 561c8a2afe325a9ee02229e65c5479bea1bb5bbe diff --git a/ostd/specs/sync/mod.rs b/ostd/specs/sync/mod.rs index e5784b635..d1ffbdd20 100644 --- a/ostd/specs/sync/mod.rs +++ b/ostd/specs/sync/mod.rs @@ -2,3 +2,4 @@ pub mod abstract_lock; pub mod mutex; //pub mod mutex_verussync; +pub mod weak_memory; diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 52a5d75e4..ecb1dd871 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -2,15 +2,21 @@ //! //! This module is a TCB boundary: executable atomic operations are connected to //! Rust atomics with `external_body`, while proofs rely only on the ghost specs -//! below. The first concrete wrapper is `AtomicUsizeW`; pointer atomics and CAS -//! should be layered on top after the view/history model stabilizes. +//! below. Concrete wrappers currently cover Rust integer atomics, `AtomicBoolW`, +//! and `AtomicPtrW`, all using the same view/history model. //! //! We focus on the repaired C11/RC11-style memory model, where relaxed behavior //! is modeled as reading from previously written messages in a location’s modi- //! fication history, subject to coherence. In particular, relaxed reads may ob- //! serve stale writes, but a thread’s view prevents it from going backwards, //! and reads do not observe future writes that have not been added to the history. -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::{ + AtomicBool, AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, + AtomicU8, AtomicUsize, Ordering, +}; + +#[cfg(target_has_atomic = "64")] +use core::sync::atomic::{AtomicI64, AtomicU64}; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::prelude::*; @@ -238,18 +244,164 @@ impl MsgSnap { } } -/// Predicate adapter stored inside `AtomicInvariant`. +} // verus! +/// Generate the proof-facing wrapper for one concrete weak-memory atomic type. /// -/// The invariant contains the authoritative history and user ghost state. The -/// constant pairs the user key `K` with the logical atomic id. -pub struct WeakAtomicPredUsize { +/// The generated type keeps the executable TCB wrapper separate from the +/// invariant protocol. Adding `AtomicU32W` or `AtomicBoolW` later should require +/// a new executable wrapper plus one macro invocation, not another copy of the +/// invariant glue. +macro_rules! declare_weak_atomic_type { + ($weak_atomic:ident, $pred_adapter:ident, $raw_atomic:ident, $value_ty:ty) => { + verus! { + /// Predicate adapter stored inside `AtomicInvariant`. + /// + /// The invariant contains the authoritative history and user ghost + /// state. The constant pairs the user key `K` with the logical atomic id. + pub struct $pred_adapter { + p: Pred, + } + + impl InvariantPredicate<(K, AtomicId), (HistAuth<$value_ty>, G)> for $pred_adapter< + Pred, + > where Pred: WeakAtomicInvariantPredicate { + open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<$value_ty>, G)) -> bool { + let (k, id) = k_id; + let (hist, g) = hist_g; + &&& hist.id() == id + &&& hist.wf() + &&& Pred::atomic_inv(k, hist.history(), g) + } + } + + /// A weak-memory atomic with an `atomic_ghost`-style invariant. + /// + /// The executable atomic remains the TCB wrapper. This proof-facing + /// wrapper stores the authoritative history in an `AtomicInvariant` and + /// exposes `well_formed`/`type_inv` predicates tying that history to the + /// executable atomic id. As in `vstd::atomic_ghost`, outer data + /// structures put this predicate in their own + /// `#[verifier::type_invariant]`. + pub struct $weak_atomic { + #[doc(hidden)] + pub atomic: $raw_atomic, + #[doc(hidden)] + pub atomic_inv: Tracked< + AtomicInvariant<(K, AtomicId), (HistAuth<$value_ty>, G), $pred_adapter>, + >, + } + + impl $weak_atomic { + pub closed spec fn constant(&self) -> K { + self.atomic_inv@.constant().0 + } + + pub closed spec fn well_formed(&self) -> bool { + self.atomic_inv@.constant().1 == self.atomic.id() + } + + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } + } + + impl $weak_atomic where + Pred: WeakAtomicInvariantPredicate, + { + #[inline(always)] + pub const fn new( + Ghost(k): Ghost, + init: $value_ty, + Tracked(g): Tracked, + ) -> (res: Self) + requires + Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), + ensures + res.well_formed(), + res.constant() == k, + { + let (atomic, Tracked(hist)) = $raw_atomic::new(init); + let tracked pair = (hist, g); + assert($pred_adapter::::inv((k, atomic.id()), pair)); + let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); + $weak_atomic { atomic, atomic_inv: Tracked(atomic_inv) } + } + + #[inline(always)] + pub fn load_relaxed( + &self, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) + requires + self.well_formed(), + { + let result; + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } + + #[inline(always)] + pub fn load_acquire( + &self, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) + requires + self.well_formed(), + { + let result; + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } + } + } + }; +} + +declare_weak_atomic_type!(WeakAtomicU8, WeakAtomicPredU8, AtomicU8W, u8); +declare_weak_atomic_type!(WeakAtomicU16, WeakAtomicPredU16, AtomicU16W, u16); +declare_weak_atomic_type!(WeakAtomicU32, WeakAtomicPredU32, AtomicU32W, u32); +declare_weak_atomic_type!(WeakAtomicUsize, WeakAtomicPredUsize, AtomicUsizeW, usize); +declare_weak_atomic_type!(WeakAtomicBool, WeakAtomicPredBool, AtomicBoolW, bool); + +#[cfg(target_has_atomic = "64")] +declare_weak_atomic_type!(WeakAtomicU64, WeakAtomicPredU64, AtomicU64W, u64); + +declare_weak_atomic_type!(WeakAtomicI8, WeakAtomicPredI8, AtomicI8W, i8); +declare_weak_atomic_type!(WeakAtomicI16, WeakAtomicPredI16, AtomicI16W, i16); +declare_weak_atomic_type!(WeakAtomicI32, WeakAtomicPredI32, AtomicI32W, i32); +declare_weak_atomic_type!(WeakAtomicIsize, WeakAtomicPredIsize, AtomicIsizeW, isize); + +#[cfg(target_has_atomic = "64")] +declare_weak_atomic_type!(WeakAtomicI64, WeakAtomicPredI64, AtomicI64W, i64); + +verus! { + +/// Predicate adapter for weak-memory pointer atomics. +/// +/// The history stores raw pointer values. This tracks the atomic pointer value +/// itself; ownership of the pointee must be modeled by the user ghost state `G`. +pub struct WeakAtomicPredPtr { + t: T, p: Pred, } -impl InvariantPredicate<(K, AtomicId), (HistAuth, G)> for WeakAtomicPredUsize< +impl InvariantPredicate<(K, AtomicId), (HistAuth<*mut T>, G)> for WeakAtomicPredPtr< + T, Pred, -> where Pred: WeakAtomicInvariantPredicate { - open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth, G)) -> bool { +> where Pred: WeakAtomicInvariantPredicate { + open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<*mut T>, G)) -> bool { let (k, id) = k_id; let (hist, g) = hist_g; &&& hist.id() == id @@ -258,23 +410,23 @@ impl InvariantPredicate<(K, AtomicId), (HistAuth, G)> for Wea } } -/// A weak-memory atomic with an `atomic_ghost`-style invariant. +/// Weak-memory atomic pointer with an `atomic_ghost`-style invariant. /// -/// `AtomicUsizeW` remains the TCB executable wrapper. This type is the proof -/// facing wrapper: it stores the authoritative history in an `AtomicInvariant` -/// and exposes `well_formed`/`type_inv` predicates tying that history to the -/// executable atomic id. As in `vstd::atomic_ghost`, outer data structures put -/// this predicate in their own `#[verifier::type_invariant]`. -pub struct WeakAtomicUsize { +/// This is the pointer analogue of [`WeakAtomicUsize`]. It deliberately models +/// only the atomic pointer value and its release/acquire synchronization history; +/// any ownership or validity claim about the pointed-to allocation belongs in +/// the user-supplied ghost state `G` and invariant predicate. +#[verifier::accept_recursive_types(T)] +pub struct WeakAtomicPtr { #[doc(hidden)] - pub atomic: AtomicUsizeW, + pub atomic: AtomicPtrW, #[doc(hidden)] pub atomic_inv: Tracked< - AtomicInvariant<(K, AtomicId), (HistAuth, G), WeakAtomicPredUsize>, + AtomicInvariant<(K, AtomicId), (HistAuth<*mut T>, G), WeakAtomicPredPtr>, >, } -impl WeakAtomicUsize { +impl WeakAtomicPtr { pub closed spec fn constant(&self) -> K { self.atomic_inv@.constant().0 } @@ -288,25 +440,27 @@ impl WeakAtomicUsize { } } -impl WeakAtomicUsize where Pred: WeakAtomicInvariantPredicate { +impl WeakAtomicPtr where + Pred: WeakAtomicInvariantPredicate, + { #[inline(always)] - pub const fn new(Ghost(k): Ghost, init: usize, Tracked(g): Tracked) -> (res: Self) + pub const fn new(Ghost(k): Ghost, init: *mut T, Tracked(g): Tracked) -> (res: Self) requires Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), ensures res.well_formed(), res.constant() == k, { - let (atomic, Tracked(hist)) = AtomicUsizeW::new(init); + let (atomic, Tracked(hist)) = AtomicPtrW::::new(init); let tracked pair = (hist, g); - assert(WeakAtomicPredUsize::::inv((k, atomic.id()), pair)); + assert(WeakAtomicPredPtr::::inv((k, atomic.id()), pair)); let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); - WeakAtomicUsize { atomic, atomic_inv: Tracked(atomic_inv) } + WeakAtomicPtr { atomic, atomic_inv: Tracked(atomic_inv) } } #[inline(always)] pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( - usize, + *mut T, Ghost, )) requires @@ -325,7 +479,7 @@ impl WeakAtomicUsize where Pred: WeakAtomicInvariantPred #[inline(always)] pub fn load_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( - usize, + *mut T, Ghost, )) requires @@ -361,6 +515,48 @@ impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { /// authoritative history. #[macro_export] macro_rules! weak_atomic_with_ghost { + ( + $atomic:expr => compare_exchange_acqrel_acquire($current:expr, $new:expr, $tv:expr); + update $prev:ident -> $next:ident; + returning $ret:ident; + timestamp $ts:ident; + message $msg:ident; + snapshot $snap:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let result; + let atomic = &($atomic); + let current = $current; + let new = $new; + ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { + #[allow(unused_mut)] + let tracked (mut hist, mut $g) = pair; + let ghost $prev = hist.history(); + let cas_result = atomic.atomic.compare_exchange_acqrel_acquire( + Tracked(&mut hist), + $tv, + current, + new, + ); + result = (cas_result.0, cas_result.1); + let ghost $next = hist.history(); + let ghost $ret = cas_result.0; + let ghost $ts = cas_result.1@; + let ghost $msg = $prev[$ts as int]; + + proof { + let tracked $snap = cas_result.2.get(); + $b + } + + proof { + pair = (hist, $g); + } + }); + result + }} + }; ( $atomic:expr => load_acquire($tv:expr); returning $ret:ident; @@ -597,6 +793,69 @@ impl AtomicUsizeW { (value, Ghost::assume_new()) } + /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` + /// failure ordering. + /// + /// This first CAS model is intentionally conservative: the operation reads + /// the latest message in the location's modification history. On success it + /// appends a new release message; on failure it behaves like an acquire load. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + current: usize, + new: usize, + ) -> (res: (Result, Ghost, Tracked>>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& read_ts + 1 == old(auth).history().len() + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& v == current + &&& read_msg.value == current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& v == read_msg.value + &&& read_msg.value != current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); + (result, Ghost::assume_new(), Tracked::assume_new()) + } + /// Relaxed store: append a new message whose published view contains only /// this store's own timestamp. #[inline(always)] @@ -666,6 +925,674 @@ impl AtomicUsizeW { } } +} // verus! +/// Generate a TCB executable wrapper around one Rust integer atomic type. +/// +/// All integer atomics share the same weak-memory history shape: load chooses a +/// readable message, stores append a message, and CAS reads the latest message +/// before either appending a new one or failing as an acquire read. +macro_rules! declare_integer_atomic_wrapper { + ($wrapper:ident, $rust_atomic:ident, $value_ty:ty) => { + verus! { + #[repr(transparent)] + #[verifier::external_body] + /// TCB wrapper around a Rust integer atomic. + pub struct $wrapper { + value: $rust_atomic, + } + + impl $wrapper { + /// Logical identity of this atomic object. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: $value_ty) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = $wrapper { value: $rust_atomic::new(init) }; + (atomic, Tracked::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + current: $value_ty, + new: $value_ty, + ) -> (res: ( + Result<$value_ty, $value_ty>, + Ghost, + Tracked>>, + )) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& read_ts + 1 == old(auth).history().len() + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& v == current + &&& read_msg.value == current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& v == read_msg.value + &&& read_msg.value != current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange( + current, + new, + Ordering::AcqRel, + Ordering::Acquire, + ); + (result, Ghost::assume_new(), Tracked::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: $value_ty, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: $value_ty, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } + } + } + }; +} + +declare_integer_atomic_wrapper!(AtomicU8W, AtomicU8, u8); + +declare_integer_atomic_wrapper!(AtomicU16W, AtomicU16, u16); + +declare_integer_atomic_wrapper!(AtomicU32W, AtomicU32, u32); + +declare_integer_atomic_wrapper!(AtomicIsizeW, AtomicIsize, isize); + +declare_integer_atomic_wrapper!(AtomicI8W, AtomicI8, i8); + +declare_integer_atomic_wrapper!(AtomicI16W, AtomicI16, i16); + +declare_integer_atomic_wrapper!(AtomicI32W, AtomicI32, i32); + +#[cfg(target_has_atomic = "64")] +declare_integer_atomic_wrapper!(AtomicU64W, AtomicU64, u64); + +#[cfg(target_has_atomic = "64")] +declare_integer_atomic_wrapper!(AtomicI64W, AtomicI64, i64); + +verus! { + +#[repr(transparent)] +#[verifier::external_body] +/// TCB wrapper around Rust's `AtomicBool`. +/// +/// Bool atomics share the load/store/CAS weak-memory protocol with integer +/// atomics, but they are not numeric atomics: this wrapper intentionally exposes +/// no arithmetic or bitwise fetch operations. +pub struct AtomicBoolW { + value: AtomicBool, +} + +impl AtomicBoolW { + /// Logical identity of this atomic object. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: bool) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = AtomicBoolW { value: AtomicBool::new(init) }; + (atomic, Tracked::assume_new()) + } + + /// Relaxed load: choose a readable bool message and advance only this + /// location's timestamp in the caller's thread view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (bool, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + /// Acquire load: same bool choice as relaxed, plus import the release view + /// carried by the selected message. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (bool, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` + /// failure ordering. + /// + /// On success it appends `new`; on failure it only imports the acquired view + /// from the message it read. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + current: bool, + new: bool, + ) -> (res: (Result, Ghost, Tracked>>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& read_ts + 1 == old(auth).history().len() + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& v == current + &&& read_msg.value == current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& v == read_msg.value + &&& read_msg.value != current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); + (result, Ghost::assume_new(), Tracked::assume_new()) + } + + /// Relaxed store: append a bool-valued message whose published view contains + /// only this store's own timestamp. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: bool, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + /// Release store: append a bool-valued message carrying the writer's current + /// view, then advance the writer's view for this location. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: bool, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } +} + +#[repr(transparent)] +#[verifier::accept_recursive_types(T)] +#[verifier::external_body] +/// TCB wrapper around Rust's `AtomicPtr`. +/// +/// This wrapper tracks the pointer value in the weak-memory history, but it +/// does not claim ownership of, or permission to dereference, the pointee. A +/// higher-level invariant must connect pointer values to `PointsTo`, refcount, +/// hazard-pointer, RCU, or other ownership ghost state when dereference safety +/// matters. +pub struct AtomicPtrW { + value: AtomicPtr, +} + +impl AtomicPtrW { + /// Logical identity of this atomic pointer object. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: *mut T) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = AtomicPtrW { value: AtomicPtr::new(init) }; + (atomic, Tracked::assume_new()) + } + + /// Relaxed load: choose a readable pointer message and advance only this + /// location's timestamp in the caller's thread view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (*mut T, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& equal(res.0, auth.msg_at(ts).value) + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + /// Acquire load: same pointer choice as relaxed, plus import the release + /// view carried by the selected message. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (*mut T, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& equal(res.0, auth.msg_at(ts).value) + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + /// Relaxed store: append a pointer-valued message whose published view + /// contains only this store's own timestamp. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: *mut T, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + /// Release store: append a pointer-valued message carrying the writer's + /// current view, then advance the writer's view for this location. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: *mut T, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } +} + +impl AtomicPtrW { + /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` + /// failure ordering. + /// + /// Pointer CAS compares runtime pointer identity, which Verus models as + /// address equality for sized pointers. The returned pointer and written + /// message still carry the full pointer value, including provenance. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + current: *mut T, + new: *mut T, + ) -> (res: (Result<*mut T, *mut T>, Ghost, Tracked>>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& read_ts + 1 == old(auth).history().len() + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& current.addr() == read_msg.value.addr() + &&& equal(v, read_msg.value) + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& current.addr() != read_msg.value.addr() + &&& equal(v, read_msg.value) + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); + (result, Ghost::assume_new(), Tracked::assume_new()) + } +} + #[cfg(verus_keep_ghost)] fn smoke_test_weak_atomic_with_ghost() { let atomic = WeakAtomicUsize::<(), (), TrueWeakAtomicInv>::new(Ghost(()), 0, Tracked(())); @@ -714,6 +1641,370 @@ fn smoke_test_weak_atomic_with_ghost() { assert(ts < history.len()); } }; + let _ = + weak_atomic_with_ghost! { + atomic => compare_exchange_acqrel_acquire(2, 3, Tracked(&mut tv)); + update prev -> next; + returning ret; + timestamp ts; + message msg; + snapshot snap; + ghost g => { + assert(ts + 1 == prev.len()); + match ret { + Result::Ok(v) => { + assert(v == 2); + assert(msg.value == 2); + match snap { + Option::Some(s) => { + assert(next == prev.push(s.msg())); + assert(s.msg().value == 3); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(v) => { + assert(v == msg.value); + assert(msg.value != 2); + match snap { + Option::Some(_) => { + assert(false); + }, + Option::None => {}, + } + assert(next == prev); + }, + } + } + }; + + let bool_atomic = WeakAtomicBool::<(), (), TrueWeakAtomicInv>::new( + Ghost(()), + false, + Tracked(()), + ); + let tracked mut bool_tv = ThreadView::new(); + weak_atomic_with_ghost! { + bool_atomic => store_release(true, Tracked(&mut bool_tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(snap.msg().value == true); + } + } + let _ = + weak_atomic_with_ghost! { + bool_atomic => load_acquire(Tracked(&mut bool_tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(ret == msg.value); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + bool_atomic => compare_exchange_acqrel_acquire(true, false, Tracked(&mut bool_tv)); + update prev -> next; + returning ret; + timestamp ts; + message msg; + snapshot snap; + ghost g => { + assert(ts + 1 == prev.len()); + match ret { + Result::Ok(v) => { + assert(v == true); + assert(msg.value == true); + match snap { + Option::Some(s) => { + assert(next == prev.push(s.msg())); + assert(s.msg().value == false); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(v) => { + assert(v == msg.value); + assert(msg.value != true); + match snap { + Option::Some(_) => { + assert(false); + }, + Option::None => {}, + } + assert(next == prev); + }, + } + } + }; + + let null = core::ptr::null_mut::(); + let ptr_atomic = WeakAtomicPtr::::new( + Ghost(()), + null, + Tracked(()), + ); + let tracked mut ptr_tv = ThreadView::new(); + weak_atomic_with_ghost! { + ptr_atomic => store_release(null, Tracked(&mut ptr_tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(equal(snap.msg().value, null)); + } + } + let _ = + weak_atomic_with_ghost! { + ptr_atomic => load_acquire(Tracked(&mut ptr_tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(equal(ret, msg.value)); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + ptr_atomic => compare_exchange_acqrel_acquire(null, null, Tracked(&mut ptr_tv)); + update prev -> next; + returning ret; + timestamp ts; + message msg; + snapshot snap; + ghost g => { + assert(ts + 1 == prev.len()); + match ret { + Result::Ok(v) => { + assert(equal(v, msg.value)); + assert(msg.value.addr() == null.addr()); + match snap { + Option::Some(s) => { + assert(next == prev.push(s.msg())); + assert(equal(s.msg().value, null)); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(v) => { + assert(equal(v, msg.value)); + assert(msg.value.addr() != null.addr()); + match snap { + Option::Some(_) => { + assert(false); + }, + Option::None => {}, + } + assert(next == prev); + }, + } + } + }; +} + +#[cfg(verus_keep_ghost)] +pub struct MessagePassingDataInv; + +#[cfg(verus_keep_ghost)] +impl WeakAtomicInvariantPredicate<(), usize, ()> for MessagePassingDataInv { + open spec fn atomic_inv(k: (), history: History, g: ()) -> bool { + &&& history.len() >= 1 + &&& history[0].value == 0 + &&& forall|i: int| 1 <= i < history.len() ==> #[trigger] history[i].value == 1 + } +} + +#[cfg(verus_keep_ghost)] +pub struct MessagePassingFlagInv; + +#[cfg(verus_keep_ghost)] +impl WeakAtomicInvariantPredicate for MessagePassingFlagInv { + open spec fn atomic_inv(data_id: AtomicId, history: History, g: ()) -> bool { + &&& history.len() >= 1 + &&& history[0].value == 0 + &&& forall|i: int| + 1 <= i < history.len() ==> { + &&& #[trigger] history[i].value == 1 + &&& history[i].view.seen_at(data_id) >= 1 + } + } +} + +#[cfg(verus_keep_ghost)] +proof fn preserve_message_passing_data_inv_on_push( + prev: History, + next: History, + msg: Msg, +) + requires + MessagePassingDataInv::atomic_inv((), prev, ()), + next == prev.push(msg), + msg.value == 1, + ensures + MessagePassingDataInv::atomic_inv((), next, ()), +{ + assert(next.len() >= 1); + assert(next[0].value == 0); + assert forall|i: int| 1 <= i < next.len() implies #[trigger] next[i].value == 1 by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + } + }; +} + +#[cfg(verus_keep_ghost)] +proof fn preserve_message_passing_flag_inv_on_push( + data_id: AtomicId, + prev: History, + next: History, + msg: Msg, +) + requires + MessagePassingFlagInv::atomic_inv(data_id, prev, ()), + next == prev.push(msg), + msg.value == 1, + msg.view.seen_at(data_id) >= 1, + ensures + MessagePassingFlagInv::atomic_inv(data_id, next, ()), +{ + assert(next.len() >= 1); + assert(next[0].value == 0); + assert forall|i: int| 1 <= i < next.len() implies { + &&& #[trigger] next[i].value == 1 + &&& next[i].view.seen_at(data_id) >= 1 + } by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + } + }; +} + +#[cfg(verus_keep_ghost)] +proof fn prove_message_passing_data_read( + data_id: AtomicId, + history: History, + ret: usize, + ts: Timestamp, + msg: Msg, + tv: WmView, +) + requires + MessagePassingDataInv::atomic_inv((), history, ()), + ts < history.len(), + ret == msg.value, + msg == history[ts as int], + tv.seen_at(data_id) >= 1, + tv.seen_at(data_id) <= ts, + ensures + ret == 1, +{ + assert(ts >= 1); + assert(history[ts as int].value == 1); } +// #[cfg(verus_keep_ghost)] +// fn message_passing_release_acquire_threads_can_prove() { +// let data = std::sync::Arc::new( +// WeakAtomicUsize::<(), (), MessagePassingDataInv>::new(Ghost(()), 0, Tracked(())), +// ); +// let ghost data_id = data.atomic.id(); +// let flag = std::sync::Arc::new( +// WeakAtomicUsize::::new(Ghost(data_id), 0, Tracked(())), +// ); +// let data_writer = data.clone(); +// let flag_writer = flag.clone(); +// let data_reader = data.clone(); +// let flag_reader = flag.clone(); +// let writer = vstd::thread::spawn( +// move || +// { +// let tracked mut writer_tv = ThreadView::new(); +// weak_atomic_with_ghost! { +// *data_writer => store_release(1, Tracked(&mut writer_tv)); +// update prev -> next; +// snapshot snap; +// ghost g => { +// preserve_message_passing_data_inv_on_push(prev, next, snap.msg()); +// assert(writer_tv.view.seen_at(data_id) >= 1); +// } +// } +// let ghost before_flag = writer_tv.view; +// assert(before_flag.seen_at(data_id) >= 1); +// weak_atomic_with_ghost! { +// *flag_writer => store_release(1, Tracked(&mut writer_tv)); +// update prev -> next; +// snapshot snap; +// ghost g => { +// assert(snap.msg().view == before_flag.observe(flag_writer.atomic.id(), snap.ts())); +// assert(snap.msg().view.seen_at(data_id) >= 1); +// preserve_message_passing_flag_inv_on_push(data_id, prev, next, snap.msg()); +// } +// } +// }, +// ); +// let reader = vstd::thread::spawn( +// move || +// { +// let tracked mut reader_tv = ThreadView::new(); +// let flag_result = +// weak_atomic_with_ghost! { +// *flag_reader => load_acquire(Tracked(&mut reader_tv)); +// returning ret; +// timestamp ts; +// message msg; +// history history; +// ghost g => { +// if ret == 1 { +// assert(msg.value == 1); +// if ts == 0 { +// assert(history[0].value == 0); +// assert(false); +// } +// assert(ts >= 1); +// assert(history[ts as int].value == 1); +// assert(history[ts as int].view.seen_at(data_id) >= 1); +// assert(msg == history[ts as int]); +// assert(msg.view.seen_at(data_id) >= 1); +// assert(reader_tv.view.seen_at(data_id) >= 1); +// } +// } +// }; +// if flag_result.0 == 1 { +// assert(reader_tv.view.seen_at(data_id) >= 1); +// let data_result = +// weak_atomic_with_ghost! { +// *data_reader => load_relaxed(Tracked(&mut reader_tv)); +// returning ret; +// timestamp ts; +// message msg; +// history history; +// ghost g => { +// prove_message_passing_data_read(data_id, history, ret, ts, msg, reader_tv.view); +// } +// }; +// assert(data_result.0 == 1); +// } +// }, +// ); +// let _ = writer.join(); +// let _ = reader.join(); +// } } // verus! From 9d3c0382fc1a7b975c60c718c563e6f51a6d8bf8 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 3 Jun 2026 04:20:29 -0400 Subject: [PATCH 05/47] more rcu models --- ostd/specs/sync/mod.rs | 2 + ostd/specs/sync/rcu.rs | 439 +++++++++++++ ostd/specs/sync/sc_model.rs | 11 + ostd/specs/sync/weak_memory.rs | 28 + ostd/src/sync/mod.rs | 1 + ostd/src/sync/rcu/__mod.rs | 1128 ++++++++++++++++++++++++++++++++ 6 files changed, 1609 insertions(+) create mode 100644 ostd/specs/sync/rcu.rs create mode 100644 ostd/specs/sync/sc_model.rs create mode 100644 ostd/src/sync/rcu/__mod.rs diff --git a/ostd/specs/sync/mod.rs b/ostd/specs/sync/mod.rs index d1ffbdd20..9a433356c 100644 --- a/ostd/specs/sync/mod.rs +++ b/ostd/specs/sync/mod.rs @@ -2,4 +2,6 @@ pub mod abstract_lock; pub mod mutex; //pub mod mutex_verussync; +pub mod rcu; +pub mod sc_model; pub mod weak_memory; diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs new file mode 100644 index 000000000..eca2d8736 --- /dev/null +++ b/ostd/specs/sync/rcu.rs @@ -0,0 +1,439 @@ +//! Specification skeleton for RCU traversal safety. +//! +//! This module models the shape of the traversal specification from the RCU +//! relaxed-memory paper: +//! +//! - the base layer provides read-side guards, protected pointers, and retire +//! permissions; +//! - the traversal layer reasons about link histories (`RcuPointsTo`) and +//! incoming-link histories (`RcuPointedBy`); +//! - concrete data structures instantiate the traversal trait. +//! +//! The module is intentionally proof-only for now. The executable RCU +//! implementation should later connect its real guard/token state to these +//! abstract ghost tokens. +use vstd::prelude::*; +use vstd::resource::Loc; + +verus! { + +pub type LinkIndex = nat; + +pub type LinkEdge = (*mut T, LinkIndex); + +/// Link view carried by an RCU read-side guard. +/// +/// `seen_at(p) = n` means the guard has observed link-history events for source +/// node `p` up to at least `n`. Following a loaded link at index `k` is allowed +/// only when `seen_at(p) <= k`; otherwise the pointer may be too stale. +#[verifier::reject_recursive_types(T)] +pub ghost struct RcuLinkView { + pub seen: Map<*mut T, LinkIndex>, +} + +impl RcuLinkView { + pub open spec fn empty() -> Self { + RcuLinkView { seen: Map::empty() } + } + + pub open spec fn seen_at(self, p: *mut T) -> LinkIndex { + if self.seen.contains_key(p) { + self.seen[p] + } else { + 0nat + } + } + + pub open spec fn observe(self, p: *mut T, n: LinkIndex) -> Self { + RcuLinkView { + seen: self.seen.insert( + p, + if self.seen_at(p) <= n { + n + } else { + self.seen_at(p) + }, + ), + } + } +} + +/// Paper-style `SeenRemoved(D, LV)`. +/// +/// `removed` is the set `D` observed by the guard; `link_view` is `LV`. +/// A dead incoming edge is either from a removed predecessor or overwritten by a +/// later observed link event. +#[verifier::reject_recursive_types(T)] +pub ghost struct RcuSeenRemoved { + pub removed: Set<*mut T>, + pub link_view: RcuLinkView, +} + +impl RcuSeenRemoved { + pub open spec fn empty() -> Self { + RcuSeenRemoved { removed: Set::empty(), link_view: RcuLinkView::empty() } + } + + pub open spec fn seen_at(self, p: *mut T) -> LinkIndex { + self.link_view.seen_at(p) + } + + pub open spec fn dead_edge(self, edge: LinkEdge) -> bool { + self.removed.contains(edge.0) || self.seen_at(edge.0) > edge.1 + } +} + +/// Authoritative ghost handle for one RCU protection domain. +/// +/// The concrete implementation owns this token in its invariant. We keep the +/// fields private so clients cannot manufacture domain authority. +pub tracked struct RcuDomainAuth { + ghost id: Loc, +} + +impl RcuDomainAuth { + pub closed spec fn id(self) -> Loc { + self.id + } +} + +/// Low-level base retire permission. +/// +/// This is the paper's `BaseRetirePerm`. By itself it is not enough to reclaim; +/// it must be combined with a `SeenRemoved` observation for the retired object. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuBaseRetirePerm { + ghost domain: Loc, + ghost ptr: *mut T, +} + +impl RcuBaseRetirePerm { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } +} + +/// High-level retire permission. +/// +/// This corresponds to `RetirePerm(l, a) = BaseRetirePerm(l, a) * +/// exists D LV. SeenRemoved(D, LV) * a in D`. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetirePerm { + ghost domain: Loc, + ghost ptr: *mut T, + ghost seen_removed: RcuSeenRemoved, +} + +impl RcuRetirePerm { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.seen_removed + } + + pub open spec fn ready_to_reclaim(self) -> bool { + self.seen_removed().removed.contains(self.ptr()) + } +} + +/// Lift a base retire permission once the caller has observed the object in the +/// removed set. +pub proof fn lift_retire_perm( + tracked base: RcuBaseRetirePerm, + seen_removed: RcuSeenRemoved, +) -> (tracked perm: RcuRetirePerm) + requires + seen_removed.removed.contains(base.ptr()), + ensures + perm.domain() == base.domain(), + perm.ptr() == base.ptr(), + perm.seen_removed() == seen_removed, + perm.ready_to_reclaim(), +{ + RcuRetirePerm { domain: base.domain(), ptr: base.ptr(), seen_removed } +} + +/// Read-side guard token for one critical section. +/// +/// This is the traversal-level guard: it includes the base guard protection and +/// the `SeenRemoved(D, LV)` observation used to rule out stale links. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuReadGuardToken { + ghost domain: Loc, + ghost seen_removed: RcuSeenRemoved, +} + +impl RcuReadGuardToken { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.seen_removed + } + + pub closed spec fn link_view(self) -> RcuLinkView { + self.seen_removed().link_view + } + + pub open spec fn seen_at(self, p: *mut T) -> LinkIndex { + self.seen_removed().seen_at(p) + } + + pub open spec fn is_for(self, domain: RcuDomainAuth) -> bool { + self.domain() == domain.id() + } + + pub open spec fn can_protect(self, p: *mut T) -> bool { + !self.seen_removed().removed.contains(p) + } +} + +/// A pointer protected by a live read-side guard. +/// +/// It records the same `SeenRemoved` snapshot as the guard. This lets traversal +/// proofs preserve the fact that the protected pointer is not in the guard's +/// removed set. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuProtectedPtr { + ghost domain: Loc, + ghost ptr: *mut T, + ghost seen_removed: RcuSeenRemoved, +} + +impl RcuProtectedPtr { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.seen_removed + } + + pub open spec fn protected_by(self, guard: RcuReadGuardToken) -> bool { + &&& self.domain() == guard.domain() + &&& self.seen_removed() == guard.seen_removed() + &&& !self.seen_removed().removed.contains(self.ptr()) + } +} + +/// Traversal specification for an RCU-protected data structure. +/// +/// `link_inv(from, n, to, g)` is the client-facing analogue of a +/// `RcuPointsTo(from, ...)` snapshot containing the `n`th link event from +/// `from` to `to`. `seen_removed_sound` is the client-facing analogue of the +/// `RcuPointedBy`/`SeenRemoved` invariant for partially ordered link histories. +pub trait RcuTraversalSafety: Sized { + type Node; + + type Ghost; + + spec fn root_inv(p: *mut Self::Node, g: Self::Ghost) -> bool; + + spec fn node_inv(p: *mut Self::Node, g: Self::Ghost) -> bool; + + spec fn link_inv( + from: *mut Self::Node, + n: LinkIndex, + to: *mut Self::Node, + g: Self::Ghost, + ) -> bool; + + spec fn seen_removed_sound(seen_removed: RcuSeenRemoved, g: Self::Ghost) -> bool; + + proof fn root_is_node_inv(p: *mut Self::Node, g: Self::Ghost) + requires + Self::root_inv(p, g), + ensures + Self::node_inv(p, g), + ; + + proof fn link_preserves_protection( + from: *mut Self::Node, + n: LinkIndex, + to: *mut Self::Node, + seen_removed: RcuSeenRemoved, + g: Self::Ghost, + ) + requires + Self::node_inv(from, g), + Self::link_inv(from, n, to, g), + Self::seen_removed_sound(seen_removed, g), + !seen_removed.removed.contains(from), + seen_removed.seen_at(from) <= n, + ensures + Self::node_inv(to, g), + !seen_removed.removed.contains(to), + ; +} + +/// Protect a freshly acquired root pointer. +pub proof fn protect_root( + tracked domain: &RcuDomainAuth, + tracked guard: &RcuReadGuardToken, + p: *mut S::Node, + g: S::Ghost, +) -> (tracked root: RcuProtectedPtr) + requires + guard.is_for(*domain), + guard.can_protect(p), + S::root_inv(p, g), + ensures + root.ptr() == p, + root.domain() == domain.id(), + root.protected_by(*guard), + S::node_inv(p, g), +{ + S::root_is_node_inv(p, g); + RcuProtectedPtr { domain: domain.id(), ptr: p, seen_removed: guard.seen_removed() } +} + +/// Protect a child reached by following a non-stale link-history event. +pub proof fn protect_link( + tracked guard: &RcuReadGuardToken, + tracked from: &RcuProtectedPtr, + n: LinkIndex, + to: *mut S::Node, + g: S::Ghost, +) -> (tracked to_protected: RcuProtectedPtr) + requires + from.protected_by(*guard), + S::node_inv(from.ptr(), g), + S::link_inv(from.ptr(), n, to, g), + S::seen_removed_sound(guard.seen_removed(), g), + guard.seen_at(from.ptr()) <= n, + ensures + to_protected.ptr() == to, + to_protected.domain() == from.domain(), + to_protected.protected_by(*guard), + S::node_inv(to, g), +{ + S::link_preserves_protection(from.ptr(), n, to, guard.seen_removed(), g); + RcuProtectedPtr { domain: from.domain(), ptr: to, seen_removed: guard.seen_removed() } +} + +/// Minimal ghost-only node used to demonstrate the traversal contract. +pub struct LinkedListNode; + +/// Paper-style ghost state for a linked list. +/// +/// `successors[p]` is the successor history for `p`, corresponding to +/// `RcuPointsTo(p, s)`. +/// +/// `incoming_all[p]` is the set of all incoming edges that have ever pointed to +/// `p`, corresponding to the authoritative incoming set in `RcuPointedBy(p, B)`. +/// +/// `current_incoming[p]` is the current incoming set `B`. It is not required for +/// the simple one-step traversal proof below, but keeping it in the ghost state +/// makes the example match the paper's predicate shape. +pub ghost struct LinkedListGhost { + pub root: *mut LinkedListNode, + pub successors: Map<*mut LinkedListNode, Seq>>, + pub incoming_all: Map<*mut LinkedListNode, Set>>, + pub current_incoming: Map<*mut LinkedListNode, Set>>, +} + +pub struct LinkedListTraversalSpec; + +impl RcuTraversalSafety for LinkedListTraversalSpec { + type Node = LinkedListNode; + + type Ghost = LinkedListGhost; + + open spec fn root_inv(p: *mut LinkedListNode, g: LinkedListGhost) -> bool { + &&& p == g.root + &&& g.successors.contains_key(p) + &&& g.incoming_all.contains_key(p) + } + + open spec fn node_inv(p: *mut LinkedListNode, g: LinkedListGhost) -> bool { + &&& g.successors.contains_key(p) + &&& g.incoming_all.contains_key(p) + } + + open spec fn link_inv( + from: *mut LinkedListNode, + n: LinkIndex, + to: *mut LinkedListNode, + g: LinkedListGhost, + ) -> bool { + &&& g.successors.contains_key(from) + &&& n < g.successors[from].len() + &&& g.successors[from][n as int] == Some(to) + &&& g.successors.contains_key(to) + &&& g.incoming_all.contains_key(to) + &&& g.incoming_all[to].contains((from, n)) + } + + open spec fn seen_removed_sound( + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + ) -> bool { + forall|to: *mut LinkedListNode| #[trigger] + seen_removed.removed.contains(to) ==> { + &&& g.incoming_all.contains_key(to) + &&& forall|edge: LinkEdge| #[trigger] + g.incoming_all[to].contains(edge) ==> seen_removed.dead_edge(edge) + } + } + + proof fn root_is_node_inv(p: *mut LinkedListNode, g: LinkedListGhost) { + } + + proof fn link_preserves_protection( + from: *mut LinkedListNode, + n: LinkIndex, + to: *mut LinkedListNode, + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + ) { + if seen_removed.removed.contains(to) { + assert(g.incoming_all[to].contains((from, n))); + assert(seen_removed.dead_edge((from, n))); + assert(false); + } + } +} + +/// Example: after protecting the root, following a non-stale successor-history +/// event protects the next node under the same guard. +pub proof fn linked_list_protect_next_example( + tracked domain: &RcuDomainAuth, + tracked guard: &RcuReadGuardToken, + root: *mut LinkedListNode, + n: LinkIndex, + next: *mut LinkedListNode, + g: LinkedListGhost, +) -> (tracked next_protected: RcuProtectedPtr) + requires + guard.is_for(*domain), + guard.can_protect(root), + LinkedListTraversalSpec::root_inv(root, g), + LinkedListTraversalSpec::link_inv(root, n, next, g), + LinkedListTraversalSpec::seen_removed_sound(guard.seen_removed(), g), + guard.seen_at(root) <= n, + ensures + next_protected.ptr() == next, + next_protected.domain() == domain.id(), + next_protected.protected_by(*guard), + LinkedListTraversalSpec::node_inv(next, g), +{ + let tracked root_protected = protect_root::(domain, guard, root, g); + protect_link::(guard, &root_protected, n, next, g) +} + +} // verus! diff --git a/ostd/specs/sync/sc_model.rs b/ostd/specs/sync/sc_model.rs new file mode 100644 index 000000000..bf926e364 --- /dev/null +++ b/ostd/specs/sync/sc_model.rs @@ -0,0 +1,11 @@ +//! This module models the semantics for the SC fences and global SC views +//! coupled with relaxed accesses. It is based on the semantics of the C11 +//! memory model but we exclude the SC access mode which is unnecessary. +//! +//! This is important to verifying some sync primitives like raw RCU. +use vstd::prelude::*; + +verus! { + + +} // verus! diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index ecb1dd871..5c615e1a9 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -10,6 +10,10 @@ //! fication history, subject to coherence. In particular, relaxed reads may ob- //! serve stale writes, but a thread’s view prevents it from going backwards, //! and reads do not observe future writes that have not been added to the history. +//! +//! # References +//! +//! - [RCU Verification](https://dl.acm.org/doi/pdf/10.1145/3729246) use core::sync::atomic::{ AtomicBool, AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8, AtomicUsize, Ordering, @@ -26,6 +30,16 @@ use vstd::seq::Seq; verus! { +// The "global" memory is defined wihtin the invariant we need to preserve and, +// by the definition of Iris operations, invariant can be opened by a thread +// provided that the invariant holds and it can close afterwards provided that +// the invariant holds as well. +// +// Thanks to Verus' native support for the semantics, we only need to define +// what means for `atomic` and we can freely open the invariant and provide +// customized macros for doing ergonomic updates on both the physical resources +// and the ghost tokens like message histories, views, etc. +/// An `AtomicId` is just an abstract identifier (memory location) of one atomic object. pub type AtomicId = Loc; /// Logical timestamp into one atomic object's message history. @@ -36,11 +50,16 @@ pub type Timestamp = nat; /// /// `seen[id] = ts` means this thread has advanced past all messages for `id` /// older than `ts`; future reads from that atomic must not go backwards. +/// +/// Typically, if another thread has published a message with timestamp `ts` for `id`, +/// and the reader reads the message via some atomic operations, then the reader's +/// thread view will advance to at least `ts` for `id`. pub ghost struct WmView { pub seen: Map, } impl WmView { + /// Creates an empty view. pub open spec fn empty() -> Self { WmView { seen: Map::empty() } } @@ -55,6 +74,14 @@ impl WmView { } /// Monotonically advance the view for one atomic object. + /// + /// Just as the name indicates, `observe` means that the current thread has observed + /// a message written by another thread with a specific timestamp; because the atomic + /// operation never "goes back", the thread's view for that atomic must advance to at + /// least that timestamp. + /// + /// "During a read from `l`, a thread can observe any message `m` from `M(l)` where + /// `m.time >= V(l)`, and updates its view to incorporate `m.time`." pub open spec fn observe(self, id: AtomicId, ts: Timestamp) -> Self { WmView { seen: self.seen.insert( @@ -86,6 +113,7 @@ impl WmView { } } + /// Partial ordering two threads' views. pub open spec fn le(self, other: Self) -> bool { forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) } diff --git a/ostd/src/sync/mod.rs b/ostd/src/sync/mod.rs index 4908e75f4..cd33ecf21 100644 --- a/ostd/src/sync/mod.rs +++ b/ostd/src/sync/mod.rs @@ -10,6 +10,7 @@ mod rwlock; mod rwmutex; mod spin; mod wait; + //pub(crate) use self::rcu::finish_grace_period; pub use self::{ atomic_data::*, diff --git a/ostd/src/sync/rcu/__mod.rs b/ostd/src/sync/rcu/__mod.rs new file mode 100644 index 000000000..f0c4e2db1 --- /dev/null +++ b/ostd/src/sync/rcu/__mod.rs @@ -0,0 +1,1128 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Read-copy update (RCU). +//! +//! # Note +//! +//! Currently this RCU model assumes a sequential consistency (SC) memory model. +//! We may explore weak memory models in the future. +use vstd::{ + atomic_ghost::AtomicPtr, atomic_with_ghost, map::Map, modes::tracked_static_ref, prelude::*, + resource::Loc, +}; + +use vstd_extra::{ + prelude::*, + resource::ghost_resource::{count::Count, tokens::CountResource}, +}; + +use core::{ + marker::PhantomData, mem::ManuallyDrop, ops::Deref, + ptr::NonNull, + /* + sync::atomic::{ + AtomicPtr, + Ordering::{AcqRel, Acquire}, + }, + */ +}; + +use non_null::{NonNullPtr, NonNullPtrRef}; +// use spin::once::Once; +use super::Once; + +use self::monitor::{RcuMonitor, RcuMonitorOwner, RcuMonitorPred}; +use crate::task::{ + //atomic_mode::{AsAtomicModeGuard, InAtomicMode}, + disable_preempt, + DisabledPreemptGuard, +}; + +mod monitor; +pub mod non_null; + +use crate::specs::task::InAtomicMode; + +verus! { + +broadcast use vstd_extra::external::nonnull::group_nonull_axioms; +// Verification-only budget for splitting read-side ghost tokens. +// +// This is not a runtime reader counter and does not model an overflow condition +// in the RCU implementation. It is a temporary bounded approximation needed by +// `CountResource`; the final RCU proof should discharge the admission assumption +// with an unbounded ghost registry or a CPU/epoch-based sharding model. + +const RCU_READER_SLOTS: u64 = 1u64 << 60; + +type RcuReadPool

= CountResource<

::Permission, RCU_READER_SLOTS>; + +type RcuReadToken

= Count<

::Permission, RCU_READER_SLOTS>; + +type RcuRetiredEntry

= (Ghost<*mut

::Target>, RcuReadPool

); + +/// Called by `drop` of the read guard to track the retired read permissions. +type RcuRetiredPools

= Map>; + +type RcuReturnedTokens

= Map>; + +tracked struct RcuPtrGhost { + tracked current: Option>, + tracked retired: RcuRetiredPools

, + tracked returned: RcuReturnedTokens

, +} + +closed spec fn retired_pools_inv(retired: RcuRetiredPools

) -> bool { + forall|id: Loc| #[trigger] + retired.contains_key(id) ==> { + let entry = retired[id]; + &&& !(entry.0@).is_null() + &&& entry.1.id() == id + &&& P::ptr_perm_match(entry.0@, entry.1@) + &&& entry.1@.inv() + &&& entry.1.wf() + &&& entry.1.not_empty() + } +} + +closed spec fn returned_tokens_inv(returned: RcuReturnedTokens

) -> bool { + forall|id: Loc| #[trigger] + returned.contains_key(id) ==> { + let token = returned[id]; + &&& token.id() == id + &&& token.resource().inv() + &&& token.frac() > 0 + } +} + +/// A Read-Copy Update (RCU) cell for sharing a pointer between threads. +/// +/// The pointer should be a non-null pointer with type `P`, which implements +/// [`NonNullPtr`]. For example, `P` can be `Box` or `Arc`. +/// +/// # Overview +/// +/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high- +/// performance, low-latency read operations in concurrent systems. It allows +/// multiple readers to access shared data simultaneously without contention, +/// while writers can update the data safely in a way that does not disrupt +/// ongoing reads. RCU is particularly suited for situations where reads are +/// far more frequent than writes. +/// +/// The original design and implementation of RCU is described in paper _The +/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared- +/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal +/// 47.2 (2008). +/// +/// # Examples +/// +/// ``` +/// use ostd::sync::Rcu; +/// +/// let rcu = Rcu::new(Box::new(42)); +/// +/// let rcu_guard = rcu.read(); +/// +/// assert_eq!(*rcu_guard, Some(&42)); +/// +/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); +/// +/// let rcu_guard = rcu.read(); +/// +/// assert_eq!(*rcu_guard, Some(&43)); +/// ``` +pub struct Rcu(RcuInner

); + +/// A guard that allows access to the pointed data protected by a [`Rcu`]. +#[clippy::has_significant_drop] +#[must_use] +pub struct RcuReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); + +/// A Read-Copy Update (RCU) cell for sharing a _ghost_nullable_ pointer. +/// +/// This is a variant of [`Rcu`] that allows the contained pointer to be null. +/// So that it can implement `Rcu>` where `P` is not a ghost_nullable +/// pointer. It is the same as [`Rcu`] in other aspects. +/// +/// # Examples +/// +/// ``` +/// use ostd::sync::RcuOption; +/// +/// static RCU: RcuOption> = RcuOption::new_none(); +/// +/// assert!(RCU.read().is_none()); +/// +/// RCU.update(Box::new(42)); +/// +/// // Read the data protected by RCU +/// { +/// let rcu_guard = RCU.read().try_get().unwrap(); +/// assert_eq!(*rcu_guard, 42); +/// } +/// +/// // Update the data protected by RCU +/// { +/// let rcu_guard = RCU.read().try_get().unwrap(); +/// +/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); +/// +/// let rcu_guard = RCU.read().try_get().unwrap(); +/// assert_eq!(*rcu_guard, 43); +/// } +/// ``` +pub struct RcuOption(RcuInner

); + +/// A guard that allows access to the pointed data protected by a [`RcuOption`]. +#[clippy::has_significant_drop] +#[must_use] +pub struct RcuOptionReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); + +struct_with_invariants! { +/// The inner implementation of both [`Rcu`] and [`RcuOption`]. +struct RcuInner { + ptr: AtomicPtr<

::Target,_,RcuPtrGhost

,_>, + // We want to implement Send and Sync explicitly. + // Having a pointer field prevents them from being implemented + // automatically by the compiler. + _marker: PhantomData<*const

::Target>, + ghost_nullable: Ghost, +} + +closed spec fn wf(self) -> bool { + invariant on ptr with (ghost_nullable, _marker) is ( + v: *mut

::Target, + g: RcuPtrGhost

, + ) { + &&& retired_pools_inv::

(g.retired) + &&& returned_tokens_inv::

(g.returned) + &&& match g.current { + Some(perm) => { + &&& !v.is_null() + &&& P::ptr_perm_match(v, perm@) + &&& perm@.inv() + &&& perm.wf() + &&& perm.not_empty() + }, + None => ghost_nullable@ && v.is_null(), + } + } +} +} +// SAFETY: It is apparent that if `P` is `Send`, then `Rcu

` is `Send`. + + +#[verifier::external] +unsafe impl Send for RcuInner

where P: Send { + +} + +// SAFETY: To implement `Sync` for `Rcu

`, we need to meet two conditions: +// 1. `P` must be `Sync` because `Rcu::get` allows concurrent access. +// 2. `P` must be `Send` because `Rcu::update` may obtain an object +// of `P` created on another thread. +#[verifier::external] +unsafe impl Sync for RcuInner

where P: Send + Sync { + +} + +impl RcuInner

{ + /// Whether the contained pointer can be null. Used to distinguish `Rcu` and `RcuOption`. + pub closed spec fn is_nullable(self) -> bool { + self.ghost_nullable@ + } +} + +#[verus_verify] +impl RcuInner

{ + #[inline(always)] + const fn new_none() -> (res: Self) + ensures + res.is_nullable(), + res.wf(), + { + proof_decl! { + let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { + current: None, + retired: Map::tracked_empty(), + returned: Map::tracked_empty(), + }; + } + Self { + ptr: AtomicPtr::new( + Ghost((Ghost(true), PhantomData::<*const

::Target>)), + core::ptr::null_mut(), + Tracked(ptr_ghost), + ), + _marker: PhantomData::<*const

::Target>, + ghost_nullable: Ghost(true), + } + } + + /// Creates a new RCU primitive with the given pointer `pointer`. + #[inline(always)] + #[verus_spec(r => + with + Ghost(ghost_nullable): Ghost, + ensures + r.type_inv(), + r.is_nullable() == ghost_nullable, + )] + fn new(pointer: P) -> Self { + // let ptr =

::into_raw(pointer).as_ptr(); + let (ptr, Tracked(ptr_perm)) =

::into_raw(pointer); + let ptr = ptr.as_ptr(); + proof_decl! { + let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { + current: Some(CountResource::alloc(ptr_perm)), + retired: Map::tracked_empty(), + returned: Map::tracked_empty(), + }; + } + + let ptr = AtomicPtr::new( + Ghost((Ghost(ghost_nullable), PhantomData)), + ptr, + Tracked(ptr_ghost), + ); + Self { ptr, _marker: PhantomData, ghost_nullable: Ghost(ghost_nullable) } + } + + #[verus_spec( + requires + self.is_nullable() || new_ptr is Some, + )] + fn update(&self, new_ptr: Option

) { + let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { + //

::into_raw(new_ptr).as_ptr() + let (ptr, Tracked(perm)) =

::into_raw(new_ptr); + let ptr = ptr.as_ptr(); + (ptr, Tracked(Some(perm))) + } else { + (core::ptr::null_mut(), Tracked(None)) + }; + + proof_decl! { + let tracked mut old_perm: Option> = None; + } + proof { + use_type_invariant(self); + } + let old_raw_ptr = + atomic_with_ghost! { + self.ptr => swap(new_ptr); + update prev -> next; + ghost g => { + old_perm = g.current; + if old_perm is Some { + let tracked mut pool = old_perm.tracked_unwrap(); + let ghost id = pool.id(); + if g.retired.contains_key(id) { + // Use tracked_borrow_mut instead + let tracked entry = g.retired.tracked_remove(id); + let tracked mut retired_pool = entry.1; + let tracked pool_token = pool.split(pool.frac()); + retired_pool.validate_with_frac(&pool_token); + retired_pool.combine(pool_token); + g.retired.tracked_insert(id, (entry.0, retired_pool)); + } else { + g.retired.tracked_insert(id, (Ghost(prev), pool)); + } + } + g.current = match new_perm { + Some(perm) => Some(CountResource::alloc(perm)), + None => None, + }; + assert(retired_pools_inv::

(g.retired)); + } + }; + + if let Some(p) = NonNull::new(old_raw_ptr) { + // SAFETY: + // 1. The pointer was previously returned by `into_raw`. + // 2. The pointer is removed from the RCU slot so that no one will + // use it after the end of the current grace period. The removal + // is done atomically, so it will only be dropped once. + // unsafe { delay_drop::

(p) }; + } + } + + #[verus_spec(obj_ptr => + with + -> tracked_ref_perm: Tracked>>, + ensures + !self.is_nullable() ==> tracked_ref_perm@ is Some, + match tracked_ref_perm@ { + Some(perm) => { + &&& !obj_ptr.is_null() + &&& P::ptr_perm_match(obj_ptr, perm.resource()) + &&& perm.resource().inv() + &&& perm.frac() == 1 + }, + None => obj_ptr.is_null(), + }, + )] + fn load_read_token(&self) -> *mut

::Target { + proof_decl! { + let tracked mut tracked_ref_perm: Option> = None; + } + proof { + use_type_invariant(self); + } + let obj_ptr = + atomic_with_ghost! { + self.ptr => load(); + update prev -> _next; + returning loaded; + ghost g => { + if g.current is Some { + let tracked mut perm = g.current.tracked_unwrap(); + assert(loaded == prev); + assert(!loaded.is_null()); + assert(P::ptr_perm_match(loaded, perm@)); + assert(perm@.inv()); + let ghost perm_snapshot = perm@; + + // Verification-only admission for the bounded read-token pool. + // This is not a runtime reader limit; it only reflects that + // `CountResource` uses a Rust const-generic `u64` budget rather + // than an unbounded mathematical `nat`. + assume(perm.not_empty()); + assume(1 < perm.frac()); + let tracked token = perm.split_one(); + assert(perm@ == perm_snapshot); + assert(token.frac() == 1); + tracked_ref_perm = Some(token); + g.current = Some(perm); + } else { + } + assert(retired_pools_inv::

(g.retired)); + } + }; + proof_with! { |= Tracked(tracked_ref_perm) } + obj_ptr + } + + #[verus_spec(r => + ensures + r.type_inv(), + r.rcu.is_nullable() == self.is_nullable(), + !self.is_nullable() ==> r.tracked_ref_perm@ is Some, + )] + fn read(&self) -> RcuReadGuardInner<'_, P> { + let guard = disable_preempt(); + proof_decl! { + let tracked mut tracked_ref_perm: Option> = None; + } + let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] + self.load_read_token(); + RcuReadGuardInner { + obj_ptr, + rcu: self, + _inner_guard: guard, + tracked_ref_perm: Tracked(tracked_ref_perm), + } + } + + #[verus_spec] + pub fn read_with<'a, A: InAtomicMode>( + &'a self, + _guard: &'a A, // &'a dyn InAtomicMode is not well-supported in Verus. + ) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + proof_decl! { + let tracked mut tracked_ref_perm: Option> = None; + } + let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] + self.load_read_token(); + if obj_ptr.is_null() { + return None; + } + proof_decl! { + // `read_with` returns only the reference and has no guard object to + // store the read token. For this temporary skeleton, leak the + // verification-only token so the returned ref can borrow it for + // `'a`. The final RCU proof should attach this token to the + // atomic-mode/CPU epoch state instead. + let tracked tracked_ref_perm = tracked_ref_perm.tracked_unwrap(); + let tracked tracked_ref_perm = tracked_static_ref(tracked_ref_perm); + let tracked tracked_ref_perm:

>::RefPermission = + P::borrow_perm_as_ref_perm(tracked_ref_perm.borrow()); + } + // SAFETY: + // 1. This pointer is not NULL. + // 2. The `_guard` guarantees atomic mode for the duration of lifetime + // `'a`, the pointer is valid because other writers won't release the + // allocation until this task passes the quiescent state. + NonNull::new(obj_ptr).map( + |ptr| + requires + P::ptr_perm_match( + ptr.view_ptr_mut(), + P::ref_perm_view_permission(tracked_ref_perm), + ), + { + unsafe { P::raw_as_ref(ptr, Tracked(tracked_ref_perm)) } + }, + ) + } +} + +/* +impl Drop for RcuInner

{ + fn drop(&mut self) { + let ptr = self.ptr.load(Acquire); + if let Some(p) = NonNull::new(ptr) { + // SAFETY: It was previously returned by `into_raw` when creating + // the RCU primitive. + let pointer = unsafe {

::from_raw(p) }; + // It is OK not to delay the drop because the RCU primitive is + // owned by nobody else. + drop(pointer); + } + } +} +*/ + +/// The inner implementation of both [`RcuReadGuard`] and [`RcuOptionReadGuard`]. +struct RcuReadGuardInner<'a, P: NonNullPtr> { + obj_ptr: *mut

::Target, + rcu: &'a RcuInner

, + _inner_guard: DisabledPreemptGuard, + tracked_ref_perm: Tracked>>, +} + +#[verus_verify] +impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { + #[inline] + #[verus_spec(r => + ensures + self.tracked_ref_perm@ is Some ==> r is Some, + )] + fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> { + proof { + use_type_invariant(self); + } + + // SAFETY: The guard ensures that `P` will not be dropped. Thus, `P` + // outlives the lifetime of `&self`. Additionally, during this period, + // it is impossible to create a mutable reference to `P`. + NonNull::new(self.obj_ptr).map( + |ptr| + requires + self.tracked_ref_perm@ is Some, + P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_ref_perm->0.resource()), + { + unsafe { + P::raw_as_ref( + ptr, + Tracked( + P::borrow_perm_as_ref_perm( + self.tracked_ref_perm.tracked_borrow().borrow(), + ), + ), + ) + } + }, + ) + } + + #[verus_spec(r => + requires + self.rcu.is_nullable() || new_ptr is Some, + ensures + new_ptr is Some && r is Err ==> r->Err_0 is Some, + )] + fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { + let obj_ptr = self.obj_ptr; + proof { + use_type_invariant(&self); + use_type_invariant(self.rcu); + } + proof_decl! { + let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + let ghost new_ptr_is_some = new_ptr is Some; + let tracked mut old_perm: Option> = None; + let tracked mut err_new_perm: Option::Permission>> = None; + } + let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { + //

::into_raw(new_ptr).as_ptr() + let (ptr, Tracked(perm)) =

::into_raw(new_ptr); + (ptr.as_ptr(), Tracked(Some(perm))) + } else { + (core::ptr::null_mut(), Tracked(None)) + }; + let res = + atomic_with_ghost! { + self.rcu.ptr => compare_exchange(obj_ptr, new_ptr); + update _prev -> next; + returning res; + ghost g => { + if res is Ok { + old_perm = g.current; + if old_perm is Some { + let tracked mut pool = old_perm.tracked_unwrap(); + let ghost id = pool.id(); + if g.retired.contains_key(id) { + // use tracked_borrow_mut instead + let tracked entry = g.retired.tracked_remove(id); + let tracked mut retired_pool = entry.1; + let tracked pool_token = pool.split(pool.frac()); + retired_pool.validate_with_frac(&pool_token); + retired_pool.combine(pool_token); + g.retired.tracked_insert(id, (entry.0, retired_pool)); + } else { + g.retired.tracked_insert(id, (Ghost(_prev), pool)); + } + } + g.current = match new_perm { + Some(perm) => Some(CountResource::alloc(perm)), + None => None, + }; + } else { + err_new_perm = Some(new_perm); + } + if tracked_ref_perm is Some { + let tracked token = tracked_ref_perm.tracked_unwrap(); + let ghost id = token.id(); + if g.retired.contains_key(id) { + let tracked entry = g.retired.tracked_remove(id); + let tracked mut pool = entry.1; + pool.combine(token); + g.retired.tracked_insert(id, (entry.0, pool)); + } else if g.current is Some { + let tracked mut pool = g.current.tracked_unwrap(); + if pool.id() == id { + pool.combine(token); + g.current = Some(pool); + } else { + g.current = Some(pool); + assume(false); + } + } else { + assume(false); + } + } + assert(retired_pools_inv::

(g.retired)); + } + }; + if res.is_ok() { + if let Some(p) = NonNull::new(obj_ptr) { + // SAFETY: + // 1. The pointer was previously returned by `into_raw`. + // 2. The pointer is removed from the RCU slot so that no one will + // use it after the end of the current grace period. The removal + // is done atomically, so it will only be dropped once. + // unsafe { delay_drop::

(p) }; + } + Ok(()) + } else { + let Some(new_nonnull) = NonNull::new(new_ptr) else { + return Err(None); + }; + proof_decl! { + let tracked new_perm = err_new_perm.tracked_unwrap().tracked_unwrap(); + } + // SAFETY: + // 1. It was previously returned by `into_raw`. + // 2. The `compare_exchange` fails so the pointer will not + // be used by other threads via reading the RCU primitive. + Err(Some(unsafe {

::from_raw(new_nonnull, Tracked(new_perm)) })) + } + } + + /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. + #[inline] + #[verus_spec] + fn drop(self) { + let rcu = self.rcu; + let obj_ptr = self.obj_ptr; + proof { + use_type_invariant(&self); + use_type_invariant(rcu); + } + proof_decl! { + let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + } + atomic_with_ghost! { + rcu.ptr => load(); + update prev -> _next; + returning _loaded; + ghost g => { + if tracked_ref_perm is Some { + let tracked token = tracked_ref_perm.tracked_unwrap(); + let ghost id = token.id(); + if g.current is Some { + let tracked mut pool = g.current.tracked_unwrap(); + if prev == obj_ptr && pool.id() == id { + assert(!obj_ptr.is_null()); + assert(P::ptr_perm_match(obj_ptr, token.resource())); + assert(token.resource().inv()); + assert(token.frac() == 1); + assert(P::ptr_perm_match(prev, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + pool.combine(token); + assert(P::ptr_perm_match(prev, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + g.current = Some(pool); + } else { + g.current = Some(pool); + if g.retired.contains_key(id) { + let tracked entry = g.retired.tracked_remove(id); + let tracked mut pool = entry.1; + assert(pool.id() == id); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + pool.combine(token); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + g.retired.tracked_insert(id, (entry.0, pool)); + } else { + assume(false); + } + } + } else { + if g.retired.contains_key(id) { + let tracked entry = g.retired.tracked_remove(id); + let tracked mut pool = entry.1; + assert(pool.id() == id); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + pool.combine(token); + assert(P::ptr_perm_match(entry.0@, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + g.retired.tracked_insert(id, (entry.0, pool)); + } else { + assume(false); + } + } + } + match &g.current { + Some(pool) => { + assert(!prev.is_null()); + assert(P::ptr_perm_match(prev, pool@)); + assert(pool@.inv()); + assert(pool.wf()); + assert(pool.not_empty()); + }, + None => { + assert(rcu.ghost_nullable@); + assert(prev.is_null()); + }, + } + assert(retired_pools_inv::

(g.retired)); + } + }; + } +} + +#[verus_verify] +impl Rcu

{ + /// Creates a new RCU primitive with the given pointer `pointer`. + #[verus_spec] + pub fn new(pointer: P) -> Self { + Self( + #[verus_spec(with Ghost(false))] + RcuInner::new(pointer), + ) + } + + /// Replaces the current pointer with a null pointer. + /// + /// This function updates the pointer to the new pointer regardless of the + /// original pointer. The original pointer will be dropped after the grace + /// period. + /// + /// Oftentimes this function is not recommended unless you have serialized + /// writes with locks. Otherwise, you can use [`Self::read`] and then + /// [`RcuReadGuard::compare_exchange`] to update the pointer. + #[inline] + pub fn update(&self, new_ptr: P) { + self.0.update(Some(new_ptr)); + } + + /// Retrieves a read guard for the RCU primitive. + /// + /// The guard allows read access to the data protected by RCU, as well + /// as the ability to do compare-and-exchange. + #[inline] + pub fn read(&self) -> RcuReadGuard<'_, P> { + proof { + use_type_invariant(self); + } + RcuReadGuard(self.0.read()) + } + // #[inline] + // pub fn read_with<'a, G: AsAtomicModeGuard + ?Sized>(&'a self, guard: &'a G) -> P::Ref<'a> where + // P: NonNullPtrRef<'a>, + // { + // self.0.read_with(guard.as_atomic_mode_guard()).unwrap() + // } + +} + +#[verus_verify] +impl RcuOption

{ + /// Creates a new RCU primitive with the given pointer. + #[verus_spec] + pub fn new(pointer: Option

) -> Self { + if let Some(pointer) = pointer { + Self( + #[verus_spec(with Ghost(true))] + RcuInner::new(pointer), + ) + } else { + Self(RcuInner::new_none()) + } + } + + /// Creates a new RCU primitive that contains nothing. + /// + /// This is a constant equivalence to [`RcuOption::new(None)`]. + #[inline(always)] + pub const fn new_none() -> Self { + Self(RcuInner::new_none()) + } + + /// Replaces the current pointer with a null pointer. + /// + /// This function updates the pointer to the new pointer regardless of the + /// original pointer. If the original pointer is not NULL, it will be + /// dropped after the grace period. + /// + /// Oftentimes this function is not recommended unless you have + /// synchronized writes with locks. Otherwise, you can use [`Self::read`] + /// and then [`RcuOptionReadGuard::compare_exchange`] to update the pointer. + #[inline] + pub fn update(&self, new_ptr: Option

) { + proof { + use_type_invariant(self); + } + self.0.update(new_ptr); + } + + /// Retrieves a read guard for the RCU primitive. + /// + /// The guard allows read access to the data protected by RCU, as well + /// as the ability to do compare-and-exchange. + /// + /// The contained pointer can be NULL and you can only get a reference + /// (if checked non-NULL) via [`RcuOptionReadGuard::get`]. + #[inline] + pub fn read(&self) -> RcuOptionReadGuard<'_, P> { + proof { + use_type_invariant(self); + } + RcuOptionReadGuard(self.0.read()) + } +} + +#[verus_verify] +impl RcuReadGuard<'_, P> { + /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. + #[inline] + pub fn drop(self) { + self.0.drop(); + } + + /// Gets the reference of the protected data. + #[inline] + pub fn get<'a>(&'a self) ->

>::Ref where P: NonNullPtrRef<'a> { + proof { + use_type_invariant(self); + } + self.0.get().unwrap() + } + + /// Tries to replace the already read pointer with a new pointer. + /// + /// If another thread has updated the pointer after the read, this + /// function will fail, and returns the given pointer back. Otherwise, + /// it will replace the pointer with the new one and drop the old pointer + /// after the grace period. + /// + /// If spinning on [`Rcu::read`] and this function, it is recommended + /// to relax the CPU or yield the task on failure. Otherwise contention + /// will occur. + /// + /// This API does not help to avoid + /// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem). + #[inline] + pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { + self.0.compare_exchange(Some(new_ptr)).map_err( + |err| + requires + err is Some, + { err.unwrap() }, + ) + } +} + +/* +impl AsAtomicModeGuard for RcuReadGuard<'_, P> { + fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { + self.0.inner_guard.as_atomic_mode_guard() + } +}*/ + +#[verus_verify] +impl RcuOptionReadGuard<'_, P> { + /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. + #[inline] + pub fn drop(self) { + self.0.drop(); + } + + #[inline] + pub fn get<'a>(&'a self) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + self.0.get() + } + + #[inline] + pub fn is_none(&self) -> bool { + self.0.obj_ptr.is_null() + } + + #[inline] + pub fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { + proof { + use_type_invariant(&self); + } + self.0.compare_exchange(new_ptr) + } +} + +/* +impl AsAtomicModeGuard for RcuOptionReadGuard<'_, P> { + fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { + self.0.inner_guard.as_atomic_mode_guard() + } +} +*/ + +/* +/// Delays the dropping of a [`NonNullPtr`] after the RCU grace period. +/// +/// This is internally needed for implementing [`Rcu`] and [`RcuOption`] +/// because we cannot alias a [`Box`]. Restoring `P` and use [`RcuDrop`] for it +/// can lead to multiple [`Box`]es simultaneously pointing to the same +/// content. +/// +/// # Safety +/// +/// The pointer must be previously returned by `into_raw`, will not be used +/// after the end of the current grace period, and will only be dropped once. +/// +/// [`Box`]: alloc::boxed::Box +unsafe fn delay_drop(pointer: NonNull<

::Target>) { + struct ForceSend(NonNull<

::Target>); + // SAFETY: Sending a raw pointer to another task is safe as long as + // the pointer access in another task is safe (guaranteed by the trait + // bound `P: Send`). + unsafe impl Send for ForceSend

{} + + let pointer: ForceSend

= ForceSend(pointer); + + let rcu_monitor = RCU_MONITOR.get().unwrap(); + rcu_monitor.after_grace_period(move || { + // This is necessary to make the Rust compiler to move the entire + // `ForceSend` structure into the closure. + let pointer = pointer; + + // SAFETY: + // 1. The pointer was previously returned by `into_raw`. + // 2. The pointer won't be used anymore since the grace period has + // finished and this is the only time the pointer gets dropped. + let p = unsafe {

::from_raw(pointer.0) }; + drop(p); + }); +} */ + +/// A wrapper to delay calling destructor of `T` after the RCU grace period. +/// +/// Upon dropping this structure, a callback will be registered to the global +/// RCU monitor and the destructor of `T` will be delayed until the callback. +/// +/// [`RcuDrop`] is guaranteed to have the same layout as `T`. You can also +/// access the inner value safely via [`RcuDrop`]. +#[repr(transparent)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RcuDrop { + value: ManuallyDrop, +} + +impl View for RcuDrop { + type V = T; + + closed spec fn view(&self) -> T { + self.value@ + } +} + +#[verus_verify] +impl RcuDrop { + /// Creates a new [`RcuDrop`] that wraps the given value. + #[inline] + #[verus_spec(r => + ensures + r@ == value, + )] + pub fn new(value: T) -> Self { + Self { value: ManuallyDrop::new(value) } + }/* + /// Extracts the value from the `RcuDrop` container. + /// + /// # Safety + /// + /// The caller must ensure that the returned value will be dropped after + /// all the threads cannot access it anymore. Specifically, dropping it + /// after the RCU grace period is guaranteed to be safe. + /// + /// Note that panic unwinding may cause the returned value to be dropped + /// immediately, which is not sound. Therefore, the caller must forget the + /// [`PanicGuard`] after it ensures that the value will be dropped at the + /// correct time. + pub(crate) unsafe fn into_inner(slot: RcuDrop) -> (T, PanicGuard) { + let mut slot = ManuallyDrop::new(slot); + let panic_guard = PanicGuard::new(); + // SAFETY: The `slot` will not be used after this point. + let val = unsafe { ManuallyDrop::take(&mut slot.value) }; + (val, panic_guard) + } + */ + +} + +#[verus_verify] +impl Deref for RcuDrop { + type Target = T; + + #[verus_spec(r => + ensures + *r == self@, + )] + #[inline] + fn deref(&self) -> &Self::Target { + &self.value + } +} + +/* +impl Drop for RcuDrop { + fn drop(&mut self) { + // SAFETY: The `ManuallyDrop` will not be used after this point. + let taken = unsafe { ManuallyDrop::take(&mut self.value) }; + let rcu_monitor = RCU_MONITOR.get().unwrap(); + rcu_monitor.after_grace_period(|| { + drop(taken); + }); + } +} + +/// Finishes the current grace period. +/// +/// This function is called when the current grace period on current CPU is +/// finished. If this CPU is the last CPU to finish the current grace period, +/// it takes all the current callbacks and invokes them. +/// +/// # Safety +/// +/// The caller must ensure that this CPU is not executing in a RCU read-side +/// critical section. +pub unsafe fn finish_grace_period() { + let rcu_monitor = RCU_MONITOR.get().unwrap(); + // SAFETY: The caller ensures safety. + unsafe { + rcu_monitor.finish_grace_period(); + } +} + +*/ + +exec static RCU_MONITOR: Once + ensures + RCU_MONITOR.wf(), + RCU_MONITOR.inv() == RcuMonitorPred, +{ + Once::new(Ghost(RcuMonitorPred)) +} + +pub fn init() { + RCU_MONITOR.init(RcuMonitor::new_data()); +} + +} // verus! +verus! { + +impl RcuInner

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +impl RcuOption

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& self.0.is_nullable() + } +} + +impl Rcu

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& !self.0.is_nullable() + } +} + +impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& !self.0.rcu.is_nullable() + &&& self.0.tracked_ref_perm@ is Some + } +} + +impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.0.type_inv() + &&& self.0.rcu.is_nullable() + } +} + +impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + match self.tracked_ref_perm@ { + Some(perm) => { + &&& !self.obj_ptr.is_null() + &&& P::ptr_perm_match(self.obj_ptr, perm.resource()) + &&& perm.resource().inv() + &&& perm.frac() == 1 + }, + None => self.obj_ptr.is_null(), + } + } +} + +impl Inv for Rcu

{ + closed spec fn inv(self) -> bool { + self.type_inv() + } +} + +} // verus! From 74f87303c79b9425eafabcc24122a6d4fa2eb505 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 3 Jun 2026 22:07:28 -0400 Subject: [PATCH 06/47] experiment with callbacks --- dv | 2 +- ostd/specs/lib.rs | 2 +- ostd/specs/sync/rcu.rs | 69 ++ ostd/specs/sync/weak_memory.rs | 72 ++- ostd/src/sync/mod.rs | 7 +- ostd/src/sync/rcu/__mod.rs | 2 +- ostd/src/sync/rcu/mod.rs | 1109 +++++++++----------------------- 7 files changed, 440 insertions(+), 823 deletions(-) diff --git a/dv b/dv index 561c8a2af..5de8011be 160000 --- a/dv +++ b/dv @@ -1 +1 @@ -Subproject commit 561c8a2afe325a9ee02229e65c5479bea1bb5bbe +Subproject commit 5de8011be6322698f986be02de2858af8756fc59 diff --git a/ostd/specs/lib.rs b/ostd/specs/lib.rs index f316bf901..c66f9100f 100644 --- a/ostd/specs/lib.rs +++ b/ostd/specs/lib.rs @@ -11,7 +11,7 @@ pub mod arch; pub mod mm; #[allow(unused_parens)] #[allow(unused_braces)] -mod sync; +pub mod sync; #[allow(unused_parens)] #[allow(unused_braces)] pub mod task; diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index eca2d8736..06024b48d 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -12,15 +12,84 @@ //! The module is intentionally proof-only for now. The executable RCU //! implementation should later connect its real guard/token state to these //! abstract ghost tokens. +use super::weak_memory::{History, Msg, WeakAtomicInvariantPredicate}; use vstd::prelude::*; use vstd::resource::Loc; verus! { +/// Workaround for supporting `dyn Fn() + 'static + Send`. +/// +/// A closure basically is just a function pointer +/// along with the pointer to the captured context. +#[verifier::external_body] +pub struct RawRcuCallback { + data: *mut (), + run: unsafe fn (*mut ()), +} + pub type LinkIndex = nat; pub type LinkEdge = (*mut T, LinkIndex); +/// The weak-memory invariant for the root pointer stored in an executable RCU +/// cell. +/// +/// The key is the cell's nullability: `true` for `RcuOption`, `false` for +/// `Rcu`. At this layer we only connect the atomic message history to the +/// public nullability contract. Ownership, read tokens, and reclamation are +/// deliberately modeled by the traversal/reclaim tokens below and will be wired +/// into this predicate in later steps. +pub struct RcuWeakAtomicInv; + +pub open spec fn rcu_history_inv(nullable: bool, history: History<*mut T>) -> bool { + &&& history.len() >= 1 + &&& !nullable ==> forall|i: int| + 0 <= i < history.len() ==> #[trigger] history[i].value.addr() != 0 +} + +impl WeakAtomicInvariantPredicate for RcuWeakAtomicInv { + open spec fn atomic_inv(nullable: bool, history: History<*mut T>, _g: ()) -> bool { + rcu_history_inv(nullable, history) + } +} + +pub proof fn preserve_rcu_history_inv_on_push( + nullable: bool, + prev: History<*mut T>, + next: History<*mut T>, + msg: Msg<*mut T>, +) + requires + rcu_history_inv(nullable, prev), + next == prev.push(msg), + nullable || msg.value.addr() != 0, + ensures + rcu_history_inv(nullable, next), +{ + assert(next.len() >= 1); + if !nullable { + assert forall|i: int| 0 <= i < next.len() implies #[trigger] next[i].value.addr() != 0 by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + } + }; + } +} + +pub proof fn rcu_history_inv_read_nonnull(history: History<*mut T>, ts: nat) + requires + rcu_history_inv(false, history), + ts < history.len(), + ensures + history[ts as int].value.addr() != 0, + !history[ts as int].value.is_null(), +{ + assert(history[ts as int].value.addr() != 0); +} + /// Link view carried by an RCU read-side guard. /// /// `seen_at(p) = n` means the guard has observed link-history events for source diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 5c615e1a9..390d80d96 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -15,8 +15,8 @@ //! //! - [RCU Verification](https://dl.acm.org/doi/pdf/10.1145/3729246) use core::sync::atomic::{ - AtomicBool, AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, - AtomicU8, AtomicUsize, Ordering, + AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, + AtomicU32, AtomicUsize, Ordering, }; #[cfg(target_has_atomic = "64")] @@ -24,8 +24,8 @@ use core::sync::atomic::{AtomicI64, AtomicU64}; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::prelude::*; -use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; use vstd::resource::Loc; +use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; use vstd::seq::Seq; verus! { @@ -328,6 +328,7 @@ macro_rules! declare_weak_atomic_type { self.atomic_inv@.constant().1 == self.atomic.id() } + #[verifier::type_invariant] pub closed spec fn type_inv(&self) -> bool { self.well_formed() } @@ -345,7 +346,6 @@ macro_rules! declare_weak_atomic_type { requires Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), ensures - res.well_formed(), res.constant() == k, { let (atomic, Tracked(hist)) = $raw_atomic::new(init); @@ -359,10 +359,7 @@ macro_rules! declare_weak_atomic_type { pub fn load_relaxed( &self, Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - requires - self.well_formed(), - { + ) -> (res: ($value_ty, Ghost)) { let result; vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; @@ -378,10 +375,7 @@ macro_rules! declare_weak_atomic_type { pub fn load_acquire( &self, Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - requires - self.well_formed(), - { + ) -> (res: ($value_ty, Ghost)) { let result; vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; @@ -463,6 +457,7 @@ impl WeakAtomicPtr { self.atomic_inv@.constant().1 == self.atomic.id() } + #[verifier::type_invariant] pub closed spec fn type_inv(&self) -> bool { self.well_formed() } @@ -490,10 +485,7 @@ impl WeakAtomicPtr where pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( *mut T, Ghost, - )) - requires - self.well_formed(), - { + )) { let result; vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; @@ -509,10 +501,7 @@ impl WeakAtomicPtr where pub fn load_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( *mut T, Ghost, - )) - requires - self.well_formed(), - { + )) { let result; vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; @@ -533,6 +522,49 @@ impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { } } +impl WeakAtomicPtr { + // TODO: Move exec code into, `vstd_extra`? + /// Release-store helper for users with the trivial atomic invariant. + /// + /// This keeps early weak-memory clients from depending on the macro while + /// we are still shaping the client-specific ghost state. + #[inline(always)] + pub fn store_release_simple(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) { + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, g) = pair; + let _snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); + proof { + pair = (hist, g); + } + }); + } + + /// Strong AcqRel/Acquire CAS helper for users with the trivial invariant. + #[inline(always)] + pub fn compare_exchange_acqrel_acquire_simple( + &self, + current: *mut T, + new: *mut T, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (Result<*mut T, *mut T>, Ghost)) { + let result; + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, g) = pair; + let cas_result = self.atomic.compare_exchange_acqrel_acquire( + Tracked(&mut hist), + Tracked(tv), + current, + new, + ); + result = (cas_result.0, cas_result.1); + proof { + pair = (hist, g); + } + }); + result + } +} + /// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, /// but for weak-memory atomics with per-thread view tokens and message histories. /// diff --git a/ostd/src/sync/mod.rs b/ostd/src/sync/mod.rs index cd33ecf21..0e84af234 100644 --- a/ostd/src/sync/mod.rs +++ b/ostd/src/sync/mod.rs @@ -11,20 +11,19 @@ mod rwmutex; mod spin; mod wait; -//pub(crate) use self::rcu::finish_grace_period; +pub(crate) use self::rcu::finish_grace_period; pub use self::{ atomic_data::*, guard::{GuardTransfer, LocalIrqDisabled, PreemptDisabled, SpinGuardian, /*WriteIrqDisabled*/}, mutex::{Mutex, MutexGuard}, once::{Once, OnceImpl, TrivialPred}, - rcu::{non_null /*, Rcu, RcuDrop, RcuOption, RcuOptionReadGuard, RcuReadGuard*/}, + rcu::{Rcu, RcuDrop, RcuOption, RcuOptionReadGuard, RcuReadGuard, non_null}, rwarc::{RoArc, RwArc}, rwlock::{RwLock, RwLockReadGuard, RwLockUpgradeableGuard, RwLockWriteGuard}, rwmutex::{RwMutex, RwMutexReadGuard, RwMutexUpgradeableGuard, RwMutexWriteGuard}, spin::{SpinLock, SpinLockGuard}, wait::{WaitQueue, Waiter, Waker}, }; -/* pub(crate) fn init() { rcu::init(); -}*/ +} diff --git a/ostd/src/sync/rcu/__mod.rs b/ostd/src/sync/rcu/__mod.rs index f0c4e2db1..e5b454780 100644 --- a/ostd/src/sync/rcu/__mod.rs +++ b/ostd/src/sync/rcu/__mod.rs @@ -32,9 +32,9 @@ use super::Once; use self::monitor::{RcuMonitor, RcuMonitorOwner, RcuMonitorPred}; use crate::task::{ + DisabledPreemptGuard, //atomic_mode::{AsAtomicModeGuard, InAtomicMode}, disable_preempt, - DisabledPreemptGuard, }; mod monitor; diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index e5b454780..d909e57b0 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -1,735 +1,331 @@ // SPDX-License-Identifier: MPL-2.0 //! Read-copy update (RCU). //! -//! # Note -//! -//! Currently this RCU model assumes a sequential consistency (SC) memory model. -//! We may explore weak memory models in the future. -use vstd::{ - atomic_ghost::AtomicPtr, atomic_with_ghost, map::Map, modes::tracked_static_ref, prelude::*, - resource::Loc, -}; - -use vstd_extra::{ - prelude::*, - resource::ghost_resource::{count::Count, tokens::CountResource}, -}; - -use core::{ - marker::PhantomData, mem::ManuallyDrop, ops::Deref, - ptr::NonNull, - /* - sync::atomic::{ - AtomicPtr, - Ordering::{AcqRel, Acquire}, +//! This is the new weak-memory RCU skeleton. The previous SC proof-oriented +//! implementation is kept in `__mod.rs` as reference and is not compiled. +use alloc::boxed::Box; +use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; + +use vstd::prelude::*; +use vstd_extra::prelude::*; + +use crate::{ + specs::{ + sync::{ + rcu as rcu_spec, + weak_memory::{ThreadView, WeakAtomicPtr}, }, - */ + task::InAtomicMode, + }, + task::{DisabledPreemptGuard, disable_preempt}, }; use non_null::{NonNullPtr, NonNullPtrRef}; -// use spin::once::Once; -use super::Once; - -use self::monitor::{RcuMonitor, RcuMonitorOwner, RcuMonitorPred}; -use crate::task::{ - DisabledPreemptGuard, - //atomic_mode::{AsAtomicModeGuard, InAtomicMode}, - disable_preempt, -}; -mod monitor; pub mod non_null; -use crate::specs::task::InAtomicMode; - verus! { broadcast use vstd_extra::external::nonnull::group_nonull_axioms; -// Verification-only budget for splitting read-side ghost tokens. -// -// This is not a runtime reader counter and does not model an overflow condition -// in the RCU implementation. It is a temporary bounded approximation needed by -// `CountResource`; the final RCU proof should discharge the admission assumption -// with an unbounded ghost registry or a CPU/epoch-based sharding model. - -const RCU_READER_SLOTS: u64 = 1u64 << 60; - -type RcuReadPool

= CountResource<

::Permission, RCU_READER_SLOTS>; -type RcuReadToken

= Count<

::Permission, RCU_READER_SLOTS>; - -type RcuRetiredEntry

= (Ghost<*mut

::Target>, RcuReadPool

); - -/// Called by `drop` of the read guard to track the retired read permissions. -type RcuRetiredPools

= Map>; - -type RcuReturnedTokens

= Map>; - -tracked struct RcuPtrGhost { - tracked current: Option>, - tracked retired: RcuRetiredPools

, - tracked returned: RcuReturnedTokens

, -} - -closed spec fn retired_pools_inv(retired: RcuRetiredPools

) -> bool { - forall|id: Loc| #[trigger] - retired.contains_key(id) ==> { - let entry = retired[id]; - &&& !(entry.0@).is_null() - &&& entry.1.id() == id - &&& P::ptr_perm_match(entry.0@, entry.1@) - &&& entry.1@.inv() - &&& entry.1.wf() - &&& entry.1.not_empty() - } -} - -closed spec fn returned_tokens_inv(returned: RcuReturnedTokens

) -> bool { - forall|id: Loc| #[trigger] - returned.contains_key(id) ==> { - let token = returned[id]; - &&& token.id() == id - &&& token.resource().inv() - &&& token.frac() > 0 - } -} - -/// A Read-Copy Update (RCU) cell for sharing a pointer between threads. -/// -/// The pointer should be a non-null pointer with type `P`, which implements -/// [`NonNullPtr`]. For example, `P` can be `Box` or `Arc`. -/// -/// # Overview -/// -/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high- -/// performance, low-latency read operations in concurrent systems. It allows -/// multiple readers to access shared data simultaneously without contention, -/// while writers can update the data safely in a way that does not disrupt -/// ongoing reads. RCU is particularly suited for situations where reads are -/// far more frequent than writes. -/// -/// The original design and implementation of RCU is described in paper _The -/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared- -/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal -/// 47.2 (2008). +/// The weak-memory atomic slot used by RCU. /// -/// # Examples -/// -/// ``` -/// use ostd::sync::Rcu; -/// -/// let rcu = Rcu::new(Box::new(42)); -/// -/// let rcu_guard = rcu.read(); -/// -/// assert_eq!(*rcu_guard, Some(&42)); -/// -/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); -/// -/// let rcu_guard = rcu.read(); -/// -/// assert_eq!(*rcu_guard, Some(&43)); -/// ``` +/// `bool` is the constant key: `true` means the public cell may contain null +/// (`RcuOption`), and `false` means the public cell is non-null (`Rcu`). The +/// ghost state is still empty for this cut, but the predicate is now +/// RCU-specific: non-null `Rcu` cells require every atomic-history message to +/// contain a non-null pointer. Later revisions should extend the ghost state to +/// carry read permissions, retired pools, and traversal snapshots. +type RcuAtomicPtr

= WeakAtomicPtr< +

::Target, + bool, + (), + rcu_spec::RcuWeakAtomicInv, +>; + +/// A Read-Copy Update cell for sharing a non-null pointer. pub struct Rcu(RcuInner

); -/// A guard that allows access to the pointed data protected by a [`Rcu`]. +/// A read-side guard for [`Rcu`]. #[clippy::has_significant_drop] #[must_use] pub struct RcuReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); -/// A Read-Copy Update (RCU) cell for sharing a _ghost_nullable_ pointer. -/// -/// This is a variant of [`Rcu`] that allows the contained pointer to be null. -/// So that it can implement `Rcu>` where `P` is not a ghost_nullable -/// pointer. It is the same as [`Rcu`] in other aspects. -/// -/// # Examples -/// -/// ``` -/// use ostd::sync::RcuOption; -/// -/// static RCU: RcuOption> = RcuOption::new_none(); -/// -/// assert!(RCU.read().is_none()); -/// -/// RCU.update(Box::new(42)); -/// -/// // Read the data protected by RCU -/// { -/// let rcu_guard = RCU.read().try_get().unwrap(); -/// assert_eq!(*rcu_guard, 42); -/// } -/// -/// // Update the data protected by RCU -/// { -/// let rcu_guard = RCU.read().try_get().unwrap(); -/// -/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); -/// -/// let rcu_guard = RCU.read().try_get().unwrap(); -/// assert_eq!(*rcu_guard, 43); -/// } -/// ``` +/// A Read-Copy Update cell that may contain null. pub struct RcuOption(RcuInner

); -/// A guard that allows access to the pointed data protected by a [`RcuOption`]. +/// A read-side guard for [`RcuOption`]. #[clippy::has_significant_drop] #[must_use] pub struct RcuOptionReadGuard<'a, P: NonNullPtr>(RcuReadGuardInner<'a, P>); -struct_with_invariants! { -/// The inner implementation of both [`Rcu`] and [`RcuOption`]. -struct RcuInner { - ptr: AtomicPtr<

::Target,_,RcuPtrGhost

,_>, - // We want to implement Send and Sync explicitly. - // Having a pointer field prevents them from being implemented - // automatically by the compiler. - _marker: PhantomData<*const

::Target>, +pub struct RcuInner { + ptr: RcuAtomicPtr

, ghost_nullable: Ghost, + _marker: PhantomData<*const

::Target>, } -closed spec fn wf(self) -> bool { - invariant on ptr with (ghost_nullable, _marker) is ( - v: *mut

::Target, - g: RcuPtrGhost

, - ) { - &&& retired_pools_inv::

(g.retired) - &&& returned_tokens_inv::

(g.returned) - &&& match g.current { - Some(perm) => { - &&& !v.is_null() - &&& P::ptr_perm_match(v, perm@) - &&& perm@.inv() - &&& perm.wf() - &&& perm.not_empty() - }, - None => ghost_nullable@ && v.is_null(), - } - } -} +struct RcuReadGuardInner<'a, P: NonNullPtr> { + obj_ptr: *mut

::Target, + rcu: &'a RcuInner

, + _inner_guard: DisabledPreemptGuard, + /// Thread-local weak-memory view for this read-side critical section. + /// + /// For now this only records the load/CAS view effects of the RCU pointer + /// itself. Traversal-safe RCU should later expose this to the paper-style + /// `SeenRemoved(D, LV)` proof layer in `specs::sync::rcu`. + view: Tracked, } -// SAFETY: It is apparent that if `P` is `Send`, then `Rcu

` is `Send`. +impl RcuInner

{ + closed spec fn is_nullable(self) -> bool { + self.ghost_nullable@ + } + + closed spec fn wf(self) -> bool { + &&& self.ptr.well_formed() + &&& self.ptr.constant() == self.ghost_nullable@ + } +} +// SAFETY: `RcuInner` only shares a raw pointer through an atomic slot. Sending +// the cell follows the same requirement as sending the managed pointer wrapper. #[verifier::external] unsafe impl Send for RcuInner

where P: Send { } -// SAFETY: To implement `Sync` for `Rcu

`, we need to meet two conditions: -// 1. `P` must be `Sync` because `Rcu::get` allows concurrent access. -// 2. `P` must be `Send` because `Rcu::update` may obtain an object -// of `P` created on another thread. +// SAFETY: Readers may obtain shared references, so `P` must be `Sync`; writers +// may install pointers created on another thread, so `P` must be `Send`. #[verifier::external] unsafe impl Sync for RcuInner

where P: Send + Sync { } -impl RcuInner

{ - /// Whether the contained pointer can be null. Used to distinguish `Rcu` and `RcuOption`. - pub closed spec fn is_nullable(self) -> bool { - self.ghost_nullable@ - } -} - #[verus_verify] impl RcuInner

{ #[inline(always)] const fn new_none() -> (res: Self) ensures + res.type_inv(), res.is_nullable(), - res.wf(), { - proof_decl! { - let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { - current: None, - retired: Map::tracked_empty(), - returned: Map::tracked_empty(), - }; - } + let ptr = WeakAtomicPtr::new(Ghost(true), core::ptr::null_mut(), Tracked(())); Self { - ptr: AtomicPtr::new( - Ghost((Ghost(true), PhantomData::<*const

::Target>)), - core::ptr::null_mut(), - Tracked(ptr_ghost), - ), - _marker: PhantomData::<*const

::Target>, + ptr, ghost_nullable: Ghost(true), + _marker: PhantomData::<*const

::Target>, } } - /// Creates a new RCU primitive with the given pointer `pointer`. #[inline(always)] - #[verus_spec(r => + #[verus_spec(res => with - Ghost(ghost_nullable): Ghost, + Ghost(nullable): Ghost, ensures - r.type_inv(), - r.is_nullable() == ghost_nullable, + res.type_inv(), + res.is_nullable() == nullable, )] fn new(pointer: P) -> Self { - // let ptr =

::into_raw(pointer).as_ptr(); - let (ptr, Tracked(ptr_perm)) =

::into_raw(pointer); - let ptr = ptr.as_ptr(); - proof_decl! { - let tracked ptr_ghost: RcuPtrGhost

= RcuPtrGhost { - current: Some(CountResource::alloc(ptr_perm)), - retired: Map::tracked_empty(), - returned: Map::tracked_empty(), - }; + let (raw, Tracked(_perm)) = P::into_raw(pointer); + let raw_ptr = raw.as_ptr(); + proof { + assert(!raw_ptr.is_null()); } - - let ptr = AtomicPtr::new( - Ghost((Ghost(ghost_nullable), PhantomData)), + let ptr = WeakAtomicPtr::new(Ghost(nullable), raw_ptr, Tracked(())); + Self { ptr, - Tracked(ptr_ghost), - ); - Self { ptr, _marker: PhantomData, ghost_nullable: Ghost(ghost_nullable) } + ghost_nullable: Ghost(nullable), + _marker: PhantomData::<*const

::Target>, + } + } + + #[inline(always)] + fn load_ptr_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: + *mut

::Target) + requires + self.type_inv(), + ensures + !self.is_nullable() ==> !res.is_null(), + { + proof { + assert(self.ptr.constant() == self.is_nullable()); + } + let res = self.ptr.load_acquire_rcu(Tracked(tv)); + proof { + if !self.is_nullable() { + assert(!self.ptr.constant()); + assert(!res.0.is_null()); + } + } + res.0 + } + + #[inline(always)] + fn store_ptr_release( + &self, + new_ptr: *mut

::Target, + Tracked(tv): Tracked<&mut ThreadView>, + ) + requires + self.type_inv(), + self.is_nullable() || !new_ptr.is_null(), + { + proof { + assert(self.ptr.constant() == self.is_nullable()); + assert(self.ptr.constant() || !new_ptr.is_null()); + } + self.ptr.store_release_rcu(new_ptr, Tracked(tv)); } #[verus_spec( requires + self.type_inv(), self.is_nullable() || new_ptr is Some, )] fn update(&self, new_ptr: Option

) { - let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { - //

::into_raw(new_ptr).as_ptr() - let (ptr, Tracked(perm)) =

::into_raw(new_ptr); - let ptr = ptr.as_ptr(); - (ptr, Tracked(Some(perm))) + proof_decl! { + let ghost new_ptr_is_some = new_ptr is Some; + } + let (raw, Tracked(_perm)) = if let Some(new_ptr) = new_ptr { + let (ptr, Tracked(perm)) = P::into_raw(new_ptr); + (ptr.as_ptr(), Tracked(Some(perm))) } else { (core::ptr::null_mut(), Tracked(None)) }; proof_decl! { - let tracked mut old_perm: Option> = None; + let tracked mut tv = ThreadView::new(); } proof { - use_type_invariant(self); - } - let old_raw_ptr = - atomic_with_ghost! { - self.ptr => swap(new_ptr); - update prev -> next; - ghost g => { - old_perm = g.current; - if old_perm is Some { - let tracked mut pool = old_perm.tracked_unwrap(); - let ghost id = pool.id(); - if g.retired.contains_key(id) { - // Use tracked_borrow_mut instead - let tracked entry = g.retired.tracked_remove(id); - let tracked mut retired_pool = entry.1; - let tracked pool_token = pool.split(pool.frac()); - retired_pool.validate_with_frac(&pool_token); - retired_pool.combine(pool_token); - g.retired.tracked_insert(id, (entry.0, retired_pool)); - } else { - g.retired.tracked_insert(id, (Ghost(prev), pool)); - } - } - g.current = match new_perm { - Some(perm) => Some(CountResource::alloc(perm)), - None => None, - }; - assert(retired_pools_inv::

(g.retired)); + if !self.is_nullable() { + assert(new_ptr_is_some); } - }; - - if let Some(p) = NonNull::new(old_raw_ptr) { - // SAFETY: - // 1. The pointer was previously returned by `into_raw`. - // 2. The pointer is removed from the RCU slot so that no one will - // use it after the end of the current grace period. The removal - // is done atomically, so it will only be dropped once. - // unsafe { delay_drop::

(p) }; - } - } - - #[verus_spec(obj_ptr => - with - -> tracked_ref_perm: Tracked>>, - ensures - !self.is_nullable() ==> tracked_ref_perm@ is Some, - match tracked_ref_perm@ { - Some(perm) => { - &&& !obj_ptr.is_null() - &&& P::ptr_perm_match(obj_ptr, perm.resource()) - &&& perm.resource().inv() - &&& perm.frac() == 1 - }, - None => obj_ptr.is_null(), - }, - )] - fn load_read_token(&self) -> *mut

::Target { - proof_decl! { - let tracked mut tracked_ref_perm: Option> = None; - } - proof { - use_type_invariant(self); + assert(self.is_nullable() || !raw.is_null()); } - let obj_ptr = - atomic_with_ghost! { - self.ptr => load(); - update prev -> _next; - returning loaded; - ghost g => { - if g.current is Some { - let tracked mut perm = g.current.tracked_unwrap(); - assert(loaded == prev); - assert(!loaded.is_null()); - assert(P::ptr_perm_match(loaded, perm@)); - assert(perm@.inv()); - let ghost perm_snapshot = perm@; - - // Verification-only admission for the bounded read-token pool. - // This is not a runtime reader limit; it only reflects that - // `CountResource` uses a Rust const-generic `u64` budget rather - // than an unbounded mathematical `nat`. - assume(perm.not_empty()); - assume(1 < perm.frac()); - let tracked token = perm.split_one(); - assert(perm@ == perm_snapshot); - assert(token.frac() == 1); - tracked_ref_perm = Some(token); - g.current = Some(perm); - } else { - } - assert(retired_pools_inv::

(g.retired)); - } - }; - proof_with! { |= Tracked(tracked_ref_perm) } - obj_ptr + self.store_ptr_release(raw, Tracked(&mut tv)); } - #[verus_spec(r => + #[verus_spec(res => + requires + self.type_inv(), ensures - r.type_inv(), - r.rcu.is_nullable() == self.is_nullable(), - !self.is_nullable() ==> r.tracked_ref_perm@ is Some, + res.type_inv(), + res.rcu.is_nullable() == self.is_nullable(), )] fn read(&self) -> RcuReadGuardInner<'_, P> { - let guard = disable_preempt(); + let inner_guard = disable_preempt(); proof_decl! { - let tracked mut tracked_ref_perm: Option> = None; - } - let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] - self.load_read_token(); - RcuReadGuardInner { - obj_ptr, - rcu: self, - _inner_guard: guard, - tracked_ref_perm: Tracked(tracked_ref_perm), + let tracked mut tv = ThreadView::new(); } + let obj_ptr = self.load_ptr_acquire(Tracked(&mut tv)); + RcuReadGuardInner { obj_ptr, rcu: self, _inner_guard: inner_guard, view: Tracked(tv) } } - #[verus_spec] - pub fn read_with<'a, A: InAtomicMode>( - &'a self, - _guard: &'a A, // &'a dyn InAtomicMode is not well-supported in Verus. - ) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { - proof_decl! { - let tracked mut tracked_ref_perm: Option> = None; - } - let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] - self.load_read_token(); - if obj_ptr.is_null() { - return None; - } + #[inline] + #[verus_spec( + requires + self.type_inv(), + )] + pub fn read_with<'a, A: InAtomicMode>(&'a self, _guard: &'a A) -> Option< +

>::Ref, + > where P: NonNullPtrRef<'a> { proof_decl! { - // `read_with` returns only the reference and has no guard object to - // store the read token. For this temporary skeleton, leak the - // verification-only token so the returned ref can borrow it for - // `'a`. The final RCU proof should attach this token to the - // atomic-mode/CPU epoch state instead. - let tracked tracked_ref_perm = tracked_ref_perm.tracked_unwrap(); - let tracked tracked_ref_perm = tracked_static_ref(tracked_ref_perm); - let tracked tracked_ref_perm:

>::RefPermission = - P::borrow_perm_as_ref_perm(tracked_ref_perm.borrow()); - } - // SAFETY: - // 1. This pointer is not NULL. - // 2. The `_guard` guarantees atomic mode for the duration of lifetime - // `'a`, the pointer is valid because other writers won't release the - // allocation until this task passes the quiescent state. - NonNull::new(obj_ptr).map( - |ptr| - requires - P::ptr_perm_match( - ptr.view_ptr_mut(), - P::ref_perm_view_permission(tracked_ref_perm), - ), - { - unsafe { P::raw_as_ref(ptr, Tracked(tracked_ref_perm)) } - }, - ) - } -} - -/* -impl Drop for RcuInner

{ - fn drop(&mut self) { - let ptr = self.ptr.load(Acquire); - if let Some(p) = NonNull::new(ptr) { - // SAFETY: It was previously returned by `into_raw` when creating - // the RCU primitive. - let pointer = unsafe {

::from_raw(p) }; - // It is OK not to delay the drop because the RCU primitive is - // owned by nobody else. - drop(pointer); + let tracked mut tv = ThreadView::new(); } + let obj_ptr = self.load_ptr_acquire(Tracked(&mut tv)); + NonNull::new(obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }) } } -*/ - -/// The inner implementation of both [`RcuReadGuard`] and [`RcuOptionReadGuard`]. -struct RcuReadGuardInner<'a, P: NonNullPtr> { - obj_ptr: *mut

::Target, - rcu: &'a RcuInner

, - _inner_guard: DisabledPreemptGuard, - tracked_ref_perm: Tracked>>, -} #[verus_verify] impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { #[inline] - #[verus_spec(r => + #[verus_spec(res => ensures - self.tracked_ref_perm@ is Some ==> r is Some, + !self.rcu.is_nullable() ==> res is Some, )] fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> { + let res = NonNull::new(self.obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }); proof { use_type_invariant(self); + if !self.rcu.is_nullable() { + assert(!self.obj_ptr.is_null()); + assert(res is Some); + } } - - // SAFETY: The guard ensures that `P` will not be dropped. Thus, `P` - // outlives the lifetime of `&self`. Additionally, during this period, - // it is impossible to create a mutable reference to `P`. - NonNull::new(self.obj_ptr).map( - |ptr| - requires - self.tracked_ref_perm@ is Some, - P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_ref_perm->0.resource()), - { - unsafe { - P::raw_as_ref( - ptr, - Tracked( - P::borrow_perm_as_ref_perm( - self.tracked_ref_perm.tracked_borrow().borrow(), - ), - ), - ) - } - }, - ) + res } - #[verus_spec(r => + #[verus_spec(res => requires self.rcu.is_nullable() || new_ptr is Some, ensures - new_ptr is Some && r is Err ==> r->Err_0 is Some, + new_ptr is Some && res is Err ==> res->Err_0 is Some, )] fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { - let obj_ptr = self.obj_ptr; + let expected = self.obj_ptr; + let rcu = self.rcu; + proof { use_type_invariant(&self); - use_type_invariant(self.rcu); } + proof_decl! { - let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + let tracked mut tv = self.view.get(); let ghost new_ptr_is_some = new_ptr is Some; - let tracked mut old_perm: Option> = None; - let tracked mut err_new_perm: Option::Permission>> = None; } - let (new_ptr, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { - //

::into_raw(new_ptr).as_ptr() - let (ptr, Tracked(perm)) =

::into_raw(new_ptr); + + let (new_raw, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { + let (ptr, Tracked(perm)) = P::into_raw(new_ptr); (ptr.as_ptr(), Tracked(Some(perm))) } else { (core::ptr::null_mut(), Tracked(None)) }; - let res = - atomic_with_ghost! { - self.rcu.ptr => compare_exchange(obj_ptr, new_ptr); - update _prev -> next; - returning res; - ghost g => { - if res is Ok { - old_perm = g.current; - if old_perm is Some { - let tracked mut pool = old_perm.tracked_unwrap(); - let ghost id = pool.id(); - if g.retired.contains_key(id) { - // use tracked_borrow_mut instead - let tracked entry = g.retired.tracked_remove(id); - let tracked mut retired_pool = entry.1; - let tracked pool_token = pool.split(pool.frac()); - retired_pool.validate_with_frac(&pool_token); - retired_pool.combine(pool_token); - g.retired.tracked_insert(id, (entry.0, retired_pool)); - } else { - g.retired.tracked_insert(id, (Ghost(_prev), pool)); - } - } - g.current = match new_perm { - Some(perm) => Some(CountResource::alloc(perm)), - None => None, - }; - } else { - err_new_perm = Some(new_perm); - } - if tracked_ref_perm is Some { - let tracked token = tracked_ref_perm.tracked_unwrap(); - let ghost id = token.id(); - if g.retired.contains_key(id) { - let tracked entry = g.retired.tracked_remove(id); - let tracked mut pool = entry.1; - pool.combine(token); - g.retired.tracked_insert(id, (entry.0, pool)); - } else if g.current is Some { - let tracked mut pool = g.current.tracked_unwrap(); - if pool.id() == id { - pool.combine(token); - g.current = Some(pool); - } else { - g.current = Some(pool); - assume(false); - } - } else { - assume(false); - } - } - assert(retired_pools_inv::

(g.retired)); - } - }; - if res.is_ok() { - if let Some(p) = NonNull::new(obj_ptr) { - // SAFETY: - // 1. The pointer was previously returned by `into_raw`. - // 2. The pointer is removed from the RCU slot so that no one will - // use it after the end of the current grace period. The removal - // is done atomically, so it will only be dropped once. - // unsafe { delay_drop::

(p) }; - } - Ok(()) - } else { - let Some(new_nonnull) = NonNull::new(new_ptr) else { - return Err(None); - }; - proof_decl! { - let tracked new_perm = err_new_perm.tracked_unwrap().tracked_unwrap(); + proof { + if !rcu.is_nullable() { + assert(new_ptr_is_some); } - // SAFETY: - // 1. It was previously returned by `into_raw`. - // 2. The `compare_exchange` fails so the pointer will not - // be used by other threads via reading the RCU primitive. - Err(Some(unsafe {

::from_raw(new_nonnull, Tracked(new_perm)) })) + assert(rcu.is_nullable() || !new_raw.is_null()); } - } - /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. - #[inline] - #[verus_spec] - fn drop(self) { - let rcu = self.rcu; - let obj_ptr = self.obj_ptr; proof { - use_type_invariant(&self); - use_type_invariant(rcu); - } - proof_decl! { - let tracked mut tracked_ref_perm = self.tracked_ref_perm.get(); + assert(rcu.ptr.constant() == rcu.is_nullable()); + assert(rcu.ptr.constant() || !new_raw.is_null()); } - atomic_with_ghost! { - rcu.ptr => load(); - update prev -> _next; - returning _loaded; - ghost g => { - if tracked_ref_perm is Some { - let tracked token = tracked_ref_perm.tracked_unwrap(); - let ghost id = token.id(); - if g.current is Some { - let tracked mut pool = g.current.tracked_unwrap(); - if prev == obj_ptr && pool.id() == id { - assert(!obj_ptr.is_null()); - assert(P::ptr_perm_match(obj_ptr, token.resource())); - assert(token.resource().inv()); - assert(token.frac() == 1); - assert(P::ptr_perm_match(prev, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - pool.combine(token); - assert(P::ptr_perm_match(prev, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - g.current = Some(pool); - } else { - g.current = Some(pool); - if g.retired.contains_key(id) { - let tracked entry = g.retired.tracked_remove(id); - let tracked mut pool = entry.1; - assert(pool.id() == id); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - pool.combine(token); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - g.retired.tracked_insert(id, (entry.0, pool)); - } else { - assume(false); - } - } - } else { - if g.retired.contains_key(id) { - let tracked entry = g.retired.tracked_remove(id); - let tracked mut pool = entry.1; - assert(pool.id() == id); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - pool.combine(token); - assert(P::ptr_perm_match(entry.0@, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - g.retired.tracked_insert(id, (entry.0, pool)); - } else { - assume(false); - } - } - } - match &g.current { - Some(pool) => { - assert(!prev.is_null()); - assert(P::ptr_perm_match(prev, pool@)); - assert(pool@.inv()); - assert(pool.wf()); - assert(pool.not_empty()); - }, - None => { - assert(rcu.ghost_nullable@); - assert(prev.is_null()); - }, + let res = rcu.ptr.compare_exchange_acqrel_acquire_rcu(expected, new_raw, Tracked(&mut tv)); + + match res.0 { + Result::Ok(_) => Ok(()), + Result::Err(_) => { + let Some(new_nonnull) = NonNull::new(new_raw) else { + return Err(None); + }; + proof_decl! { + let tracked perm = new_perm.tracked_unwrap(); } - assert(retired_pools_inv::

(g.retired)); - } - }; + Err(Some(unsafe { P::from_raw(new_nonnull, Tracked(perm)) })) + }, + } + } + + #[inline] + fn drop(self) { + } +} + +#[verifier::external_body] +unsafe fn assume_shared_ref<'a, P: NonNullPtrRef<'a>>(ptr: NonNull) -> P::Ref { + proof_decl! { + let tracked perm: P::RefPermission = Tracked::::assume_new().get(); } + unsafe { P::raw_as_ref(ptr, Tracked(perm)) } } #[verus_verify] impl Rcu

{ - /// Creates a new RCU primitive with the given pointer `pointer`. - #[verus_spec] + /// Creates a new RCU primitive with the given pointer. + #[inline] pub fn new(pointer: P) -> Self { Self( #[verus_spec(with Ghost(false))] @@ -737,24 +333,16 @@ impl Rcu

{ ) } - /// Replaces the current pointer with a null pointer. - /// - /// This function updates the pointer to the new pointer regardless of the - /// original pointer. The original pointer will be dropped after the grace - /// period. - /// - /// Oftentimes this function is not recommended unless you have serialized - /// writes with locks. Otherwise, you can use [`Self::read`] and then - /// [`RcuReadGuard::compare_exchange`] to update the pointer. + /// Replaces the current pointer with `new_ptr` using a release store. #[inline] pub fn update(&self, new_ptr: P) { + proof { + use_type_invariant(self); + } self.0.update(Some(new_ptr)); } - /// Retrieves a read guard for the RCU primitive. - /// - /// The guard allows read access to the data protected by RCU, as well - /// as the ability to do compare-and-exchange. + /// Starts a read-side critical section and acquires the current pointer. #[inline] pub fn read(&self) -> RcuReadGuard<'_, P> { proof { @@ -762,19 +350,12 @@ impl Rcu

{ } RcuReadGuard(self.0.read()) } - // #[inline] - // pub fn read_with<'a, G: AsAtomicModeGuard + ?Sized>(&'a self, guard: &'a G) -> P::Ref<'a> where - // P: NonNullPtrRef<'a>, - // { - // self.0.read_with(guard.as_atomic_mode_guard()).unwrap() - // } - } #[verus_verify] impl RcuOption

{ - /// Creates a new RCU primitive with the given pointer. - #[verus_spec] + /// Creates a nullable RCU primitive. + #[inline] pub fn new(pointer: Option

) -> Self { if let Some(pointer) = pointer { Self( @@ -786,23 +367,13 @@ impl RcuOption

{ } } - /// Creates a new RCU primitive that contains nothing. - /// - /// This is a constant equivalence to [`RcuOption::new(None)`]. + /// Creates an empty nullable RCU primitive. #[inline(always)] pub const fn new_none() -> Self { Self(RcuInner::new_none()) } - /// Replaces the current pointer with a null pointer. - /// - /// This function updates the pointer to the new pointer regardless of the - /// original pointer. If the original pointer is not NULL, it will be - /// dropped after the grace period. - /// - /// Oftentimes this function is not recommended unless you have - /// synchronized writes with locks. Otherwise, you can use [`Self::read`] - /// and then [`RcuOptionReadGuard::compare_exchange`] to update the pointer. + /// Replaces the current pointer using a release store. #[inline] pub fn update(&self, new_ptr: Option

) { proof { @@ -811,13 +382,7 @@ impl RcuOption

{ self.0.update(new_ptr); } - /// Retrieves a read guard for the RCU primitive. - /// - /// The guard allows read access to the data protected by RCU, as well - /// as the ability to do compare-and-exchange. - /// - /// The contained pointer can be NULL and you can only get a reference - /// (if checked non-NULL) via [`RcuOptionReadGuard::get`]. + /// Starts a read-side critical section and acquires the current pointer. #[inline] pub fn read(&self) -> RcuOptionReadGuard<'_, P> { proof { @@ -825,38 +390,35 @@ impl RcuOption

{ } RcuOptionReadGuard(self.0.read()) } + + #[inline] + pub fn read_with<'a, A: InAtomicMode>(&'a self, guard: &'a A) -> Option< +

>::Ref, + > where P: NonNullPtrRef<'a> { + proof { + use_type_invariant(self); + } + self.0.read_with(guard) + } } #[verus_verify] impl RcuReadGuard<'_, P> { - /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. #[inline] pub fn drop(self) { self.0.drop(); } - /// Gets the reference of the protected data. #[inline] pub fn get<'a>(&'a self) ->

>::Ref where P: NonNullPtrRef<'a> { proof { use_type_invariant(self); } - self.0.get().unwrap() + let res = self.0.get(); + res.unwrap() } - /// Tries to replace the already read pointer with a new pointer. - /// - /// If another thread has updated the pointer after the read, this - /// function will fail, and returns the given pointer back. Otherwise, - /// it will replace the pointer with the new one and drop the old pointer - /// after the grace period. - /// - /// If spinning on [`Rcu::read`] and this function, it is recommended - /// to relax the CPU or yield the task on failure. Otherwise contention - /// will occur. - /// - /// This API does not help to avoid - /// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem). + /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { self.0.compare_exchange(Some(new_ptr)).map_err( @@ -868,16 +430,8 @@ impl RcuReadGuard<'_, P> { } } -/* -impl AsAtomicModeGuard for RcuReadGuard<'_, P> { - fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { - self.0.inner_guard.as_atomic_mode_guard() - } -}*/ - #[verus_verify] impl RcuOptionReadGuard<'_, P> { - /// VERUS LIMITATION: We implement `drop` and call it manually because Verus's support for `Drop` is incomplete for now. #[inline] pub fn drop(self) { self.0.drop(); @@ -893,6 +447,7 @@ impl RcuOptionReadGuard<'_, P> { self.0.obj_ptr.is_null() } + /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] pub fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { proof { @@ -902,59 +457,86 @@ impl RcuOptionReadGuard<'_, P> { } } -/* -impl AsAtomicModeGuard for RcuOptionReadGuard<'_, P> { - fn as_atomic_mode_guard(&self) -> &dyn InAtomicMode { - self.0.inner_guard.as_atomic_mode_guard() +} // verus! +/// A one-shot, type-erased RCU callback. +/// +/// This is the small executable wrapper we use instead of `Box`. +/// `data` is a thin pointer to a sized heap payload, and `run` is the +/// monomorphized shim that knows how to consume that payload. The proof layer +/// treats the pair as a TCB boundary; later monitor proofs should reason about +/// which retired resource each callback represents, not about arbitrary closure +/// bodies. +#[must_use] +#[allow(dead_code)] +pub(crate) struct RawRcuCallback { + data: *mut (), + run: unsafe fn(*mut ()), +} + +struct RawRcuCallbackPayload { + context: C, + run: unsafe fn(C), +} + +// SAFETY: `RawRcuCallback::new` only accepts `C: Send + 'static` payloads, and +// `call_once` consumes the payload through the matching monomorphized runner. +unsafe impl Send for RawRcuCallback {} + +#[allow(dead_code)] +impl RawRcuCallback { + /// Builds a callback from an explicit captured context and a typed one-shot + /// runner. + #[inline] + pub(crate) fn new(context: C, run: unsafe fn(C)) -> Self { + let payload = Box::new(RawRcuCallbackPayload { context, run }); + Self { + data: Box::into_raw(payload).cast::<()>(), + run: run_raw_callback::, + } + } + + /// Builds the common "drop this value after the grace period" callback. + #[inline] + pub(crate) fn defer_drop(value: T) -> Self { + Self::new(value, drop_context::) + } + + /// Runs the callback exactly once. + /// + /// # Safety + /// + /// The caller must ensure this callback has not already been run and will + /// not be run again. This is the executable one-shot invariant that the + /// monitor queue will eventually own. + #[inline] + pub(crate) unsafe fn call_once(self) { + unsafe { + (self.run)(self.data); + } } } -*/ -/* -/// Delays the dropping of a [`NonNullPtr`] after the RCU grace period. -/// -/// This is internally needed for implementing [`Rcu`] and [`RcuOption`] -/// because we cannot alias a [`Box`]. Restoring `P` and use [`RcuDrop`] for it -/// can lead to multiple [`Box`]es simultaneously pointing to the same -/// content. -/// -/// # Safety -/// -/// The pointer must be previously returned by `into_raw`, will not be used -/// after the end of the current grace period, and will only be dropped once. -/// -/// [`Box`]: alloc::boxed::Box -unsafe fn delay_drop(pointer: NonNull<

::Target>) { - struct ForceSend(NonNull<

::Target>); - // SAFETY: Sending a raw pointer to another task is safe as long as - // the pointer access in another task is safe (guaranteed by the trait - // bound `P: Send`). - unsafe impl Send for ForceSend

{} - - let pointer: ForceSend

= ForceSend(pointer); - - let rcu_monitor = RCU_MONITOR.get().unwrap(); - rcu_monitor.after_grace_period(move || { - // This is necessary to make the Rust compiler to move the entire - // `ForceSend` structure into the closure. - let pointer = pointer; - - // SAFETY: - // 1. The pointer was previously returned by `into_raw`. - // 2. The pointer won't be used anymore since the grace period has - // finished and this is the only time the pointer gets dropped. - let p = unsafe {

::from_raw(pointer.0) }; - drop(p); - }); -} */ - -/// A wrapper to delay calling destructor of `T` after the RCU grace period. -/// -/// Upon dropping this structure, a callback will be registered to the global -/// RCU monitor and the destructor of `T` will be delayed until the callback. +#[inline] +unsafe fn run_raw_callback(data: *mut ()) { + let payload = unsafe { Box::from_raw(data.cast::>()) }; + let RawRcuCallbackPayload { context, run } = *payload; + unsafe { + run(context); + } +} + +#[inline] +fn drop_context(value: T) { + drop(value); +} + +verus! { + +/// A wrapper whose destructor will eventually be delayed until after an RCU +/// grace period. /// -/// [`RcuDrop`] is guaranteed to have the same layout as `T`. You can also -/// access the inner value safely via [`RcuDrop`]. +/// The delayed-drop path is deliberately not restored in this first weak-memory +/// cut; `__mod.rs` contains the old callback-monitor reference. #[repr(transparent)] #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct RcuDrop { @@ -971,94 +553,37 @@ impl View for RcuDrop { #[verus_verify] impl RcuDrop { - /// Creates a new [`RcuDrop`] that wraps the given value. #[inline] - #[verus_spec(r => + #[verus_spec(res => ensures - r@ == value, + res@ == value, )] pub fn new(value: T) -> Self { Self { value: ManuallyDrop::new(value) } - }/* - /// Extracts the value from the `RcuDrop` container. - /// - /// # Safety - /// - /// The caller must ensure that the returned value will be dropped after - /// all the threads cannot access it anymore. Specifically, dropping it - /// after the RCU grace period is guaranteed to be safe. - /// - /// Note that panic unwinding may cause the returned value to be dropped - /// immediately, which is not sound. Therefore, the caller must forget the - /// [`PanicGuard`] after it ensures that the value will be dropped at the - /// correct time. - pub(crate) unsafe fn into_inner(slot: RcuDrop) -> (T, PanicGuard) { - let mut slot = ManuallyDrop::new(slot); - let panic_guard = PanicGuard::new(); - // SAFETY: The `slot` will not be used after this point. - let val = unsafe { ManuallyDrop::take(&mut slot.value) }; - (val, panic_guard) } - */ - } #[verus_verify] impl Deref for RcuDrop { type Target = T; - #[verus_spec(r => + #[inline] + #[verus_spec(res => ensures - *r == self@, + *res == self@, )] - #[inline] fn deref(&self) -> &Self::Target { &self.value } } -/* -impl Drop for RcuDrop { - fn drop(&mut self) { - // SAFETY: The `ManuallyDrop` will not be used after this point. - let taken = unsafe { ManuallyDrop::take(&mut self.value) }; - let rcu_monitor = RCU_MONITOR.get().unwrap(); - rcu_monitor.after_grace_period(|| { - drop(taken); - }); - } -} - -/// Finishes the current grace period. -/// -/// This function is called when the current grace period on current CPU is -/// finished. If this CPU is the last CPU to finish the current grace period, -/// it takes all the current callbacks and invokes them. -/// -/// # Safety +/// Finishes a grace period on the current CPU. /// -/// The caller must ensure that this CPU is not executing in a RCU read-side -/// critical section. +/// No-op until the weak-memory monitor/reclamation path is rebuilt. pub unsafe fn finish_grace_period() { - let rcu_monitor = RCU_MONITOR.get().unwrap(); - // SAFETY: The caller ensures safety. - unsafe { - rcu_monitor.finish_grace_period(); - } -} - -*/ - -exec static RCU_MONITOR: Once - ensures - RCU_MONITOR.wf(), - RCU_MONITOR.inv() == RcuMonitorPred, -{ - Once::new(Ghost(RcuMonitorPred)) } pub fn init() { - RCU_MONITOR.init(RcuMonitor::new_data()); } } // verus! @@ -1066,24 +591,24 @@ verus! { impl RcuInner

{ #[verifier::type_invariant] - closed spec fn type_inv(self) -> bool { + pub closed spec fn type_inv(self) -> bool { self.wf() } } -impl RcuOption

{ +impl Rcu

{ #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() - &&& self.0.is_nullable() + &&& !self.0.is_nullable() } } -impl Rcu

{ +impl RcuOption

{ #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() - &&& !self.0.is_nullable() + &&& self.0.is_nullable() } } @@ -1092,7 +617,6 @@ impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() &&& !self.0.rcu.is_nullable() - &&& self.0.tracked_ref_perm@ is Some } } @@ -1107,15 +631,8 @@ impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { - match self.tracked_ref_perm@ { - Some(perm) => { - &&& !self.obj_ptr.is_null() - &&& P::ptr_perm_match(self.obj_ptr, perm.resource()) - &&& perm.resource().inv() - &&& perm.frac() == 1 - }, - None => self.obj_ptr.is_null(), - } + &&& self.rcu.type_inv() + &&& !self.rcu.is_nullable() ==> !self.obj_ptr.is_null() } } From eb7520370665593809f4e7411378017f7c619ae1 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 4 Jun 2026 02:34:12 -0400 Subject: [PATCH 07/47] more rcu stuff --- ostd/specs/sync/rcu.rs | 117 ++++++- ostd/specs/sync/weak_memory.rs | 236 +++++++++++++- ostd/src/sync/rcu/mod.rs | 77 +---- ostd/src/sync/rcu/monitor.rs | 553 ++++++++++++++++++--------------- vstd_extra/src/lib.rs | 1 + vstd_extra/src/raw_callback.rs | 114 +++++++ 6 files changed, 748 insertions(+), 350 deletions(-) create mode 100644 vstd_extra/src/raw_callback.rs diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 06024b48d..5263e5d80 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -12,22 +12,12 @@ //! The module is intentionally proof-only for now. The executable RCU //! implementation should later connect its real guard/token state to these //! abstract ghost tokens. -use super::weak_memory::{History, Msg, WeakAtomicInvariantPredicate}; +use super::weak_memory::{History, Msg, WeakAtomicInvariantPredicate, WmView}; use vstd::prelude::*; use vstd::resource::Loc; verus! { -/// Workaround for supporting `dyn Fn() + 'static + Send`. -/// -/// A closure basically is just a function pointer -/// along with the pointer to the captured context. -#[verifier::external_body] -pub struct RawRcuCallback { - data: *mut (), - run: unsafe fn (*mut ()), -} - pub type LinkIndex = nat; pub type LinkEdge = (*mut T, LinkIndex); @@ -54,6 +44,111 @@ impl WeakAtomicInvariantPredicate for RcuWeakAtomicInv { } } +/// Ghost summary paired with the RCU monitor's `is_monitoring` flag. +/// +/// `pending[i]` summarizes whether the monitor state represented at flag +/// message `i` has queued callbacks or an active grace period that must still be +/// observed. This is intentionally a summary: the concrete callback vectors +/// live in the monitor state protected by its lock. +pub ghost struct RcuMonitorFlagGhost { + pub pending: Seq, +} + +impl RcuMonitorFlagGhost { + pub open spec fn initial() -> Self { + RcuMonitorFlagGhost { pending: seq![false] } + } + + pub open spec fn push(self, pending: bool) -> Self { + RcuMonitorFlagGhost { pending: self.pending.push(pending) } + } +} + +/// Weak-memory invariant for the monitor's fast-path flag. +/// +/// The invariant is deliberately one-way: a `false` flag message certifies no +/// pending monitor work for that message's ghost summary. A `true` flag is +/// conservative and may over-approximate pending work. +pub open spec fn rcu_monitor_flag_history_inv( + history: History, + ghost: RcuMonitorFlagGhost, +) -> bool { + &&& history.len() >= 1 + &&& ghost.pending.len() == history.len() + &&& forall|i: int| + 0 <= i < history.len() ==> { + &&& !#[trigger] history[i].value ==> !ghost.pending[i] + } +} + +pub struct RcuMonitorFlagInv; + +impl WeakAtomicInvariantPredicate<(), bool, RcuMonitorFlagGhost> for RcuMonitorFlagInv { + open spec fn atomic_inv( + _k: (), + history: History, + ghost: RcuMonitorFlagGhost, + ) -> bool { + rcu_monitor_flag_history_inv(history, ghost) + } +} + +pub proof fn rcu_monitor_flag_initial_inv() + ensures + RcuMonitorFlagInv::atomic_inv( + (), + seq![Msg { value: false, view: WmView::empty() }], + RcuMonitorFlagGhost::initial(), + ), +{ +} + +pub proof fn preserve_rcu_monitor_flag_inv_on_push( + prev: History, + next: History, + msg: Msg, + prev_ghost: RcuMonitorFlagGhost, + next_ghost: RcuMonitorFlagGhost, + pending: bool, +) + requires + rcu_monitor_flag_history_inv(prev, prev_ghost), + next == prev.push(msg), + next_ghost == prev_ghost.push(pending), + !msg.value ==> !pending, + ensures + rcu_monitor_flag_history_inv(next, next_ghost), +{ + assert(next.len() >= 1); + assert(next_ghost.pending.len() == next.len()); + assert forall|i: int| 0 <= i < next.len() implies { + &&& !#[trigger] next[i].value ==> !next_ghost.pending[i] + } by { + if i == prev.len() { + assert(next[i] == msg); + assert(next_ghost.pending[i] == pending); + } else { + assert(i < prev.len()); + assert(next[i] == prev[i]); + assert(next_ghost.pending[i] == prev_ghost.pending[i]); + } + }; +} + +pub proof fn rcu_monitor_flag_false_has_no_pending( + history: History, + ghost: RcuMonitorFlagGhost, + ts: nat, +) + requires + rcu_monitor_flag_history_inv(history, ghost), + ts < history.len(), + !history[ts as int].value, + ensures + !ghost.pending[ts as int], +{ +} + pub proof fn preserve_rcu_history_inv_on_push( nullable: bool, prev: History<*mut T>, diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 390d80d96..eff13b4c1 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -19,6 +19,8 @@ use core::sync::atomic::{ AtomicU32, AtomicUsize, Ordering, }; +use super::rcu as rcu_spec; + #[cfg(target_has_atomic = "64")] use core::sync::atomic::{AtomicI64, AtomicU64}; @@ -312,9 +314,9 @@ macro_rules! declare_weak_atomic_type { /// `#[verifier::type_invariant]`. pub struct $weak_atomic { #[doc(hidden)] - pub atomic: $raw_atomic, + atomic: $raw_atomic, #[doc(hidden)] - pub atomic_inv: Tracked< + atomic_inv: Tracked< AtomicInvariant<(K, AtomicId), (HistAuth<$value_ty>, G), $pred_adapter>, >, } @@ -361,8 +363,16 @@ macro_rules! declare_weak_atomic_type { Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ($value_ty, Ghost)) { let result; + proof { + use_type_invariant(self); + } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); proof { pair = (hist, g); @@ -377,8 +387,16 @@ macro_rules! declare_weak_atomic_type { Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ($value_ty, Ghost)) { let result; + proof { + use_type_invariant(self); + } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { pair = (hist, g); @@ -441,9 +459,9 @@ impl InvariantPredicate<(K, AtomicId), (HistAuth<*mut T>, G)> for #[verifier::accept_recursive_types(T)] pub struct WeakAtomicPtr { #[doc(hidden)] - pub atomic: AtomicPtrW, + atomic: AtomicPtrW, #[doc(hidden)] - pub atomic_inv: Tracked< + atomic_inv: Tracked< AtomicInvariant<(K, AtomicId), (HistAuth<*mut T>, G), WeakAtomicPredPtr>, >, } @@ -487,8 +505,16 @@ impl WeakAtomicPtr where Ghost, )) { let result; + proof { + use_type_invariant(self); + } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); proof { pair = (hist, g); @@ -503,8 +529,16 @@ impl WeakAtomicPtr where Ghost, )) { let result; + proof { + use_type_invariant(self); + } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { pair = (hist, g); @@ -530,8 +564,16 @@ impl WeakAtomicPtr { /// we are still shaping the client-specific ghost state. #[inline(always)] pub fn store_release_simple(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) { + proof { + use_type_invariant(self); + } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (mut hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } let _snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); proof { pair = (hist, g); @@ -548,8 +590,127 @@ impl WeakAtomicPtr { Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: (Result<*mut T, *mut T>, Ghost)) { let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + let cas_result = self.atomic.compare_exchange_acqrel_acquire( + Tracked(&mut hist), + Tracked(tv), + current, + new, + ); + result = (cas_result.0, cas_result.1); + proof { + pair = (hist, g); + } + }); + result + } +} + +impl WeakAtomicPtr { + /// Acquire-load helper for RCU root pointers. + #[inline(always)] + pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + *mut T, + Ghost, + )) + requires + self.well_formed(), + ensures + !self.constant() ==> !res.0.is_null(), + { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + assert(rcu_spec::rcu_history_inv(self.constant(), hist.history())); + if !self.constant() { + rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), result.1@); + assert(!result.0.is_null()); + } + pair = (hist, g); + } + }); + result + } + + /// Release-store helper for RCU root pointers. + #[inline(always)] + pub fn store_release_rcu(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) + requires + self.well_formed(), + self.constant() || !value.is_null(), + { + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + let ghost prev = hist.history(); + let snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); + let ghost next = hist.history(); + proof { + if !self.constant() { + assert(!value.is_null()); + assert(snap@.msg().value.addr() != 0); + } + rcu_spec::preserve_rcu_history_inv_on_push( + self.constant(), + prev, + next, + snap@.msg(), + ); + pair = (hist, g); + } + }); + } + + /// Strong AcqRel/Acquire CAS helper for RCU root pointers. + #[inline(always)] + pub fn compare_exchange_acqrel_acquire_rcu( + &self, + current: *mut T, + new: *mut T, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (Result<*mut T, *mut T>, Ghost)) + requires + self.well_formed(), + self.constant() || !new.is_null(), + { + let result; + proof { + use_type_invariant(self); + } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { let tracked (mut hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + let ghost prev = hist.history(); let cas_result = self.atomic.compare_exchange_acqrel_acquire( Tracked(&mut hist), Tracked(tv), @@ -557,7 +718,34 @@ impl WeakAtomicPtr { new, ); result = (cas_result.0, cas_result.1); + let ghost next = hist.history(); proof { + match cas_result.0 { + Result::Ok(_) => { + let tracked snap_opt = cas_result.2.get(); + match snap_opt { + Option::Some(snap) => { + if !self.constant() { + assert(!new.is_null()); + assert(snap.msg().value.addr() != 0); + } + rcu_spec::preserve_rcu_history_inv_on_push( + self.constant(), + prev, + next, + snap.msg(), + ); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(_) => { + assert(next == prev); + assert(rcu_spec::rcu_history_inv(self.constant(), next)); + }, + } pair = (hist, g); } }); @@ -589,9 +777,17 @@ macro_rules! weak_atomic_with_ghost { let atomic = &($atomic); let current = $current; let new = $new; + proof { + use_type_invariant(atomic); + } ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { #[allow(unused_mut)] let tracked (mut hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.atomic_inv@.constant().1); + assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); + assert(hist.id() == atomic.atomic.id()); + } let ghost $prev = hist.history(); let cas_result = atomic.atomic.compare_exchange_acqrel_acquire( Tracked(&mut hist), @@ -628,9 +824,17 @@ macro_rules! weak_atomic_with_ghost { ::vstd::prelude::verus_exec_expr! {{ let result; let atomic = &($atomic); + proof { + use_type_invariant(atomic); + } ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { #[allow(unused_mut)] let tracked (hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.atomic_inv@.constant().1); + assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); + assert(hist.id() == atomic.atomic.id()); + } let ghost $history = hist.history(); result = atomic.atomic.load_acquire(Tracked(&hist), $tv); let ghost $ret = result.0; @@ -657,9 +861,17 @@ macro_rules! weak_atomic_with_ghost { ::vstd::prelude::verus_exec_expr! {{ let result; let atomic = &($atomic); + proof { + use_type_invariant(atomic); + } ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { #[allow(unused_mut)] let tracked (hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.atomic_inv@.constant().1); + assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); + assert(hist.id() == atomic.atomic.id()); + } let ghost $history = hist.history(); result = atomic.atomic.load_relaxed(Tracked(&hist), $tv); let ghost $ret = result.0; @@ -684,9 +896,17 @@ macro_rules! weak_atomic_with_ghost { ::vstd::prelude::verus_exec_expr! {{ let atomic = &($atomic); let value = $value; + proof { + use_type_invariant(atomic); + } ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { #[allow(unused_mut)] let tracked (mut hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.atomic_inv@.constant().1); + assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); + assert(hist.id() == atomic.atomic.id()); + } let ghost $prev = hist.history(); let snap_tracked = atomic.atomic.store_release(Tracked(&mut hist), $tv, value); let ghost $next = hist.history(); @@ -711,9 +931,17 @@ macro_rules! weak_atomic_with_ghost { ::vstd::prelude::verus_exec_expr! {{ let atomic = &($atomic); let value = $value; + proof { + use_type_invariant(atomic); + } ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { #[allow(unused_mut)] let tracked (mut hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.atomic_inv@.constant().1); + assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); + assert(hist.id() == atomic.atomic.id()); + } let ghost $prev = hist.history(); let snap_tracked = atomic.atomic.store_relaxed(Tracked(&mut hist), $tv, value); let ghost $next = hist.history(); diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index d909e57b0..59bf792f3 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -3,7 +3,6 @@ //! //! This is the new weak-memory RCU skeleton. The previous SC proof-oriented //! implementation is kept in `__mod.rs` as reference and is not compiled. -use alloc::boxed::Box; use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; @@ -23,6 +22,7 @@ use crate::{ use non_null::{NonNullPtr, NonNullPtrRef}; pub mod non_null; +pub mod monitor; verus! { @@ -457,81 +457,6 @@ impl RcuOptionReadGuard<'_, P> { } } -} // verus! -/// A one-shot, type-erased RCU callback. -/// -/// This is the small executable wrapper we use instead of `Box`. -/// `data` is a thin pointer to a sized heap payload, and `run` is the -/// monomorphized shim that knows how to consume that payload. The proof layer -/// treats the pair as a TCB boundary; later monitor proofs should reason about -/// which retired resource each callback represents, not about arbitrary closure -/// bodies. -#[must_use] -#[allow(dead_code)] -pub(crate) struct RawRcuCallback { - data: *mut (), - run: unsafe fn(*mut ()), -} - -struct RawRcuCallbackPayload { - context: C, - run: unsafe fn(C), -} - -// SAFETY: `RawRcuCallback::new` only accepts `C: Send + 'static` payloads, and -// `call_once` consumes the payload through the matching monomorphized runner. -unsafe impl Send for RawRcuCallback {} - -#[allow(dead_code)] -impl RawRcuCallback { - /// Builds a callback from an explicit captured context and a typed one-shot - /// runner. - #[inline] - pub(crate) fn new(context: C, run: unsafe fn(C)) -> Self { - let payload = Box::new(RawRcuCallbackPayload { context, run }); - Self { - data: Box::into_raw(payload).cast::<()>(), - run: run_raw_callback::, - } - } - - /// Builds the common "drop this value after the grace period" callback. - #[inline] - pub(crate) fn defer_drop(value: T) -> Self { - Self::new(value, drop_context::) - } - - /// Runs the callback exactly once. - /// - /// # Safety - /// - /// The caller must ensure this callback has not already been run and will - /// not be run again. This is the executable one-shot invariant that the - /// monitor queue will eventually own. - #[inline] - pub(crate) unsafe fn call_once(self) { - unsafe { - (self.run)(self.data); - } - } -} - -#[inline] -unsafe fn run_raw_callback(data: *mut ()) { - let payload = unsafe { Box::from_raw(data.cast::>()) }; - let RawRcuCallbackPayload { context, run } = *payload; - unsafe { - run(context); - } -} - -#[inline] -fn drop_context(value: T) { - drop(value); -} - -verus! { - /// A wrapper whose destructor will eventually be delayed until after an RCU /// grace period. /// diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index c50b073ef..d23117301 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -1,290 +1,325 @@ // SPDX-License-Identifier: MPL-2.0 -use vstd::{ - atomic_ghost::AtomicBool, atomic_with_ghost, predicate::Predicate as DataPredicate, prelude::*, -}; -use vstd_extra::ownership::Inv; +use alloc::vec::Vec; + +use vstd::prelude::*; +use vstd_extra::raw_callback::RawCallback; -use crate::{ - specs::mm::cpu::{AtomicCpuSet, CpuSet}, - sync::{AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate}, +use crate::specs::{ + mm::cpu::{AtomicCpuSet, CpuSet}, + sync::{ + rcu as rcu_spec, + weak_memory::WeakAtomicBool, + }, }; +use crate::sync::{LocalIrqDisabled, SpinLock}; verus! { -// This thing can be very tricky to deal with -// type Callbacks = VecDeque>; +pub type Callbacks = Vec; + +type MonitorAtomicBool = + WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv>; + pub(super) struct GracePeriod { - // callbacks: Callbacks, + callbacks: Callbacks, cpu_mask: AtomicCpuSet, is_complete: bool, } + pub(super) struct State { current_gp: GracePeriod, - // next_callbacks: Callbacks, + next_callbacks: Callbacks, } -/// Owner of this [`RcuMonitor`]. -pub(super) tracked struct RcuMonitorOwner {} - -struct_with_invariants! { /// A RCU monitor ensures the completion of _grace periods_ by keeping track /// of each CPU's passing _quiescent states_. pub(super) struct RcuMonitor { - pub(super) is_monitoring: AtomicBool<_, bool, _>, + pub(super) is_monitoring: MonitorAtomicBool, pub(super) state: SpinLock, } -closed spec fn wf(self) -> bool { - invariant on is_monitoring with (state) is (v: bool, g: bool) { - &&& v == g - &&& state.type_inv() - } -} -} - -impl DataPredicate for RcuMonitorOwner { - closed spec fn predicate(&self, v: RcuMonitor) -> bool { - true +impl RcuMonitor { + closed spec fn wf(self) -> bool { + &&& self.is_monitoring.well_formed() + &&& self.state.type_inv() } -} -impl RcuMonitor { #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { self.wf() } } -pub(super) struct RcuMonitorPred; - -impl OncePredicate> for RcuMonitorPred { - closed spec fn inv(self, v: AtomicDataWithOwner) -> bool { - &&& v.permission@.predicate(v.data) - &&& v.data.inv() - } -} - -impl Inv for RcuMonitor { - closed spec fn inv(self) -> bool { - self.wf() - } -} - -impl Inv for State { - closed spec fn inv(self) -> bool { - self.current_gp.inv() - } -} - -impl Inv for GracePeriod { - closed spec fn inv(self) -> bool { - true - } -} - -#[verus_verify] -impl RcuMonitor { - /// Creates a new RCU monitor. - pub(super) fn new() -> Self { - let state = SpinLock::new(State::new()); - proof { - use_type_invariant(&state); - assert(state.type_inv()); - } - let res = RcuMonitor { - is_monitoring: AtomicBool::new(Ghost(state), false, Tracked(false)), - state, - }; - - proof { - use_type_invariant(&res.state); - assert(res.state.type_inv()); - assert(res.inv()); - } - res - } - - /// Creates a new RCU monitor together with its tracked owner for `Once`. - #[verus_spec(r => - ensures - r.inv(), - r.data.inv(), - RcuMonitorPred.inv(r), - )] - pub(super) fn new_data() -> AtomicDataWithOwner { - let data = Self::new(); - proof { - use_type_invariant(&data); - assert(data.inv()); - } - AtomicDataWithOwner { data, permission: Tracked(RcuMonitorOwner { }) } - } - - fn is_monitoring(&self) -> bool { - proof { - use_type_invariant(self); - } - self.is_monitoring.load() - } - - fn set_monitoring(&self, value: bool) { - proof { - use_type_invariant(self); - } - atomic_with_ghost! { - self.is_monitoring => store(value); - ghost g => { - g = value; - } - } - } -} - -impl State { - fn new() -> (res: Self) - ensures - res.inv(), - { - Self { current_gp: GracePeriod::new() } - } -} - -impl GracePeriod { - fn new() -> (res: Self) - ensures - res.inv(), - { - Self { cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), is_complete: true } - } -} - } // verus! -/*use alloc::collections::VecDeque; -use core::sync::atomic::{ - AtomicBool, - Ordering::{self, Relaxed}, -}; - -use crate::{ - cpu::{AtomicCpuSet, CpuId, CpuSet, PinCurrentCpu}, - prelude::*, - sync::SpinLock, - task::atomic_mode::AsAtomicModeGuard, -}; - - - -impl RcuMonitor { - /// Creates a new RCU monitor. - /// - /// This function is used to initialize a singleton instance of `RcuMonitor`. - /// The singleton instance is globally accessible via the `RCU_MONITOR`. - pub(super) fn new() -> Self { - Self { - is_monitoring: AtomicBool::new(false), - state: SpinLock::new(State::new()), - } - } - - pub(super) unsafe fn finish_grace_period(&self) { - // Fast path - if !self.is_monitoring.load(Relaxed) { - return; - } - - // Check if the current GP is complete after passing the quiescent state - // on the current CPU. If GP is complete, take the callbacks of the current - // GP. - let callbacks = { - let mut state = self.state.disable_irq().lock(); - let cpu = state.as_atomic_mode_guard().current_cpu(); - if state.current_gp.is_complete() { - return; - } - - state.current_gp.finish_grace_period(cpu); - if !state.current_gp.is_complete() { - return; - } - - // Now that the current GP is complete, take its callbacks - let current_callbacks = state.current_gp.take_callbacks(); - - // Check if we need to watch for a next GP - if !state.next_callbacks.is_empty() { - let callbacks = core::mem::take(&mut state.next_callbacks); - state.current_gp.restart(callbacks); - } else { - self.is_monitoring.store(false, Relaxed); - } - - current_callbacks - }; - - // Invoke the callbacks to notify the completion of GP - for f in callbacks { - (f)(); - } - } - - pub(super) fn after_grace_period(&self, f: F) - where - F: FnOnce() + Send + 'static, - { - let mut state = self.state.disable_irq().lock(); - - state.next_callbacks.push_back(Box::new(f)); - - if !state.current_gp.is_complete() { - return; - } - let callbacks = core::mem::take(&mut state.next_callbacks); - state.current_gp.restart(callbacks); - self.is_monitoring.store(true, Relaxed); - } -} - - -impl State { - fn new() -> Self { - Self { - current_gp: GracePeriod::new(), - next_callbacks: VecDeque::new(), - } - } -} - -impl GracePeriod { - fn new() -> Self { - Self { - callbacks: Callbacks::new(), - cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), - is_complete: true, - } - } - - fn is_complete(&self) -> bool { - self.is_complete - } - - fn finish_grace_period(&mut self, this_cpu: CpuId) { - self.cpu_mask.add(this_cpu, Ordering::Relaxed); - - if self.cpu_mask.load(Ordering::Relaxed).is_full() { - self.is_complete = true; - } - } - - fn take_callbacks(&mut self) -> Callbacks { - core::mem::take(&mut self.callbacks) - } - - fn restart(&mut self, callbacks: Callbacks) { - self.is_complete = false; - self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); - self.callbacks = callbacks; - } -} -*/ +// use vstd::{ +// atomic_ghost::AtomicBool, atomic_with_ghost, predicate::Predicate as DataPredicate, prelude::*, +// }; +// use vstd_extra::ownership::Inv; + +// use crate::{ +// specs::mm::cpu::{AtomicCpuSet, CpuSet}, +// sync::{AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate}, +// }; + +// verus! { + +// // This thing can be very tricky to deal with +// // type Callbacks = VecDeque>; +// pub(super) struct GracePeriod { +// // callbacks: Callbacks, +// cpu_mask: AtomicCpuSet, +// is_complete: bool, +// } + +// pub(super) struct State { +// current_gp: GracePeriod, +// // next_callbacks: Callbacks, +// } + +// /// Owner of this [`RcuMonitor`]. +// pub(super) tracked struct RcuMonitorOwner {} + +// impl DataPredicate for RcuMonitorOwner { +// closed spec fn predicate(&self, v: RcuMonitor) -> bool { +// true +// } +// } + +// impl RcuMonitor { +// #[verifier::type_invariant] +// closed spec fn type_inv(self) -> bool { +// self.wf() +// } +// } + +// pub(super) struct RcuMonitorPred; + +// impl OncePredicate> for RcuMonitorPred { +// closed spec fn inv(self, v: AtomicDataWithOwner) -> bool { +// &&& v.permission@.predicate(v.data) +// &&& v.data.inv() +// } +// } + +// impl Inv for RcuMonitor { +// closed spec fn inv(self) -> bool { +// self.wf() +// } +// } + +// impl Inv for State { +// closed spec fn inv(self) -> bool { +// self.current_gp.inv() +// } +// } + +// impl Inv for GracePeriod { +// closed spec fn inv(self) -> bool { +// true +// } +// } + +// #[verus_verify] +// impl RcuMonitor { +// /// Creates a new RCU monitor. +// pub(super) fn new() -> Self { +// let state = SpinLock::new(State::new()); +// proof { +// use_type_invariant(&state); +// assert(state.type_inv()); +// } +// let res = RcuMonitor { +// is_monitoring: AtomicBool::new(Ghost(state), false, Tracked(false)), +// state, +// }; + +// proof { +// use_type_invariant(&res.state); +// assert(res.state.type_inv()); +// assert(res.inv()); +// } +// res +// } + +// /// Creates a new RCU monitor together with its tracked owner for `Once`. +// #[verus_spec(r => +// ensures +// r.inv(), +// r.data.inv(), +// RcuMonitorPred.inv(r), +// )] +// pub(super) fn new_data() -> AtomicDataWithOwner { +// let data = Self::new(); +// proof { +// use_type_invariant(&data); +// assert(data.inv()); +// } +// AtomicDataWithOwner { data, permission: Tracked(RcuMonitorOwner { }) } +// } + +// fn is_monitoring(&self) -> bool { +// proof { +// use_type_invariant(self); +// } +// self.is_monitoring.load() +// } + +// fn set_monitoring(&self, value: bool) { +// proof { +// use_type_invariant(self); +// } +// atomic_with_ghost! { +// self.is_monitoring => store(value); +// ghost g => { +// g = value; +// } +// } +// } +// } + +// impl State { +// fn new() -> (res: Self) +// ensures +// res.inv(), +// { +// Self { current_gp: GracePeriod::new() } +// } +// } + +// impl GracePeriod { +// fn new() -> (res: Self) +// ensures +// res.inv(), +// { +// Self { cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), is_complete: true } +// } +// } + +// } // verus! +// /*use alloc::collections::VecDeque; +// use core::sync::atomic::{ +// AtomicBool, +// Ordering::{self, Relaxed}, +// }; + +// use crate::{ +// cpu::{AtomicCpuSet, CpuId, CpuSet, PinCurrentCpu}, +// prelude::*, +// sync::SpinLock, +// task::atomic_mode::AsAtomicModeGuard, +// }; + +// impl RcuMonitor { +// /// Creates a new RCU monitor. +// /// +// /// This function is used to initialize a singleton instance of `RcuMonitor`. +// /// The singleton instance is globally accessible via the `RCU_MONITOR`. +// pub(super) fn new() -> Self { +// Self { +// is_monitoring: AtomicBool::new(false), +// state: SpinLock::new(State::new()), +// } +// } + +// pub(super) unsafe fn finish_grace_period(&self) { +// // Fast path +// if !self.is_monitoring.load(Relaxed) { +// return; +// } + +// // Check if the current GP is complete after passing the quiescent state +// // on the current CPU. If GP is complete, take the callbacks of the current +// // GP. +// let callbacks = { +// let mut state = self.state.disable_irq().lock(); +// let cpu = state.as_atomic_mode_guard().current_cpu(); +// if state.current_gp.is_complete() { +// return; +// } + +// state.current_gp.finish_grace_period(cpu); +// if !state.current_gp.is_complete() { +// return; +// } + +// // Now that the current GP is complete, take its callbacks +// let current_callbacks = state.current_gp.take_callbacks(); + +// // Check if we need to watch for a next GP +// if !state.next_callbacks.is_empty() { +// let callbacks = core::mem::take(&mut state.next_callbacks); +// state.current_gp.restart(callbacks); +// } else { +// self.is_monitoring.store(false, Relaxed); +// } + +// current_callbacks +// }; + +// // Invoke the callbacks to notify the completion of GP +// for f in callbacks { +// (f)(); +// } +// } + +// pub(super) fn after_grace_period(&self, f: F) +// where +// F: FnOnce() + Send + 'static, +// { +// let mut state = self.state.disable_irq().lock(); + +// state.next_callbacks.push_back(Box::new(f)); + +// if !state.current_gp.is_complete() { +// return; +// } + +// let callbacks = core::mem::take(&mut state.next_callbacks); +// state.current_gp.restart(callbacks); +// self.is_monitoring.store(true, Relaxed); +// } +// } + +// impl State { +// fn new() -> Self { +// Self { +// current_gp: GracePeriod::new(), +// next_callbacks: VecDeque::new(), +// } +// } +// } + +// impl GracePeriod { +// fn new() -> Self { +// Self { +// callbacks: Callbacks::new(), +// cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), +// is_complete: true, +// } +// } + +// fn is_complete(&self) -> bool { +// self.is_complete +// } + +// fn finish_grace_period(&mut self, this_cpu: CpuId) { +// self.cpu_mask.add(this_cpu, Ordering::Relaxed); + +// if self.cpu_mask.load(Ordering::Relaxed).is_full() { +// self.is_complete = true; +// } +// } + +// fn take_callbacks(&mut self) -> Callbacks { +// core::mem::take(&mut self.callbacks) +// } + +// fn restart(&mut self, callbacks: Callbacks) { +// self.is_complete = false; +// self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); +// self.callbacks = callbacks; +// } +// } +// */ diff --git a/vstd_extra/src/lib.rs b/vstd_extra/src/lib.rs index 519f29abf..18a418564 100644 --- a/vstd_extra/src/lib.rs +++ b/vstd_extra/src/lib.rs @@ -32,6 +32,7 @@ pub mod trans_macros; pub mod map_extra; pub mod prelude; +pub mod raw_callback; pub mod raw_ptr_extra; pub mod seq_extra; pub mod set_extra; diff --git a/vstd_extra/src/raw_callback.rs b/vstd_extra/src/raw_callback.rs new file mode 100644 index 000000000..b97cdbe38 --- /dev/null +++ b/vstd_extra/src/raw_callback.rs @@ -0,0 +1,114 @@ +//! Type-erased one-shot callback wrapper for Verus code. +//! +//! Verus does not support function pointer types in public API signatures. This +//! module therefore exposes callbacks as sized capture objects implementing +//! [`RawCallbackContext`], while the executable runner erasure is kept behind +//! trusted `external_body` wrappers. + +use alloc::boxed::Box; + +use vstd::prelude::*; + +verus! { + +/// A one-shot, type-erased callback. +/// +/// `data` is a thin pointer to a heap payload, and `run` stores the address of +/// the monomorphized shim that knows how to consume that payload. The pair is a +/// TCB boundary: proofs should reason about the resource represented by the +/// callback, not about arbitrary closure bodies. +#[must_use] +pub struct RawCallback { + data: *mut (), + run: usize, +} + +/// Captured context for a [`RawCallback`]. +/// +/// Implement this trait on a sized capture object instead of passing +/// `unsafe fn(C)` or `dyn FnOnce()`, both of which are outside Verus' supported +/// executable API surface. +pub trait RawCallbackContext: Send + 'static { + fn run(self); +} + +struct RawDropContext { + value: T, +} + +// SAFETY: `RawCallback::new` only accepts `C: Send + 'static` payloads, and +// `call_once` consumes the payload through the matching monomorphized runner. +#[verifier::external] +unsafe impl Send for RawCallback {} + +impl RawCallbackContext for RawDropContext { + #[inline] + #[verifier::external_body] + fn run(self) { + let RawDropContext { value } = self; + drop(value); + } +} + +impl RawCallback { + /// Builds a callback from an explicit captured context. + #[inline] + #[verifier::external_body] + pub fn new(context: C) -> Self { + let payload = Box::new(RawCallbackPayload { context }); + Self { + data: Box::into_raw(payload).cast::<()>(), + run: raw_callback_runner::(), + } + } + + /// Builds the common "drop this value later" callback. + #[inline] + #[verifier::external_body] + pub fn defer_drop(value: T) -> Self { + Self::new(RawDropContext { value }) + } + + /// Runs the callback exactly once. + /// + /// # Safety + /// + /// The caller must ensure this callback has not already been run and will + /// not be run again. + #[inline] + #[verifier::external_body] + pub unsafe fn call_once(self) { + unsafe { + call_raw_callback(self.data, self.run); + } + } +} + +struct RawCallbackPayload { + context: C, +} + +#[inline] +#[verifier::external_body] +fn raw_callback_runner() -> usize { + run_raw_callback:: as *const () as usize +} + +#[inline] +#[verifier::external_body] +unsafe fn call_raw_callback(data: *mut (), run: usize) { + let run: unsafe fn(*mut ()) = unsafe { core::mem::transmute(run) }; + unsafe { + run(data); + } +} + +#[inline] +#[verifier::external_body] +unsafe fn run_raw_callback(data: *mut ()) { + let payload = unsafe { Box::from_raw(data.cast::>()) }; + let RawCallbackPayload { context } = *payload; + context.run(); +} + +} // verus! From bc31ae51d563c76e68f7fd0736fc9285d99420ba Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 9 Jun 2026 03:19:23 -0400 Subject: [PATCH 08/47] experiment with sync --- ostd/specs/sync/rcu.rs | 96 ++++++++++++++++++++++++++++++++++++ ostd/src/sync/rcu/mod.rs | 56 +++++++++++++++++++++ ostd/src/sync/rcu/monitor.rs | 71 +++++++++++++++++++++++++- 3 files changed, 222 insertions(+), 1 deletion(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 5263e5d80..64055dd7b 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -22,6 +22,22 @@ pub type LinkIndex = nat; pub type LinkEdge = (*mut T, LinkIndex); +/// Proof summary for a type-erased RCU callback. +/// +/// The executable callback may close over any sized Rust value, but the RCU +/// proof only needs to know which logical object it will reclaim and which +/// grace-period generation retired that object. `domain` identifies the RCU +/// protection domain, and `obj` identifies the reclaimed allocation/object +/// inside that domain. +pub ghost struct RcuCallbackSummary { + /// The RCU protection domain whose grace period governs this callback. + pub domain: Loc, + /// Logical identity of the retired object inside `domain`. + pub obj: Loc, + /// The domain-local epoch in which `obj` was retired. + pub retire_epoch: nat, +} + /// The weak-memory invariant for the root pointer stored in an executable RCU /// cell. /// @@ -261,6 +277,32 @@ impl RcuDomainAuth { } } +/// Logical identity of one RCU-managed object. +/// +/// Traversal proofs are typed and pointer-based (`*mut T`), while the callback +/// monitor stores type-erased callbacks. This token bridges the two worlds: it +/// says that `obj` is the logical identity of `ptr` inside `domain`. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuObjectId { + ghost domain: Loc, + ghost obj: Loc, + ghost ptr: *mut T, +} + +impl RcuObjectId { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn obj(self) -> Loc { + self.obj + } + + pub closed spec fn ptr(self) -> *mut T { + self.ptr + } +} + /// Low-level base retire permission. /// /// This is the paper's `BaseRetirePerm`. By itself it is not enough to reclaim; @@ -327,6 +369,60 @@ pub proof fn lift_retire_perm( RcuRetirePerm { domain: base.domain(), ptr: base.ptr(), seen_removed } } +/// Non-generic proof certificate carried across the type-erasure boundary. +/// +/// A certificate can only be produced from a typed traversal retire permission, +/// but after that point the monitor only needs the erased callback summary. +pub tracked struct RcuCallbackSafety { + ghost summary: RcuCallbackSummary, +} + +impl View for RcuCallbackSafety { + type V = RcuCallbackSummary; + + closed spec fn view(&self) -> RcuCallbackSummary { + self.summary + } +} + +pub open spec fn callback_safety_from_traversal( + cert: RcuCallbackSafety, + object: RcuObjectId, + retire_epoch: nat, +) -> bool { + &&& cert@.domain == object.domain() + &&& cert@.obj == object.obj() + &&& cert@.retire_epoch == retire_epoch +} + +/// Consume a typed traversal retire permission and compress it into the +/// non-generic summary needed by the type-erased callback monitor. +pub proof fn certify_callback_from_retire_perm( + tracked object: &RcuObjectId, + tracked retire: RcuRetirePerm, + retire_epoch: nat, +) -> (tracked cert: RcuCallbackSafety) + requires + object.domain() == retire.domain(), + object.ptr() == retire.ptr(), + retire.ready_to_reclaim(), + ensures + cert@ == (RcuCallbackSummary { + domain: retire.domain(), + obj: object.obj(), + retire_epoch, + }), + callback_safety_from_traversal(cert, *object, retire_epoch), +{ + RcuCallbackSafety { + summary: RcuCallbackSummary { + domain: retire.domain(), + obj: object.obj(), + retire_epoch, + }, + } +} + /// Read-side guard token for one critical section. /// /// This is the traversal-level guard: it includes the base guard protection and diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 59bf792f3..dde449e29 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -3,6 +3,62 @@ //! //! This is the new weak-memory RCU skeleton. The previous SC proof-oriented //! implementation is kept in `__mod.rs` as reference and is not compiled. +//! +//! # Verification model +//! +//! The executable RCU API is being rebuilt around an explicit weak-memory +//! history model. The atomic root pointer is a trusted executable wrapper around +//! Rust atomics, while proofs only rely on the specification in +//! [`specs::sync::weak_memory`]. Each RCU root pointer is represented by a +//! `WeakAtomicPtr` whose history records the messages that may be observed by +//! relaxed/acquire loads and CAS operations. +//! +//! The current root-pointer invariant is intentionally small: `Rcu` roots are +//! non-null in every atomic-history message, while `RcuOption` roots may be +//! null. Ownership, reader permissions, traversal snapshots, and reclamation are +//! modeled separately in [`specs::sync::rcu`] and are being connected +//! incrementally. +//! +//! The traversal layer follows the paper's shape: +//! +//! - [`RcuReadGuardToken`] represents a read-side critical section together +//! with its `SeenRemoved(D, LV)` observation. +//! - [`RcuProtectedPtr`] records that a typed pointer is protected by that +//! guard and has not been observed removed. +//! - [`RcuBaseRetirePerm`] becomes [`RcuRetirePerm`] only after the caller has +//! observed enough traversal state to prove the retired object is in the +//! removed set. +//! - `RcuCallbackSafety` compresses that typed retire proof into an erased +//! `RcuCallbackSummary { domain, obj, retire_epoch }`, which is what the +//! monitor stores next to a type-erased executable callback. +//! +//! # Callback boundary +//! +//! Executable callbacks are represented by `vstd_extra::raw_callback::RawCallback`. +//! `RawCallback` is proof-opaque: it only stores a thin data pointer plus a +//! monomorphized runner pointer. The RCU monitor wraps it in `monitor::RcuCallback`, +//! which can only be constructed from a `RcuCallbackSafety` certificate. This +//! prevents the proof layer from treating an arbitrary type-erased callback as a +//! safe reclamation callback. +//! +//! The monitor also has a weak-memory `is_monitoring` flag with an RCU-specific +//! invariant. Today that invariant records whether a flag-history message may +//! correspond to pending monitor work. The next step is to tie that flag summary +//! to `monitor::State::pending_summaries()` and to prove callback execution only +//! after the relevant grace period has completed. +//! +//! # Usage outline +//! +//! Use `Rcu

` when the root pointer is always non-null, and `RcuOption

` +//! when the root may be null. `P` must implement `NonNullPtr`; the common cases +//! are sized thin-pointer owners such as `Box` and `Arc`. Readers call +//! `read()` to obtain a guard and then use `get()` while the guard is live. +//! Writers install a new pointer with `update()` or use the read guard's +//! `compare_exchange()` to replace the value they observed. +//! +//! Delayed reclamation is still being wired into the weak-memory proof. For now, +//! `RcuDrop` preserves the public wrapper API, while the monitor/callback +//! path carries the new proof summary and safety certificate skeleton. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index d23117301..38fa252c1 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -15,23 +15,92 @@ use crate::sync::{LocalIrqDisabled, SpinLock}; verus! { -pub type Callbacks = Vec; +pub type Callbacks = Vec; type MonitorAtomicBool = WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv>; +/// RCU-specific wrapper around a type-erased executable callback. +/// +/// `RawCallback` is intentionally proof-opaque. The summary records the object +/// identity that the monitor invariant will use to decide when this callback is +/// safe to run after a grace period. +#[must_use] +pub struct RcuCallback { + raw: RawCallback, + summary: Ghost, +} + +impl View for RcuCallback { + type V = rcu_spec::RcuCallbackSummary; + + closed spec fn view(&self) -> rcu_spec::RcuCallbackSummary { + self.summary@ + } +} + +impl RcuCallback { + /// Converts a raw callback into an RCU callback, given a proof that the callback is + /// safe to run after a grace period. + #[inline] + pub fn from_raw( + raw: RawCallback, + Tracked(cert): Tracked, + ) -> (res: Self) + ensures + res@ == cert@, + { + let ghost summary = cert@; + Self { raw, summary: Ghost(summary) } + } + + /// Runs the underlying callback. The reclaim-safety precondition is not + /// encoded here yet; the next layer will prove it from the monitor/global + /// RCU invariant before invoking this method. + #[inline] + #[verifier::external_body] + pub unsafe fn call_once(self) { + unsafe { + self.raw.call_once(); + } + } +} + +pub open spec fn callback_summaries(callbacks: Callbacks) -> Seq { + Seq::new(callbacks@.len(), |i: int| callbacks@[i]@) +} + pub(super) struct GracePeriod { callbacks: Callbacks, cpu_mask: AtomicCpuSet, is_complete: bool, } +impl GracePeriod { + closed spec fn callback_summaries(self) -> Seq { + callback_summaries(self.callbacks) + } + + closed spec fn has_pending_work(self) -> bool { + !self.is_complete || self.callback_summaries().len() > 0 + } +} pub(super) struct State { current_gp: GracePeriod, next_callbacks: Callbacks, } +impl State { + closed spec fn pending_summaries(self) -> Seq { + self.current_gp.callback_summaries().add(callback_summaries(self.next_callbacks)) + } + + closed spec fn has_pending_work(self) -> bool { + self.current_gp.has_pending_work() || callback_summaries(self.next_callbacks).len() > 0 + } +} + /// A RCU monitor ensures the completion of _grace periods_ by keeping track /// of each CPU's passing _quiescent states_. pub(super) struct RcuMonitor { From 7c874a0b95a640d6ad5631b0f6dcdc3225365bbe Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 10 Jun 2026 05:32:27 -0400 Subject: [PATCH 09/47] fix unsoundness when dealing with old messages --- ostd/specs/sync/weak_memory.rs | 83 ++++++++++++++++---------- ostd/src/sync/rcu/mod.rs | 10 ++++ ostd/src/sync/rcu/monitor.rs | 105 +++++++++++++++++++++++++++++++-- 3 files changed, 163 insertions(+), 35 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index eff13b4c1..912aaa59f 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -150,7 +150,9 @@ pub trait WeakAtomicInvariantPredicate { /// `AtomicInvariant` next to the executable atomic. pub tracked struct HistAuth { auth: GhostMapAuth>, - pub ghost len: nat, + // Private: code outside this TCB module must not forge the history length, + // which would desynchronize `len` from the authoritative map domain. + ghost len: nat, } impl HistAuth { @@ -963,39 +965,46 @@ macro_rules! weak_atomic_with_ghost { /// /// Passing this token through atomic operations makes the weak-memory effects /// visible in specs instead of hiding them in global or thread-local state. +/// +/// # Soundness +/// +/// The wrapped view is private and can only evolve through the TCB atomic +/// operations in this module. Those operations maintain the invariant that a +/// view never claims a timestamp at or beyond the length of that location's +/// history: loads observe an existing message, stores observe the message they +/// just appended, and acquire joins only import message views that were built +/// from existing timestamps. This keeps the `readable`-based postconditions of +/// loads satisfiable. Do not add raw mutators (e.g. an unconditional +/// `observe`/`join` proof fn): a forged view claiming an unwritten timestamp +/// would make the next load's postcondition vacuously false. pub tracked struct ThreadView { - pub ghost view: WmView, + ghost view: WmView, } impl View for ThreadView { type V = WmView; - open spec fn view(&self) -> WmView { + closed spec fn view(&self) -> WmView { self.view } } impl ThreadView { + /// Creates a fresh token holding the empty view. + /// + /// The empty view is the weakest token: it lower-bounds every location at + /// timestamp 0, so minting one is always sound — the holder merely + /// forfeits all ordering knowledge. Note that minting a fresh view + /// mid-thread over-approximates real executions (it forgets per-location + /// coherence the thread has already observed) and publishes nothing useful + /// through release stores, so executable code should thread one token per + /// logical operation or critical section, and eventually one per task. pub proof fn new() -> (tracked res: Self) ensures res@ == WmView::empty(), { ThreadView { view: WmView::empty() } } - - pub proof fn observe(tracked &mut self, id: AtomicId, ts: Timestamp) - ensures - final(self)@ == old(self)@.observe(id, ts), - { - self.view = self.view.observe(id, ts); - } - - pub proof fn join(tracked &mut self, view: WmView) - ensures - final(self)@ == old(self)@.join(view), - { - self.view = self.view.join(view); - } } #[repr(transparent)] @@ -1084,9 +1093,12 @@ impl AtomicUsizeW { /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` /// failure ordering. /// - /// This first CAS model is intentionally conservative: the operation reads - /// the latest message in the location's modification history. On success it - /// appends a new release message; on failure it behaves like an acquire load. + /// On success, RMW atomicity forces the read to be the latest message in + /// the modification history, and the new release message is appended + /// immediately after it. On failure, the operation is only an acquire + /// load: it may read *any* readable message whose value differs from + /// `current`, not necessarily the latest one. A strong CAS merely never + /// fails after reading a value equal to `current`. #[inline(always)] #[verifier::external_body] #[verifier::atomic] @@ -1106,7 +1118,6 @@ impl AtomicUsizeW { let read_msg = old(auth).msg_at(read_ts); let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); &&& old(auth).readable(old(tv)@, read_ts) - &&& read_ts + 1 == old(auth).history().len() &&& match res.0 { Ok(v) => { let write_ts = old(auth).history().len(); @@ -1114,6 +1125,7 @@ impl AtomicUsizeW { value: new, view: after_read.observe(self.id(), write_ts), }; + &&& read_ts + 1 == old(auth).history().len() &&& v == current &&& read_msg.value == current &&& final(auth).id() == old(auth).id() @@ -1217,8 +1229,9 @@ impl AtomicUsizeW { /// Generate a TCB executable wrapper around one Rust integer atomic type. /// /// All integer atomics share the same weak-memory history shape: load chooses a -/// readable message, stores append a message, and CAS reads the latest message -/// before either appending a new one or failing as an acquire read. +/// readable message, stores append a message, and CAS either reads the latest +/// message and appends its write right after it (success), or acts as an +/// acquire read of any readable message with a different value (failure). macro_rules! declare_integer_atomic_wrapper { ($wrapper:ident, $rust_atomic:ident, $value_ty:ty) => { verus! { @@ -1318,7 +1331,6 @@ macro_rules! declare_integer_atomic_wrapper { let read_msg = old(auth).msg_at(read_ts); let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); &&& old(auth).readable(old(tv)@, read_ts) - &&& read_ts + 1 == old(auth).history().len() &&& match res.0 { Ok(v) => { let write_ts = old(auth).history().len(); @@ -1326,6 +1338,7 @@ macro_rules! declare_integer_atomic_wrapper { value: new, view: after_read.observe(self.id(), write_ts), }; + &&& read_ts + 1 == old(auth).history().len() &&& v == current &&& read_msg.value == current &&& final(auth).id() == old(auth).id() @@ -1535,8 +1548,9 @@ impl AtomicBoolW { /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` /// failure ordering. /// - /// On success it appends `new`; on failure it only imports the acquired view - /// from the message it read. + /// On success it reads the latest message and appends `new` immediately + /// after it; on failure it acts as an acquire load that may read any + /// readable message with a different value, importing that message's view. #[inline(always)] #[verifier::external_body] #[verifier::atomic] @@ -1556,7 +1570,6 @@ impl AtomicBoolW { let read_msg = old(auth).msg_at(read_ts); let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); &&& old(auth).readable(old(tv)@, read_ts) - &&& read_ts + 1 == old(auth).history().len() &&& match res.0 { Ok(v) => { let write_ts = old(auth).history().len(); @@ -1564,6 +1577,7 @@ impl AtomicBoolW { value: new, view: after_read.observe(self.id(), write_ts), }; + &&& read_ts + 1 == old(auth).history().len() &&& v == current &&& read_msg.value == current &&& final(auth).id() == old(auth).id() @@ -1823,6 +1837,10 @@ impl AtomicPtrW { /// Pointer CAS compares runtime pointer identity, which Verus models as /// address equality for sized pointers. The returned pointer and written /// message still carry the full pointer value, including provenance. + /// + /// On success the read is the latest message and the write is appended + /// immediately after it; on failure the operation is an acquire load that + /// may read any readable message whose address differs from `current`. #[inline(always)] #[verifier::external_body] #[verifier::atomic] @@ -1842,7 +1860,6 @@ impl AtomicPtrW { let read_msg = old(auth).msg_at(read_ts); let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); &&& old(auth).readable(old(tv)@, read_ts) - &&& read_ts + 1 == old(auth).history().len() &&& match res.0 { Ok(v) => { let write_ts = old(auth).history().len(); @@ -1850,6 +1867,7 @@ impl AtomicPtrW { value: new, view: after_read.observe(self.id(), write_ts), }; + &&& read_ts + 1 == old(auth).history().len() &&& current.addr() == read_msg.value.addr() &&& equal(v, read_msg.value) &&& final(auth).id() == old(auth).id() @@ -1938,9 +1956,10 @@ fn smoke_test_weak_atomic_with_ghost() { message msg; snapshot snap; ghost g => { - assert(ts + 1 == prev.len()); + assert(ts < prev.len()); match ret { Result::Ok(v) => { + assert(ts + 1 == prev.len()); assert(v == 2); assert(msg.value == 2); match snap { @@ -2005,9 +2024,10 @@ fn smoke_test_weak_atomic_with_ghost() { message msg; snapshot snap; ghost g => { - assert(ts + 1 == prev.len()); + assert(ts < prev.len()); match ret { Result::Ok(v) => { + assert(ts + 1 == prev.len()); assert(v == true); assert(msg.value == true); match snap { @@ -2073,9 +2093,10 @@ fn smoke_test_weak_atomic_with_ghost() { message msg; snapshot snap; ghost g => { - assert(ts + 1 == prev.len()); + assert(ts < prev.len()); match ret { Result::Ok(v) => { + assert(ts + 1 == prev.len()); assert(equal(v, msg.value)); assert(msg.value.addr() == null.addr()); match snap { diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index dde449e29..ac36cf453 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -250,6 +250,11 @@ impl RcuInner

{ (core::ptr::null_mut(), Tracked(None)) }; + // TODO: A freshly minted empty view makes this release store publish no + // prior observations, so in the model it degrades to relaxed-strength + // publication. This is sound (the empty view is the weakest token) but + // lossy; switch to a per-task/per-CPU view once task-attached ghost + // state is available. proof_decl! { let tracked mut tv = ThreadView::new(); } @@ -271,6 +276,9 @@ impl RcuInner

{ )] fn read(&self) -> RcuReadGuardInner<'_, P> { let inner_guard = disable_preempt(); + // TODO: The guard's view starts empty instead of inheriting the + // thread's accumulated view; sound but forgets prior observations. + // Replace with a per-task/per-CPU view once available. proof_decl! { let tracked mut tv = ThreadView::new(); } @@ -286,6 +294,8 @@ impl RcuInner

{ pub fn read_with<'a, A: InAtomicMode>(&'a self, _guard: &'a A) -> Option<

>::Ref, > where P: NonNullPtrRef<'a> { + // TODO: Same as `read`: a fresh empty view forgets the thread's prior + // observations; replace with a per-task/per-CPU view once available. proof_decl! { let tracked mut tv = ThreadView::new(); } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 38fa252c1..7fe906346 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -70,19 +70,67 @@ pub open spec fn callback_summaries(callbacks: Callbacks) -> Seq, + pub is_complete: bool, +} + +impl GracePeriodView { + pub open spec fn has_pending_work(self) -> bool { + !self.is_complete || self.callbacks.len() > 0 + } +} + pub(super) struct GracePeriod { callbacks: Callbacks, cpu_mask: AtomicCpuSet, is_complete: bool, } +impl View for GracePeriod { + type V = GracePeriodView; + + closed spec fn view(&self) -> GracePeriodView { + GracePeriodView { + callbacks: callback_summaries(self.callbacks), + is_complete: self.is_complete, + } + } +} + impl GracePeriod { closed spec fn callback_summaries(self) -> Seq { - callback_summaries(self.callbacks) + self@.callbacks } closed spec fn has_pending_work(self) -> bool { - !self.is_complete || self.callback_summaries().len() > 0 + self@.has_pending_work() + } +} + +/// Proof-facing summary of the monitor state protected by `RcuMonitor::state`. +pub ghost struct MonitorStateView { + pub current_gp: GracePeriodView, + pub next_callbacks: Seq, +} + +impl MonitorStateView { + pub open spec fn pending_summaries(self) -> Seq { + self.current_gp.callbacks.add(self.next_callbacks) + } + + pub open spec fn has_pending_work(self) -> bool { + self.current_gp.has_pending_work() || self.next_callbacks.len() > 0 + } + + pub open spec fn no_pending_work(self) -> bool { + !self.has_pending_work() } } @@ -91,14 +139,63 @@ pub(super) struct State { next_callbacks: Callbacks, } +impl View for State { + type V = MonitorStateView; + + closed spec fn view(&self) -> MonitorStateView { + MonitorStateView { + current_gp: self.current_gp@, + next_callbacks: callback_summaries(self.next_callbacks), + } + } +} + impl State { closed spec fn pending_summaries(self) -> Seq { - self.current_gp.callback_summaries().add(callback_summaries(self.next_callbacks)) + self@.pending_summaries() } closed spec fn has_pending_work(self) -> bool { - self.current_gp.has_pending_work() || callback_summaries(self.next_callbacks).len() > 0 + self@.has_pending_work() } + + closed spec fn no_pending_work(self) -> bool { + self@.no_pending_work() + } +} + +/// Relationship later maintained between the weak `is_monitoring` flag and the +/// lock-protected monitor state. A `true` flag may over-approximate work, but a +/// `false` flag must be precise enough to certify no pending callbacks or grace +/// period. +pub open spec fn monitor_flag_matches_state(flag: bool, state: MonitorStateView) -> bool { + !flag ==> state.no_pending_work() +} + +/// Bridge used by the future `set_monitoring` helper: the weak atomic flag +/// invariant already requires `!flag ==> !pending`; once the caller proves that +/// `pending` exactly summarizes the lock-protected state, we get the stronger +/// state-level fact needed by the monitor fast path. +proof fn monitor_flag_matches_state_from_pending( + flag: bool, + pending: bool, + state: State, +) + requires + !flag ==> !pending, + pending == state.has_pending_work(), + ensures + monitor_flag_matches_state(flag, state@), +{ +} + +proof fn monitor_flag_false_has_no_pending_state(flag: bool, state: State) + requires + monitor_flag_matches_state(flag, state@), + !flag, + ensures + state.no_pending_work(), +{ } /// A RCU monitor ensures the completion of _grace periods_ by keeping track From e16147df9c9c184729f8ff721dc37d928ee71106 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 11 Jun 2026 05:12:37 -0400 Subject: [PATCH 10/47] more --- ostd/specs/sync/rcu.rs | 171 ++++++++++++++++++++++++++++++----- ostd/src/sync/rcu/mod.rs | 10 +- ostd/src/sync/rcu/monitor.rs | 124 ++++++++++++++----------- 3 files changed, 226 insertions(+), 79 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 64055dd7b..d38e3d585 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -60,40 +60,146 @@ impl WeakAtomicInvariantPredicate for RcuWeakAtomicInv { } } +/// Proof-facing summary of one grace period. +/// +/// The executable CPU mask is intentionally not part of this first state model: +/// the current proof cut only needs to connect pending callbacks to the monitor +/// flag. A later epoch/quiescent-state invariant should refine this view with +/// CPU progress. +pub ghost struct GracePeriodView { + pub callbacks: Seq, + pub is_complete: bool, +} + +impl GracePeriodView { + /// The state of the grace period when the monitor is created: complete, + /// with no callbacks attached. + pub open spec fn initial() -> Self { + GracePeriodView { callbacks: Seq::empty(), is_complete: true } + } + + pub open spec fn has_pending_work(self) -> bool { + !self.is_complete || self.callbacks.len() > 0 + } + + /// Lock-protected well-formedness: a completed grace period has already + /// had its callbacks taken. The monitor may break this transiently inside + /// a critical section (between completing a grace period and taking its + /// callbacks), but it must hold whenever the monitor lock is released. + pub open spec fn wf(self) -> bool { + self.is_complete ==> self.callbacks.len() == 0 + } +} + +/// Proof-facing summary of the monitor state protected by the RCU monitor's +/// lock. +pub ghost struct MonitorStateView { + pub current_gp: GracePeriodView, + pub next_callbacks: Seq, +} + +impl MonitorStateView { + /// The monitor state at creation: a complete grace period and no queued + /// callbacks. + pub open spec fn initial() -> Self { + MonitorStateView { + current_gp: GracePeriodView::initial(), + next_callbacks: Seq::empty(), + } + } + + /// All callback summaries the monitor is still responsible for. + pub open spec fn pending_summaries(self) -> Seq { + self.current_gp.callbacks.add(self.next_callbacks) + } + + pub open spec fn has_pending_work(self) -> bool { + self.current_gp.has_pending_work() || self.next_callbacks.len() > 0 + } + + pub open spec fn no_pending_work(self) -> bool { + !self.has_pending_work() + } + + /// Lock-protected well-formedness: when the current grace period is + /// complete, the monitor has either restarted it with the queued callbacks + /// or stopped monitoring, so both callback lists are empty. + pub open spec fn wf(self) -> bool { + &&& self.current_gp.wf() + &&& self.current_gp.is_complete ==> self.next_callbacks.len() == 0 + } +} + +/// Under the lock-protected invariant, "has pending work" collapses to "the +/// current grace period is incomplete": a complete grace period implies both +/// callback lists are empty. +pub proof fn monitor_state_pending_iff_incomplete(state: MonitorStateView) + requires + state.wf(), + ensures + state.has_pending_work() <==> !state.current_gp.is_complete, + state.no_pending_work() <==> state.current_gp.is_complete, +{ +} + +/// `no_pending_work` certifies that the pending-summary sequence is empty. +pub proof fn monitor_state_no_pending_no_summaries(state: MonitorStateView) + requires + state.no_pending_work(), + ensures + state.pending_summaries() == Seq::::empty(), +{ +} + /// Ghost summary paired with the RCU monitor's `is_monitoring` flag. /// -/// `pending[i]` summarizes whether the monitor state represented at flag -/// message `i` has queued callbacks or an active grace period that must still be -/// observed. This is intentionally a summary: the concrete callback vectors -/// live in the monitor state protected by its lock. +/// `states[i]` summarizes the lock-protected monitor state at the moment flag +/// message `i` was appended. This is intentionally a summary: the concrete +/// callback vectors live in the monitor state protected by its lock, and the +/// agreement between `states[i]` and that state is established by the writer, +/// which performs every flag store while holding the monitor lock. pub ghost struct RcuMonitorFlagGhost { - pub pending: Seq, + pub states: Seq, } impl RcuMonitorFlagGhost { pub open spec fn initial() -> Self { - RcuMonitorFlagGhost { pending: seq![false] } + RcuMonitorFlagGhost { states: seq![MonitorStateView::initial()] } + } + + pub open spec fn push(self, state: MonitorStateView) -> Self { + RcuMonitorFlagGhost { states: self.states.push(state) } } - pub open spec fn push(self, pending: bool) -> Self { - RcuMonitorFlagGhost { pending: self.pending.push(pending) } + /// Whether the state recorded at flag message `i` still had work pending. + pub open spec fn pending_at(self, i: int) -> bool { + self.states[i].has_pending_work() } } /// Weak-memory invariant for the monitor's fast-path flag. /// -/// The invariant is deliberately one-way: a `false` flag message certifies no -/// pending monitor work for that message's ghost summary. A `true` flag is -/// conservative and may over-approximate pending work. +/// Every flag message carries a well-formed snapshot of the monitor state, and +/// the invariant is deliberately one-way: a `false` flag message certifies that +/// the state recorded at that message had no pending monitor work. A `true` +/// flag is conservative and may over-approximate pending work. +/// +/// Note the weak-memory reading: a relaxed load may observe a stale message, +/// so a `false` read only certifies "no pending work as of that message", not +/// "no pending work now". That is exactly what the monitor fast path needs: +/// callbacks enqueued after that message were published together with a `true` +/// flag message, so skipping the slow path can only delay their grace period, +/// never lose them. pub open spec fn rcu_monitor_flag_history_inv( history: History, ghost: RcuMonitorFlagGhost, ) -> bool { &&& history.len() >= 1 - &&& ghost.pending.len() == history.len() + &&& ghost.states.len() == history.len() + &&& forall|i: int| 0 <= i < history.len() ==> (#[trigger] ghost.states[i]).wf() &&& forall|i: int| 0 <= i < history.len() ==> { - &&& !#[trigger] history[i].value ==> !ghost.pending[i] + !(#[trigger] history[i].value) ==> ghost.states[i].no_pending_work() } } @@ -119,38 +225,57 @@ pub proof fn rcu_monitor_flag_initial_inv() { } +/// Pushing one flag message preserves the history invariant, provided the +/// writer records a well-formed state snapshot and only writes `false` when +/// that snapshot has no pending work. +/// +/// This is the proof obligation of the future `set_monitoring` helper: it +/// stores the flag while holding the monitor lock, so it can supply the +/// lock-protected state view as the snapshot. pub proof fn preserve_rcu_monitor_flag_inv_on_push( prev: History, next: History, msg: Msg, prev_ghost: RcuMonitorFlagGhost, next_ghost: RcuMonitorFlagGhost, - pending: bool, + state: MonitorStateView, ) requires rcu_monitor_flag_history_inv(prev, prev_ghost), next == prev.push(msg), - next_ghost == prev_ghost.push(pending), - !msg.value ==> !pending, + next_ghost == prev_ghost.push(state), + state.wf(), + !msg.value ==> state.no_pending_work(), ensures rcu_monitor_flag_history_inv(next, next_ghost), { assert(next.len() >= 1); - assert(next_ghost.pending.len() == next.len()); + assert(next_ghost.states.len() == next.len()); + assert forall|i: int| 0 <= i < next.len() implies (#[trigger] next_ghost.states[i]).wf() by { + if i == prev.len() { + assert(next_ghost.states[i] == state); + } else { + assert(i < prev.len()); + assert(next_ghost.states[i] == prev_ghost.states[i]); + } + }; assert forall|i: int| 0 <= i < next.len() implies { - &&& !#[trigger] next[i].value ==> !next_ghost.pending[i] + !(#[trigger] next[i].value) ==> next_ghost.states[i].no_pending_work() } by { if i == prev.len() { assert(next[i] == msg); - assert(next_ghost.pending[i] == pending); + assert(next_ghost.states[i] == state); } else { assert(i < prev.len()); assert(next[i] == prev[i]); - assert(next_ghost.pending[i] == prev_ghost.pending[i]); + assert(next_ghost.states[i] == prev_ghost.states[i]); } }; } +/// The key safety fact behind the monitor fast path: observing a `false` flag +/// message certifies that the monitor state recorded at that message had no +/// queued callbacks and no incomplete grace period. pub proof fn rcu_monitor_flag_false_has_no_pending( history: History, ghost: RcuMonitorFlagGhost, @@ -161,8 +286,12 @@ pub proof fn rcu_monitor_flag_false_has_no_pending( ts < history.len(), !history[ts as int].value, ensures - !ghost.pending[ts as int], + ghost.states[ts as int].no_pending_work(), + ghost.states[ts as int].pending_summaries() =~= Seq::::empty(), + ghost.states[ts as int].current_gp.is_complete, { + monitor_state_pending_iff_incomplete(ghost.states[ts as int]); + monitor_state_no_pending_no_summaries(ghost.states[ts as int]); } pub proof fn preserve_rcu_history_inv_on_push( diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index ac36cf453..b89db7119 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -42,10 +42,12 @@ //! safe reclamation callback. //! //! The monitor also has a weak-memory `is_monitoring` flag with an RCU-specific -//! invariant. Today that invariant records whether a flag-history message may -//! correspond to pending monitor work. The next step is to tie that flag summary -//! to `monitor::State::pending_summaries()` and to prove callback execution only -//! after the relevant grace period has completed. +//! invariant: every flag-history message records a snapshot of the +//! lock-protected monitor state (`specs::sync::rcu::MonitorStateView`), and a +//! `false` message certifies that its snapshot has no pending callbacks and no +//! incomplete grace period. The next step is to implement +//! `set_monitoring`/`finish_grace_period` against this invariant and to prove +//! callback execution only after the relevant grace period has completed. //! //! # Usage outline //! diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 7fe906346..be540e293 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -8,7 +8,8 @@ use crate::specs::{ mm::cpu::{AtomicCpuSet, CpuSet}, sync::{ rcu as rcu_spec, - weak_memory::WeakAtomicBool, + rcu::{GracePeriodView, MonitorStateView}, + weak_memory::{History, WeakAtomicBool}, }, }; use crate::sync::{LocalIrqDisabled, SpinLock}; @@ -70,23 +71,9 @@ pub open spec fn callback_summaries(callbacks: Callbacks) -> Seq, - pub is_complete: bool, -} - -impl GracePeriodView { - pub open spec fn has_pending_work(self) -> bool { - !self.is_complete || self.callbacks.len() > 0 - } -} - +// The proof-facing views `GracePeriodView` and `MonitorStateView` live in +// `specs::sync::rcu` so that the monitor flag's weak-memory ghost state can +// record a state snapshot per flag message without depending on this module. pub(super) struct GracePeriod { callbacks: Callbacks, cpu_mask: AtomicCpuSet, @@ -112,25 +99,13 @@ impl GracePeriod { closed spec fn has_pending_work(self) -> bool { self@.has_pending_work() } -} -/// Proof-facing summary of the monitor state protected by `RcuMonitor::state`. -pub ghost struct MonitorStateView { - pub current_gp: GracePeriodView, - pub next_callbacks: Seq, -} - -impl MonitorStateView { - pub open spec fn pending_summaries(self) -> Seq { - self.current_gp.callbacks.add(self.next_callbacks) - } - - pub open spec fn has_pending_work(self) -> bool { - self.current_gp.has_pending_work() || self.next_callbacks.len() > 0 - } - - pub open spec fn no_pending_work(self) -> bool { - !self.has_pending_work() + /// Lock-protected invariant: a completed grace period has already had its + /// callbacks taken. Monitor methods may break this transiently inside a + /// critical section (between completing a grace period and taking its + /// callbacks), but must restore it before releasing the monitor lock. + closed spec fn wf(self) -> bool { + self@.wf() } } @@ -162,39 +137,80 @@ impl State { closed spec fn no_pending_work(self) -> bool { self@.no_pending_work() } + + /// Lock-protected invariant of the whole monitor state: the current grace + /// period is well-formed, and a complete grace period implies an empty + /// next-callback queue (the monitor either restarted the grace period with + /// the queued callbacks or stopped monitoring). Holds whenever the monitor + /// lock is free. + closed spec fn wf(self) -> bool { + self@.wf() + } } -/// Relationship later maintained between the weak `is_monitoring` flag and the -/// lock-protected monitor state. A `true` flag may over-approximate work, but a -/// `false` flag must be precise enough to certify no pending callbacks or grace -/// period. +/// Relationship maintained between one message of the weak `is_monitoring` +/// flag and the monitor-state snapshot recorded with it. A `true` flag may +/// over-approximate work, but a `false` flag must be precise enough to certify +/// no pending callbacks or grace period. pub open spec fn monitor_flag_matches_state(flag: bool, state: MonitorStateView) -> bool { !flag ==> state.no_pending_work() } -/// Bridge used by the future `set_monitoring` helper: the weak atomic flag -/// invariant already requires `!flag ==> !pending`; once the caller proves that -/// `pending` exactly summarizes the lock-protected state, we get the stronger -/// state-level fact needed by the monitor fast path. -proof fn monitor_flag_matches_state_from_pending( - flag: bool, - pending: bool, - state: State, +/// Every message of the monitor flag matches the state snapshot recorded with +/// it: the weak-memory history invariant implies the per-message relation +/// above, for stale messages as well as the latest one. +proof fn monitor_flag_message_matches_state( + history: History, + flag_ghost: rcu_spec::RcuMonitorFlagGhost, + ts: nat, ) requires - !flag ==> !pending, - pending == state.has_pending_work(), + rcu_spec::rcu_monitor_flag_history_inv(history, flag_ghost), + ts < history.len(), ensures - monitor_flag_matches_state(flag, state@), + monitor_flag_matches_state(history[ts as int].value, flag_ghost.states[ts as int]), { + if !history[ts as int].value { + rcu_spec::rcu_monitor_flag_false_has_no_pending(history, flag_ghost, ts); + } } -proof fn monitor_flag_false_has_no_pending_state(flag: bool, state: State) +/// The fast-path certificate at the executable `State` level: reading a +/// `false` flag message whose snapshot agrees with the lock-protected state +/// proves that this state has no queued callbacks and no incomplete grace +/// period. The agreement precondition is discharged by the writer protocol: +/// every flag store happens under the monitor lock and records the +/// lock-protected state as its snapshot. +proof fn monitor_flag_false_certifies_no_pending( + history: History, + flag_ghost: rcu_spec::RcuMonitorFlagGhost, + ts: nat, + state: State, +) requires - monitor_flag_matches_state(flag, state@), - !flag, + rcu_spec::rcu_monitor_flag_history_inv(history, flag_ghost), + ts < history.len(), + !history[ts as int].value, + flag_ghost.states[ts as int] == state@, ensures state.no_pending_work(), + state.pending_summaries() =~= Seq::::empty(), +{ + rcu_spec::rcu_monitor_flag_false_has_no_pending(history, flag_ghost, ts); +} + +/// Bridge for the future `set_monitoring` helper: while holding the monitor +/// lock with a well-formed state, writing any flag value that over-approximates +/// the state's pending work discharges the push obligation of +/// [`rcu_spec::preserve_rcu_monitor_flag_inv_on_push`]. +proof fn monitor_flag_push_obligation(flag: bool, state: State) + requires + state.wf(), + state.has_pending_work() ==> flag, + ensures + state@.wf(), + !flag ==> state@.no_pending_work(), + monitor_flag_matches_state(flag, state@), { } From 3cbf462e0d0d8ca70d81ac041824954ac6129277 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 11 Jun 2026 22:29:22 -0400 Subject: [PATCH 11/47] test monitor --- ostd/specs/sync/rcu.rs | 22 +++- ostd/specs/sync/weak_memory.rs | 50 +++++++++ ostd/src/sync/rcu/monitor.rs | 179 ++++++++++++++++++++++++++++++++- 3 files changed, 245 insertions(+), 6 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index d38e3d585..30dc1bbeb 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -158,8 +158,8 @@ pub proof fn monitor_state_no_pending_no_summaries(state: MonitorStateView) /// callback vectors live in the monitor state protected by its lock, and the /// agreement between `states[i]` and that state is established by the writer, /// which performs every flag store while holding the monitor lock. -pub ghost struct RcuMonitorFlagGhost { - pub states: Seq, +pub tracked struct RcuMonitorFlagGhost { + pub ghost states: Seq, } impl RcuMonitorFlagGhost { @@ -167,10 +167,26 @@ impl RcuMonitorFlagGhost { RcuMonitorFlagGhost { states: seq![MonitorStateView::initial()] } } + /// Proof-mode constructor for the tracked ghost state stored inside the + /// monitor flag's weak atomic invariant. + pub proof fn tracked_initial() -> (tracked res: Self) + ensures + res == Self::initial(), + { + RcuMonitorFlagGhost { states: seq![MonitorStateView::initial()] } + } + pub open spec fn push(self, state: MonitorStateView) -> Self { RcuMonitorFlagGhost { states: self.states.push(state) } } + pub proof fn tracked_push(tracked self, state: MonitorStateView) -> (tracked res: Self) + ensures + res == self.push(state), + { + RcuMonitorFlagGhost { states: self.states.push(state) } + } + /// Whether the state recorded at flag message `i` still had work pending. pub open spec fn pending_at(self, i: int) -> bool { self.states[i].has_pending_work() @@ -287,7 +303,7 @@ pub proof fn rcu_monitor_flag_false_has_no_pending( !history[ts as int].value, ensures ghost.states[ts as int].no_pending_work(), - ghost.states[ts as int].pending_summaries() =~= Seq::::empty(), + ghost.states[ts as int].pending_summaries() == Seq::::empty(), ghost.states[ts as int].current_gp.is_complete, { monitor_state_pending_iff_incomplete(ghost.states[ts as int]); diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 912aaa59f..6f962a979 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -350,6 +350,7 @@ macro_rules! declare_weak_atomic_type { requires Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), ensures + res.well_formed(), res.constant() == k, { let (atomic, Tracked(hist)) = $raw_atomic::new(init); @@ -755,6 +756,55 @@ impl WeakAtomicPtr { } } +impl WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv> { + /// Relaxed-store helper for the RCU monitor flag. + /// + /// The executable flag remains a relaxed atomic flag, matching the old + /// monitor protocol. The proof-side effect is stronger: each stored flag + /// message appends the lock-protected monitor-state snapshot supplied by + /// the writer. + #[inline(always)] + pub fn store_relaxed_rcu_monitor( + &self, + value: bool, + Ghost(state): Ghost, + Tracked(tv): Tracked<&mut ThreadView>, + ) + requires + self.well_formed(), + state.wf(), + !value ==> state.no_pending_work(), + { + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, mut g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + let ghost prev = hist.history(); + let snap = self.atomic.store_relaxed(Tracked(&mut hist), Tracked(tv), value); + let ghost next = hist.history(); + proof { + assert(snap@.msg().value == value); + rcu_spec::preserve_rcu_monitor_flag_inv_on_push( + prev, + next, + snap@.msg(), + g, + g.push(state), + state, + ); + g = g.tracked_push(state); + pair = (hist, g); + } + }); + } +} + /// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, /// but for weak-memory atomics with per-thread view tokens and message histories. /// diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index be540e293..853ba94d9 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -9,7 +9,7 @@ use crate::specs::{ sync::{ rcu as rcu_spec, rcu::{GracePeriodView, MonitorStateView}, - weak_memory::{History, WeakAtomicBool}, + weak_memory::{History, ThreadView, WeakAtomicBool}, }, }; use crate::sync::{LocalIrqDisabled, SpinLock}; @@ -71,6 +71,18 @@ pub open spec fn callback_summaries(callbacks: Callbacks) -> Seq::empty(), + ensures + callback_summaries(callbacks) == Seq::::empty(), +{ + assert(callback_summaries(callbacks).len() == 0); + vstd::seq_lib::assert_seqs_equal!( + callback_summaries(callbacks) == Seq::::empty() + ); +} + // The proof-facing views `GracePeriodView` and `MonitorStateView` live in // `specs::sync::rcu` so that the monitor flag's weak-memory ghost state can // record a state snapshot per flag message without depending on this module. @@ -92,6 +104,35 @@ impl View for GracePeriod { } impl GracePeriod { + /// Creates the initial completed grace period. A completed grace period may + /// not retain callbacks, so both executable callback storage and the proof + /// summary start empty. + pub(super) fn new() -> (res: Self) + ensures + res@ == GracePeriodView::initial(), + { + let callbacks = Vec::new(); + let cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); + let res = Self { callbacks, cpu_mask, is_complete: true }; + proof { + callback_summaries_empty(res.callbacks); + } + res + } + + /// Starts a new incomplete grace period with the callbacks that should run + /// after it completes. The CPU mask is reset because all CPUs must pass a + /// fresh quiescent state for this new batch. + fn restart(&mut self, callbacks: Callbacks) + ensures + final(self).callback_summaries() == callback_summaries(callbacks), + !final(self).is_complete, + { + self.is_complete = false; + self.callbacks = callbacks; + self.cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); + } + closed spec fn callback_summaries(self) -> Seq { self@.callbacks } @@ -107,6 +148,11 @@ impl GracePeriod { closed spec fn wf(self) -> bool { self@.wf() } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } } pub(super) struct State { @@ -126,6 +172,45 @@ impl View for State { } impl State { + /// Creates the lock-protected initial monitor state: there is no active + /// grace period and no callbacks waiting to be attached to the next one. + pub(super) fn new() -> (res: Self) + ensures + res@ == MonitorStateView::initial(), + res.no_pending_work(), + { + let current_gp = GracePeriod::new(); + let next_callbacks = Vec::new(); + let res = Self { current_gp, next_callbacks }; + proof { + callback_summaries_empty(res.next_callbacks); + } + res + } + + /// Enqueues one callback and starts a grace period if the monitor was idle. + /// + /// The method preserves the lock-protected state invariant. Returning with + /// pending work lets the caller publish `is_monitoring = true` with the + /// resulting state view. + fn enqueue_after_grace_period(&mut self, callback: RcuCallback) + ensures + final(self).wf(), + final(self).has_pending_work(), + { + if self.current_gp.is_complete { + let mut callbacks = Vec::new(); + callbacks.push(callback); + let cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); + self.current_gp = GracePeriod { callbacks, cpu_mask, is_complete: false }; + } else { + let mut next_callbacks = Vec::new(); + core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); + next_callbacks.push(callback); + self.next_callbacks = next_callbacks; + } + } + closed spec fn pending_summaries(self) -> Seq { self@.pending_summaries() } @@ -134,7 +219,7 @@ impl State { self@.has_pending_work() } - closed spec fn no_pending_work(self) -> bool { + pub closed spec fn no_pending_work(self) -> bool { self@.no_pending_work() } @@ -146,6 +231,11 @@ impl State { closed spec fn wf(self) -> bool { self@.wf() } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } } /// Relationship maintained between one message of the weak `is_monitoring` @@ -156,6 +246,20 @@ pub open spec fn monitor_flag_matches_state(flag: bool, state: MonitorStateView) !flag ==> state.no_pending_work() } +/// View-level form of the monitor flag write obligation. Executable monitor +/// code can call this while holding a guard by passing the protected state's +/// view, without moving the `State` value out of the lock. +proof fn monitor_flag_view_push_obligation(flag: bool, state: MonitorStateView) + requires + state.wf(), + state.has_pending_work() ==> flag, + ensures + state.wf(), + !flag ==> state.no_pending_work(), + monitor_flag_matches_state(flag, state), +{ +} + /// Every message of the monitor flag matches the state snapshot recorded with /// it: the weak-memory history invariant implies the per-message relation /// above, for stale messages as well as the latest one. @@ -194,7 +298,7 @@ proof fn monitor_flag_false_certifies_no_pending( flag_ghost.states[ts as int] == state@, ensures state.no_pending_work(), - state.pending_summaries() =~= Seq::::empty(), + state.pending_summaries() == Seq::::empty(), { rcu_spec::rcu_monitor_flag_false_has_no_pending(history, flag_ghost, ts); } @@ -212,6 +316,7 @@ proof fn monitor_flag_push_obligation(flag: bool, state: State) !flag ==> state@.no_pending_work(), monitor_flag_matches_state(flag, state@), { + monitor_flag_view_push_obligation(flag, state@); } /// A RCU monitor ensures the completion of _grace periods_ by keeping track @@ -222,6 +327,74 @@ pub(super) struct RcuMonitor { } impl RcuMonitor { + /// Creates the monitor with an initially false weak flag. The flag ghost + /// records the same initial lock-protected state snapshot, so every + /// possible read of the initial `false` message certifies that there is no + /// pending monitor work. + pub(super) fn new() -> (res: Self) { + let state = State::new(); + proof { + rcu_spec::rcu_monitor_flag_initial_inv(); + } + proof_decl! { + let tracked flag_ghost = rcu_spec::RcuMonitorFlagGhost::tracked_initial(); + } + let is_monitoring = MonitorAtomicBool::new( + Ghost(()), + false, + Tracked(flag_ghost), + ); + let state = SpinLock::new(state); + proof { + use_type_invariant(&is_monitoring); + assert(is_monitoring.well_formed()); + use_type_invariant(&state); + } + let res = Self { is_monitoring, state }; + res + } + + /// Stores the monitor fast-path flag together with the monitor-state + /// snapshot that justifies the new flag message. Callers should hold the + /// monitor lock and pass the view of the lock-protected state. + fn set_monitoring( + &self, + value: bool, + Ghost(state): Ghost, + Tracked(tv): Tracked<&mut ThreadView>, + ) + requires + self.wf(), + state.wf(), + state.has_pending_work() ==> value, + { + proof { + use_type_invariant(self); + monitor_flag_view_push_obligation(value, state); + } + self.is_monitoring.store_relaxed_rcu_monitor(value, Ghost(state), Tracked(tv)); + } + + /// Schedules `callback` to run after a future grace period and publishes a + /// conservative `true` monitor flag message for the resulting state. + pub(super) fn after_grace_period(&self, callback: RcuCallback) { + proof { + use_type_invariant(self); + } + let mut state = self.state.lock(); + state.enqueue_after_grace_period(callback); + proof { + assert(state.view().wf()); + assert(state.view().has_pending_work()); + use_type_invariant(self); + } + proof_decl! { + let tracked mut tv = ThreadView::new(); + } + self.set_monitoring(true, Ghost(state.view()@), Tracked(&mut tv)); + state.drop(); + } + closed spec fn wf(self) -> bool { &&& self.is_monitoring.well_formed() &&& self.state.type_inv() From fc3f983c4d0fe98afe3990eb9ece7ca5b8991ee0 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Sun, 14 Jun 2026 22:53:29 -0400 Subject: [PATCH 12/47] Update Cargo.lock --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35318fb49..59fc8d2ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,7 +576,7 @@ version = "0.0.0-2026-05-17-0151" [[package]] name = "verus_builtin_macros" -version = "0.0.0-2026-05-31-0205" +version = "0.0.0-2026-06-14-0213" dependencies = [ "proc-macro2", "quote", @@ -596,7 +596,7 @@ dependencies = [ [[package]] name = "verus_state_machines_macros" -version = "0.0.0-2026-05-31-0205" +version = "0.0.0-2026-06-14-0213" dependencies = [ "indexmap", "proc-macro2", @@ -627,7 +627,7 @@ checksum = "af8ca9a5d4debca0633e697c88269395493cebf2e10db21ca2dbde37c1356452" [[package]] name = "vstd" -version = "0.0.0-2026-05-31-0205" +version = "0.0.0-2026-06-14-0213" dependencies = [ "verus_builtin", "verus_builtin_macros", From 80ae11eaf0037058da3fb86d838c9b008b7fbed7 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Sun, 14 Jun 2026 23:01:42 -0400 Subject: [PATCH 13/47] fix merge issues and vstd version --- ostd/specs/sync/weak_memory.rs | 10 +++++----- ostd/src/sync/rcu/mod.rs | 11 ----------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 6f962a979..e99cf9397 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -57,13 +57,13 @@ pub type Timestamp = nat; /// and the reader reads the message via some atomic operations, then the reader's /// thread view will advance to at least `ts` for `id`. pub ghost struct WmView { - pub seen: Map, + pub seen: IMap, } impl WmView { /// Creates an empty view. pub open spec fn empty() -> Self { - WmView { seen: Map::empty() } + WmView { seen: IMap::empty() } } pub open spec fn seen_at(self, id: AtomicId) -> Timestamp { @@ -103,7 +103,7 @@ impl WmView { /// release view carried by the message it read. pub open spec fn join(self, other: Self) -> Self { WmView { - seen: Map::new( + seen: IMap::new( |id: AtomicId| self.seen.contains_key(id) || other.seen.contains_key(id), |id: AtomicId| if self.seen_at(id) <= other.seen_at(id) { @@ -160,7 +160,7 @@ impl HistAuth { self.auth.id() } - pub closed spec fn map(self) -> Map> { + pub closed spec fn map(self) -> IMap> { self.auth@ } @@ -177,7 +177,7 @@ impl HistAuth { pub open spec fn wf(self) -> bool { &&& self.len() > 0 - &&& self.map().dom() =~= Set::new(|ts: Timestamp| ts < self.len()) + &&& self.map().dom() =~= ISet::new(|ts: Timestamp| ts < self.len()) } pub open spec fn valid_ts(self, ts: Timestamp) -> bool { diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 2c714a032..b89db7119 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -175,7 +175,6 @@ impl RcuInner

{ _marker: PhantomData::<*const

::Target>, } } -} #[inline(always)] #[verus_spec(res => @@ -421,16 +420,6 @@ impl Rcu

{ } } -impl RcuOption

{ - /// Creates a new RCU primitive that contains nothing. - /// - /// This is a constant equivalence to [`RcuOption::new(None)`]. - #[inline(always)] - pub const fn new_none() -> Self { - Self(RcuInner::new_none()) - } -} - #[verus_verify] impl RcuOption

{ /// Creates a nullable RCU primitive. From 7f62937bba35560b8133b39c6ebd68b7ef788296 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Sun, 14 Jun 2026 23:03:22 -0400 Subject: [PATCH 14/47] formatting --- ostd/specs/sync/rcu.rs | 23 ++------ ostd/src/sync/rcu/mod.rs | 2 +- ostd/src/sync/rcu/monitor.rs | 60 +++----------------- verified_libs/vstd_extra/src/raw_callback.rs | 12 ++-- 4 files changed, 18 insertions(+), 79 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 30dc1bbeb..9f14b063e 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -102,10 +102,7 @@ impl MonitorStateView { /// The monitor state at creation: a complete grace period and no queued /// callbacks. pub open spec fn initial() -> Self { - MonitorStateView { - current_gp: GracePeriodView::initial(), - next_callbacks: Seq::empty(), - } + MonitorStateView { current_gp: GracePeriodView::initial(), next_callbacks: Seq::empty() } } /// All callback summaries the monitor is still responsible for. @@ -222,11 +219,7 @@ pub open spec fn rcu_monitor_flag_history_inv( pub struct RcuMonitorFlagInv; impl WeakAtomicInvariantPredicate<(), bool, RcuMonitorFlagGhost> for RcuMonitorFlagInv { - open spec fn atomic_inv( - _k: (), - history: History, - ghost: RcuMonitorFlagGhost, - ) -> bool { + open spec fn atomic_inv(_k: (), history: History, ghost: RcuMonitorFlagGhost) -> bool { rcu_monitor_flag_history_inv(history, ghost) } } @@ -552,19 +545,11 @@ pub proof fn certify_callback_from_retire_perm( object.ptr() == retire.ptr(), retire.ready_to_reclaim(), ensures - cert@ == (RcuCallbackSummary { - domain: retire.domain(), - obj: object.obj(), - retire_epoch, - }), + cert@ == (RcuCallbackSummary { domain: retire.domain(), obj: object.obj(), retire_epoch }), callback_safety_from_traversal(cert, *object, retire_epoch), { RcuCallbackSafety { - summary: RcuCallbackSummary { - domain: retire.domain(), - obj: object.obj(), - retire_epoch, - }, + summary: RcuCallbackSummary { domain: retire.domain(), obj: object.obj(), retire_epoch }, } } diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index b89db7119..be0501709 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -79,8 +79,8 @@ use crate::{ use non_null::{NonNullPtr, NonNullPtrRef}; -pub mod non_null; pub mod monitor; +pub mod non_null; verus! { diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 853ba94d9..1464bc760 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -18,8 +18,11 @@ verus! { pub type Callbacks = Vec; -type MonitorAtomicBool = - WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv>; +type MonitorAtomicBool = WeakAtomicBool< + (), + rcu_spec::RcuMonitorFlagGhost, + rcu_spec::RcuMonitorFlagInv, +>; /// RCU-specific wrapper around a type-erased executable callback. /// @@ -44,10 +47,8 @@ impl RcuCallback { /// Converts a raw callback into an RCU callback, given a proof that the callback is /// safe to run after a grace period. #[inline] - pub fn from_raw( - raw: RawCallback, - Tracked(cert): Tracked, - ) -> (res: Self) + pub fn from_raw(raw: RawCallback, Tracked(cert): Tracked) -> (res: + Self) ensures res@ == cert@, { @@ -339,11 +340,7 @@ impl RcuMonitor { proof_decl! { let tracked flag_ghost = rcu_spec::RcuMonitorFlagGhost::tracked_initial(); } - let is_monitoring = MonitorAtomicBool::new( - Ghost(()), - false, - Tracked(flag_ghost), - ); + let is_monitoring = MonitorAtomicBool::new(Ghost(()), false, Tracked(flag_ghost)); let state = SpinLock::new(state); proof { use_type_invariant(&is_monitoring); @@ -407,19 +404,15 @@ impl RcuMonitor { } } // verus! - // use vstd::{ // atomic_ghost::AtomicBool, atomic_with_ghost, predicate::Predicate as DataPredicate, prelude::*, // }; // use vstd_extra::ownership::Inv; - // use crate::{ // specs::mm::cpu::{AtomicCpuSet, CpuSet}, // sync::{AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate}, // }; - // verus! { - // // This thing can be very tricky to deal with // // type Callbacks = VecDeque>; // pub(super) struct GracePeriod { @@ -427,55 +420,45 @@ impl RcuMonitor { // cpu_mask: AtomicCpuSet, // is_complete: bool, // } - // pub(super) struct State { // current_gp: GracePeriod, // // next_callbacks: Callbacks, // } - // /// Owner of this [`RcuMonitor`]. // pub(super) tracked struct RcuMonitorOwner {} - // impl DataPredicate for RcuMonitorOwner { // closed spec fn predicate(&self, v: RcuMonitor) -> bool { // true // } // } - // impl RcuMonitor { // #[verifier::type_invariant] // closed spec fn type_inv(self) -> bool { // self.wf() // } // } - // pub(super) struct RcuMonitorPred; - // impl OncePredicate> for RcuMonitorPred { // closed spec fn inv(self, v: AtomicDataWithOwner) -> bool { // &&& v.permission@.predicate(v.data) // &&& v.data.inv() // } // } - // impl Inv for RcuMonitor { // closed spec fn inv(self) -> bool { // self.wf() // } // } - // impl Inv for State { // closed spec fn inv(self) -> bool { // self.current_gp.inv() // } // } - // impl Inv for GracePeriod { // closed spec fn inv(self) -> bool { // true // } // } - // #[verus_verify] // impl RcuMonitor { // /// Creates a new RCU monitor. @@ -489,7 +472,6 @@ impl RcuMonitor { // is_monitoring: AtomicBool::new(Ghost(state), false, Tracked(false)), // state, // }; - // proof { // use_type_invariant(&res.state); // assert(res.state.type_inv()); @@ -497,7 +479,6 @@ impl RcuMonitor { // } // res // } - // /// Creates a new RCU monitor together with its tracked owner for `Once`. // #[verus_spec(r => // ensures @@ -513,14 +494,12 @@ impl RcuMonitor { // } // AtomicDataWithOwner { data, permission: Tracked(RcuMonitorOwner { }) } // } - // fn is_monitoring(&self) -> bool { // proof { // use_type_invariant(self); // } // self.is_monitoring.load() // } - // fn set_monitoring(&self, value: bool) { // proof { // use_type_invariant(self); @@ -533,7 +512,6 @@ impl RcuMonitor { // } // } // } - // impl State { // fn new() -> (res: Self) // ensures @@ -542,7 +520,6 @@ impl RcuMonitor { // Self { current_gp: GracePeriod::new() } // } // } - // impl GracePeriod { // fn new() -> (res: Self) // ensures @@ -551,21 +528,18 @@ impl RcuMonitor { // Self { cpu_mask: AtomicCpuSet::new(CpuSet::new_empty()), is_complete: true } // } // } - // } // verus! // /*use alloc::collections::VecDeque; // use core::sync::atomic::{ // AtomicBool, // Ordering::{self, Relaxed}, // }; - // use crate::{ // cpu::{AtomicCpuSet, CpuId, CpuSet, PinCurrentCpu}, // prelude::*, // sync::SpinLock, // task::atomic_mode::AsAtomicModeGuard, // }; - // impl RcuMonitor { // /// Creates a new RCU monitor. // /// @@ -577,13 +551,11 @@ impl RcuMonitor { // state: SpinLock::new(State::new()), // } // } - // pub(super) unsafe fn finish_grace_period(&self) { // // Fast path // if !self.is_monitoring.load(Relaxed) { // return; // } - // // Check if the current GP is complete after passing the quiescent state // // on the current CPU. If GP is complete, take the callbacks of the current // // GP. @@ -593,15 +565,12 @@ impl RcuMonitor { // if state.current_gp.is_complete() { // return; // } - // state.current_gp.finish_grace_period(cpu); // if !state.current_gp.is_complete() { // return; // } - // // Now that the current GP is complete, take its callbacks // let current_callbacks = state.current_gp.take_callbacks(); - // // Check if we need to watch for a next GP // if !state.next_callbacks.is_empty() { // let callbacks = core::mem::take(&mut state.next_callbacks); @@ -609,34 +578,27 @@ impl RcuMonitor { // } else { // self.is_monitoring.store(false, Relaxed); // } - // current_callbacks // }; - // // Invoke the callbacks to notify the completion of GP // for f in callbacks { // (f)(); // } // } - // pub(super) fn after_grace_period(&self, f: F) // where // F: FnOnce() + Send + 'static, // { // let mut state = self.state.disable_irq().lock(); - // state.next_callbacks.push_back(Box::new(f)); - // if !state.current_gp.is_complete() { // return; // } - // let callbacks = core::mem::take(&mut state.next_callbacks); // state.current_gp.restart(callbacks); // self.is_monitoring.store(true, Relaxed); // } // } - // impl State { // fn new() -> Self { // Self { @@ -645,7 +607,6 @@ impl RcuMonitor { // } // } // } - // impl GracePeriod { // fn new() -> Self { // Self { @@ -654,23 +615,18 @@ impl RcuMonitor { // is_complete: true, // } // } - // fn is_complete(&self) -> bool { // self.is_complete // } - // fn finish_grace_period(&mut self, this_cpu: CpuId) { // self.cpu_mask.add(this_cpu, Ordering::Relaxed); - // if self.cpu_mask.load(Ordering::Relaxed).is_full() { // self.is_complete = true; // } // } - // fn take_callbacks(&mut self) -> Callbacks { // core::mem::take(&mut self.callbacks) // } - // fn restart(&mut self, callbacks: Callbacks) { // self.is_complete = false; // self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); diff --git a/verified_libs/vstd_extra/src/raw_callback.rs b/verified_libs/vstd_extra/src/raw_callback.rs index b97cdbe38..034659e91 100644 --- a/verified_libs/vstd_extra/src/raw_callback.rs +++ b/verified_libs/vstd_extra/src/raw_callback.rs @@ -4,7 +4,6 @@ //! module therefore exposes callbacks as sized capture objects implementing //! [`RawCallbackContext`], while the executable runner erasure is kept behind //! trusted `external_body` wrappers. - use alloc::boxed::Box; use vstd::prelude::*; @@ -39,7 +38,9 @@ struct RawDropContext { // SAFETY: `RawCallback::new` only accepts `C: Send + 'static` payloads, and // `call_once` consumes the payload through the matching monomorphized runner. #[verifier::external] -unsafe impl Send for RawCallback {} +unsafe impl Send for RawCallback { + +} impl RawCallbackContext for RawDropContext { #[inline] @@ -56,10 +57,7 @@ impl RawCallback { #[verifier::external_body] pub fn new(context: C) -> Self { let payload = Box::new(RawCallbackPayload { context }); - Self { - data: Box::into_raw(payload).cast::<()>(), - run: raw_callback_runner::(), - } + Self { data: Box::into_raw(payload).cast::<()>(), run: raw_callback_runner::() } } /// Builds the common "drop this value later" callback. @@ -97,7 +95,7 @@ fn raw_callback_runner() -> usize { #[inline] #[verifier::external_body] unsafe fn call_raw_callback(data: *mut (), run: usize) { - let run: unsafe fn(*mut ()) = unsafe { core::mem::transmute(run) }; + let run: unsafe fn (*mut ()) = unsafe { core::mem::transmute(run) }; unsafe { run(data); } From eece4b87e758d74cd142973fa7083a111ce7add6 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 15 Jun 2026 01:42:09 -0400 Subject: [PATCH 15/47] tighten specs for monitor and align with upstream --- ostd/specs/mm/cpu.rs | 34 ++++ ostd/src/sync/rcu/monitor.rs | 308 +++++++++++++++++++++++++++++++---- 2 files changed, 314 insertions(+), 28 deletions(-) diff --git a/ostd/specs/mm/cpu.rs b/ostd/specs/mm/cpu.rs index 0072227e5..9398a1b56 100644 --- a/ostd/specs/mm/cpu.rs +++ b/ostd/specs/mm/cpu.rs @@ -1,9 +1,18 @@ +use core::sync::atomic::Ordering; + use vstd::prelude::*; verus! { pub struct CpuId(u32); +impl CpuId { + #[verifier::external_body] + pub fn current() -> Self { + unimplemented!() + } +} + pub struct AtomicCpuSet; impl AtomicCpuSet { @@ -11,6 +20,23 @@ impl AtomicCpuSet { pub fn new(_initial: CpuSet) -> Self { unimplemented!() } + + #[verifier::external_body] + pub fn load(&self, _ordering: Ordering) -> CpuSet + no_unwind + { + unimplemented!() + } + + pub fn store(&self, _value: &CpuSet, _ordering: Ordering) + no_unwind + { + } + + pub fn add(&self, _cpu: CpuId, _ordering: Ordering) + no_unwind + { + } } pub struct CpuSet { @@ -27,6 +53,14 @@ impl CpuSet { pub fn new_empty() -> Self returns Self::new_empty_spec(), + no_unwind + { + unimplemented!() + } + + #[verifier::external_body] + pub fn is_full(&self) -> bool + no_unwind { unimplemented!() } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 1464bc760..502e135bc 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: MPL-2.0 -use alloc::vec::Vec; +use alloc::collections::VecDeque; +use core::sync::atomic::Ordering; use vstd::prelude::*; use vstd_extra::raw_callback::RawCallback; use crate::specs::{ - mm::cpu::{AtomicCpuSet, CpuSet}, + mm::cpu::{AtomicCpuSet, CpuId, CpuSet}, sync::{ rcu as rcu_spec, rcu::{GracePeriodView, MonitorStateView}, @@ -16,7 +17,7 @@ use crate::sync::{LocalIrqDisabled, SpinLock}; verus! { -pub type Callbacks = Vec; +pub type Callbacks = VecDeque; type MonitorAtomicBool = WeakAtomicBool< (), @@ -56,22 +57,94 @@ impl RcuCallback { Self { raw, summary: Ghost(summary) } } - /// Runs the underlying callback. The reclaim-safety precondition is not - /// encoded here yet; the next layer will prove it from the monitor/global - /// RCU invariant before invoking this method. + /// Runs the underlying callback once the monitor has completed the grace + /// period that contained this callback's retire summary. #[inline] #[verifier::external_body] - pub unsafe fn call_once(self) { + unsafe fn call_once(self, Tracked(completed): Tracked<&CompletedGracePeriod>) + requires + completed.covers(self@), + { unsafe { self.raw.call_once(); } } } +/// Proof token produced by the monitor when a grace period finishes. +/// +/// The token is private to the monitor implementation. External code can +/// certify that a callback is safe to enqueue, but cannot manufacture the +/// completion fact needed to execute the callback. +tracked struct CompletedGracePeriod { + ghost callbacks: Seq, +} + +impl View for CompletedGracePeriod { + type V = Seq; + + closed spec fn view(&self) -> Seq { + self.callbacks + } +} + +impl CompletedGracePeriod { + closed spec fn covers(self, callback: rcu_spec::RcuCallbackSummary) -> bool { + self@.contains(callback) + } +} + pub open spec fn callback_summaries(callbacks: Callbacks) -> Seq { Seq::new(callbacks@.len(), |i: int| callbacks@[i]@) } +fn run_completed_callbacks( + mut callbacks: Callbacks, + Tracked(completed): Tracked, +) + requires + completed@ == callback_summaries(callbacks), +{ + proof { + assert forall|i: int| + 0 <= i < callbacks@.len() implies completed.covers((#[trigger] callbacks@[i])@) by + { + let summaries = callback_summaries(callbacks); + assert(completed@ == summaries); + assert(summaries[i] == callbacks@[i]@); + summaries.lemma_index_contains(i); + } + } + while callbacks.len() > 0 + invariant + forall|i: int| + 0 <= i < callbacks@.len() ==> completed.covers((#[trigger] callbacks@[i])@), + decreases callbacks@.len(), + { + proof { + assert(callbacks@.len() > 0); + assert(completed.covers(callbacks@[0]@)); + } + let ghost before = callbacks@; + let callback = callbacks.pop_front().unwrap(); + proof { + assert(callback == before[0]); + assert(callback@ == before[0]@); + assert(completed.covers(callback@)); + assert forall|i: int| + 0 <= i < callbacks@.len() implies completed.covers((#[trigger] callbacks@[i])@) by + { + assert(callbacks@ == before.subrange(1, before.len() as int)); + assert(callbacks@[i] == before[i + 1]); + assert(0 <= i + 1 < before.len()); + } + } + unsafe { + callback.call_once(Tracked(&completed)); + } + } +} + proof fn callback_summaries_empty(callbacks: Callbacks) requires callbacks@ == Seq::::empty(), @@ -84,6 +157,12 @@ proof fn callback_summaries_empty(callbacks: Callbacks) ); } +proof fn callback_summaries_len(callbacks: Callbacks) + ensures + callback_summaries(callbacks).len() == callbacks@.len(), +{ +} + // The proof-facing views `GracePeriodView` and `MonitorStateView` live in // `specs::sync::rcu` so that the monitor flag's weak-memory ghost state can // record a state snapshot per flag message without depending on this module. @@ -112,7 +191,7 @@ impl GracePeriod { ensures res@ == GracePeriodView::initial(), { - let callbacks = Vec::new(); + let callbacks = Callbacks::new(); let cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); let res = Self { callbacks, cpu_mask, is_complete: true }; proof { @@ -123,15 +202,30 @@ impl GracePeriod { /// Starts a new incomplete grace period with the callbacks that should run /// after it completes. The CPU mask is reset because all CPUs must pass a - /// fresh quiescent state for this new batch. + /// fresh quiescent state for this new batch. Keep the same atomic object so + /// later weak-memory ghost state can attach stable identity to this mask. fn restart(&mut self, callbacks: Callbacks) ensures final(self).callback_summaries() == callback_summaries(callbacks), !final(self).is_complete, + no_unwind { self.is_complete = false; self.callbacks = callbacks; - self.cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); + self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); + } + + /// Records that `this_cpu` has passed a quiescent state for this grace + /// period and returns whether the executable CPU mask now covers all CPUs. + /// + /// The CPU-mask contents are not part of the current proof view, so this + /// method refines only the executable monitor protocol. The higher-level + /// proof still treats the returned completion bit abstractly. + fn record_quiescent_state(&self, this_cpu: CpuId) -> (complete: bool) + no_unwind + { + self.cpu_mask.add(this_cpu, Ordering::Relaxed); + self.cpu_mask.load(Ordering::Relaxed).is_full() } closed spec fn callback_summaries(self) -> Seq { @@ -181,7 +275,7 @@ impl State { res.no_pending_work(), { let current_gp = GracePeriod::new(); - let next_callbacks = Vec::new(); + let next_callbacks = Callbacks::new(); let res = Self { current_gp, next_callbacks }; proof { callback_summaries_empty(res.next_callbacks); @@ -191,25 +285,129 @@ impl State { /// Enqueues one callback and starts a grace period if the monitor was idle. /// - /// The method preserves the lock-protected state invariant. Returning with - /// pending work lets the caller publish `is_monitoring = true` with the - /// resulting state view. - fn enqueue_after_grace_period(&mut self, callback: RcuCallback) + /// This follows the upstream monitor protocol at the observable boundary: + /// an idle monitor starts a new current grace period and the caller must + /// publish `is_monitoring = true`; an already active monitor only appends + /// the callback to the next batch and does not publish another flag + /// message. The upstream implementation transiently stages the idle + /// callback in `next_callbacks` before promoting it, but our type invariant + /// keeps `next_callbacks` empty whenever the current grace period is + /// complete, so the idle case constructs the current batch directly. + fn enqueue_after_grace_period(&mut self, callback: RcuCallback) -> (started_gp: bool) ensures final(self).wf(), final(self).has_pending_work(), + started_gp ==> !final(self)@.current_gp.is_complete, + !started_gp ==> !old(self)@.current_gp.is_complete, { if self.current_gp.is_complete { - let mut callbacks = Vec::new(); - callbacks.push(callback); - let cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); - self.current_gp = GracePeriod { callbacks, cpu_mask, is_complete: false }; + let mut callbacks = Callbacks::new(); + callbacks.push_back(callback); + self.current_gp.restart(callbacks); + true } else { - let mut next_callbacks = Vec::new(); + let mut next_callbacks = Callbacks::new(); core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); - next_callbacks.push(callback); + next_callbacks.push_back(callback); self.next_callbacks = next_callbacks; + false + } + } + + /// Records a quiescent state for the current CPU, returns the callbacks + /// that become reclaimable if this completes the grace period, and + /// immediately starts the next grace period if callbacks accumulated while + /// the current one was running. + /// + /// This mirrors the upstream state machine: an incomplete CPU mask leaves + /// the current grace period running and returns no completed callbacks. + /// The exact CPU-mask contents are still outside the proof view; the proof + /// treats `record_quiescent_state`'s boolean result as the completion cut. + fn finish_grace_period( + &mut self, + this_cpu: CpuId, + ) -> ((completed_gp, completed_callbacks, completed_token): ( + bool, + Callbacks, + Tracked, + )) + ensures + final(self).wf(), + completed_token@@ == callback_summaries(completed_callbacks), + completed_gp ==> !old(self)@.current_gp.is_complete, + completed_gp ==> completed_token@@ == old(self)@.current_gp.callbacks, + !completed_gp ==> completed_token@@ == Seq::::empty(), + (!completed_gp && !(old(self)@.current_gp.is_complete)) ==> !( + final(self)@.current_gp.is_complete + ), + { + proof { + use_type_invariant(&*self); + } + let ghost initially_complete = self.current_gp.is_complete; + let ghost initial_current_callbacks = self.current_gp@.callbacks; + let mut completed_callbacks = Callbacks::new(); + let mut completed_gp = false; + if !self.current_gp.is_complete { + let is_complete = self.current_gp.record_quiescent_state(this_cpu); + if is_complete { + completed_gp = true; + core::mem::swap(&mut completed_callbacks, &mut self.current_gp.callbacks); + proof { + assert(callback_summaries(completed_callbacks) == initial_current_callbacks); + callback_summaries_empty(self.current_gp.callbacks); + } + if self.next_callbacks.len() > 0 { + let mut next_callbacks = Callbacks::new(); + core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); + proof { + callback_summaries_empty(self.next_callbacks); + } + self.current_gp.restart(next_callbacks); + } else { + self.current_gp.is_complete = true; + proof { + callback_summaries_empty(self.current_gp.callbacks); + callback_summaries_len(self.next_callbacks); + assert(self.next_callbacks@.len() == 0); + assert(callback_summaries(self.next_callbacks).len() == 0); + } + } + } + } + proof_decl! { + let tracked completed = CompletedGracePeriod { + callbacks: callback_summaries(completed_callbacks), + }; } + proof { + callback_summaries_len(self.current_gp.callbacks); + callback_summaries_len(self.next_callbacks); + assert(completed@ == callback_summaries(completed_callbacks)); + if !completed_gp { + callback_summaries_empty(completed_callbacks); + } else { + assert(!initially_complete); + assert(callback_summaries(completed_callbacks) == initial_current_callbacks); + } + if initially_complete { + assert(initial_current_callbacks.len() == 0); + vstd::seq_lib::assert_seqs_equal!( + initial_current_callbacks == Seq::::empty() + ); + } + if self.current_gp.is_complete { + if initially_complete { + assert(self.wf()); + } + assert(self.current_gp@.callbacks.len() == 0); + assert(self.next_callbacks@.len() == 0); + assert(callback_summaries(self.next_callbacks).len() == 0); + } + assert(self.current_gp.wf()); + assert(self.wf()); + } + (completed_gp, completed_callbacks, Tracked(completed)) } closed spec fn pending_summaries(self) -> Seq { @@ -372,24 +570,78 @@ impl RcuMonitor { self.is_monitoring.store_relaxed_rcu_monitor(value, Ghost(state), Tracked(tv)); } - /// Schedules `callback` to run after a future grace period and publishes a - /// conservative `true` monitor flag message for the resulting state. + /// Schedules `callback` to run after a future grace period. + /// + /// Matches the upstream protocol: callbacks are first queued for the next + /// grace period. Only an idle monitor promotes that queue into a new + /// current grace period and publishes a `true` flag; if a grace period is + /// already running, the existing monitor flag is left unchanged. pub(super) fn after_grace_period(&self, callback: RcuCallback) { proof { use_type_invariant(self); } let mut state = self.state.lock(); - state.enqueue_after_grace_period(callback); + let started_gp = state.enqueue_after_grace_period(callback); + if started_gp { + proof { + assert(state.view().wf()); + assert(state.view().has_pending_work()); + use_type_invariant(self); + } + proof_decl! { + let tracked mut tv = ThreadView::new(); + } + self.set_monitoring(true, Ghost(state.view()@), Tracked(&mut tv)); + } + state.drop(); + } + + /// Reports this CPU's quiescent state and runs a completed callback batch + /// outside the monitor lock. + /// + /// The control flow is aligned with upstream: a relaxed false flag returns + /// immediately, a stale true flag may still find a completed state under + /// the lock and return, an incomplete CPU mask keeps monitoring without + /// touching the flag, and callback bodies run outside the monitor lock. + pub(super) unsafe fn finish_grace_period(&self) { proof { - assert(state.view().wf()); - assert(state.view().has_pending_work()); use_type_invariant(self); } proof_decl! { - let tracked mut tv = ThreadView::new(); + let tracked mut fast_tv = ThreadView::new(); + } + let is_monitoring = self.is_monitoring.load_relaxed(Tracked(&mut fast_tv)).0; + if !is_monitoring { + return; + } + + let mut state = self.state.lock(); + if state.current_gp.is_complete { + state.drop(); + return; + } + + let this_cpu = CpuId::current(); + let (completed_gp, completed_callbacks, Tracked(completed)) = + state.finish_grace_period(this_cpu); + if !completed_gp { + state.drop(); + return; + } + if state.current_gp.is_complete { + proof { + assert(state.view().wf()); + rcu_spec::monitor_state_pending_iff_incomplete(state.view()@); + assert(state.view()@.no_pending_work()); + use_type_invariant(self); + } + proof_decl! { + let tracked mut tv = ThreadView::new(); + } + self.set_monitoring(false, Ghost(state.view()@), Tracked(&mut tv)); } - self.set_monitoring(true, Ghost(state.view()@), Tracked(&mut tv)); state.drop(); + run_completed_callbacks(completed_callbacks, Tracked(completed)); } closed spec fn wf(self) -> bool { From 9507696261755ff37ced4169d5673acdf1adea73 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 1 Jul 2026 23:09:10 -0400 Subject: [PATCH 16/47] format and fix verus `map` --- ostd/specs/sync/weak_memory.rs | 44 ++++++++++++++++++++++++++++++++-- ostd/src/sync/rcu/monitor.rs | 35 ++++++++++++--------------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index e99cf9397..be5de0ffe 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -24,6 +24,7 @@ use super::rcu as rcu_spec; #[cfg(target_has_atomic = "64")] use core::sync::atomic::{AtomicI64, AtomicU64}; +use vstd::assert_sets_equal; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::prelude::*; use vstd::resource::Loc; @@ -155,12 +156,41 @@ pub tracked struct HistAuth { ghost len: nat, } +proof fn lemma_timestamp_range_insert_last(hi: Timestamp) + ensures + Set::range(0nat, hi).insert(hi) == Set::range(0nat, hi + 1), +{ + broadcast use vstd::set_lib::range_set_properties; + + assert_sets_equal!(Set::range(0nat, hi).insert(hi), Set::range(0nat, hi + 1), ts: Timestamp => { + if Set::range(0nat, hi).insert(hi).contains(ts) { + if ts != hi { + assert(Set::range(0nat, hi).contains(ts)); + assert(ts < hi); + } + assert(ts < hi + 1); + assert(Set::range(0nat, hi + 1).contains(ts)); + } + + if Set::range(0nat, hi + 1).contains(ts) { + assert(ts < hi + 1); + if ts == hi { + assert(Set::range(0nat, hi).insert(hi).contains(ts)); + } else { + assert(ts < hi); + assert(Set::range(0nat, hi).contains(ts)); + assert(Set::range(0nat, hi).insert(hi).contains(ts)); + } + } + }); +} + impl HistAuth { pub closed spec fn id(self) -> AtomicId { self.auth.id() } - pub closed spec fn map(self) -> IMap> { + pub closed spec fn map(self) -> Map> { self.auth@ } @@ -177,7 +207,7 @@ impl HistAuth { pub open spec fn wf(self) -> bool { &&& self.len() > 0 - &&& self.map().dom() =~= ISet::new(|ts: Timestamp| ts < self.len()) + &&& self.map().dom() == Set::range(0nat, self.len()) } pub open spec fn valid_ts(self, ts: Timestamp) -> bool { @@ -213,10 +243,20 @@ impl HistAuth { snap.agrees_with(*final(self)), { let ghost ts = self.len(); + let ghost old_dom = self.map().dom(); let tracked pt = self.auth.insert(ts, msg); self.len = self.len + 1; + // Full-crate verification does not reliably rediscover this range/domain + // fact after the ghost-map insert, so keep the append step explicit. + lemma_timestamp_range_insert_last(ts); + assert(old_dom == Set::range(0nat, ts)); + assert(self.map().dom() == old_dom.insert(ts)); + assert(self.map().dom() == Set::range(0nat, ts + 1)); + assert(ts + 1 == self.len()); + assert(self.map().dom() == Set::range(0nat, self.len())); + let tracked psnap = pt.persist(); MsgSnap { snap: psnap } } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 502e135bc..b3d4baaff 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -106,9 +106,9 @@ fn run_completed_callbacks( completed@ == callback_summaries(callbacks), { proof { - assert forall|i: int| - 0 <= i < callbacks@.len() implies completed.covers((#[trigger] callbacks@[i])@) by - { + assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( + (#[trigger] callbacks@[i])@, + ) by { let summaries = callback_summaries(callbacks); assert(completed@ == summaries); assert(summaries[i] == callbacks@[i]@); @@ -131,9 +131,9 @@ fn run_completed_callbacks( assert(callback == before[0]); assert(callback@ == before[0]@); assert(completed.covers(callback@)); - assert forall|i: int| - 0 <= i < callbacks@.len() implies completed.covers((#[trigger] callbacks@[i])@) by - { + assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( + (#[trigger] callbacks@[i])@, + ) by { assert(callbacks@ == before.subrange(1, before.len() as int)); assert(callbacks@[i] == before[i + 1]); assert(0 <= i + 1 < before.len()); @@ -323,14 +323,11 @@ impl State { /// the current grace period running and returns no completed callbacks. /// The exact CPU-mask contents are still outside the proof view; the proof /// treats `record_quiescent_state`'s boolean result as the completion cut. - fn finish_grace_period( - &mut self, - this_cpu: CpuId, - ) -> ((completed_gp, completed_callbacks, completed_token): ( - bool, - Callbacks, - Tracked, - )) + fn finish_grace_period(&mut self, this_cpu: CpuId) -> (( + completed_gp, + completed_callbacks, + completed_token, + ): (bool, Callbacks, Tracked)) ensures final(self).wf(), completed_token@@ == callback_summaries(completed_callbacks), @@ -338,8 +335,7 @@ impl State { completed_gp ==> completed_token@@ == old(self)@.current_gp.callbacks, !completed_gp ==> completed_token@@ == Seq::::empty(), (!completed_gp && !(old(self)@.current_gp.is_complete)) ==> !( - final(self)@.current_gp.is_complete - ), + final(self)@.current_gp.is_complete), { proof { use_type_invariant(&*self); @@ -614,16 +610,15 @@ impl RcuMonitor { if !is_monitoring { return; } - let mut state = self.state.lock(); if state.current_gp.is_complete { state.drop(); return; } - let this_cpu = CpuId::current(); - let (completed_gp, completed_callbacks, Tracked(completed)) = - state.finish_grace_period(this_cpu); + let (completed_gp, completed_callbacks, Tracked(completed)) = state.finish_grace_period( + this_cpu, + ); if !completed_gp { state.drop(); return; From 9033604039a28ec23a9a4eb663a1e817b8d5c6d4 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Fri, 3 Jul 2026 04:13:31 -0400 Subject: [PATCH 17/47] add specs for scheduler for next steps --- ostd/specs/sync/weak_memory.rs | 3 + ostd/src/task/mod.rs | 48 +++- ostd/src/task/preempt/guard.rs | 13 +- ostd/src/task/scheduler/mod.rs | 472 ++++++++++++++++++++++++++++++--- 4 files changed, 484 insertions(+), 52 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index be5de0ffe..23aa8e7f8 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -1089,6 +1089,9 @@ impl ThreadView { /// coherence the thread has already observed) and publishes nothing useful /// through release stores, so executable code should thread one token per /// logical operation or critical section, and eventually one per task. + /// + /// TODO: This API should not be exposed as a public because the only way + /// to create this is via critical section markers such as `disable_preempt`. pub proof fn new() -> (tracked res: Self) ensures res@ == WmView::empty(), diff --git a/ostd/src/task/mod.rs b/ostd/src/task/mod.rs index 1bf2f5a5e..fc8daecd9 100644 --- a/ostd/src/task/mod.rs +++ b/ostd/src/task/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 //! Tasks are the unit of code execution. use vstd::prelude::*; +use vstd::resource::Loc; /* pub mod atomic_mode; mod kernel_stack; */ @@ -44,6 +45,8 @@ pub fn inject_post_schedule_handler(handler: fn()) { POST_SCHEDULE_HANDLER.call_once(|| handler); }*/ +verus! { + /// A task that executes a function to the end. /// /// Each task is associated with per-task data and an optional user space. @@ -71,17 +74,46 @@ pub struct Task { schedule_info: TaskScheduleInfo,*/ } -/* + +#[verus_verify] impl Task { + /// Returns the unique identifier of the task. + pub uninterp spec fn id(&self) -> Loc; + /// Gets the current task. /// /// It returns `None` if the function is called in the bootstrap context. pub fn current() -> Option { - let current_task = current_task()?; + Some(CurrentTask { }) + // let current_task = current_task()?; + // // SAFETY: `current_task` is the current task. + // Some(unsafe { CurrentTask::new(current_task) }) - // SAFETY: `current_task` is the current task. - Some(unsafe { CurrentTask::new(current_task) }) } +} + +/// The current task. +/// +/// This type is not `Send`, so it cannot outlive the current task. +/// +/// This type is also not `Sync`, so it can provide access to the local data of the current task. +pub struct CurrentTask; + +#[verifier::external] +impl !Send for Task { + +} + +#[verifier::external] +impl !Sync for Task { + +} + +} // verus! +/* +impl Task { + + pub(super) fn ctx(&self) -> &SyncUnsafeCell { &self.ctx @@ -267,13 +299,7 @@ impl TaskOptions { } } -/// The current task. -/// -/// This type is not `Send`, so it cannot outlive the current task. -/// -/// This type is also not `Sync`, so it can provide access to the local data of the current task. -#[derive(Debug)] -pub struct CurrentTask(NonNull); + // The intern `NonNull` contained by `CurrentTask` implies that `CurrentTask` is `!Send` and // `!Sync`. But it is still good to do this explicitly because these properties are key for diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 8e44ce12b..d6ce5ad69 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: MPL-2.0 use vstd::prelude::*; -use crate::{sync::GuardTransfer /*, task::atomic_mode::InAtomicMode*/}; +use crate::{ + specs::sync::weak_memory::ThreadView, + sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ +}; /// A guard for disable preempt. #[verus_verify] @@ -11,6 +14,14 @@ use crate::{sync::GuardTransfer /*, task::atomic_mode::InAtomicMode*/}; pub struct DisabledPreemptGuard { // This private field prevents user from constructing values of this type directly. _private: (), + + // Weak-memory lower-bound view carried by this guard. + // + // Short-term: minted when preemption is disabled. + // Long-term: borrowed/moved from task-local state instead (but how?). + // + // This should be "provided" by the "current" task. + wm_view: Tracked, } /* impl !Send for DisabledPreemptGuard {} diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 0f31b0af1..d738db8c3 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -63,6 +63,436 @@ //! Violating this invariant—e.g., running the same task on two CPUs concurrently— //! can have catastrophic consequences, //! as the task's stack and internal state may be corrupted by concurrent modifications. + +use vstd::{map::Map, prelude::*, resource::Loc}; + +use super::Task; +use crate::{ + specs::mm::cpu::CpuId, + specs::sync::weak_memory::{ThreadView, WmView}, + sync::{OnceImpl, RoArc, TrivialPred}, +}; + +verus! { + +/// A task-like object that can be identified in scheduler ghost state. +pub trait Schedulable { + spec fn sched_id(&self) -> Loc; +} + +impl Schedulable for Task { + open spec fn sched_id(&self) -> Loc { + self.id() + } +} + +#[derive(PartialEq, Eq, Clone, Copy)] +pub ghost enum TaskSchedState { + New, + Runnable, + Blocked, + Running, + Exited, +} + +/// Abstract proof view of the scheduler state. +/// +/// `runqueues` contains runnable-but-not-current tasks, `current` contains +/// the task currently running on each CPU, and `state` records the scheduler +/// state for every known task. `task_views` is the logical per-task +/// weak-memory view. `stored_views` records views still owned by the scheduler +/// resource; `checked_out_views` records views temporarily held by guards. +pub ghost struct SchedulerView { + pub runqueues: Map>, + pub current: Map>, + pub state: Map, + pub task_views: Map, + pub stored_views: Map, + pub checked_out_views: Map, +} + +impl SchedulerView { + pub open spec fn task_is_known(self, task: Loc) -> bool { + self.state.contains_key(task) + } + + pub open spec fn task_has_thread_view(self, task: Loc) -> bool { + self.task_views.contains_key(task) + } + + pub open spec fn task_thread_view(self, task: Loc) -> WmView + recommends + self.task_has_thread_view(task), + { + self.task_views[task] + } + + pub open spec fn task_view_is_stored(self, task: Loc) -> bool { + self.stored_views.contains_key(task) + } + + pub open spec fn task_view_is_checked_out(self, task: Loc) -> bool { + self.checked_out_views.contains_key(task) + } + + pub open spec fn task_in_runqueue(self, task: Loc) -> bool { + exists|cpu: CpuId, idx: int| + self.runqueues.contains_key(cpu) + && 0 <= idx + && idx < self.runqueues[cpu].len() + && (#[trigger] self.runqueues[cpu][idx]) == task + } + + pub open spec fn task_is_current(self, task: Loc) -> bool { + exists|cpu: CpuId| + #[trigger] self.current.contains_key(cpu) + && self.current[cpu] is Some + && self.current[cpu]->0 == task + } + + pub open spec fn task_is_runnable(self, task: Loc) -> bool { + &&& self.state.contains_key(task) + &&& (self.state[task] is Runnable || self.state[task] is Running) + } + + pub open spec fn task_is_live(self, task: Loc) -> bool { + &&& self.state.contains_key(task) + &&& !(self.state[task] is Exited) + } + + /// Prevent that two tasks happen to be the same task, which would be a violation of the + /// scheduler's safety invariant. + pub open spec fn no_duplicate_task(self, task: Loc) -> bool { + &&& forall|cpu1: CpuId, cpu2: CpuId, idx1: int, idx2: int| + #![trigger self.runqueues[cpu1][idx1], self.runqueues[cpu2][idx2]] + self.runqueues.contains_key(cpu1) + && self.runqueues.contains_key(cpu2) + && 0 <= idx1 + && idx1 < self.runqueues[cpu1].len() + && 0 <= idx2 + && idx2 < self.runqueues[cpu2].len() + && self.runqueues[cpu1][idx1] == task + && self.runqueues[cpu2][idx2] == task + ==> cpu1 == cpu2 && idx1 == idx2 + &&& forall|cpu1: CpuId, cpu2: CpuId| + #![trigger self.current.contains_key(cpu1), self.current.contains_key(cpu2)] + self.current.contains_key(cpu1) + && self.current.contains_key(cpu2) + && self.current[cpu1] is Some + && self.current[cpu2] is Some + && self.current[cpu1]->0 == task + && self.current[cpu2]->0 == task + ==> cpu1 == cpu2 + &&& !(self.task_in_runqueue(task) && self.task_is_current(task)) + } + + pub open spec fn checkout_task_view(self, cpu: CpuId) -> SchedulerView + recommends + self.current.contains_key(cpu), + self.current[cpu] is Some, + self.task_view_is_stored(self.current[cpu]->0), + !self.task_view_is_checked_out(self.current[cpu]->0), + { + let task = self.current[cpu]->0; + SchedulerView { + stored_views: self.stored_views.remove(task), + checked_out_views: self.checked_out_views.insert(task, self.stored_views[task]), + ..self + } + } + + pub open spec fn update_checked_out_task_view(self, task: Loc, view: WmView) -> SchedulerView + recommends + self.task_view_is_checked_out(task), + { + SchedulerView { + task_views: self.task_views.insert(task, view), + checked_out_views: self.checked_out_views.insert(task, view), + ..self + } + } + + pub open spec fn checkin_task_view(self, task: Loc, view: WmView) -> SchedulerView + recommends + self.task_view_is_checked_out(task), + !self.task_view_is_stored(task), + self.checked_out_views[task] == view, + { + SchedulerView { + task_views: self.task_views.insert(task, view), + stored_views: self.stored_views.insert(task, view), + checked_out_views: self.checked_out_views.remove(task), + ..self + } + } + + pub open spec fn wf(self) -> bool { + &&& forall|cpu: CpuId| + #[trigger] self.runqueues.contains_key(cpu) ==> valid_cpu(cpu) + &&& forall|cpu: CpuId| + #[trigger] self.current.contains_key(cpu) ==> valid_cpu(cpu) + &&& forall|cpu: CpuId, idx: int| + #![trigger self.runqueues[cpu][idx]] + self.runqueues.contains_key(cpu) + && 0 <= idx + && idx < self.runqueues[cpu].len() + ==> self.state.contains_key(self.runqueues[cpu][idx]) + && self.state[self.runqueues[cpu][idx]] is Runnable + &&& forall|cpu: CpuId| + #[trigger] self.current.contains_key(cpu) + && self.current[cpu] is Some + ==> self.state.contains_key(self.current[cpu]->0) + && self.state[self.current[cpu]->0] is Running + &&& forall|task: Loc| + #[trigger] self.state.contains_key(task) ==> self.no_duplicate_task(task) + &&& forall|task: Loc| + #[trigger] self.state.contains_key(task) + && !(self.state[task] is Exited) + ==> self.task_views.contains_key(task) + &&& forall|task: Loc| + #[trigger] self.task_views.contains_key(task) ==> self.state.contains_key(task) + && !(self.state[task] is Exited) + &&& forall|task: Loc| + #[trigger] self.stored_views.contains_key(task) ==> self.task_views.contains_key(task) + && self.stored_views[task] == self.task_views[task] + && !self.checked_out_views.contains_key(task) + && self.task_is_live(task) + &&& forall|task: Loc| + #[trigger] self.checked_out_views.contains_key(task) ==> self.task_views.contains_key( + task, + ) + && self.checked_out_views[task] == self.task_views[task] + && !self.stored_views.contains_key(task) + && self.task_is_live(task) + && self.task_is_current(task) + &&& forall|task: Loc| + #[trigger] self.task_views.contains_key(task) ==> (self.stored_views.contains_key(task) + || self.checked_out_views.contains_key(task)) + } +} + +/// Tracked owner of per-task weak-memory views. +/// +/// This is the resource that should eventually back `disable_preempt()`: +/// a guard borrows or moves out the current task's `ThreadView`, atomic +/// operations update it, and guard drop writes it back to the same task entry. +pub tracked struct SchedulerThreadViews { + tracked views: Map, +} + +/// A checked-out per-task `ThreadView`. +/// +/// The `task` field records where the linear `ThreadView` must be written +/// back. This prevents proofs from taking one task's view and re-inserting it +/// under another task id. +pub tracked struct TaskThreadView { + ghost task: Loc, + tracked thread_view: ThreadView, +} + +impl TaskThreadView { + pub proof fn new(task: Loc, tracked thread_view: ThreadView) -> (tracked res: Self) + ensures + res.task() == task, + res.view() == thread_view@, + { + TaskThreadView { task, thread_view } + } + + pub closed spec fn task(self) -> Loc { + self.task + } + + pub closed spec fn view(self) -> WmView { + self.thread_view@ + } + + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + &&& sched_view.wf() + &&& sched_view.checked_out_views.contains_key(self.task()) + &&& sched_view.checked_out_views[self.task()] == self.view() + &&& sched_view.task_views.contains_key(self.task()) + &&& sched_view.task_views[self.task()] == self.view() + } + + pub proof fn borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).view() == (*final(tv))@, + { + &mut self.thread_view + } +} + +impl SchedulerThreadViews { + pub proof fn empty() -> (tracked res: Self) + ensures + res.view() == Map::::empty(), + { + let tracked views = Map::::tracked_empty(); + SchedulerThreadViews { views } + } + + pub closed spec fn view(self) -> Map { + Map::new( + self.views.dom(), + |task: Loc| self.views[task]@, + ) + } + + pub closed spec fn contains(self, task: Loc) -> bool { + self.views.contains_key(task) + } + + pub closed spec fn thread_view(self, task: Loc) -> WmView + recommends + self.contains(task), + { + self.views[task]@ + } + + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + self.view() == sched_view.stored_views + } + + pub proof fn insert_initial_thread_view(tracked &mut self, tracked token: TaskThreadView) + requires + !old(self).contains(token.task()), + ensures + final(self).view() == old(self).view().insert(token.task(), token.view()), + { + let tracked TaskThreadView { task, thread_view } = token; + self.views.tracked_insert(task, thread_view); + } + + pub proof fn take_current_thread_view( + tracked &mut self, + sched_view: SchedulerView, + cpu: CpuId, + ) -> (tracked token: TaskThreadView) + requires + old(self).wf(sched_view), + sched_view.wf(), + sched_view.current.contains_key(cpu), + sched_view.current[cpu] is Some, + sched_view.task_view_is_stored(sched_view.current[cpu]->0), + old(self).contains(sched_view.current[cpu]->0), + ensures + token.task() == sched_view.current[cpu]->0, + token.view() == old(self).thread_view(sched_view.current[cpu]->0), + final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), + { + let task = sched_view.current[cpu]->0; + let tracked thread_view = self.views.tracked_remove(task); + TaskThreadView { task, thread_view } + } + + pub proof fn put_checked_out_thread_view( + tracked &mut self, + sched_view: SchedulerView, + tracked token: TaskThreadView, + ) + requires + token.wf(sched_view), + !old(self).contains(token.task()), + ensures + final(self).view() == old(self).view().insert(token.task(), token.view()), + { + let tracked TaskThreadView { task, thread_view } = token; + self.views.tracked_insert(task, thread_view); + } +} + +/// Logical identity of a runnable task handle. +/// +/// `RoArc` does not yet expose a proof-level view of its pointee. Keep this +/// abstract at the scheduler boundary and connect it to `RoArc`'s internals +/// later when the task registry is introduced. +pub uninterp spec fn runnable_id(runnable: &RoArc) -> Loc; + +pub open spec fn valid_cpu(_cpu: CpuId) -> bool { + true +} + +pub open spec fn can_enqueue(view: SchedulerView, task: Loc, flags: EnqueueFlags) -> bool { + match flags { + EnqueueFlags::Spawn => !view.state.contains_key(task) || view.state[task] is New, + EnqueueFlags::Wake => view.state.contains_key(task) && !(view.state[task] is Exited), + } +} + +/// An SMP-aware task scheduler. +pub trait Scheduler: Sync + Send { + spec fn view(&self) -> SchedulerView; + + spec fn wf(&self) -> bool; + + /// Enqueues a runnable task. + /// + /// The scheduler implementer can perform load-balancing or some time accounting work here. + /// + /// The newly-enqueued task may have a higher priority than the currently running one on a CPU + /// and thus should preempt the latter. + /// In this case, this method returns the ID of that CPU. + fn enqueue(&self, runnable: RoArc, flags: EnqueueFlags) -> (r: Option) + requires + self.wf(), + self.view().wf(), + can_enqueue(self.view(), runnable_id(&runnable), flags), + ensures + self.wf(), + self.view().wf(), + self.view().task_is_runnable(runnable_id(&runnable)), + self.view().no_duplicate_task(runnable_id(&runnable)), + r matches Some(cpu) ==> valid_cpu(cpu), + ; +} + +exec static SCHEDULER: OnceImpl<&'static dyn Scheduler, TrivialPred> + ensures + SCHEDULER.wf(), + SCHEDULER.inv() == TrivialPred, +{ + OnceImpl::new(Ghost(TrivialPred)) +} + +/// Possible actions of a rescheduling. +enum ReschedAction { + /// Keep running current task and do nothing. + DoNothing, + /// Loop until finding a task to swap out the current. + Retry, + /// Switch to target task. + SwitchTo(RoArc), +} + + +/// Possible triggers of an `enqueue` action. +#[derive(PartialEq, Copy, Clone)] +pub enum EnqueueFlags { + /// Spawn a new task. + Spawn, + /// Wake a sleeping task. + Wake, +} + +/// Possible triggers of an `update_current` action. +#[derive(PartialEq, Copy, Clone)] +pub enum UpdateFlags { + /// Timer interrupt. + Tick, + /// Task waiting. + Wait, + /// Task yielding. + Yield, + /// Task exiting. + Exit, +} + +} // verus! + +/* // mod fifo_scheduler; // pub mod info; use alloc::sync::Arc; @@ -95,18 +525,9 @@ pub fn inject_scheduler(scheduler: &'static dyn Scheduler) { }); */ } -static SCHEDULER: Once<&'static dyn Scheduler> = Once::new(); /// A SMP-aware task scheduler. pub trait Scheduler: Sync + Send { - /// Enqueues a runnable task. - /// - /// The scheduler implementer can perform load-balancing or some time accounting work here. - /// - /// The newly-enqueued task may have a higher priority than the currently running one on a CPU - /// and thus should preempt the latter. - /// In this case, this method returns the ID of that CPU. - fn enqueue(&self, runnable: Arc, flags: EnqueueFlags) -> Option; /// Gets an immutable access to the local runqueue of the current CPU. fn local_rq_with(&self, f: &mut dyn FnMut(&dyn LocalRunQueue)); @@ -357,28 +778,6 @@ pub trait LocalRunQueue { fn dequeue_current(&mut self) -> Option>; } -/// Possible triggers of an `enqueue` action. -#[derive(PartialEq, Copy, Clone)] -pub enum EnqueueFlags { - /// Spawn a new task. - Spawn, - /// Wake a sleeping task. - Wake, -} - -/// Possible triggers of an `update_current` action. -#[derive(PartialEq, Copy, Clone)] -pub enum UpdateFlags { - /// Timer interrupt. - Tick, - /// Task waiting. - Wait, - /// Task yielding. - Yield, - /// Task exiting. - Exit, -} - /// Preempts the current task. #[track_caller] pub(crate) fn might_preempt() { @@ -579,12 +978,5 @@ where // processor::switch_to_task(next_task); } -/// Possible actions of a rescheduling. -enum ReschedAction { - /// Keep running current task and do nothing. - DoNothing, - /// Loop until finding a task to swap out the current. - Retry, - /// Switch to target task. - SwitchTo(Arc), -} + +*/ From 2230edcd1869bace21b1f45237d3b4ab9c8db63e Mon Sep 17 00:00:00 2001 From: Hiroki Date: Fri, 3 Jul 2026 05:01:46 -0400 Subject: [PATCH 18/47] format --- ostd/src/task/scheduler/mod.rs | 198 +++++++++++++++++++++------------ 1 file changed, 129 insertions(+), 69 deletions(-) diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index d738db8c3..1bbef9e79 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -63,7 +63,6 @@ //! Violating this invariant—e.g., running the same task on two CPUs concurrently— //! can have catastrophic consequences, //! as the task's stack and internal state may be corrupted by concurrent modifications. - use vstd::{map::Map, prelude::*, resource::Loc}; use super::Task; @@ -95,6 +94,27 @@ pub ghost enum TaskSchedState { Exited, } +/// High-level model for scheduler-owned weak-memory views. +/// +/// The scheduler proof state has two layers. `SchedulerView` is the copyable +/// ghost snapshot used in specifications: it records scheduling state and the +/// current weak-memory view for each live task. `SchedulerThreadViews` is the +/// tracked owner that stores the actual linear `ThreadView` resources for +/// tasks whose views are still owned by the scheduler. +/// +/// When the outermost preemption-disable guard is created, the current task's +/// `ThreadView` is checked out of `SchedulerThreadViews` and moved into a +/// `TaskThreadView` token held by the guard. While it is checked out, +/// `SchedulerView::task_views` remains the logical source of truth, but the +/// ownership partition records that the view is in `checked_out_views` rather +/// than `stored_views`. Weak-memory operations mutate the token's +/// `ThreadView`; the scheduler view must be updated with +/// `update_checked_out_task_view` to keep the logical snapshot synchronized. +/// Dropping the guard checks the token back into `stored_views`. +/// +/// In short, the resource flow is: +/// `stored_views -> TaskThreadView -> checked_out_views update -> stored_views`. +/// /// Abstract proof view of the scheduler state. /// /// `runqueues` contains runnable-but-not-current tasks, `current` contains @@ -137,17 +157,14 @@ impl SchedulerView { pub open spec fn task_in_runqueue(self, task: Loc) -> bool { exists|cpu: CpuId, idx: int| - self.runqueues.contains_key(cpu) - && 0 <= idx - && idx < self.runqueues[cpu].len() - && (#[trigger] self.runqueues[cpu][idx]) == task + self.runqueues.contains_key(cpu) && 0 <= idx && idx < self.runqueues[cpu].len() && ( + #[trigger] self.runqueues[cpu][idx]) == task } pub open spec fn task_is_current(self, task: Loc) -> bool { - exists|cpu: CpuId| - #[trigger] self.current.contains_key(cpu) - && self.current[cpu] is Some - && self.current[cpu]->0 == task + exists|cpu: CpuId| #[trigger] + self.current.contains_key(cpu) && self.current[cpu] is Some && self.current[cpu]->0 + == task } pub open spec fn task_is_runnable(self, task: Loc) -> bool { @@ -160,32 +177,30 @@ impl SchedulerView { &&& !(self.state[task] is Exited) } - /// Prevent that two tasks happen to be the same task, which would be a violation of the - /// scheduler's safety invariant. + /// States the scheduler's main safety invariant for one task id. + /// + /// A task may occur at most once across all runqueues, at most once in the + /// `current` map, and never in both places at the same time. pub open spec fn no_duplicate_task(self, task: Loc) -> bool { &&& forall|cpu1: CpuId, cpu2: CpuId, idx1: int, idx2: int| #![trigger self.runqueues[cpu1][idx1], self.runqueues[cpu2][idx2]] - self.runqueues.contains_key(cpu1) - && self.runqueues.contains_key(cpu2) - && 0 <= idx1 - && idx1 < self.runqueues[cpu1].len() - && 0 <= idx2 - && idx2 < self.runqueues[cpu2].len() - && self.runqueues[cpu1][idx1] == task - && self.runqueues[cpu2][idx2] == task - ==> cpu1 == cpu2 && idx1 == idx2 + self.runqueues.contains_key(cpu1) && self.runqueues.contains_key(cpu2) && 0 <= idx1 + && idx1 < self.runqueues[cpu1].len() && 0 <= idx2 && idx2 + < self.runqueues[cpu2].len() && self.runqueues[cpu1][idx1] == task + && self.runqueues[cpu2][idx2] == task ==> cpu1 == cpu2 && idx1 == idx2 &&& forall|cpu1: CpuId, cpu2: CpuId| #![trigger self.current.contains_key(cpu1), self.current.contains_key(cpu2)] - self.current.contains_key(cpu1) - && self.current.contains_key(cpu2) - && self.current[cpu1] is Some - && self.current[cpu2] is Some - && self.current[cpu1]->0 == task - && self.current[cpu2]->0 == task - ==> cpu1 == cpu2 + self.current.contains_key(cpu1) && self.current.contains_key(cpu2) + && self.current[cpu1] is Some && self.current[cpu2] is Some && self.current[cpu1]->0 + == task && self.current[cpu2]->0 == task ==> cpu1 == cpu2 &&& !(self.task_in_runqueue(task) && self.task_is_current(task)) } + /// Moves the current task's weak-memory view out of scheduler storage. + /// + /// This is the logical transition for the outermost preemption-disable + /// guard. The total logical snapshot `task_views` is unchanged; only the + /// ownership partition changes from `stored_views` to `checked_out_views`. pub open spec fn checkout_task_view(self, cpu: CpuId) -> SchedulerView recommends self.current.contains_key(cpu), @@ -201,6 +216,11 @@ impl SchedulerView { } } + /// Records a mutation to a checked-out `ThreadView`. + /// + /// Weak-memory operations mutate the linear `ThreadView` carried by the + /// guard. This transition keeps the logical snapshot and checked-out + /// partition synchronized with that updated view. pub open spec fn update_checked_out_task_view(self, task: Loc, view: WmView) -> SchedulerView recommends self.task_view_is_checked_out(task), @@ -212,6 +232,10 @@ impl SchedulerView { } } + /// Writes a checked-out task view back to scheduler storage. + /// + /// The caller must provide the same view that is recorded as checked out; + /// this prevents check-in from overwriting the task with an unrelated view. pub open spec fn checkin_task_view(self, task: Loc, view: WmView) -> SchedulerView recommends self.task_view_is_checked_out(task), @@ -227,46 +251,59 @@ impl SchedulerView { } pub open spec fn wf(self) -> bool { - &&& forall|cpu: CpuId| - #[trigger] self.runqueues.contains_key(cpu) ==> valid_cpu(cpu) - &&& forall|cpu: CpuId| - #[trigger] self.current.contains_key(cpu) ==> valid_cpu(cpu) + // CPU-indexed maps may only mention valid CPUs. + &&& forall|cpu: CpuId| #[trigger] self.runqueues.contains_key(cpu) ==> valid_cpu(cpu) + &&& forall|cpu: CpuId| #[trigger] + self.current.contains_key(cpu) ==> valid_cpu( + cpu, + ) + // Runqueues contain exactly runnable tasks; current slots contain + // running tasks. &&& forall|cpu: CpuId, idx: int| #![trigger self.runqueues[cpu][idx]] - self.runqueues.contains_key(cpu) - && 0 <= idx - && idx < self.runqueues[cpu].len() + self.runqueues.contains_key(cpu) && 0 <= idx && idx < self.runqueues[cpu].len() ==> self.state.contains_key(self.runqueues[cpu][idx]) - && self.state[self.runqueues[cpu][idx]] is Runnable - &&& forall|cpu: CpuId| - #[trigger] self.current.contains_key(cpu) - && self.current[cpu] is Some - ==> self.state.contains_key(self.current[cpu]->0) - && self.state[self.current[cpu]->0] is Running - &&& forall|task: Loc| - #[trigger] self.state.contains_key(task) ==> self.no_duplicate_task(task) - &&& forall|task: Loc| - #[trigger] self.state.contains_key(task) - && !(self.state[task] is Exited) + && self.state[self.runqueues[cpu][idx]] is Runnable + &&& forall|cpu: CpuId| #[trigger] + self.current.contains_key(cpu) && self.current[cpu] is Some ==> self.state.contains_key( + self.current[cpu]->0, + ) + && self.state[self.current[cpu]->0] is Running + // No known task may be duplicated across scheduler positions. + &&& forall|task: Loc| #[trigger] + self.state.contains_key(task) ==> self.no_duplicate_task( + task, + ) + // Live tasks have a weak-memory view, while exited tasks do not have + // to keep one around. + &&& forall|task: Loc| #[trigger] + self.state.contains_key(task) && !(self.state[task] is Exited) ==> self.task_views.contains_key(task) - &&& forall|task: Loc| - #[trigger] self.task_views.contains_key(task) ==> self.state.contains_key(task) - && !(self.state[task] is Exited) - &&& forall|task: Loc| - #[trigger] self.stored_views.contains_key(task) ==> self.task_views.contains_key(task) + &&& forall|task: Loc| #[trigger] + self.task_views.contains_key(task) ==> self.state.contains_key(task) && !( + self.state[task] is Exited) + // `stored_views` is the part of `task_views` still owned by the + // scheduler resource. + &&& forall|task: Loc| #[trigger] + self.stored_views.contains_key(task) ==> self.task_views.contains_key(task) && self.stored_views[task] == self.task_views[task] - && !self.checked_out_views.contains_key(task) - && self.task_is_live(task) - &&& forall|task: Loc| - #[trigger] self.checked_out_views.contains_key(task) ==> self.task_views.contains_key( + && !self.checked_out_views.contains_key(task) && self.task_is_live( task, ) + // `checked_out_views` is the part temporarily held by guards. For the + // preemption-disable path, only the current running task may be checked + // out. + &&& forall|task: Loc| #[trigger] + self.checked_out_views.contains_key(task) ==> self.task_views.contains_key(task) && self.checked_out_views[task] == self.task_views[task] - && !self.stored_views.contains_key(task) - && self.task_is_live(task) - && self.task_is_current(task) - &&& forall|task: Loc| - #[trigger] self.task_views.contains_key(task) ==> (self.stored_views.contains_key(task) + && !self.stored_views.contains_key(task) && self.task_is_live(task) + && self.task_is_current( + task, + ) + // Together, the stored and checked-out partitions cover all logical + // task views. + &&& forall|task: Loc| #[trigger] + self.task_views.contains_key(task) ==> (self.stored_views.contains_key(task) || self.checked_out_views.contains_key(task)) } } @@ -307,6 +344,10 @@ impl TaskThreadView { self.thread_view@ } + /// Connects the checked-out token to the scheduler view that owns it. + /// + /// The token's linear `ThreadView` must agree with both the checked-out + /// partition and the total logical `task_views` snapshot. pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { &&& sched_view.wf() &&& sched_view.checked_out_views.contains_key(self.task()) @@ -315,7 +356,12 @@ impl TaskThreadView { &&& sched_view.task_views[self.task()] == self.view() } - pub proof fn borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + /// Borrows the linear `ThreadView` for weak-memory operations. + /// + /// After the borrow mutates the view, the caller must use + /// `update_checked_out_task_view` on the scheduler view before relying on + /// `TaskThreadView::wf` again. + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) ensures (*tv)@ == old(self).view(), final(self).task() == old(self).task(), @@ -335,10 +381,7 @@ impl SchedulerThreadViews { } pub closed spec fn view(self) -> Map { - Map::new( - self.views.dom(), - |task: Loc| self.views[task]@, - ) + Map::new(self.views.dom(), |task: Loc| self.views[task]@) } pub closed spec fn contains(self, task: Loc) -> bool { @@ -352,11 +395,21 @@ impl SchedulerThreadViews { self.views[task]@ } + /// The tracked owner contains exactly the views still stored in scheduler + /// state. Checked-out views are represented by `TaskThreadView` tokens + /// instead, so they are intentionally absent here. pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { self.view() == sched_view.stored_views } - pub proof fn insert_initial_thread_view(tracked &mut self, tracked token: TaskThreadView) + /// Inserts a task view created during task registration. + /// + /// This is separate from check-in: initial insertion creates a new stored + /// entry, while check-in returns an existing checked-out view. + pub proof fn tracked_insert_initial_thread_view( + tracked &mut self, + tracked token: TaskThreadView, + ) requires !old(self).contains(token.task()), ensures @@ -366,7 +419,11 @@ impl SchedulerThreadViews { self.views.tracked_insert(task, thread_view); } - pub proof fn take_current_thread_view( + /// Checks out the current CPU's task view from the tracked owner. + /// + /// The proof-side owner loses this task entry and returns the linear token + /// that a guard will carry until check-in. + pub proof fn tracked_take_current_thread_view( tracked &mut self, sched_view: SchedulerView, cpu: CpuId, @@ -388,7 +445,12 @@ impl SchedulerThreadViews { TaskThreadView { task, thread_view } } - pub proof fn put_checked_out_thread_view( + /// Returns a checked-out task view to the tracked owner. + /// + /// The `token.wf(sched_view)` precondition ties this write-back to the + /// scheduler's checked-out partition, so the task id and view cannot be + /// swapped with another task. + pub proof fn tracked_put_checked_out_thread_view( tracked &mut self, sched_view: SchedulerView, tracked token: TaskThreadView, @@ -467,7 +529,6 @@ enum ReschedAction { SwitchTo(RoArc), } - /// Possible triggers of an `enqueue` action. #[derive(PartialEq, Copy, Clone)] pub enum EnqueueFlags { @@ -491,7 +552,6 @@ pub enum UpdateFlags { } } // verus! - /* // mod fifo_scheduler; // pub mod info; From 9c2a001354f87bca67cc2cb61b85563ad285173e Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 6 Jul 2026 02:59:33 -0400 Subject: [PATCH 19/47] add netested preemption token for resource tracking --- ostd/src/task/preempt/guard.rs | 92 +++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index d6ce5ad69..027c00d67 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -2,10 +2,70 @@ use vstd::prelude::*; use crate::{ - specs::sync::weak_memory::ThreadView, sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ + task::scheduler::{SchedulerView, TaskThreadView}, }; +verus! { + +/// Proof token carried by a nested preemption-disable guard. +/// +/// Nested guards deliberately do not carry a `TaskThreadView`; the current +/// task's weak-memory view has already been checked out by the outermost +/// guard. This token only records that the guard was created while preemption +/// was already disabled. +pub tracked struct NestedPreemptToken { + ghost depth_before: nat, +} + +impl NestedPreemptToken { + pub proof fn new(depth_before: nat) -> (tracked res: Self) + requires + depth_before > 0, + ensures + res.depth_before() == depth_before, + res.wf(), + { + NestedPreemptToken { depth_before } + } + + pub closed spec fn depth_before(self) -> nat { + self.depth_before + } + + pub closed spec fn wf(self) -> bool { + self.depth_before() > 0 + } +} + +/// Proof resource carried by a `DisabledPreemptGuard`. +/// +/// Only the outermost guard owns the checked-out task view. Nested guards own +/// a separate token that cannot be used to borrow or synthesize a `ThreadView`. +pub tracked enum PreemptGuardResource { + Outermost(TaskThreadView), + Nested(NestedPreemptToken), +} + +impl PreemptGuardResource { + pub closed spec fn is_outermost(self) -> bool { + self is Outermost + } + + pub closed spec fn is_nested(self) -> bool { + self is Nested + } + + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + match self { + PreemptGuardResource::Outermost(task_view) => task_view.wf(sched_view), + PreemptGuardResource::Nested(token) => token.wf(), + } + } +} + +} // verus! + /// A guard for disable preempt. #[verus_verify] #[clippy::has_significant_drop] @@ -15,13 +75,12 @@ pub struct DisabledPreemptGuard { // This private field prevents user from constructing values of this type directly. _private: (), - // Weak-memory lower-bound view carried by this guard. + // Proof-only guard resource. // - // Short-term: minted when preemption is disabled. - // Long-term: borrowed/moved from task-local state instead (but how?). - // - // This should be "provided" by the "current" task. - wm_view: Tracked, + // The outermost guard owns the current task's checked-out `TaskThreadView`. + // Nested guards only own a nesting token, so they cannot mint another + // `ThreadView` for the same task. + tracked_resource: Tracked, } /* impl !Send for DisabledPreemptGuard {} @@ -37,6 +96,25 @@ impl DisabledPreemptGuard { } } */ + +verus! { + +impl DisabledPreemptGuard { + pub closed spec fn is_outermost(&self) -> bool { + self.tracked_resource@.is_outermost() + } + + pub closed spec fn is_nested(&self) -> bool { + self.tracked_resource@.is_nested() + } + + pub closed spec fn wf(&self, sched_view: SchedulerView) -> bool { + self.tracked_resource@.wf(sched_view) + } +} + +} // verus! + #[verus_verify] impl GuardTransfer for DisabledPreemptGuard { #[verifier::external_body] From 0dfeb342b3ebd8d3dd3e6249dc11d6a3f1d0ff74 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 7 Jul 2026 02:20:52 -0400 Subject: [PATCH 20/47] bridge the use of preempt guard --- ostd/src/task/preempt/guard.rs | 334 ++++++++++++++++++++++++++++++--- ostd/src/task/preempt/mod.rs | 4 +- ostd/src/task/scheduler/mod.rs | 20 +- 3 files changed, 325 insertions(+), 33 deletions(-) diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 027c00d67..2811f73f0 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -1,13 +1,20 @@ // SPDX-License-Identifier: MPL-2.0 -use vstd::prelude::*; +use vstd::{prelude::*, resource::Loc}; +use vstd_extra::resource::ghost_resource::{ + count::CountGhost, + tokens::CountGhostResource, +}; use crate::{ + specs::sync::weak_memory::{ThreadView, WmView}, sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ task::scheduler::{SchedulerView, TaskThreadView}, }; verus! { +pub const PREEMPT_SESSION_FRACTIONS: u64 = 1 << 31; + /// Proof token carried by a nested preemption-disable guard. /// /// Nested guards deliberately do not carry a `TaskThreadView`; the current @@ -38,16 +45,219 @@ impl NestedPreemptToken { } } +/// A shareable proof token tying a guard to the active preemption session. +/// +/// The token is a fractional resource-algebra fragment. It records only stable +/// session identity, currently the running task id. The mutable weak-memory +/// view is intentionally not stored here because weak atomic operations update +/// that view while guard fragments may be outstanding. +pub tracked struct PreemptSessionToken { + tracked token: CountGhost, +} + +impl PreemptSessionToken { + proof fn new_placeholder() -> (tracked res: Self) + ensures + res.wf(), + { + let tracked mut tokens = CountGhostResource::::alloc( + arbitrary(), + ); + let tracked token = tokens.split_one(); + PreemptSessionToken { token } + } + + pub closed spec fn id(self) -> Loc { + self.token.id() + } + + pub closed spec fn task(self) -> Loc { + self.token@ + } + + pub closed spec fn frac(self) -> int { + self.token.frac() + } + + pub closed spec fn wf(self) -> bool { + &&& self.frac() == 1 + &&& 0 < self.frac() <= PREEMPT_SESSION_FRACTIONS + } + + pub proof fn agree(tracked &self, tracked other: &Self) + requires + self.id() == other.id(), + ensures + self.task() == other.task(), + { + self.token.agree(&other.token); + } +} + +/// The active preemption-disable session that owns the task-local view. +/// +/// Nested preemption-disable guards do not own this resource. They can only +/// use it by borrowing the session that was established by the outermost +/// preemption-disable scope. This keeps the model to one linear +/// `ThreadView` per running task while still allowing nested RCU code to +/// perform weak atomic operations. +pub tracked struct PreemptThreadViewSession { + tracked task_view: TaskThreadView, + tracked tokens: CountGhostResource, +} + +impl PreemptThreadViewSession { + pub proof fn new(tracked task_view: TaskThreadView) -> (tracked res: Self) + ensures + res.task() == task_view.task(), + res.view() == task_view.view(), + res.session_task() == task_view.task(), + res.available_fractions() == PREEMPT_SESSION_FRACTIONS, + res.wf_session_resource(), + { + let task = task_view.task(); + let tracked tokens = CountGhostResource::alloc(task); + PreemptThreadViewSession { task_view, tokens } + } + + pub closed spec fn task(self) -> Loc { + self.task_view.task() + } + + pub closed spec fn view(self) -> WmView { + self.task_view.view() + } + + pub closed spec fn session_id(self) -> Loc { + self.tokens.id() + } + + pub closed spec fn session_task(self) -> Loc { + self.tokens@ + } + + pub closed spec fn available_fractions(self) -> int { + self.tokens.frac() + } + + pub closed spec fn wf_session_resource(self) -> bool { + &&& self.tokens.wf() + &&& self.session_task() == self.task() + &&& 0 < self.available_fractions() <= PREEMPT_SESSION_FRACTIONS + } + + pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + &&& self.task_view.wf(sched_view) + &&& self.wf_session_resource() + } + + pub closed spec fn token_matches(self, token: PreemptSessionToken) -> bool { + &&& token.wf() + &&& token.id() == self.session_id() + &&& token.task() == self.session_task() + } + + /// Splits one guard fragment from the active session. + /// + /// The session keeps at least one fraction after the split so future + /// agreement checks can still relate guard fragments back to the session. + pub proof fn tracked_split_guard_token( + tracked &mut self, + ) -> (tracked token: PreemptSessionToken) + requires + old(self).wf_session_resource(), + old(self).available_fractions() > 1, + ensures + final(self).task() == old(self).task(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).available_fractions() + 1 == old(self).available_fractions(), + final(self).wf_session_resource(), + token.wf(), + final(self).token_matches(token), + { + let tracked token = self.tokens.split_one(); + PreemptSessionToken { token } + } + + /// Returns a guard fragment when a preemption-disable guard is dropped. + pub proof fn tracked_return_guard_token( + tracked &mut self, + tracked token: PreemptSessionToken, + ) + requires + old(self).wf_session_resource(), + old(self).token_matches(token), + ensures + final(self).task() == old(self).task(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).available_fractions() == old(self).available_fractions() + token.frac(), + final(self).wf_session_resource(), + { + let tracked PreemptSessionToken { token } = token; + self.tokens.combine(token); + } + + /// Borrows the single task-local `ThreadView` for weak atomic operations. + /// + /// After the borrow mutates the view, the caller must update the scheduler + /// snapshot with `SchedulerView::update_checked_out_task_view` before + /// relying on `wf` again. + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).wf_session_resource() == old(self).wf_session_resource(), + final(self).view() == (*final(tv))@, + { + self.task_view.tracked_borrow_thread_view_mut() + } + + /// Returns the checked-out view to the caller for scheduler check-in. + /// + /// This is the proof-side counterpart of dropping the outermost + /// preemption-disable scope: the session stops owning the task view, and + /// the caller can write it back with + /// `SchedulerThreadViews::tracked_put_checked_out_thread_view`. + pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) + ensures + res.task() == self.task(), + res.view() == self.view(), + { + self.task_view + } +} + /// Proof resource carried by a `DisabledPreemptGuard`. /// -/// Only the outermost guard owns the checked-out task view. Nested guards own -/// a separate token that cannot be used to borrow or synthesize a `ThreadView`. +/// The guard records whether this preemption-disable scope is the outermost +/// one or a nested one. It deliberately does not own the checked-out +/// `TaskThreadView`; that linear resource lives in `PreemptThreadViewSession`. +/// This keeps nested guards from minting another `ThreadView` while allowing +/// them to borrow the active session. pub tracked enum PreemptGuardResource { - Outermost(TaskThreadView), - Nested(NestedPreemptToken), + Outermost(PreemptSessionToken), + Nested { + tracked session: PreemptSessionToken, + tracked nested: NestedPreemptToken, + }, } impl PreemptGuardResource { + proof fn new_placeholder() -> (tracked res: Self) + ensures + res.wf(arbitrary()), + { + let tracked token = PreemptSessionToken::new_placeholder(); + PreemptGuardResource::Outermost(token) + } + pub closed spec fn is_outermost(self) -> bool { self is Outermost } @@ -56,18 +266,41 @@ impl PreemptGuardResource { self is Nested } - pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { + pub closed spec fn session_token(self) -> PreemptSessionToken + recommends + self is Outermost || self is Nested, + { match self { - PreemptGuardResource::Outermost(task_view) => task_view.wf(sched_view), - PreemptGuardResource::Nested(token) => token.wf(), + PreemptGuardResource::Outermost(token) => token, + PreemptGuardResource::Nested { session, nested: _ } => session, } } -} -} // verus! + pub closed spec fn session_id(self) -> Loc { + self.session_token().id() + } + + pub closed spec fn task(self) -> Loc { + self.session_token().task() + } + + pub closed spec fn wf(self, _sched_view: SchedulerView) -> bool { + match self { + PreemptGuardResource::Outermost(token) => token.wf(), + PreemptGuardResource::Nested { session, nested } => { + &&& session.wf() + &&& nested.wf() + }, + } + } + + pub closed spec fn matches_session(self, session: PreemptThreadViewSession) -> bool { + &&& self.wf(arbitrary()) + &&& session.token_matches(self.session_token()) + } +} /// A guard for disable preempt. -#[verus_verify] #[clippy::has_significant_drop] #[must_use] #[derive(Debug)] @@ -77,9 +310,8 @@ pub struct DisabledPreemptGuard { // Proof-only guard resource. // - // The outermost guard owns the current task's checked-out `TaskThreadView`. - // Nested guards only own a nesting token, so they cannot mint another - // `ThreadView` for the same task. + // The guard only records whether this scope is outermost or nested. The + // checked-out `TaskThreadView` is owned by `PreemptThreadViewSession`. tracked_resource: Tracked, } @@ -88,16 +320,27 @@ pub struct DisabledPreemptGuard { // SAFETY: The guard disables preemptions, which meets the second // sufficient condition for atomic mode. unsafe impl InAtomicMode for DisabledPreemptGuard {} +*/ impl DisabledPreemptGuard { - fn new() -> Self { - super::cpu_local::inc_guard_count(); - Self { _private: () } + fn new( + Tracked(tracked_resource): Tracked, + ) -> (res: DisabledPreemptGuard) + requires + tracked_resource.wf(arbitrary()), + ensures + res.wf(arbitrary()), + { + // The current verification slice does not include the CPU-local + // runtime preemption counter backend. This body verifies construction + // of the guard resource; wiring the real counter increment back in + // should happen when that backend is part of this dependency closure. + Self { + _private: (), + tracked_resource: Tracked(tracked_resource), + } } } -*/ - -verus! { impl DisabledPreemptGuard { pub closed spec fn is_outermost(&self) -> bool { @@ -111,10 +354,38 @@ impl DisabledPreemptGuard { pub closed spec fn wf(&self, sched_view: SchedulerView) -> bool { self.tracked_resource@.wf(sched_view) } + + pub closed spec fn matches_session(&self, session: PreemptThreadViewSession) -> bool { + self.tracked_resource@.matches_session(session) + } + + /// Lets a nested preemption-disable scope use the outer/session view. + /// + /// The `nested` guard is only a witness that this call happens under a + /// nested preemption-disable scope. The linear `ThreadView` is borrowed + /// from `session`, so nested RCU can perform weak atomic operations + /// without owning or synthesizing another per-task view. + pub proof fn tracked_borrow_thread_view_mut_from_session<'session>( + tracked session: &'session mut PreemptThreadViewSession, + nested: &DisabledPreemptGuard, + ) -> (tracked tv: &'session mut ThreadView) + requires + nested.is_nested(), + nested.matches_session(*old(session)), + ensures + (*tv)@ == old(session).view(), + final(session).task() == old(session).task(), + final(session).session_id() == old(session).session_id(), + final(session).session_task() == old(session).session_task(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).wf_session_resource() == old(session).wf_session_resource(), + final(session).view() == (*final(tv))@, + { + session.tracked_borrow_thread_view_mut() + } } } // verus! - #[verus_verify] impl GuardTransfer for DisabledPreemptGuard { #[verifier::external_body] @@ -123,16 +394,17 @@ impl GuardTransfer for DisabledPreemptGuard { } } -/* -impl Drop for DisabledPreemptGuard { - fn drop(&mut self) { - super::cpu_local::dec_guard_count(); - } -} */ +verus! { /// Disables preemption. -#[verifier::external_body] -pub fn disable_preempt() -> DisabledPreemptGuard { - // DisabledPreemptGuard::new() - unimplemented!() +pub fn disable_preempt() -> (res: DisabledPreemptGuard) + ensures + res.wf(arbitrary()), +{ + proof_decl! { + let tracked tracked_resource = PreemptGuardResource::new_placeholder(); + } + DisabledPreemptGuard::new(Tracked(tracked_resource)) } + +} // verus! diff --git a/ostd/src/task/preempt/mod.rs b/ostd/src/task/preempt/mod.rs index 101aac5c3..08cb34284 100644 --- a/ostd/src/task/preempt/mod.rs +++ b/ostd/src/task/preempt/mod.rs @@ -2,7 +2,9 @@ // pub(super) mod cpu_local; mod guard; -pub use self::guard::{DisabledPreemptGuard, disable_preempt}; +pub use self::guard::{ + DisabledPreemptGuard, PreemptSessionToken, PreemptThreadViewSession, disable_preempt, +}; /* /// Halts the CPU until interrupts if no preemption is required. /// diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 1bbef9e79..3e0b2ae41 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -439,10 +439,18 @@ impl SchedulerThreadViews { token.task() == sched_view.current[cpu]->0, token.view() == old(self).thread_view(sched_view.current[cpu]->0), final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), + final(self).view() == sched_view.checkout_task_view(cpu).stored_views, + final(self).wf(sched_view.checkout_task_view(cpu)), + token.wf(sched_view.checkout_task_view(cpu)), { let task = sched_view.current[cpu]->0; let tracked thread_view = self.views.tracked_remove(task); - TaskThreadView { task, thread_view } + let tracked token = TaskThreadView { task, thread_view }; + let next = sched_view.checkout_task_view(cpu); + assert(final(self).view() == next.stored_views); + assert(final(self).wf(next)); + assert(token.wf(next)); + token } /// Returns a checked-out task view to the tracked owner. @@ -456,13 +464,23 @@ impl SchedulerThreadViews { tracked token: TaskThreadView, ) requires + old(self).wf(sched_view), token.wf(sched_view), !old(self).contains(token.task()), ensures final(self).view() == old(self).view().insert(token.task(), token.view()), + final(self).view() == sched_view.checkin_task_view( + token.task(), + token.view(), + ).stored_views, + final(self).wf(sched_view.checkin_task_view(token.task(), token.view())), { + let ghost view = token.view(); let tracked TaskThreadView { task, thread_view } = token; self.views.tracked_insert(task, thread_view); + let next = sched_view.checkin_task_view(task, view); + assert(final(self).view() == next.stored_views); + assert(final(self).wf(next)); } } From f3b0498c04f04275c386d511fc2d748c5fd4bdf1 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 15 Jul 2026 02:34:39 -0400 Subject: [PATCH 21/47] bridge the task scheduler and the task token --- ostd/src/sync/rcu/mod.rs | 285 +++++++++++++++++++------ ostd/src/sync/rcu/monitor.rs | 54 +++-- ostd/src/task/mod.rs | 5 +- ostd/src/task/preempt/guard.rs | 375 ++++++++++++++++++++++++++++----- ostd/src/task/preempt/mod.rs | 4 +- ostd/src/task/scheduler/mod.rs | 109 +++++++++- 6 files changed, 695 insertions(+), 137 deletions(-) diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index be0501709..1973b16d4 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -11,7 +11,10 @@ //! Rust atomics, while proofs only rely on the specification in //! [`specs::sync::weak_memory`]. Each RCU root pointer is represented by a //! `WeakAtomicPtr` whose history records the messages that may be observed by -//! relaxed/acquire loads and CAS operations. +//! relaxed/acquire loads and CAS operations. Weak atomic operations borrow the +//! unique `ThreadView` from the current task's `RunningTaskContext`; RCU +//! never mints a fresh view and therefore preserves observations across RCU +//! operations and release publication. //! //! The current root-pointer invariant is intentionally small: `Rcu` roots are //! non-null in every atomic-history message, while `RcuOption` roots may be @@ -45,9 +48,9 @@ //! invariant: every flag-history message records a snapshot of the //! lock-protected monitor state (`specs::sync::rcu::MonitorStateView`), and a //! `false` message certifies that its snapshot has no pending callbacks and no -//! incomplete grace period. The next step is to implement -//! `set_monitoring`/`finish_grace_period` against this invariant and to prove -//! callback execution only after the relevant grace period has completed. +//! incomplete grace period. `finish_grace_period` removes the completed batch +//! under the monitor lock, produces a `CompletedGracePeriod` certificate, and +//! executes exactly that batch outside the lock. //! //! # Usage outline //! @@ -58,6 +61,13 @@ //! Writers install a new pointer with `update()` or use the read guard's //! `compare_exchange()` to replace the value they observed. //! +//! Verified callers carry one `RunningTaskContext` for the current task. RCU +//! operations receive a mutable borrow of that context through erased +//! `#[verus_spec(with ...)]` arguments. Starting a read-side critical section +//! increments its modeled preemption depth and removes one session fraction; +//! guard destruction reverses both changes. The scheduler can check the +//! updated view back in only after the context is quiescent. +//! //! Delayed reclamation is still being wired into the weak-memory proof. For now, //! `RcuDrop` preserves the public wrapper API, while the monitor/callback //! path carries the new proof summary and safety certificate skeleton. @@ -74,7 +84,7 @@ use crate::{ }, task::InAtomicMode, }, - task::{DisabledPreemptGuard, disable_preempt}, + task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, }; use non_null::{NonNullPtr, NonNullPtrRef}; @@ -127,12 +137,6 @@ struct RcuReadGuardInner<'a, P: NonNullPtr> { obj_ptr: *mut

::Target, rcu: &'a RcuInner

, _inner_guard: DisabledPreemptGuard, - /// Thread-local weak-memory view for this read-side critical section. - /// - /// For now this only records the load/CAS view effects of the RCU pointer - /// itself. Traversal-safe RCU should later expose this to the paper-style - /// `SeenRemoved(D, LV)` proof layer in `specs::sync::rcu`. - view: Tracked, } impl RcuInner

{ @@ -236,12 +240,18 @@ impl RcuInner

{ self.ptr.store_release_rcu(new_ptr, Tracked(tv)); } - #[verus_spec( + fn update(&self, new_ptr: Option

, Tracked(session): Tracked<&mut RunningTaskContext>) requires self.type_inv(), self.is_nullable() || new_ptr is Some, - )] - fn update(&self, new_ptr: Option

) { + old(session).wf(), + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + { proof_decl! { let ghost new_ptr_is_some = new_ptr is Some; } @@ -252,13 +262,8 @@ impl RcuInner

{ (core::ptr::null_mut(), Tracked(None)) }; - // TODO: A freshly minted empty view makes this release store publish no - // prior observations, so in the model it degrades to relaxed-strength - // publication. This is sound (the empty view is the weakest token) but - // lossy; switch to a per-task/per-CPU view once task-attached ghost - // state is available. proof_decl! { - let tracked mut tv = ThreadView::new(); + let tracked tv = session.tracked_borrow_thread_view_mut(); } proof { if !self.is_nullable() { @@ -266,42 +271,54 @@ impl RcuInner

{ } assert(self.is_nullable() || !raw.is_null()); } - self.store_ptr_release(raw, Tracked(&mut tv)); + self.store_ptr_release(raw, Tracked(tv)); } - #[verus_spec(res => + fn read(&self, Tracked(session): Tracked<&mut RunningTaskContext>) -> (res: RcuReadGuardInner< + '_, + P, + >) requires self.type_inv(), + old(session).wf(), + old(session).available_fractions() > 1, ensures res.type_inv(), res.rcu.is_nullable() == self.is_nullable(), - )] - fn read(&self) -> RcuReadGuardInner<'_, P> { - let inner_guard = disable_preempt(); - // TODO: The guard's view starts empty instead of inheriting the - // thread's accumulated view; sound but forgets prior observations. - // Replace with a per-task/per-CPU view once available. + final(session).wf(), + final(session).available_fractions() + 1 == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth() + 1, + res.matches_context(*final(session)), + { + let inner_guard = disable_preempt_in_context(Tracked(session)); proof_decl! { - let tracked mut tv = ThreadView::new(); + let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( + session, + &inner_guard, + ); } - let obj_ptr = self.load_ptr_acquire(Tracked(&mut tv)); - RcuReadGuardInner { obj_ptr, rcu: self, _inner_guard: inner_guard, view: Tracked(tv) } + let obj_ptr = self.load_ptr_acquire(Tracked(tv)); + RcuReadGuardInner { obj_ptr, rcu: self, _inner_guard: inner_guard } } #[inline] - #[verus_spec( + pub fn read_with<'a, A: InAtomicMode>( + &'a self, + _guard: &'a A, + Tracked(session): Tracked<&mut RunningTaskContext>, + ) -> Option<

>::Ref> where P: NonNullPtrRef<'a> requires self.type_inv(), - )] - pub fn read_with<'a, A: InAtomicMode>(&'a self, _guard: &'a A) -> Option< -

>::Ref, - > where P: NonNullPtrRef<'a> { - // TODO: Same as `read`: a fresh empty view forgets the thread's prior - // observations; replace with a per-task/per-CPU view once available. + old(session).wf(), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + { proof_decl! { - let tracked mut tv = ThreadView::new(); + let tracked tv = session.tracked_borrow_thread_view_mut(); } - let obj_ptr = self.load_ptr_acquire(Tracked(&mut tv)); + let obj_ptr = self.load_ptr_acquire(Tracked(tv)); NonNull::new(obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }) } } @@ -325,13 +342,23 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { res } - #[verus_spec(res => + fn compare_exchange( + self, + new_ptr: Option

, + Tracked(session): Tracked<&mut RunningTaskContext>, + ) -> (res: Result<(), Option

>) requires self.rcu.is_nullable() || new_ptr is Some, + old(session).wf(), + self.matches_context(*old(session)), ensures new_ptr is Some && res is Err ==> res->Err_0 is Some, - )] - fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { + final(session).wf(), + final(session).task() == old(session).task(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions() + 1, + final(session).preempt_depth() + 1 == old(session).preempt_depth(), + { let expected = self.obj_ptr; let rcu = self.rcu; @@ -340,8 +367,11 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { } proof_decl! { - let tracked mut tv = self.view.get(); let ghost new_ptr_is_some = new_ptr is Some; + let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( + session, + &self._inner_guard, + ); } let (new_raw, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { @@ -361,24 +391,39 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { assert(rcu.ptr.constant() == rcu.is_nullable()); assert(rcu.ptr.constant() || !new_raw.is_null()); } - let res = rcu.ptr.compare_exchange_acqrel_acquire_rcu(expected, new_raw, Tracked(&mut tv)); + let cas_res = rcu.ptr.compare_exchange_acqrel_acquire_rcu(expected, new_raw, Tracked(tv)); - match res.0 { + let res = match cas_res.0 { Result::Ok(_) => Ok(()), Result::Err(_) => { - let Some(new_nonnull) = NonNull::new(new_raw) else { - return Err(None); - }; - proof_decl! { - let tracked perm = new_perm.tracked_unwrap(); + if let Some(new_nonnull) = NonNull::new(new_raw) { + proof_decl! { + let tracked perm = new_perm.tracked_unwrap(); + } + Err(Some(unsafe { P::from_raw(new_nonnull, Tracked(perm)) })) + } else { + Err(None) } - Err(Some(unsafe { P::from_raw(new_nonnull, Tracked(perm)) })) }, - } + }; + self._inner_guard.release_to_context(Tracked(session)); + res } #[inline] - fn drop(self) { + fn drop(self, Tracked(session): Tracked<&mut RunningTaskContext>) + requires + old(session).wf(), + self.matches_context(*old(session)), + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).view() == old(session).view(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions() + 1, + final(session).preempt_depth() + 1 == old(session).preempt_depth(), + { + self._inner_guard.release_to_context(Tracked(session)); } } @@ -403,20 +448,42 @@ impl Rcu

{ /// Replaces the current pointer with `new_ptr` using a release store. #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub fn update(&self, new_ptr: P) { proof { use_type_invariant(self); } - self.0.update(Some(new_ptr)); + self.0.update(Some(new_ptr), Tracked(session)); } /// Starts a read-side critical section and acquires the current pointer. #[inline] + #[verus_spec(res => + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).available_fractions() > 1, + ensures + final(session).wf(), + final(session).available_fractions() + 1 == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth() + 1, + res.matches_context(*final(session)), + )] pub fn read(&self) -> RcuReadGuard<'_, P> { proof { use_type_invariant(self); } - RcuReadGuard(self.0.read()) + RcuReadGuard(self.0.read(Tracked(session))) } } @@ -443,38 +510,81 @@ impl RcuOption

{ /// Replaces the current pointer using a release store. #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub fn update(&self, new_ptr: Option

) { proof { use_type_invariant(self); } - self.0.update(new_ptr); + self.0.update(new_ptr, Tracked(session)); } /// Starts a read-side critical section and acquires the current pointer. #[inline] + #[verus_spec(res => + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).available_fractions() > 1, + ensures + final(session).wf(), + final(session).available_fractions() + 1 == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth() + 1, + res.matches_context(*final(session)), + )] pub fn read(&self) -> RcuOptionReadGuard<'_, P> { proof { use_type_invariant(self); } - RcuOptionReadGuard(self.0.read()) + RcuOptionReadGuard(self.0.read(Tracked(session))) } #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub fn read_with<'a, A: InAtomicMode>(&'a self, guard: &'a A) -> Option<

>::Ref, > where P: NonNullPtrRef<'a> { proof { use_type_invariant(self); } - self.0.read_with(guard) + self.0.read_with(guard, Tracked(session)) } } #[verus_verify] impl RcuReadGuard<'_, P> { #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + self.matches_context(*old(session)), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions() + 1, + final(session).preempt_depth() + 1 == old(session).preempt_depth(), + )] pub fn drop(self) { - self.0.drop(); + self.0.drop(Tracked(session)); } #[inline] @@ -488,8 +598,19 @@ impl RcuReadGuard<'_, P> { /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + self.matches_context(*old(session)), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions() + 1, + final(session).preempt_depth() + 1 == old(session).preempt_depth(), + )] pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { - self.0.compare_exchange(Some(new_ptr)).map_err( + self.0.compare_exchange(Some(new_ptr), Tracked(session)).map_err( |err| requires err is Some, @@ -501,8 +622,19 @@ impl RcuReadGuard<'_, P> { #[verus_verify] impl RcuOptionReadGuard<'_, P> { #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + self.matches_context(*old(session)), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions() + 1, + final(session).preempt_depth() + 1 == old(session).preempt_depth(), + )] pub fn drop(self) { - self.0.drop(); + self.0.drop(Tracked(session)); } #[inline] @@ -517,11 +649,22 @@ impl RcuOptionReadGuard<'_, P> { /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + self.matches_context(*old(session)), + ensures + final(session).wf(), + final(session).available_fractions() == old(session).available_fractions() + 1, + final(session).preempt_depth() + 1 == old(session).preempt_depth(), + )] pub fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { proof { use_type_invariant(&self); } - self.0.compare_exchange(new_ptr) + self.0.compare_exchange(new_ptr, Tracked(session)) } } @@ -606,6 +749,12 @@ impl RcuOption

{ } impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { + /// Relates this guard to the task session that supplied its weak-memory + /// view. Consuming operations require the same session. + pub closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + self.0.matches_context(session) + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() @@ -614,6 +763,12 @@ impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { } impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { + /// Relates this guard to the task session that supplied its weak-memory + /// view. Consuming operations require the same session. + pub closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + self.0.matches_context(session) + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() @@ -622,6 +777,10 @@ impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { } impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + self._inner_guard.matches_context(session) + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.rcu.type_inv() diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index b3d4baaff..8accd9a22 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -14,6 +14,7 @@ use crate::specs::{ }, }; use crate::sync::{LocalIrqDisabled, SpinLock}; +use crate::task::RunningTaskContext; verus! { @@ -109,27 +110,20 @@ fn run_completed_callbacks( assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( (#[trigger] callbacks@[i])@, ) by { - let summaries = callback_summaries(callbacks); - assert(completed@ == summaries); - assert(summaries[i] == callbacks@[i]@); - summaries.lemma_index_contains(i); + callback_summaries(callbacks).lemma_index_contains(i); } } - while callbacks.len() > 0 + + #[verus_spec( invariant forall|i: int| 0 <= i < callbacks@.len() ==> completed.covers((#[trigger] callbacks@[i])@), decreases callbacks@.len(), - { - proof { - assert(callbacks@.len() > 0); - assert(completed.covers(callbacks@[0]@)); - } + )] + while callbacks.len() > 0 { let ghost before = callbacks@; let callback = callbacks.pop_front().unwrap(); proof { - assert(callback == before[0]); - assert(callback@ == before[0]@); assert(completed.covers(callback@)); assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( (#[trigger] callbacks@[i])@, @@ -572,6 +566,18 @@ impl RcuMonitor { /// grace period. Only an idle monitor promotes that queue into a new /// current grace period and publishes a `true` flag; if a grace period is /// already running, the existing monitor flag is left unchanged. + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub(super) fn after_grace_period(&self, callback: RcuCallback) { proof { use_type_invariant(self); @@ -585,9 +591,9 @@ impl RcuMonitor { use_type_invariant(self); } proof_decl! { - let tracked mut tv = ThreadView::new(); + let tracked tv = session.tracked_borrow_thread_view_mut(); } - self.set_monitoring(true, Ghost(state.view()@), Tracked(&mut tv)); + self.set_monitoring(true, Ghost(state.view()@), Tracked(tv)); } state.drop(); } @@ -599,14 +605,26 @@ impl RcuMonitor { /// immediately, a stale true flag may still find a completed state under /// the lock and return, an incomplete CPU mask keeps monitoring without /// touching the flag, and callback bodies run outside the monitor lock. + #[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + )] pub(super) unsafe fn finish_grace_period(&self) { proof { use_type_invariant(self); } proof_decl! { - let tracked mut fast_tv = ThreadView::new(); + let tracked fast_tv = session.tracked_borrow_thread_view_mut(); } - let is_monitoring = self.is_monitoring.load_relaxed(Tracked(&mut fast_tv)).0; + let is_monitoring = self.is_monitoring.load_relaxed(Tracked(fast_tv)).0; if !is_monitoring { return; } @@ -631,9 +649,9 @@ impl RcuMonitor { use_type_invariant(self); } proof_decl! { - let tracked mut tv = ThreadView::new(); + let tracked tv = session.tracked_borrow_thread_view_mut(); } - self.set_monitoring(false, Ghost(state.view()@), Tracked(&mut tv)); + self.set_monitoring(false, Ghost(state.view()@), Tracked(tv)); } state.drop(); run_completed_callbacks(completed_callbacks, Tracked(completed)); diff --git a/ostd/src/task/mod.rs b/ostd/src/task/mod.rs index fc8daecd9..dbc8c1047 100644 --- a/ostd/src/task/mod.rs +++ b/ostd/src/task/mod.rs @@ -23,8 +23,11 @@ use kernel_stack::KernelStack; use processor::current_task;*/ use spin::Once; // use utils::ForceSync; +pub(crate) use self::preempt::disable_preempt_in_context; pub use self::{ - preempt::{DisabledPreemptGuard, disable_preempt}, + preempt::{ + DisabledPreemptGuard, PreemptThreadViewSession, RunningTaskContext, disable_preempt, + }, /* scheduler::info::{AtomicCpuId, TaskScheduleInfo}, */ }; /* diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 2811f73f0..435888599 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 use vstd::{prelude::*, resource::Loc}; -use vstd_extra::resource::ghost_resource::{ - count::CountGhost, - tokens::CountGhostResource, -}; +use vstd_extra::resource::ghost_resource::{count::CountGhost, tokens::CountGhostResource}; use crate::{ specs::sync::weak_memory::{ThreadView, WmView}, @@ -52,7 +49,7 @@ impl NestedPreemptToken { /// view is intentionally not stored here because weak atomic operations update /// that view while guard fragments may be outstanding. pub tracked struct PreemptSessionToken { - tracked token: CountGhost, + token: CountGhost, } impl PreemptSessionToken { @@ -60,11 +57,16 @@ impl PreemptSessionToken { ensures res.wf(), { + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); let tracked mut tokens = CountGhostResource::::alloc( arbitrary(), ); let tracked token = tokens.split_one(); - PreemptSessionToken { token } + assert(token.frac() == 1); + let tracked res = PreemptSessionToken { token }; + assert(res.wf()); + res } pub closed spec fn id(self) -> Loc { @@ -115,9 +117,17 @@ impl PreemptThreadViewSession { res.available_fractions() == PREEMPT_SESSION_FRACTIONS, res.wf_session_resource(), { + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); let task = task_view.task(); - let tracked tokens = CountGhostResource::alloc(task); - PreemptThreadViewSession { task_view, tokens } + let tracked tokens = CountGhostResource::::alloc(task); + assert(tokens.is_full()); + tokens.validate_full(); + assert(tokens.frac() == PREEMPT_SESSION_FRACTIONS); + let tracked res = PreemptThreadViewSession { task_view, tokens }; + assert(res.available_fractions() == PREEMPT_SESSION_FRACTIONS); + assert(res.wf_session_resource()); + res } pub closed spec fn task(self) -> Loc { @@ -161,9 +171,8 @@ impl PreemptThreadViewSession { /// /// The session keeps at least one fraction after the split so future /// agreement checks can still relate guard fragments back to the session. - pub proof fn tracked_split_guard_token( - tracked &mut self, - ) -> (tracked token: PreemptSessionToken) + pub proof fn tracked_split_guard_token(tracked &mut self) -> (tracked token: + PreemptSessionToken) requires old(self).wf_session_resource(), old(self).available_fractions() > 1, @@ -182,10 +191,7 @@ impl PreemptThreadViewSession { } /// Returns a guard fragment when a preemption-disable guard is dropped. - pub proof fn tracked_return_guard_token( - tracked &mut self, - tracked token: PreemptSessionToken, - ) + pub proof fn tracked_return_guard_token(tracked &mut self, tracked token: PreemptSessionToken) requires old(self).wf_session_resource(), old(self).token_matches(token), @@ -197,8 +203,19 @@ impl PreemptThreadViewSession { final(self).available_fractions() == old(self).available_fractions() + token.frac(), final(self).wf_session_resource(), { + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + let ghost old_frac = self.tokens.frac(); let tracked PreemptSessionToken { token } = token; + let ghost returned_frac = token.frac(); + assert(returned_frac == 1); self.tokens.combine(token); + assert(old_frac + returned_frac > PREEMPT_SESSION_FRACTIONS ==> false); + assert(old_frac + returned_frac <= PREEMPT_SESSION_FRACTIONS); + self.tokens.validate(); + assert(self.tokens.frac() == old_frac + returned_frac); + assert(0 < self.tokens.frac() <= PREEMPT_SESSION_FRACTIONS); + assert(self.tokens@ == self.task_view.task()); + assert(self.wf_session_resource()); } /// Borrows the single task-local `ThreadView` for weak atomic operations. @@ -226,6 +243,9 @@ impl PreemptThreadViewSession { /// the caller can write it back with /// `SchedulerThreadViews::tracked_put_checked_out_thread_view`. pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) + requires + self.wf_session_resource(), + self.available_fractions() == PREEMPT_SESSION_FRACTIONS, ensures res.task() == self.task(), res.view() == self.view(), @@ -234,6 +254,132 @@ impl PreemptThreadViewSession { } } +/// The proof-owned state for one task while it is running. +/// +/// The scheduler creates this context after checking out the task's +/// `TaskThreadView`. Every preemption-disable guard consumes one fractional +/// session token and increments `preempt_depth`; releasing the guard performs +/// the inverse transition. Consequently the context can only be returned to +/// the scheduler when no guard remains live. +pub tracked struct RunningTaskContext { + tracked session: PreemptThreadViewSession, + ghost preempt_depth: nat, +} + +impl RunningTaskContext { + /// Starts a running interval for a checked-out task view. + pub proof fn new( + tracked task_view: TaskThreadView, + sched_view: SchedulerView, + ) -> (tracked res: Self) + requires + task_view.wf(sched_view), + ensures + res.task() == task_view.task(), + res.view() == task_view.view(), + res.preempt_depth() == 0, + res.available_fractions() == PREEMPT_SESSION_FRACTIONS, + res.wf(), + res.is_quiescent(), + res.wf_scheduler(sched_view), + { + let tracked session = PreemptThreadViewSession::new(task_view); + let tracked res = RunningTaskContext { session, preempt_depth: 0 }; + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(res.wf()); + res + } + + pub closed spec fn task(self) -> Loc { + self.session.task() + } + + pub closed spec fn view(self) -> WmView { + self.session.view() + } + + pub closed spec fn session_id(self) -> Loc { + self.session.session_id() + } + + pub closed spec fn available_fractions(self) -> int { + self.session.available_fractions() + } + + pub closed spec fn preempt_depth(self) -> nat { + self.preempt_depth + } + + pub open spec fn wf(self) -> bool { + &&& self.session.wf_session_resource() + &&& self.available_fractions() + self.preempt_depth() + == PREEMPT_SESSION_FRACTIONS + } + + /// Relates this running context to the scheduler snapshot from which its + /// task view was checked out. + pub open spec fn wf_scheduler(self, sched_view: SchedulerView) -> bool { + &&& self.wf() + &&& self.session.wf(sched_view) + } + + pub open spec fn is_quiescent(self) -> bool { + &&& self.preempt_depth() == 0 + &&& self.available_fractions() == PREEMPT_SESSION_FRACTIONS + } + + /// Borrows the running task's persistent weak-memory view. + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + requires + old(self).wf(), + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).wf(), + final(self).view() == (*final(tv))@, + { + self.session.tracked_borrow_thread_view_mut() + } + + /// Ends a running interval and returns the updated task view to scheduler + /// ownership. The full-fraction requirement rules out live preempt guards. + pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) + requires + self.wf(), + self.preempt_depth() == 0, + ensures + res.task() == self.task(), + res.view() == self.view(), + { + assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); + self.session.tracked_into_task_view() + } + + /// Scheduler-facing form of `tracked_into_task_view` that preserves the + /// checked-out token's relation to the supplied scheduler snapshot. + pub proof fn tracked_into_task_view_for_scheduler( + tracked self, + sched_view: SchedulerView, + ) -> (tracked res: TaskThreadView) + requires + self.wf_scheduler(sched_view), + self.is_quiescent(), + ensures + res.task() == self.task(), + res.view() == self.view(), + res.wf(sched_view), + { + assert(self.preempt_depth() == 0); + assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); + let tracked res = self.session.tracked_into_task_view(); + assert(res.wf(sched_view)); + res + } +} + /// Proof resource carried by a `DisabledPreemptGuard`. /// /// The guard records whether this preemption-disable scope is the outermost @@ -243,10 +389,7 @@ impl PreemptThreadViewSession { /// them to borrow the active session. pub tracked enum PreemptGuardResource { Outermost(PreemptSessionToken), - Nested { - tracked session: PreemptSessionToken, - tracked nested: NestedPreemptToken, - }, + Nested { tracked session: PreemptSessionToken, tracked nested: NestedPreemptToken }, } impl PreemptGuardResource { @@ -298,6 +441,96 @@ impl PreemptGuardResource { &&& self.wf(arbitrary()) &&& session.token_matches(self.session_token()) } + + pub closed spec fn matches_context(self, context: RunningTaskContext) -> bool { + &&& context.wf() + &&& context.preempt_depth() > 0 + &&& self.matches_session(context.session) + } + + /// Returns this guard's session fragment to its owning session. + pub proof fn tracked_return_to_session( + tracked self, + tracked session: &mut PreemptThreadViewSession, + ) + requires + old(session).wf_session_resource(), + self.matches_session(*old(session)), + ensures + final(session).wf_session_resource(), + final(session).task() == old(session).task(), + final(session).view() == old(session).view(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions() + 1, + { + match self { + PreemptGuardResource::Outermost(token) => { + session.tracked_return_guard_token(token); + }, + PreemptGuardResource::Nested { session: token, nested: _ } => { + session.tracked_return_guard_token(token); + }, + } + } +} + +impl RunningTaskContext { + /// Performs the proof transition corresponding to incrementing the + /// executable preemption counter. + pub proof fn tracked_disable_preempt(tracked &mut self) -> (tracked resource: + PreemptGuardResource) + requires + old(self).wf(), + old(self).available_fractions() > 1, + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() + 1 == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth() + 1, + resource.matches_context(*final(self)), + resource.is_outermost() <==> old(self).preempt_depth() == 0, + resource.is_nested() <==> old(self).preempt_depth() > 0, + { + let ghost depth_before = self.preempt_depth; + let tracked token = self.session.tracked_split_guard_token(); + let tracked resource = if depth_before == 0 { + PreemptGuardResource::Outermost(token) + } else { + let tracked nested = NestedPreemptToken::new(depth_before); + PreemptGuardResource::Nested { session: token, nested } + }; + self.preempt_depth = depth_before + 1; + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(self.wf()); + resource + } + + /// Performs the inverse transition when a preemption-disable guard is + /// consumed. + pub proof fn tracked_enable_preempt( + tracked &mut self, + tracked resource: PreemptGuardResource, + ) + requires + old(self).wf(), + old(self).preempt_depth() > 0, + resource.matches_context(*old(self)), + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() == old(self).available_fractions() + 1, + final(self).preempt_depth() + 1 == old(self).preempt_depth(), + { + let ghost old_depth = self.preempt_depth; + resource.tracked_return_to_session(&mut self.session); + self.preempt_depth = (old_depth - 1) as nat; + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(self.wf()); + } } /// A guard for disable preempt. @@ -307,7 +540,6 @@ impl PreemptGuardResource { pub struct DisabledPreemptGuard { // This private field prevents user from constructing values of this type directly. _private: (), - // Proof-only guard resource. // // The guard only records whether this scope is outermost or nested. The @@ -323,22 +555,18 @@ unsafe impl InAtomicMode for DisabledPreemptGuard {} */ impl DisabledPreemptGuard { - fn new( - Tracked(tracked_resource): Tracked, - ) -> (res: DisabledPreemptGuard) + fn new(Tracked(tracked_resource): Tracked) -> (res: DisabledPreemptGuard) requires tracked_resource.wf(arbitrary()), ensures res.wf(arbitrary()), + res.tracked_resource@ == tracked_resource, { // The current verification slice does not include the CPU-local // runtime preemption counter backend. This body verifies construction // of the guard resource; wiring the real counter increment back in // should happen when that backend is part of this dependency closure. - Self { - _private: (), - tracked_resource: Tracked(tracked_resource), - } + Self { _private: (), tracked_resource: Tracked(tracked_resource) } } } @@ -359,29 +587,54 @@ impl DisabledPreemptGuard { self.tracked_resource@.matches_session(session) } - /// Lets a nested preemption-disable scope use the outer/session view. - /// - /// The `nested` guard is only a witness that this call happens under a - /// nested preemption-disable scope. The linear `ThreadView` is borrowed - /// from `session`, so nested RCU can perform weak atomic operations - /// without owning or synthesizing another per-task view. - pub proof fn tracked_borrow_thread_view_mut_from_session<'session>( - tracked session: &'session mut PreemptThreadViewSession, - nested: &DisabledPreemptGuard, - ) -> (tracked tv: &'session mut ThreadView) + pub closed spec fn matches_context(&self, context: RunningTaskContext) -> bool { + self.tracked_resource@.matches_context(context) + } + + /// Borrows the running task's view while this guard witnesses that + /// preemption is disabled. Both outermost and nested guards use the same + /// context-owned view. + pub proof fn tracked_borrow_thread_view_mut_from_context<'context>( + tracked context: &'context mut RunningTaskContext, + guard: &DisabledPreemptGuard, + ) -> (tracked tv: &'context mut ThreadView) requires - nested.is_nested(), - nested.matches_session(*old(session)), + old(context).wf(), + guard.matches_context(*old(context)), ensures - (*tv)@ == old(session).view(), - final(session).task() == old(session).task(), - final(session).session_id() == old(session).session_id(), - final(session).session_task() == old(session).session_task(), - final(session).available_fractions() == old(session).available_fractions(), - final(session).wf_session_resource() == old(session).wf_session_resource(), - final(session).view() == (*final(tv))@, + (*tv)@ == old(context).view(), + final(context).task() == old(context).task(), + final(context).session_id() == old(context).session_id(), + final(context).available_fractions() == old(context).available_fractions(), + final(context).preempt_depth() == old(context).preempt_depth(), + final(context).wf(), + final(context).view() == (*final(tv))@, + guard.matches_context(*final(context)), { - session.tracked_borrow_thread_view_mut() + context.tracked_borrow_thread_view_mut() + } + + /// Consumes this guard, returns its fractional witness, and decrements the + /// modeled preemption depth. + pub(crate) fn release_to_context(self, Tracked(context): Tracked<&mut RunningTaskContext>) + requires + old(context).wf(), + old(context).preempt_depth() > 0, + self.matches_context(*old(context)), + ensures + final(context).wf(), + final(context).task() == old(context).task(), + final(context).view() == old(context).view(), + final(context).session_id() == old(context).session_id(), + final(context).available_fractions() == old(context).available_fractions() + 1, + final(context).preempt_depth() + 1 == old(context).preempt_depth(), + { + proof_decl! { + let tracked resource = self.tracked_resource.get(); + } + proof { + context.tracked_enable_preempt(resource); + } } } @@ -407,4 +660,32 @@ pub fn disable_preempt() -> (res: DisabledPreemptGuard) DisabledPreemptGuard::new(Tracked(tracked_resource)) } +/// Disables preemption inside the current running-task context. +/// +/// This has the same executable effect as [`disable_preempt`]. Its additional +/// tracked argument updates the modeled preemption depth and ties the guard to +/// the task view checked out by the scheduler. +pub(crate) fn disable_preempt_in_context( + Tracked(context): Tracked<&mut RunningTaskContext>, +) -> (res: DisabledPreemptGuard) + requires + old(context).wf(), + old(context).available_fractions() > 1, + ensures + final(context).wf(), + final(context).task() == old(context).task(), + final(context).view() == old(context).view(), + final(context).session_id() == old(context).session_id(), + final(context).available_fractions() + 1 == old(context).available_fractions(), + final(context).preempt_depth() == old(context).preempt_depth() + 1, + res.is_outermost() <==> old(context).preempt_depth() == 0, + res.is_nested() <==> old(context).preempt_depth() > 0, + res.matches_context(*final(context)), +{ + proof_decl! { + let tracked resource = context.tracked_disable_preempt(); + } + DisabledPreemptGuard::new(Tracked(resource)) +} + } // verus! diff --git a/ostd/src/task/preempt/mod.rs b/ostd/src/task/preempt/mod.rs index 08cb34284..9ff63decc 100644 --- a/ostd/src/task/preempt/mod.rs +++ b/ostd/src/task/preempt/mod.rs @@ -2,8 +2,10 @@ // pub(super) mod cpu_local; mod guard; +pub(crate) use self::guard::disable_preempt_in_context; pub use self::guard::{ - DisabledPreemptGuard, PreemptSessionToken, PreemptThreadViewSession, disable_preempt, + DisabledPreemptGuard, PreemptSessionToken, PreemptThreadViewSession, RunningTaskContext, + disable_preempt, }; /* /// Halts the CPU until interrupts if no preemption is required. diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 3e0b2ae41..45be5ed1d 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -65,7 +65,7 @@ //! as the task's stack and internal state may be corrupted by concurrent modifications. use vstd::{map::Map, prelude::*, resource::Loc}; -use super::Task; +use super::{Task, preempt::RunningTaskContext}; use crate::{ specs::mm::cpu::CpuId, specs::sync::weak_memory::{ThreadView, WmView}, @@ -102,18 +102,18 @@ pub ghost enum TaskSchedState { /// tracked owner that stores the actual linear `ThreadView` resources for /// tasks whose views are still owned by the scheduler. /// -/// When the outermost preemption-disable guard is created, the current task's -/// `ThreadView` is checked out of `SchedulerThreadViews` and moved into a -/// `TaskThreadView` token held by the guard. While it is checked out, +/// When a task is scheduled in, its `ThreadView` is checked out of +/// `SchedulerThreadViews` and moved into a `RunningTaskContext`. While it is +/// checked out, /// `SchedulerView::task_views` remains the logical source of truth, but the /// ownership partition records that the view is in `checked_out_views` rather /// than `stored_views`. Weak-memory operations mutate the token's /// `ThreadView`; the scheduler view must be updated with /// `update_checked_out_task_view` to keep the logical snapshot synchronized. -/// Dropping the guard checks the token back into `stored_views`. +/// A quiescent schedule-out checks the updated token back into `stored_views`. /// /// In short, the resource flow is: -/// `stored_views -> TaskThreadView -> checked_out_views update -> stored_views`. +/// `stored_views -> RunningTaskContext -> checked_out_views update -> stored_views`. /// /// Abstract proof view of the scheduler state. /// @@ -232,6 +232,22 @@ impl SchedulerView { } } + /// Updating the view of the currently checked-out task preserves the + /// scheduler ownership partition and all scheduling invariants. + pub proof fn lemma_update_checked_out_task_view_preserves_wf(self, task: Loc, view: WmView) + requires + self.wf(), + self.task_view_is_checked_out(task), + ensures + self.update_checked_out_task_view(task, view).wf(), + self.update_checked_out_task_view(task, view).task_view_is_checked_out(task), + self.update_checked_out_task_view(task, view).task_views.contains_key(task), + self.update_checked_out_task_view(task, view).task_views[task] == view, + self.update_checked_out_task_view(task, view).checked_out_views[task] == view, + !self.update_checked_out_task_view(task, view).stored_views.contains_key(task), + { + } + /// Writes a checked-out task view back to scheduler storage. /// /// The caller must provide the same view that is recorded as checked out; @@ -312,7 +328,8 @@ impl SchedulerView { /// /// This is the resource that should eventually back `disable_preempt()`: /// a guard borrows or moves out the current task's `ThreadView`, atomic -/// operations update it, and guard drop writes it back to the same task entry. +/// operations update it, and schedule-out writes it back to the same task +/// entry after all preemption guards have been released. pub tracked struct SchedulerThreadViews { tracked views: Map, } @@ -356,6 +373,20 @@ impl TaskThreadView { &&& sched_view.task_views[self.task()] == self.view() } + /// Packages the public scheduler facts needed to establish ownership of a + /// checked-out task view. + pub proof fn lemma_wf(tracked &self, sched_view: SchedulerView) + requires + sched_view.wf(), + sched_view.task_view_is_checked_out(self.task()), + sched_view.checked_out_views[self.task()] == self.view(), + sched_view.task_views.contains_key(self.task()), + sched_view.task_views[self.task()] == self.view(), + ensures + self.wf(sched_view), + { + } + /// Borrows the linear `ThreadView` for weak-memory operations. /// /// After the borrow mutates the view, the caller must use @@ -453,6 +484,34 @@ impl SchedulerThreadViews { token } + /// Checks out the current task's weak-memory view and starts its running + /// context. + pub proof fn tracked_take_current_running_context( + tracked &mut self, + sched_view: SchedulerView, + cpu: CpuId, + ) -> (tracked context: RunningTaskContext) + requires + old(self).wf(sched_view), + sched_view.wf(), + sched_view.current.contains_key(cpu), + sched_view.current[cpu] is Some, + sched_view.task_view_is_stored(sched_view.current[cpu]->0), + old(self).contains(sched_view.current[cpu]->0), + ensures + context.task() == sched_view.current[cpu]->0, + context.view() == old(self).thread_view(sched_view.current[cpu]->0), + context.is_quiescent(), + context.wf_scheduler(sched_view.checkout_task_view(cpu)), + final(self).view() == sched_view.checkout_task_view(cpu).stored_views, + final(self).wf(sched_view.checkout_task_view(cpu)), + { + let ghost next = sched_view.checkout_task_view(cpu); + let tracked task_view = self.tracked_take_current_thread_view(sched_view, cpu); + let tracked context = RunningTaskContext::new(task_view, next); + context + } + /// Returns a checked-out task view to the tracked owner. /// /// The `token.wf(sched_view)` precondition ties this write-back to the @@ -482,6 +541,42 @@ impl SchedulerThreadViews { assert(final(self).view() == next.stored_views); assert(final(self).wf(next)); } + + /// Ends a quiescent running interval and checks its updated task view back + /// into scheduler ownership. + pub proof fn tracked_put_running_context( + tracked &mut self, + sched_view: SchedulerView, + tracked context: RunningTaskContext, + ) + requires + old(self).wf(sched_view), + sched_view.wf(), + sched_view.task_view_is_checked_out(context.task()), + context.wf(), + context.is_quiescent(), + !old(self).contains(context.task()), + ensures + final(self).view() == old(self).view().insert(context.task(), context.view()), + final(self).view() == sched_view.update_checked_out_task_view( + context.task(), + context.view(), + ).checkin_task_view(context.task(), context.view()).stored_views, + final(self).wf( + sched_view.update_checked_out_task_view( + context.task(), + context.view(), + ).checkin_task_view(context.task(), context.view()), + ), + { + let ghost task = context.task(); + let ghost view = context.view(); + sched_view.lemma_update_checked_out_task_view_preserves_wf(task, view); + let ghost updated = sched_view.update_checked_out_task_view(task, view); + context.lemma_wf_scheduler(updated); + let tracked task_view = context.tracked_into_task_view_for_scheduler(updated); + self.tracked_put_checked_out_thread_view(updated, task_view); + } } /// Logical identity of a runnable task handle. From dd5461cfaccf829c856928932d2f59a5d3af6f8a Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 15 Jul 2026 23:48:33 -0400 Subject: [PATCH 22/47] add more tracked resources for the rcu and weak memory --- Cargo.lock | 7 + ostd/specs/sync/rcu.rs | 993 ++++++++++++++++++++++++++++----- ostd/specs/sync/weak_memory.rs | 38 +- ostd/src/sync/rcu/mod.rs | 61 +- ostd/src/task/preempt/guard.rs | 77 ++- 5 files changed, 1004 insertions(+), 172 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff21dba00..904f203f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -86,6 +86,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "critical-section" version = "1.2.0" @@ -589,6 +595,7 @@ version = "0.0.0-2026-05-17-0151" name = "verus_builtin_macros" version = "0.0.0-2026-07-12-0122" dependencies = [ + "convert_case", "proc-macro2", "quote", "syn 2.0.101", diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 9f14b063e..79b46abee 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -1,26 +1,33 @@ -//! Specification skeleton for RCU traversal safety. +//! Weak-memory RCU base and traversal specification. //! //! This module models the shape of the traversal specification from the RCU //! relaxed-memory paper: //! -//! - the base layer provides read-side guards, protected pointers, and retire -//! permissions; +//! - the base layer provides registration-time allocation IDs, persistent +//! block information, unique retire permissions, and the +//! `Inactive(tid) <-> Guard(tid, X, G)` reader protocol; //! - the traversal layer reasons about link histories (`RcuPointsTo`) and //! incoming-link histories (`RcuPointedBy`); //! - concrete data structures instantiate the traversal trait. //! -//! The module is intentionally proof-only for now. The executable RCU -//! implementation should later connect its real guard/token state to these -//! abstract ghost tokens. +//! Allocation IDs, removed sets, link views, and incoming edges are all keyed +//! by AId, not by physical address. Physical addresses only appear in +//! `BlockInfo` and the guard's `address -> AId` protection map. This distinction +//! is required to handle stale weak-memory messages after address reuse. +//! +//! The module remains proof-only. The executable RCU must still connect its +//! preemption guard to the same domain's reader token and route the retire +//! permission released by pointer replacement into the callback monitor. use super::weak_memory::{History, Msg, WeakAtomicInvariantPredicate, WmView}; use vstd::prelude::*; use vstd::resource::Loc; +use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; verus! { pub type LinkIndex = nat; -pub type LinkEdge = (*mut T, LinkIndex); +pub type LinkEdge = (nat, LinkIndex); /// Proof summary for a type-erased RCU callback. /// @@ -33,19 +40,171 @@ pub ghost struct RcuCallbackSummary { /// The RCU protection domain whose grace period governs this callback. pub domain: Loc, /// Logical identity of the retired object inside `domain`. - pub obj: Loc, + pub obj: nat, /// The domain-local epoch in which `obj` was retired. pub retire_epoch: nat, } +/// Logical identity attached to one non-null publication in an RCU root. +/// +/// The paper distinguishes a physical address from an allocation ID because +/// an address may be reused after reclamation. `domain` identifies this RCU +/// registry, while `obj` identifies one registration within that domain. The +/// same address may therefore occur in multiple `RcuPublishedObject` values +/// without introducing an ABA-style identity collision. +pub ghost struct RcuPublishedObject { + pub domain: Loc, + pub obj: nat, + pub addr: usize, +} + +/// Publication metadata paired with an RCU root's atomic message history. +/// +/// Entry `publications[i]` describes atomic message `i`. A null message has no +/// allocation identity; a non-null message refers to an allocation ID obtained +/// from `RcuDomainAuth::tracked_register`. In particular, the allocation ID is +/// not the history index `i`. +/// +/// This state intentionally does not contain the traversal removed set or a +/// grace-period epoch. In the paper, removal belongs to `SeenRemoved` and link +/// histories, while expiration/reclamation belongs to the base RCU protocol. +/// A root store publishes a pointer, but cannot by itself prove that a node is +/// unreachable from every incoming link. +pub tracked struct RcuRootGhost { + domain: RcuDomainAuth, + ghost publications: Seq>, +} + +impl RcuRootGhost { + pub closed spec fn domain(self) -> Loc { + self.domain.id() + } + + pub closed spec fn objects(self) -> Map { + self.domain.objects() + } + + pub closed spec fn domain_wf(self) -> bool { + self.domain.wf() + } + + pub closed spec fn publications(self) -> Seq> { + self.publications + } + + pub open spec fn published_at(self, ts: nat) -> Option + recommends + ts < self.publications().len(), + { + match self.publications()[ts as int] { + Some(obj) => Some( + RcuPublishedObject { domain: self.domain(), obj, addr: self.objects()[obj] }, + ), + None => None, + } + } + + /// Allocation identity carried by the latest atomic message. + pub open spec fn current(self) -> Option + recommends + self.publications().len() > 0, + { + self.published_at((self.publications().len() - 1) as nat) + } + + /// Allocate a fresh publication registry containing the initial message. + pub proof fn tracked_initial(ptr: *mut T) -> (tracked res: Self) + ensures + rcu_root_history_inv(seq![Msg { value: ptr, view: WmView::empty() }], res), + { + let tracked mut domain = RcuDomainAuth::tracked_new(); + if ptr.addr() == 0 { + RcuRootGhost { domain, publications: seq![None] } + } else { + let tracked (block_info, _retire_perm) = domain.tracked_register(ptr); + let ghost obj = block_info.obj(); + assert(domain.objects().contains_pair(obj, ptr.addr())); + RcuRootGhost { domain, publications: seq![Some(obj)] } + } + } + + /// Extend the publication registry for one newly appended atomic message. + pub proof fn tracked_push( + tracked &mut self, + prev: History<*mut T>, + next: History<*mut T>, + msg: Msg<*mut T>, + ) + requires + rcu_root_history_inv(prev, *old(self)), + next == prev.push(msg), + ensures + rcu_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + { + let ghost ts = prev.len(); + assert(self.publications().len() == ts); + + if msg.value.addr() == 0 { + self.publications = self.publications.push(None); + } else { + let tracked (block_info, _retire_perm) = self.domain.tracked_register(msg.value); + let ghost obj = block_info.obj(); + self.publications = self.publications.push(Some(obj)); + } + + assert forall|i: int| 0 <= i < next.len() implies { + match #[trigger] self.publications()[i] { + None => next[i].value.addr() == 0, + Some(obj) => { + &&& next[i].value.addr() != 0 + &&& self.objects().contains_pair(obj, next[i].value.addr()) + }, + } + } by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + assert(next[i] == prev[i]); + assert(self.publications()[i] == old(self).publications()[i]); + match self.publications()[i] { + Some(obj) => { + assert(old(self).objects().contains_pair(obj, prev[i].value.addr())); + assert(self.objects().contains_pair(obj, next[i].value.addr())); + }, + None => {}, + } + } + }; + } +} + +/// Agreement between the weak-memory message history and RCU allocation IDs. +pub open spec fn rcu_root_history_inv(history: History<*mut T>, ghost: RcuRootGhost) -> bool { + &&& history.len() >= 1 + &&& ghost.domain_wf() + &&& ghost.publications().len() == history.len() + &&& forall|i: int| + 0 <= i < history.len() ==> { + match #[trigger] ghost.publications()[i] { + None => history[i].value.addr() == 0, + Some(obj) => { + &&& history[i].value.addr() != 0 + &&& ghost.objects().contains_pair(obj, history[i].value.addr()) + }, + } + } +} + /// The weak-memory invariant for the root pointer stored in an executable RCU /// cell. /// /// The key is the cell's nullability: `true` for `RcuOption`, `false` for -/// `Rcu`. At this layer we only connect the atomic message history to the -/// public nullability contract. Ownership, read tokens, and reclamation are -/// deliberately modeled by the traversal/reclaim tokens below and will be wired -/// into this predicate in later steps. +/// `Rcu`. The predicate connects each atomic message both to the public +/// nullability contract and to its domain-local allocation identity. Physical +/// ownership, read tokens, and reclamation are deliberately modeled by the +/// traversal/reclaim tokens below and will be wired in later steps. pub struct RcuWeakAtomicInv; pub open spec fn rcu_history_inv(nullable: bool, history: History<*mut T>) -> bool { @@ -54,9 +213,10 @@ pub open spec fn rcu_history_inv(nullable: bool, history: History<*mut T>) -> 0 <= i < history.len() ==> #[trigger] history[i].value.addr() != 0 } -impl WeakAtomicInvariantPredicate for RcuWeakAtomicInv { - open spec fn atomic_inv(nullable: bool, history: History<*mut T>, _g: ()) -> bool { - rcu_history_inv(nullable, history) +impl WeakAtomicInvariantPredicate for RcuWeakAtomicInv { + open spec fn atomic_inv(nullable: bool, history: History<*mut T>, g: RcuRootGhost) -> bool { + &&& rcu_history_inv(nullable, history) + &&& rcu_root_history_inv(history, g) } } @@ -341,37 +501,39 @@ pub proof fn rcu_history_inv_read_nonnull(history: History<*mut T>, ts: nat) /// Link view carried by an RCU read-side guard. /// -/// `seen_at(p) = n` means the guard has observed link-history events for source -/// node `p` up to at least `n`. Following a loaded link at index `k` is allowed -/// only when `seen_at(p) <= k`; otherwise the pointer may be too stale. +/// `seen_at(a) = n` means the guard has observed link-history events for source +/// AId `a` up to at least `n`. Following a loaded link at index `k` is allowed +/// only when `seen_at(a) <= k`; otherwise the pointer may be too stale. #[verifier::reject_recursive_types(T)] pub ghost struct RcuLinkView { - pub seen: Map<*mut T, LinkIndex>, + pub seen: Map, + pub marker: Option<*mut T>, } impl RcuLinkView { pub open spec fn empty() -> Self { - RcuLinkView { seen: Map::empty() } + RcuLinkView { seen: Map::empty(), marker: None } } - pub open spec fn seen_at(self, p: *mut T) -> LinkIndex { - if self.seen.contains_key(p) { - self.seen[p] + pub open spec fn seen_at(self, obj: nat) -> LinkIndex { + if self.seen.contains_key(obj) { + self.seen[obj] } else { 0nat } } - pub open spec fn observe(self, p: *mut T, n: LinkIndex) -> Self { + pub open spec fn observe(self, obj: nat, n: LinkIndex) -> Self { RcuLinkView { seen: self.seen.insert( - p, - if self.seen_at(p) <= n { + obj, + if self.seen_at(obj) <= n { n } else { - self.seen_at(p) + self.seen_at(obj) }, ), + marker: self.marker, } } } @@ -379,11 +541,11 @@ impl RcuLinkView { /// Paper-style `SeenRemoved(D, LV)`. /// /// `removed` is the set `D` observed by the guard; `link_view` is `LV`. -/// A dead incoming edge is either from a removed predecessor or overwritten by a -/// later observed link event. +/// A dead incoming edge is either from a removed predecessor AId or overwritten +/// by a later observed link event. #[verifier::reject_recursive_types(T)] pub ghost struct RcuSeenRemoved { - pub removed: Set<*mut T>, + pub removed: Set, pub link_view: RcuLinkView, } @@ -392,11 +554,11 @@ impl RcuSeenRemoved { RcuSeenRemoved { removed: Set::empty(), link_view: RcuLinkView::empty() } } - pub open spec fn seen_at(self, p: *mut T) -> LinkIndex { - self.link_view.seen_at(p) + pub open spec fn seen_at(self, obj: nat) -> LinkIndex { + self.link_view.seen_at(obj) } - pub open spec fn dead_edge(self, edge: LinkEdge) -> bool { + pub open spec fn dead_edge(self, edge: LinkEdge) -> bool { self.removed.contains(edge.0) || self.seen_at(edge.0) > edge.1 } } @@ -406,48 +568,445 @@ impl RcuSeenRemoved { /// The concrete implementation owns this token in its invariant. We keep the /// fields private so clients cannot manufacture domain authority. pub tracked struct RcuDomainAuth { - ghost id: Loc, + objects: GhostMapAuth, + retire_perms: GhostMapAuth, + readers: GhostMapAuth, + ghost next_obj: nat, + ghost next_reader: nat, + ghost retired: Set, } impl RcuDomainAuth { + /// The paper's RCU location `l` is the identity of the allocation registry. + /// Every object registered through this authority belongs to this domain. pub closed spec fn id(self) -> Loc { - self.id + self.objects.id() + } + + pub closed spec fn objects(self) -> Map { + self.objects@ + } + + pub closed spec fn next_obj(self) -> nat { + self.next_obj + } + + pub closed spec fn retired(self) -> Set { + self.retired + } + + pub closed spec fn reader_registry(self) -> Loc { + self.readers.id() + } + + pub closed spec fn retire_registry(self) -> Loc { + self.retire_perms.id() + } + + pub closed spec fn next_reader(self) -> nat { + self.next_reader + } + + /// Internal consistency of the two resource algebras used by the base RCU + /// model. The first map backs persistent `BlockInfo`; the second map backs + /// the unique retire capability. + pub closed spec fn wf(self) -> bool { + &&& self.objects@ == self.retire_perms@ + &&& forall|obj: nat| #[trigger] self.objects@.contains_key(obj) ==> obj < self.next_obj() + &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> tid < self.next_reader() + &&& self.retired().subset_of(self.objects().dom()) + } + + /// Allocates a fresh RCU protection domain. + pub proof fn tracked_new() -> (tracked res: Self) + ensures + res.wf(), + res.objects() == Map::::empty(), + res.next_obj() == 0, + { + let tracked (objects, _objects_entries) = GhostMapAuth::new(Map::empty()); + let tracked (retire_perms, _retire_entries) = GhostMapAuth::new(Map::empty()); + let tracked (readers, _reader_entries) = GhostMapAuth::new(Map::empty()); + RcuDomainAuth { + objects, + retire_perms, + readers, + next_obj: 0, + next_reader: 0, + retired: Set::empty(), + } + } + + /// Implements the paper's `rcu-register` rule. + /// + /// The allocation ID is chosen here, once per registration. It is not an + /// atomic-history timestamp. Registration returns both persistent block + /// information and the unique base retire permission for the allocation. + pub proof fn tracked_register(tracked &mut self, ptr: *mut T) -> (tracked res: ( + RcuBlockInfo, + RcuBaseRetirePerm, + )) + requires + old(self).wf(), + ptr.addr() != 0, + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).next_obj() == old(self).next_obj() + 1, + final(self).retired() == old(self).retired(), + final(self).objects() == old(self).objects().insert(old(self).next_obj(), ptr.addr()), + res.0.domain() == final(self).id(), + res.0.obj() == old(self).next_obj(), + res.0.ptr() == ptr, + res.0.addr() == ptr.addr(), + res.1.domain() == final(self).id(), + res.1.obj() == res.0.obj(), + res.1.ptr() == ptr, + res.1.belongs_to(*final(self)), + { + let ghost obj = self.next_obj; + assert(!self.objects@.contains_key(obj)); + assert(!self.retire_perms@.contains_key(obj)); + + let tracked object = self.objects.insert(obj, ptr.addr()); + let tracked block_info = object.persist(); + let tracked retire_perm = self.retire_perms.insert(obj, ptr.addr()); + self.next_obj = self.next_obj + 1; + + assert forall|registered: nat| #[trigger] + self.objects@.contains_key(registered) implies registered < self.next_obj by { + if registered != obj { + assert(old(self).objects().contains_key(registered)); + } + }; + + ( + RcuBlockInfo { info: block_info, ptr }, + RcuBaseRetirePerm { domain: self.id(), perm: retire_perm, ptr }, + ) + } + + /// Establishes agreement between domain authority and persistent block + /// information supplied by a client or an atomic-history invariant. + pub proof fn lemma_block_info_agree(tracked &self, tracked info: &RcuBlockInfo) + requires + self.wf(), + info.domain() == self.id(), + ensures + self.objects().contains_pair(info.obj(), info.addr()), + { + info.info.agree(&self.objects); + } + + /// Registers one reader slot and returns the paper's `Inactive(tid)` + /// resource for it. + pub proof fn tracked_register_reader(tracked &mut self) -> (tracked res: RcuInactive) + requires + old(self).wf(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).objects() == old(self).objects(), + final(self).retired() == old(self).retired(), + res.domain() == final(self).id(), + res.tid() == old(self).next_reader(), + res.belongs_to(*final(self)), + res.wf(), + { + let ghost tid = self.next_reader; + assert(!self.readers@.contains_key(tid)); + let tracked state = self.readers.insert(tid, false); + self.next_reader = self.next_reader + 1; + assert forall|registered: nat| #[trigger] + self.readers@.contains_key(registered) implies registered < self.next_reader by { + if registered != tid { + assert(old(self).readers@.contains_key(registered)); + } + }; + RcuInactive { domain: self.id(), state } + } + + /// Starts a read-side critical section, snapshotting the set `X` of AIds + /// that had already been retired. + pub proof fn tracked_guard_start( + tracked &mut self, + tracked mut inactive: RcuInactive, + ) -> (tracked res: RcuBaseGuard) + requires + old(self).wf(), + inactive.belongs_to(*old(self)), + inactive.wf(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).objects() == old(self).objects(), + final(self).retired() == old(self).retired(), + res.belongs_to(*final(self)), + res.tid() == inactive.tid(), + res.expired() == old(self).retired(), + res.protected() == Map::::empty(), + res.wf(), + { + let ghost tid = inactive.tid(); + inactive.state.agree(&self.readers); + assert(self.readers@.contains_key(tid)); + inactive.state.update(&mut self.readers, true); + assert forall|registered: nat| #[trigger] + self.readers@.contains_key(registered) implies registered < self.next_reader by { + if registered == tid { + assert(old(self).readers@.contains_key(registered)); + } else { + assert(old(self).readers@.contains_key(registered)); + } + }; + RcuBaseGuard { + domain: self.id(), + state: inactive.state, + expired: self.retired, + protected: Map::empty(), + } + } + + /// Ends a read-side critical section and returns the unique inactive token + /// for the same reader slot. + pub proof fn tracked_guard_stop( + tracked &mut self, + tracked mut guard: RcuBaseGuard, + ) -> (tracked res: RcuInactive) + requires + old(self).wf(), + guard.belongs_to(*old(self)), + guard.wf(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).objects() == old(self).objects(), + final(self).retired() == old(self).retired(), + res.belongs_to(*final(self)), + res.tid() == guard.tid(), + res.wf(), + { + let ghost tid = guard.tid(); + guard.state.agree(&self.readers); + assert(self.readers@.contains_key(tid)); + guard.state.update(&mut self.readers, false); + assert forall|registered: nat| #[trigger] + self.readers@.contains_key(registered) implies registered < self.next_reader by { + if registered == tid { + assert(old(self).readers@.contains_key(registered)); + } else { + assert(old(self).readers@.contains_key(registered)); + } + }; + RcuInactive { domain: self.id(), state: guard.state } + } + + /// Implements the base `rcu-retire` transition by adding the detached AId + /// to `RcuState.R` and consuming its unique traversal retire permission. + pub proof fn tracked_retire( + tracked &mut self, + tracked retire: RcuRetirePerm, + ) -> (tracked res: RcuRetired) + requires + old(self).wf(), + retire.belongs_to(*old(self)), + retire.ready_to_retire(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).objects() == old(self).objects(), + final(self).retired() == old(self).retired().insert(retire.obj()), + res.domain() == final(self).id(), + res.obj() == retire.obj(), + res.ptr() == retire.ptr(), + { + retire.base.perm.agree(&self.retire_perms); + assert(self.objects().contains_key(retire.obj())); + self.retired = self.retired.insert(retire.obj()); + assert(self.retired().subset_of(self.objects().dom())); + RcuRetired { retire } + } +} + +/// The paper's unique `Inactive(tid)` reader resource. +pub tracked struct RcuInactive { + ghost domain: Loc, + state: GhostPointsTo, +} + +impl RcuInactive { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn tid(self) -> nat { + self.state.key() + } + + pub closed spec fn wf(self) -> bool { + !self.state.value() + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + &&& self.domain() == domain.id() + &&& self.state.id() == domain.reader_registry() } } -/// Logical identity of one RCU-managed object. +/// The base paper guard `Guard(tid, X, G)`. /// -/// Traversal proofs are typed and pointer-based (`*mut T`), while the callback -/// monitor stores type-erased callbacks. This token bridges the two worlds: it -/// says that `obj` is the logical identity of `ptr` inside `domain`. -#[verifier::reject_recursive_types(T)] -pub tracked struct RcuObjectId { +/// `expired` is the start-time snapshot `X`. `protected[addr] = a` is the +/// mutable protection map `G` populated by successful protect operations. +pub tracked struct RcuBaseGuard { ghost domain: Loc, - ghost obj: Loc, - ghost ptr: *mut T, + state: GhostPointsTo, + ghost expired: Set, + ghost protected: Map, } -impl RcuObjectId { +impl RcuBaseGuard { pub closed spec fn domain(self) -> Loc { self.domain } - pub closed spec fn obj(self) -> Loc { - self.obj + pub closed spec fn tid(self) -> nat { + self.state.key() + } + + pub closed spec fn expired(self) -> Set { + self.expired + } + + pub closed spec fn protected(self) -> Map { + self.protected + } + + pub closed spec fn wf(self) -> bool { + self.state.value() + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + &&& self.domain() == domain.id() + &&& self.state.id() == domain.reader_registry() + } + + pub closed spec fn protects(self, addr: usize, obj: nat) -> bool { + self.protected().contains_pair(addr, obj) + } + + /// Implements the base `Guard-protect` update. An object already in the + /// guard's start snapshot `X` cannot be newly protected by this guard. + pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) + requires + old(self).wf(), + info.wf(), + info.domain() == old(self).domain(), + !old(self).expired().contains(info.obj()), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), + final(self).protects(info.addr(), info.obj()), + { + self.protected = self.protected.insert(info.addr(), info.obj()); + } +} + +/// Persistent `BlockInfo(l, a, P)` for one RCU-managed allocation. +/// +/// The current Verus cut records the allocation's typed pointer and physical +/// address; the client-owned block predicate `P` remains represented by the +/// corresponding `P::Permission` at the executable boundary. The resource +/// token is persistent and can therefore be copied into every weak-memory +/// history entry that publishes this allocation. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuBlockInfo { + info: GhostPersistentPointsTo, + ghost ptr: *mut T, +} + +impl RcuBlockInfo { + pub closed spec fn domain(self) -> Loc { + self.info.id() + } + + pub closed spec fn obj(self) -> nat { + self.info.key() } pub closed spec fn ptr(self) -> *mut T { self.ptr } + + pub closed spec fn addr(self) -> usize { + self.info.value() + } + + pub open spec fn wf(self) -> bool { + self.addr() == self.ptr().addr() + } + + /// Persistent block information can be retained by both the client and + /// every historical atomic message that mentions the allocation. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.domain() == self.domain(), + res.obj() == self.obj(), + res.ptr() == self.ptr(), + res.addr() == self.addr(), + res.wf() == self.wf(), + { + let tracked info = self.info.duplicate(); + RcuBlockInfo { info, ptr: self.ptr } + } +} + +/// Compatibility name used by the callback/traversal boundary. +pub type RcuObjectId = RcuBlockInfo; + +/// Regression proof for the allocation-ID discipline required by relaxed +/// memory RCU. +/// +/// `res.0` and `res.1` are two persistent copies of one registration, so they +/// carry the same AId. `res.2` is a second registration at the same physical +/// address, so it carries a fresh AId. This is the distinction that rules out +/// identifying stale weak-memory messages by address alone. +pub proof fn registration_distinguishes_reused_address(ptr: *mut T) -> (tracked res: ( + RcuBlockInfo, + RcuBlockInfo, + RcuBlockInfo, +)) + requires + ptr.addr() != 0, + ensures + res.0.domain() == res.1.domain(), + res.0.domain() == res.2.domain(), + res.0.obj() == res.1.obj(), + res.0.obj() != res.2.obj(), + res.0.addr() == ptr.addr(), + res.1.addr() == ptr.addr(), + res.2.addr() == ptr.addr(), +{ + let tracked mut domain = RcuDomainAuth::tracked_new(); + let tracked (first, _first_retire) = domain.tracked_register(ptr); + let tracked first_history_copy = first.tracked_duplicate(); + let tracked (second, _second_retire) = domain.tracked_register(ptr); + assert(first.obj() < second.obj()); + (first, first_history_copy, second) } /// Low-level base retire permission. /// -/// This is the paper's `BaseRetirePerm`. By itself it is not enough to reclaim; -/// it must be combined with a `SeenRemoved` observation for the retired object. +/// This is the paper's unique `BaseRetirePerm(l, a)`. The embedded owning +/// points-to resource makes duplication impossible. By itself it is not enough +/// to retire or reclaim an object; traversal must first establish that every +/// relevant incoming edge has been removed. #[verifier::reject_recursive_types(T)] pub tracked struct RcuBaseRetirePerm { ghost domain: Loc, + perm: GhostPointsTo, ghost ptr: *mut T, } @@ -459,6 +1018,23 @@ impl RcuBaseRetirePerm { pub closed spec fn ptr(self) -> *mut T { self.ptr } + + pub closed spec fn obj(self) -> nat { + self.perm.key() + } + + pub closed spec fn addr(self) -> usize { + self.perm.value() + } + + pub open spec fn wf(self) -> bool { + self.addr() == self.ptr().addr() + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + &&& self.domain() == domain.id() + &&& self.perm.id() == domain.retire_registry() + } } /// High-level retire permission. @@ -467,26 +1043,35 @@ impl RcuBaseRetirePerm { /// exists D LV. SeenRemoved(D, LV) * a in D`. #[verifier::reject_recursive_types(T)] pub tracked struct RcuRetirePerm { - ghost domain: Loc, - ghost ptr: *mut T, + base: RcuBaseRetirePerm, ghost seen_removed: RcuSeenRemoved, } impl RcuRetirePerm { pub closed spec fn domain(self) -> Loc { - self.domain + self.base.domain() } pub closed spec fn ptr(self) -> *mut T { - self.ptr + self.base.ptr() + } + + pub closed spec fn obj(self) -> nat { + self.base.obj() } pub closed spec fn seen_removed(self) -> RcuSeenRemoved { self.seen_removed } - pub open spec fn ready_to_reclaim(self) -> bool { - self.seen_removed().removed.contains(self.ptr()) + /// The traversal layer has established that this object may be retired. + /// Reclamation still requires a completed base-RCU grace period. + pub open spec fn ready_to_retire(self) -> bool { + self.seen_removed().removed.contains(self.obj()) + } + + pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { + self.base.belongs_to(domain) } } @@ -497,14 +1082,37 @@ pub proof fn lift_retire_perm( seen_removed: RcuSeenRemoved, ) -> (tracked perm: RcuRetirePerm) requires - seen_removed.removed.contains(base.ptr()), + seen_removed.removed.contains(base.obj()), ensures perm.domain() == base.domain(), perm.ptr() == base.ptr(), + perm.obj() == base.obj(), perm.seen_removed() == seen_removed, - perm.ready_to_reclaim(), + perm.ready_to_retire(), { - RcuRetirePerm { domain: base.domain(), ptr: base.ptr(), seen_removed } + RcuRetirePerm { base, seen_removed } +} + +/// Objective record that an allocation has passed the base `rcu-retire` +/// transition. It is safe to enqueue its callback, but not yet safe to execute +/// it; execution additionally needs monitor grace-period completion. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetired { + retire: RcuRetirePerm, +} + +impl RcuRetired { + pub closed spec fn domain(self) -> Loc { + self.retire.domain() + } + + pub closed spec fn obj(self) -> nat { + self.retire.obj() + } + + pub closed spec fn ptr(self) -> *mut T { + self.retire.ptr() + } } /// Non-generic proof certificate carried across the type-erasure boundary. @@ -535,21 +1143,21 @@ pub open spec fn callback_safety_from_traversal( /// Consume a typed traversal retire permission and compress it into the /// non-generic summary needed by the type-erased callback monitor. -pub proof fn certify_callback_from_retire_perm( +pub proof fn certify_callback_from_retired( tracked object: &RcuObjectId, - tracked retire: RcuRetirePerm, + tracked retired: RcuRetired, retire_epoch: nat, ) -> (tracked cert: RcuCallbackSafety) requires - object.domain() == retire.domain(), - object.ptr() == retire.ptr(), - retire.ready_to_reclaim(), + object.domain() == retired.domain(), + object.obj() == retired.obj(), + object.ptr() == retired.ptr(), ensures - cert@ == (RcuCallbackSummary { domain: retire.domain(), obj: object.obj(), retire_epoch }), + cert@ == (RcuCallbackSummary { domain: retired.domain(), obj: object.obj(), retire_epoch }), callback_safety_from_traversal(cert, *object, retire_epoch), { RcuCallbackSafety { - summary: RcuCallbackSummary { domain: retire.domain(), obj: object.obj(), retire_epoch }, + summary: RcuCallbackSummary { domain: retired.domain(), obj: object.obj(), retire_epoch }, } } @@ -559,13 +1167,29 @@ pub proof fn certify_callback_from_retire_perm( /// the `SeenRemoved(D, LV)` observation used to rule out stale links. #[verifier::reject_recursive_types(T)] pub tracked struct RcuReadGuardToken { - ghost domain: Loc, + base: RcuBaseGuard, ghost seen_removed: RcuSeenRemoved, } impl RcuReadGuardToken { pub closed spec fn domain(self) -> Loc { - self.domain + self.base.domain() + } + + pub closed spec fn tid(self) -> nat { + self.base.tid() + } + + pub closed spec fn expired(self) -> Set { + self.base.expired() + } + + pub closed spec fn protected(self) -> Map { + self.base.protected() + } + + pub closed spec fn protects(self, addr: usize, obj: nat) -> bool { + self.base.protects(addr, obj) } pub closed spec fn seen_removed(self) -> RcuSeenRemoved { @@ -576,16 +1200,58 @@ impl RcuReadGuardToken { self.seen_removed().link_view } - pub open spec fn seen_at(self, p: *mut T) -> LinkIndex { - self.seen_removed().seen_at(p) + pub open spec fn seen_at(self, obj: nat) -> LinkIndex { + self.seen_removed().seen_at(obj) } - pub open spec fn is_for(self, domain: RcuDomainAuth) -> bool { - self.domain() == domain.id() + pub closed spec fn wf(self) -> bool { + &&& self.base.wf() + &&& self.expired().subset_of(self.seen_removed().removed) } - pub open spec fn can_protect(self, p: *mut T) -> bool { - !self.seen_removed().removed.contains(p) + pub closed spec fn is_for(self, domain: RcuDomainAuth) -> bool { + self.base.belongs_to(domain) + } + + pub open spec fn can_protect(self, info: RcuBlockInfo) -> bool { + &&& self.wf() + &&& info.wf() + &&& info.domain() == self.domain() + &&& !self.expired().contains(info.obj()) + &&& !self.seen_removed().removed.contains(info.obj()) + } + + /// Combines the paper's base guard with traversal `SeenRemoved(D, LV)`. + pub proof fn tracked_new( + tracked base: RcuBaseGuard, + seen_removed: RcuSeenRemoved, + ) -> (tracked res: Self) + requires + base.wf(), + base.expired().subset_of(seen_removed.removed), + ensures + res.wf(), + res.domain() == base.domain(), + res.tid() == base.tid(), + res.seen_removed() == seen_removed, + { + RcuReadGuardToken { base, seen_removed } + } + + /// Records one successful base `Guard-protect` operation in `G`. + pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) + requires + old(self).can_protect(*info), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).expired() == old(self).expired(), + final(self).seen_removed() == old(self).seen_removed(), + final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), + final(self).protects(info.addr(), info.obj()), + { + self.base.tracked_protect(info); } } @@ -597,6 +1263,7 @@ impl RcuReadGuardToken { #[verifier::reject_recursive_types(T)] pub tracked struct RcuProtectedPtr { ghost domain: Loc, + ghost obj: nat, ghost ptr: *mut T, ghost seen_removed: RcuSeenRemoved, } @@ -610,6 +1277,10 @@ impl RcuProtectedPtr { self.ptr } + pub closed spec fn obj(self) -> nat { + self.obj + } + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { self.seen_removed } @@ -617,7 +1288,8 @@ impl RcuProtectedPtr { pub open spec fn protected_by(self, guard: RcuReadGuardToken) -> bool { &&& self.domain() == guard.domain() &&& self.seen_removed() == guard.seen_removed() - &&& !self.seen_removed().removed.contains(self.ptr()) + &&& !self.seen_removed().removed.contains(self.obj()) + &&& guard.protects(self.ptr().addr(), self.obj()) } } @@ -632,88 +1304,127 @@ pub trait RcuTraversalSafety: Sized { type Ghost; - spec fn root_inv(p: *mut Self::Node, g: Self::Ghost) -> bool; + spec fn root_inv(p: *mut Self::Node, obj: nat, g: Self::Ghost) -> bool; - spec fn node_inv(p: *mut Self::Node, g: Self::Ghost) -> bool; + spec fn node_inv(p: *mut Self::Node, obj: nat, g: Self::Ghost) -> bool; spec fn link_inv( from: *mut Self::Node, + from_obj: nat, n: LinkIndex, to: *mut Self::Node, + to_obj: nat, g: Self::Ghost, ) -> bool; spec fn seen_removed_sound(seen_removed: RcuSeenRemoved, g: Self::Ghost) -> bool; - proof fn root_is_node_inv(p: *mut Self::Node, g: Self::Ghost) + proof fn root_is_node_inv(p: *mut Self::Node, obj: nat, g: Self::Ghost) requires - Self::root_inv(p, g), + Self::root_inv(p, obj, g), ensures - Self::node_inv(p, g), + Self::node_inv(p, obj, g), ; proof fn link_preserves_protection( from: *mut Self::Node, + from_obj: nat, n: LinkIndex, to: *mut Self::Node, + to_obj: nat, seen_removed: RcuSeenRemoved, g: Self::Ghost, ) requires - Self::node_inv(from, g), - Self::link_inv(from, n, to, g), + Self::node_inv(from, from_obj, g), + Self::link_inv(from, from_obj, n, to, to_obj, g), Self::seen_removed_sound(seen_removed, g), - !seen_removed.removed.contains(from), - seen_removed.seen_at(from) <= n, + !seen_removed.removed.contains(from_obj), + seen_removed.seen_at(from_obj) <= n, ensures - Self::node_inv(to, g), - !seen_removed.removed.contains(to), + Self::node_inv(to, to_obj, g), + !seen_removed.removed.contains(to_obj), ; } /// Protect a freshly acquired root pointer. pub proof fn protect_root( tracked domain: &RcuDomainAuth, - tracked guard: &RcuReadGuardToken, + tracked guard: &mut RcuReadGuardToken, + tracked info: &RcuBlockInfo, p: *mut S::Node, g: S::Ghost, ) -> (tracked root: RcuProtectedPtr) requires - guard.is_for(*domain), - guard.can_protect(p), - S::root_inv(p, g), + old(guard).is_for(*domain), + old(guard).can_protect(*info), + info.ptr() == p, + S::root_inv(p, info.obj(), g), ensures root.ptr() == p, + root.obj() == info.obj(), root.domain() == domain.id(), - root.protected_by(*guard), - S::node_inv(p, g), + root.protected_by(*final(guard)), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).expired() == old(guard).expired(), + final(guard).seen_removed() == old(guard).seen_removed(), + S::node_inv(p, info.obj(), g), { - S::root_is_node_inv(p, g); - RcuProtectedPtr { domain: domain.id(), ptr: p, seen_removed: guard.seen_removed() } + S::root_is_node_inv(p, info.obj(), g); + guard.tracked_protect(info); + RcuProtectedPtr { + domain: domain.id(), + obj: info.obj(), + ptr: p, + seen_removed: guard.seen_removed(), + } } /// Protect a child reached by following a non-stale link-history event. pub proof fn protect_link( - tracked guard: &RcuReadGuardToken, + tracked guard: &mut RcuReadGuardToken, tracked from: &RcuProtectedPtr, + tracked to_info: &RcuBlockInfo, n: LinkIndex, to: *mut S::Node, g: S::Ghost, ) -> (tracked to_protected: RcuProtectedPtr) requires - from.protected_by(*guard), - S::node_inv(from.ptr(), g), - S::link_inv(from.ptr(), n, to, g), + from.protected_by(*old(guard)), + old(guard).can_protect(*to_info), + to_info.ptr() == to, + S::node_inv(from.ptr(), from.obj(), g), + S::link_inv(from.ptr(), from.obj(), n, to, to_info.obj(), g), S::seen_removed_sound(guard.seen_removed(), g), - guard.seen_at(from.ptr()) <= n, + guard.seen_at(from.obj()) <= n, ensures to_protected.ptr() == to, + to_protected.obj() == to_info.obj(), to_protected.domain() == from.domain(), - to_protected.protected_by(*guard), - S::node_inv(to, g), + to_protected.protected_by(*final(guard)), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).expired() == old(guard).expired(), + final(guard).seen_removed() == old(guard).seen_removed(), + S::node_inv(to, to_info.obj(), g), { - S::link_preserves_protection(from.ptr(), n, to, guard.seen_removed(), g); - RcuProtectedPtr { domain: from.domain(), ptr: to, seen_removed: guard.seen_removed() } + S::link_preserves_protection( + from.ptr(), + from.obj(), + n, + to, + to_info.obj(), + guard.seen_removed(), + g, + ); + guard.tracked_protect(to_info); + RcuProtectedPtr { + domain: from.domain(), + obj: to_info.obj(), + ptr: to, + seen_removed: guard.seen_removed(), + } } /// Minimal ghost-only node used to demonstrate the traversal contract. @@ -732,9 +1443,11 @@ pub struct LinkedListNode; /// makes the example match the paper's predicate shape. pub ghost struct LinkedListGhost { pub root: *mut LinkedListNode, + pub root_obj: nat, + pub objects: Map<*mut LinkedListNode, nat>, pub successors: Map<*mut LinkedListNode, Seq>>, - pub incoming_all: Map<*mut LinkedListNode, Set>>, - pub current_incoming: Map<*mut LinkedListNode, Set>>, + pub incoming_all: Map>, + pub current_incoming: Map>, } pub struct LinkedListTraversalSpec; @@ -744,56 +1457,65 @@ impl RcuTraversalSafety for LinkedListTraversalSpec { type Ghost = LinkedListGhost; - open spec fn root_inv(p: *mut LinkedListNode, g: LinkedListGhost) -> bool { + open spec fn root_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { &&& p == g.root + &&& obj == g.root_obj + &&& g.objects.contains_pair(p, obj) &&& g.successors.contains_key(p) - &&& g.incoming_all.contains_key(p) + &&& g.incoming_all.contains_key(obj) } - open spec fn node_inv(p: *mut LinkedListNode, g: LinkedListGhost) -> bool { + open spec fn node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { + &&& g.objects.contains_pair(p, obj) &&& g.successors.contains_key(p) - &&& g.incoming_all.contains_key(p) + &&& g.incoming_all.contains_key(obj) } open spec fn link_inv( from: *mut LinkedListNode, + from_obj: nat, n: LinkIndex, to: *mut LinkedListNode, + to_obj: nat, g: LinkedListGhost, ) -> bool { + &&& g.objects.contains_pair(from, from_obj) + &&& g.objects.contains_pair(to, to_obj) &&& g.successors.contains_key(from) &&& n < g.successors[from].len() &&& g.successors[from][n as int] == Some(to) &&& g.successors.contains_key(to) - &&& g.incoming_all.contains_key(to) - &&& g.incoming_all[to].contains((from, n)) + &&& g.incoming_all.contains_key(to_obj) + &&& g.incoming_all[to_obj].contains((from_obj, n)) } open spec fn seen_removed_sound( seen_removed: RcuSeenRemoved, g: LinkedListGhost, ) -> bool { - forall|to: *mut LinkedListNode| #[trigger] - seen_removed.removed.contains(to) ==> { - &&& g.incoming_all.contains_key(to) - &&& forall|edge: LinkEdge| #[trigger] - g.incoming_all[to].contains(edge) ==> seen_removed.dead_edge(edge) + forall|to_obj: nat| #[trigger] + seen_removed.removed.contains(to_obj) ==> { + &&& g.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge) } } - proof fn root_is_node_inv(p: *mut LinkedListNode, g: LinkedListGhost) { + proof fn root_is_node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) { } proof fn link_preserves_protection( from: *mut LinkedListNode, + from_obj: nat, n: LinkIndex, to: *mut LinkedListNode, + to_obj: nat, seen_removed: RcuSeenRemoved, g: LinkedListGhost, ) { - if seen_removed.removed.contains(to) { - assert(g.incoming_all[to].contains((from, n))); - assert(seen_removed.dead_edge((from, n))); + if seen_removed.removed.contains(to_obj) { + assert(g.incoming_all[to_obj].contains((from_obj, n))); + assert(seen_removed.dead_edge((from_obj, n))); assert(false); } } @@ -803,27 +1525,40 @@ impl RcuTraversalSafety for LinkedListTraversalSpec { /// event protects the next node under the same guard. pub proof fn linked_list_protect_next_example( tracked domain: &RcuDomainAuth, - tracked guard: &RcuReadGuardToken, + tracked guard: &mut RcuReadGuardToken, + tracked root_info: &RcuBlockInfo, + tracked next_info: &RcuBlockInfo, root: *mut LinkedListNode, n: LinkIndex, next: *mut LinkedListNode, g: LinkedListGhost, ) -> (tracked next_protected: RcuProtectedPtr) requires - guard.is_for(*domain), - guard.can_protect(root), - LinkedListTraversalSpec::root_inv(root, g), - LinkedListTraversalSpec::link_inv(root, n, next, g), - LinkedListTraversalSpec::seen_removed_sound(guard.seen_removed(), g), - guard.seen_at(root) <= n, + old(guard).is_for(*domain), + old(guard).can_protect(*root_info), + old(guard).can_protect(*next_info), + root_info.ptr() == root, + next_info.ptr() == next, + LinkedListTraversalSpec::root_inv(root, root_info.obj(), g), + LinkedListTraversalSpec::link_inv(root, root_info.obj(), n, next, next_info.obj(), g), + LinkedListTraversalSpec::seen_removed_sound(old(guard).seen_removed(), g), + old(guard).seen_at(root_info.obj()) <= n, ensures next_protected.ptr() == next, + next_protected.obj() == next_info.obj(), next_protected.domain() == domain.id(), - next_protected.protected_by(*guard), - LinkedListTraversalSpec::node_inv(next, g), + next_protected.protected_by(*final(guard)), + final(guard).wf(), + LinkedListTraversalSpec::node_inv(next, next_info.obj(), g), { - let tracked root_protected = protect_root::(domain, guard, root, g); - protect_link::(guard, &root_protected, n, next, g) + let tracked root_protected = protect_root::( + domain, + guard, + root_info, + root, + g, + ); + protect_link::(guard, &root_protected, next_info, n, next, g) } } // verus! diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 23aa8e7f8..4d75a9be8 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -658,17 +658,25 @@ impl WeakAtomicPtr { } } -impl WeakAtomicPtr { +impl WeakAtomicPtr { /// Acquire-load helper for RCU root pointers. #[inline(always)] pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( *mut T, Ghost, + Ghost>, )) requires self.well_formed(), ensures !self.constant() ==> !res.0.is_null(), + match res.2@ { + None => res.0.addr() == 0, + Some(object) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + }, + }, { let result; proof { @@ -681,13 +689,27 @@ impl WeakAtomicPtr { assert(self.atomic_inv@.constant().1 == self.atomic.id()); assert(hist.id() == self.atomic.id()); } - result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + let loaded = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof_decl! { + let ghost published = g.published_at(loaded.1@); + } proof { assert(rcu_spec::rcu_history_inv(self.constant(), hist.history())); + match published { + Some(object) => { + assert(object.addr == loaded.0.addr()); + }, + None => { + assert(loaded.0.addr() == 0); + }, + } if !self.constant() { - rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), result.1@); - assert(!result.0.is_null()); + rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), loaded.1@); + assert(!loaded.0.is_null()); } + } + result = (loaded.0, loaded.1, Ghost(published)); + proof { pair = (hist, g); } }); @@ -705,7 +727,7 @@ impl WeakAtomicPtr { use_type_invariant(self); } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, g) = pair; + let tracked (mut hist, mut g) = pair; proof { assert(hist.id() == self.atomic_inv@.constant().1); assert(self.atomic_inv@.constant().1 == self.atomic.id()); @@ -715,6 +737,7 @@ impl WeakAtomicPtr { let snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); let ghost next = hist.history(); proof { + assert(rcu_spec::rcu_root_history_inv(prev, g)); if !self.constant() { assert(!value.is_null()); assert(snap@.msg().value.addr() != 0); @@ -725,6 +748,7 @@ impl WeakAtomicPtr { next, snap@.msg(), ); + g.tracked_push(prev, next, snap@.msg()); pair = (hist, g); } }); @@ -747,7 +771,7 @@ impl WeakAtomicPtr { use_type_invariant(self); } vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, g) = pair; + let tracked (mut hist, mut g) = pair; proof { assert(hist.id() == self.atomic_inv@.constant().1); assert(self.atomic_inv@.constant().1 == self.atomic.id()); @@ -763,6 +787,7 @@ impl WeakAtomicPtr { result = (cas_result.0, cas_result.1); let ghost next = hist.history(); proof { + assert(rcu_spec::rcu_root_history_inv(prev, g)); match cas_result.0 { Result::Ok(_) => { let tracked snap_opt = cas_result.2.get(); @@ -778,6 +803,7 @@ impl WeakAtomicPtr { next, snap.msg(), ); + g.tracked_push(prev, next, snap.msg()); }, Option::None => { assert(false); diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 1973b16d4..a1f5fc75f 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -16,22 +16,25 @@ //! never mints a fresh view and therefore preserves observations across RCU //! operations and release publication. //! -//! The current root-pointer invariant is intentionally small: `Rcu` roots are -//! non-null in every atomic-history message, while `RcuOption` roots may be -//! null. Ownership, reader permissions, traversal snapshots, and reclamation are -//! modeled separately in [`specs::sync::rcu`] and are being connected -//! incrementally. +//! The root-pointer invariant keeps publication metadata for the complete +//! atomic history. Each non-null message has a domain-local allocation ID, so +//! stale messages remain distinguishable even if a physical address is later +//! reused. `Rcu` roots are non-null in every message, while `RcuOption` roots +//! may contain null messages without allocation IDs. Physical `P::Permission`, +//! reader permissions, traversal snapshots, and reclamation are modeled +//! separately in [`specs::sync::rcu`] and are being connected incrementally. //! //! The traversal layer follows the paper's shape: //! //! - [`RcuReadGuardToken`] represents a read-side critical section together -//! with its `SeenRemoved(D, LV)` observation. -//! - [`RcuProtectedPtr`] records that a typed pointer is protected by that -//! guard and has not been observed removed. +//! with its base `Guard(tid, X, G)` state and `SeenRemoved(D, LV)` observation. +//! - [`RcuProtectedPtr`] records an AId/address pair installed in the live +//! guard's mutable protection map `G`. //! - [`RcuBaseRetirePerm`] becomes [`RcuRetirePerm`] only after the caller has -//! observed enough traversal state to prove the retired object is in the -//! removed set. -//! - `RcuCallbackSafety` compresses that typed retire proof into an erased +//! observed enough traversal state to prove the allocation ID is in the +//! removed set. The domain's base `rcu-retire` transition then records it in +//! `RcuState.R` as `RcuRetired`. +//! - `RcuCallbackSafety` compresses that recorded retire proof into an erased //! `RcuCallbackSummary { domain, obj, retire_epoch }`, which is what the //! monitor stores next to a type-erased executable callback. //! @@ -68,9 +71,11 @@ //! guard destruction reverses both changes. The scheduler can check the //! updated view back in only after the context is quiescent. //! -//! Delayed reclamation is still being wired into the weak-memory proof. For now, -//! `RcuDrop` preserves the public wrapper API, while the monitor/callback -//! path carries the new proof summary and safety certificate skeleton. +//! Delayed reclamation is still being wired into the weak-memory proof. The +//! remaining boundary is concrete: executable preemption guards do not yet own +//! the domain's `Inactive/Guard` reader token, and pointer replacement does not +//! yet route the old allocation's unique retire permission into the monitor. +//! For now, `RcuDrop` preserves the public wrapper API. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; @@ -100,14 +105,14 @@ broadcast use vstd_extra::external::nonnull::group_nonull_axioms; /// /// `bool` is the constant key: `true` means the public cell may contain null /// (`RcuOption`), and `false` means the public cell is non-null (`Rcu`). The -/// ghost state is still empty for this cut, but the predicate is now -/// RCU-specific: non-null `Rcu` cells require every atomic-history message to -/// contain a non-null pointer. Later revisions should extend the ghost state to -/// carry read permissions, retired pools, and traversal snapshots. +/// RCU-specific predicate requires non-null `Rcu` cells to contain only +/// non-null history messages. Its publication registry also assigns every +/// non-null message a domain-local allocation identity, matching the paper's +/// distinction between physical addresses and allocation IDs. type RcuAtomicPtr

= WeakAtomicPtr<

::Target, bool, - (), + rcu_spec::RcuRootGhost, rcu_spec::RcuWeakAtomicInv, >; @@ -172,7 +177,12 @@ impl RcuInner

{ res.type_inv(), res.is_nullable(), { - let ptr = WeakAtomicPtr::new(Ghost(true), core::ptr::null_mut(), Tracked(())); + proof_decl! { + let tracked root_ghost = rcu_spec::RcuRootGhost::tracked_initial( + core::ptr::null_mut::<

::Target>(), + ); + } + let ptr = WeakAtomicPtr::new(Ghost(true), core::ptr::null_mut(), Tracked(root_ghost)); Self { ptr, ghost_nullable: Ghost(true), @@ -194,7 +204,10 @@ impl RcuInner

{ proof { assert(!raw_ptr.is_null()); } - let ptr = WeakAtomicPtr::new(Ghost(nullable), raw_ptr, Tracked(())); + proof_decl! { + let tracked root_ghost = rcu_spec::RcuRootGhost::tracked_initial(raw_ptr); + } + let ptr = WeakAtomicPtr::new(Ghost(nullable), raw_ptr, Tracked(root_ghost)); Self { ptr, ghost_nullable: Ghost(nullable), @@ -406,6 +419,9 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { } }, }; + proof { + self._inner_guard.lemma_matches_context_depth(session); + } self._inner_guard.release_to_context(Tracked(session)); res } @@ -423,6 +439,9 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), { + proof { + self._inner_guard.lemma_matches_context_depth(session); + } self._inner_guard.release_to_context(Tracked(session)); } } diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 435888599..ed50747f5 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -109,13 +109,17 @@ pub tracked struct PreemptThreadViewSession { } impl PreemptThreadViewSession { - pub proof fn new(tracked task_view: TaskThreadView) -> (tracked res: Self) + pub proof fn new(tracked task_view: TaskThreadView, sched_view: SchedulerView) -> (tracked res: + Self) + requires + task_view.wf(sched_view), ensures res.task() == task_view.task(), res.view() == task_view.view(), res.session_task() == task_view.task(), res.available_fractions() == PREEMPT_SESSION_FRACTIONS, res.wf_session_resource(), + res.wf(sched_view), { assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); @@ -127,6 +131,7 @@ impl PreemptThreadViewSession { let tracked res = PreemptThreadViewSession { task_view, tokens }; assert(res.available_fractions() == PREEMPT_SESSION_FRACTIONS); assert(res.wf_session_resource()); + assert(res.wf(sched_view)); res } @@ -252,6 +257,22 @@ impl PreemptThreadViewSession { { self.task_view } + + /// Returns the checked-out view while preserving its scheduler relation. + pub proof fn tracked_into_task_view_for_scheduler( + tracked self, + sched_view: SchedulerView, + ) -> (tracked res: TaskThreadView) + requires + self.wf(sched_view), + self.available_fractions() == PREEMPT_SESSION_FRACTIONS, + ensures + res.task() == self.task(), + res.view() == self.view(), + res.wf(sched_view), + { + self.task_view + } } /// The proof-owned state for one task while it is running. @@ -268,10 +289,8 @@ pub tracked struct RunningTaskContext { impl RunningTaskContext { /// Starts a running interval for a checked-out task view. - pub proof fn new( - tracked task_view: TaskThreadView, - sched_view: SchedulerView, - ) -> (tracked res: Self) + pub proof fn new(tracked task_view: TaskThreadView, sched_view: SchedulerView) -> (tracked res: + Self) requires task_view.wf(sched_view), ensures @@ -283,10 +302,12 @@ impl RunningTaskContext { res.is_quiescent(), res.wf_scheduler(sched_view), { - let tracked session = PreemptThreadViewSession::new(task_view); + let tracked session = PreemptThreadViewSession::new(task_view, sched_view); let tracked res = RunningTaskContext { session, preempt_depth: 0 }; assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(res.wf()); + assert(res.session.wf(sched_view)); + assert(res.wf_scheduler(sched_view)); res } @@ -310,19 +331,35 @@ impl RunningTaskContext { self.preempt_depth } - pub open spec fn wf(self) -> bool { + pub closed spec fn wf(self) -> bool { &&& self.session.wf_session_resource() - &&& self.available_fractions() + self.preempt_depth() - == PREEMPT_SESSION_FRACTIONS + &&& self.available_fractions() + self.preempt_depth() == PREEMPT_SESSION_FRACTIONS } /// Relates this running context to the scheduler snapshot from which its /// task view was checked out. - pub open spec fn wf_scheduler(self, sched_view: SchedulerView) -> bool { + pub closed spec fn wf_scheduler(self, sched_view: SchedulerView) -> bool { &&& self.wf() &&& self.session.wf(sched_view) } + /// Re-establishes the scheduler relation after the checked-out task view + /// has been updated to this context's current weak-memory view. + pub proof fn lemma_wf_scheduler(tracked &self, sched_view: SchedulerView) + requires + self.wf(), + sched_view.wf(), + sched_view.task_view_is_checked_out(self.task()), + sched_view.checked_out_views[self.task()] == self.view(), + sched_view.task_views.contains_key(self.task()), + sched_view.task_views[self.task()] == self.view(), + ensures + self.wf_scheduler(sched_view), + { + self.session.task_view.lemma_wf(sched_view); + assert(self.session.wf(sched_view)); + } + pub open spec fn is_quiescent(self) -> bool { &&& self.preempt_depth() == 0 &&& self.available_fractions() == PREEMPT_SESSION_FRACTIONS @@ -374,8 +411,8 @@ impl RunningTaskContext { { assert(self.preempt_depth() == 0); assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); - let tracked res = self.session.tracked_into_task_view(); - assert(res.wf(sched_view)); + assert(self.session.wf(sched_view)); + let tracked res = self.session.tracked_into_task_view_for_scheduler(sched_view); res } } @@ -509,10 +546,7 @@ impl RunningTaskContext { /// Performs the inverse transition when a preemption-disable guard is /// consumed. - pub proof fn tracked_enable_preempt( - tracked &mut self, - tracked resource: PreemptGuardResource, - ) + pub proof fn tracked_enable_preempt(tracked &mut self, tracked resource: PreemptGuardResource) requires old(self).wf(), old(self).preempt_depth() > 0, @@ -591,6 +625,15 @@ impl DisabledPreemptGuard { self.tracked_resource@.matches_context(context) } + /// Extracts the positive preemption depth witnessed by this guard. + pub proof fn lemma_matches_context_depth(&self, tracked context: &RunningTaskContext) + requires + self.matches_context(*context), + ensures + context.preempt_depth() > 0, + { + } + /// Borrows the running task's view while this guard witnesses that /// preemption is disabled. Both outermost and nested guards use the same /// context-owned view. @@ -650,6 +693,8 @@ impl GuardTransfer for DisabledPreemptGuard { verus! { /// Disables preemption. +/// +/// TODO: This API is still unsound. pub fn disable_preempt() -> (res: DisabledPreemptGuard) ensures res.wf(arbitrary()), From 32f517996a22555a068745a7ff94832be18c2072 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 16 Jul 2026 02:28:18 -0400 Subject: [PATCH 23/47] add utility functions/specs for traversal --- ostd/specs/sync/rcu.rs | 149 ++++++++++++++++++++++++++++++--- ostd/specs/sync/weak_memory.rs | 45 ++++++++-- ostd/src/sync/rcu/mod.rs | 31 +++++-- 3 files changed, 197 insertions(+), 28 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 79b46abee..3524999ce 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -58,6 +58,13 @@ pub ghost struct RcuPublishedObject { pub addr: usize, } +/// Resources created by one application of the paper's registration rule. +/// +/// `BlockInfo` is persistent and may justify any number of publications. The +/// base retire permission is unique and must survive until traversal proves +/// that the registered allocation has been removed. +pub type RcuRegistration = (RcuBlockInfo, RcuBaseRetirePerm); + /// Publication metadata paired with an RCU root's atomic message history. /// /// Entry `publications[i]` describes atomic message `i`. A null message has no @@ -113,45 +120,128 @@ impl RcuRootGhost { } /// Allocate a fresh publication registry containing the initial message. - pub proof fn tracked_initial(ptr: *mut T) -> (tracked res: Self) + /// + /// A non-null initial value is registered exactly once and the registration + /// resources are returned to the caller. The root history retains only the + /// allocation ID; it does not consume the unique retire permission. + pub proof fn tracked_initial(ptr: *mut T) -> (tracked res: ( + Self, + Option>, + )) ensures - rcu_root_history_inv(seq![Msg { value: ptr, view: WmView::empty() }], res), + rcu_root_history_inv(seq![Msg { value: ptr, view: WmView::empty() }], res.0), + (res.1 is Some) == (ptr.addr() != 0), + res.1 is Some ==> res.1->Some_0.0.ptr() == ptr, + res.1 is Some ==> res.1->Some_0.0.obj() == res.1->Some_0.1.obj(), + res.1 is Some ==> res.1->Some_0.0.domain() == res.0.domain(), + res.1 is Some ==> res.0.publications()[0] == Some(res.1->Some_0.0.obj()), + res.1 is Some ==> res.1->Some_0.0.wf(), { let tracked mut domain = RcuDomainAuth::tracked_new(); if ptr.addr() == 0 { - RcuRootGhost { domain, publications: seq![None] } + (RcuRootGhost { domain, publications: seq![None] }, None) } else { - let tracked (block_info, _retire_perm) = domain.tracked_register(ptr); + let tracked (block_info, retire_perm) = domain.tracked_register(ptr); let ghost obj = block_info.obj(); assert(domain.objects().contains_pair(obj, ptr.addr())); - RcuRootGhost { domain, publications: seq![Some(obj)] } + ( + RcuRootGhost { domain, publications: seq![Some(obj)] }, + Some((block_info, retire_perm)), + ) } } - /// Extend the publication registry for one newly appended atomic message. - pub proof fn tracked_push( + /// Publish a freshly introduced allocation. + /// + /// This combines the paper's registration rule with the first publication + /// of that registration. The returned resources must remain associated + /// with the allocation; in particular, the retire permission is not part + /// of the append-only atomic history. + pub proof fn tracked_push_fresh( tracked &mut self, prev: History<*mut T>, next: History<*mut T>, msg: Msg<*mut T>, - ) + ) -> (tracked res: Option>) requires rcu_root_history_inv(prev, *old(self)), next == prev.push(msg), ensures rcu_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), + (res is Some) == (msg.value.addr() != 0), + res is Some ==> res->Some_0.0.ptr() == msg.value, + res is Some ==> res->Some_0.0.obj() == res->Some_0.1.obj(), { let ghost ts = prev.len(); assert(self.publications().len() == ts); - if msg.value.addr() == 0 { + let tracked res = if msg.value.addr() == 0 { self.publications = self.publications.push(None); + None } else { - let tracked (block_info, _retire_perm) = self.domain.tracked_register(msg.value); + let tracked (block_info, retire_perm) = self.domain.tracked_register(msg.value); let ghost obj = block_info.obj(); self.publications = self.publications.push(Some(obj)); - } + Some((block_info, retire_perm)) + }; + + assert forall|i: int| 0 <= i < next.len() implies { + match #[trigger] self.publications()[i] { + None => next[i].value.addr() == 0, + Some(obj) => { + &&& next[i].value.addr() != 0 + &&& self.objects().contains_pair(obj, next[i].value.addr()) + }, + } + } by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + assert(next[i] == prev[i]); + assert(self.publications()[i] == old(self).publications()[i]); + match self.publications()[i] { + Some(obj) => { + assert(old(self).objects().contains_pair(obj, prev[i].value.addr())); + assert(self.objects().contains_pair(obj, next[i].value.addr())); + }, + None => {}, + } + } + }; + res + } + + /// Re-publish an allocation that was registered earlier. + /// + /// Unlike [`tracked_push_fresh`](Self::tracked_push_fresh), this rule does + /// not allocate a new AId. Every message published with the same persistent + /// `BlockInfo` therefore carries the same allocation identity. + pub proof fn tracked_push_registered( + tracked &mut self, + prev: History<*mut T>, + next: History<*mut T>, + msg: Msg<*mut T>, + tracked info: &RcuBlockInfo, + ) + requires + rcu_root_history_inv(prev, *old(self)), + next == prev.push(msg), + info.domain() == old(self).domain(), + info.ptr() == msg.value, + info.wf(), + ensures + rcu_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + final(self).objects() == old(self).objects(), + final(self).publications() == old(self).publications().push(Some(info.obj())), + { + let ghost ts = prev.len(); + assert(self.publications().len() == ts); + self.domain.lemma_block_info_agree(info); + self.publications = self.publications.push(Some(info.obj())); + assert(self.objects() == old(self).objects()); assert forall|i: int| 0 <= i < next.len() implies { match #[trigger] self.publications()[i] { @@ -164,6 +254,10 @@ impl RcuRootGhost { } by { if i == prev.len() { assert(next[i] == msg); + assert(info.addr() == msg.value.addr()); + assert(msg.value.addr() != 0); + assert(self.publications()[i] == Some(info.obj())); + assert(self.objects().contains_pair(info.obj(), next[i].value.addr())); } else { assert(i < prev.len()); assert(next[i] == prev[i]); @@ -659,9 +753,11 @@ impl RcuDomainAuth { res.0.obj() == old(self).next_obj(), res.0.ptr() == ptr, res.0.addr() == ptr.addr(), + res.0.wf(), res.1.domain() == final(self).id(), res.1.obj() == res.0.obj(), res.1.ptr() == ptr, + res.1.wf(), res.1.belongs_to(*final(self)), { let ghost obj = self.next_obj; @@ -945,7 +1041,8 @@ impl RcuBlockInfo { } pub open spec fn wf(self) -> bool { - self.addr() == self.ptr().addr() + &&& self.addr() == self.ptr().addr() + &&& self.ptr().addr() != 0 } /// Persistent block information can be retained by both the client and @@ -997,6 +1094,34 @@ pub proof fn registration_distinguishes_reused_address(ptr: *mut T) -> (track (first, first_history_copy, second) } +/// Regression proof for registration-time identity across re-publication. +/// +/// Both history entries are justified by the same persistent `BlockInfo`, so +/// they contain the same AId even though they have different atomic +/// timestamps. This is the paper's required separation between allocation +/// identity and modification-order position. +pub proof fn registered_republication_preserves_allocation_id(ptr: *mut T) -> (tracked res: ( + RcuRootGhost, + RcuRegistration, +)) + requires + ptr.addr() != 0, + ensures + res.0.publications().len() == 2, + res.0.publications()[0] == Some(res.1.0.obj()), + res.0.publications()[1] == Some(res.1.0.obj()), + res.1.0.domain() == res.0.domain(), + res.1.0.obj() == res.1.1.obj(), +{ + let ghost initial = seq![Msg { value: ptr, view: WmView::empty() }]; + let tracked (mut root, registration_opt) = RcuRootGhost::tracked_initial(ptr); + let tracked registration = registration_opt.tracked_unwrap(); + let ghost msg = Msg { value: ptr, view: WmView::empty() }; + let ghost next = initial.push(msg); + root.tracked_push_registered(initial, next, msg, ®istration.0); + (root, registration) +} + /// Low-level base retire permission. /// /// This is the paper's unique `BaseRetirePerm(l, a)`. The embedded owning diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 4d75a9be8..4b1d00f60 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -716,13 +716,25 @@ impl WeakAtomicPtr) + pub fn store_release_rcu(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) -> (res: + Tracked>>) requires self.well_formed(), self.constant() || !value.is_null(), + ensures + (res@ is Some) == !value.is_null(), + res@ is Some ==> res@->Some_0.0.ptr() == value, + res@ is Some ==> res@->Some_0.0.obj() == res@->Some_0.1.obj(), { + proof_decl! { + let tracked registration; + } proof { use_type_invariant(self); } @@ -748,25 +760,42 @@ impl WeakAtomicPtr, - ) -> (res: (Result<*mut T, *mut T>, Ghost)) + ) -> (res: ( + Result<*mut T, *mut T>, + Ghost, + Tracked>>, + )) requires self.well_formed(), self.constant() || !new.is_null(), + ensures + res.0 is Ok ==> ((res.2@ is Some) == !new.is_null()), + res.0 is Err ==> res.2@ is None, + res.2@ is Some ==> res.2@->Some_0.0.ptr() == new, + res.2@ is Some ==> res.2@->Some_0.0.obj() == res.2@->Some_0.1.obj(), { let result; + proof_decl! { + let tracked registration; + } proof { use_type_invariant(self); } @@ -803,14 +832,16 @@ impl WeakAtomicPtr { assert(false); + registration = None; }, } }, Result::Err(_) => { + registration = None; assert(next == prev); assert(rcu_spec::rcu_history_inv(self.constant(), next)); }, @@ -818,7 +849,7 @@ impl WeakAtomicPtr` preserves the public wrapper API. +//! the domain's `Inactive/Guard` reader token. Fresh stores and successful CAS +//! operations now produce `RcuRegistration`, but `RcuInner` does not yet retain +//! that resource together with `P::Permission` or route the replaced object's +//! retire permission into the monitor. For now, `RcuDrop` preserves the +//! public wrapper API. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; @@ -178,7 +182,8 @@ impl RcuInner

{ res.is_nullable(), { proof_decl! { - let tracked root_ghost = rcu_spec::RcuRootGhost::tracked_initial( + // TODO: retain this beside the initial `P::Permission`. + let tracked (root_ghost, _registration) = rcu_spec::RcuRootGhost::tracked_initial( core::ptr::null_mut::<

::Target>(), ); } @@ -205,7 +210,9 @@ impl RcuInner

{ assert(!raw_ptr.is_null()); } proof_decl! { - let tracked root_ghost = rcu_spec::RcuRootGhost::tracked_initial(raw_ptr); + // TODO: retain this beside the initial `P::Permission`. + let tracked (root_ghost, _registration) = + rcu_spec::RcuRootGhost::tracked_initial(raw_ptr); } let ptr = WeakAtomicPtr::new(Ghost(nullable), raw_ptr, Tracked(root_ghost)); Self { @@ -241,16 +248,19 @@ impl RcuInner

{ &self, new_ptr: *mut

::Target, Tracked(tv): Tracked<&mut ThreadView>, - ) + ) -> (res: Tracked::Target>>>) requires self.type_inv(), self.is_nullable() || !new_ptr.is_null(), + ensures + (res@ is Some) == !new_ptr.is_null(), + res@ is Some ==> res@->Some_0.0.ptr() == new_ptr, { proof { assert(self.ptr.constant() == self.is_nullable()); assert(self.ptr.constant() || !new_ptr.is_null()); } - self.ptr.store_release_rcu(new_ptr, Tracked(tv)); + self.ptr.store_release_rcu(new_ptr, Tracked(tv)) } fn update(&self, new_ptr: Option

, Tracked(session): Tracked<&mut RunningTaskContext>) @@ -284,7 +294,8 @@ impl RcuInner

{ } assert(self.is_nullable() || !raw.is_null()); } - self.store_ptr_release(raw, Tracked(tv)); + // TODO: retain the registration with `perm` until this object is detached. + let Tracked(_registration) = self.store_ptr_release(raw, Tracked(tv)); } fn read(&self, Tracked(session): Tracked<&mut RunningTaskContext>) -> (res: RcuReadGuardInner< @@ -405,6 +416,8 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { assert(rcu.ptr.constant() || !new_raw.is_null()); } let cas_res = rcu.ptr.compare_exchange_acqrel_acquire_rcu(expected, new_raw, Tracked(tv)); + // A successful CAS must route this registration into the ownership map. + let Tracked(_registration) = cas_res.2; let res = match cas_res.0 { Result::Ok(_) => Ok(()), From d57bbd004d636cb54bc23c00895cf9b121caa3b4 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 16 Jul 2026 04:19:22 -0400 Subject: [PATCH 24/47] Wire weak-memory RCU reclamation --- ostd/specs/sync/rcu.rs | 597 +++++++++++++++++++++++++++++++-- ostd/specs/sync/weak_memory.rs | 182 ++++++++-- ostd/src/sync/rcu/mod.rs | 252 ++++++++++++-- ostd/src/sync/rcu/monitor.rs | 248 ++++++++++++-- ostd/src/task/preempt/guard.rs | 24 ++ 5 files changed, 1184 insertions(+), 119 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 3524999ce..bdfcdd200 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -18,6 +18,8 @@ //! The module remains proof-only. The executable RCU must still connect its //! preemption guard to the same domain's reader token and route the retire //! permission released by pointer replacement into the callback monitor. +use core::marker::PhantomData; + use super::weak_memory::{History, Msg, WeakAtomicInvariantPredicate, WmView}; use vstd::prelude::*; use vstd::resource::Loc; @@ -65,6 +67,138 @@ pub ghost struct RcuPublishedObject { /// that the registered allocation has been removed. pub type RcuRegistration = (RcuBlockInfo, RcuBaseRetirePerm); +/// Complete linear ownership associated with one registered allocation. +/// +/// The RCU base protocol treats `ownership` abstractly. The executable OSTD +/// instance uses `P::Permission`, while proof examples may use `()` or another +/// client resource. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuOwnedObject { + registration: RcuRegistration, + ownership: O, +} + +/// Complete ownership of a detached root after the base retire transition. +/// +/// The persistent object identity justifies the erased callback summary, +/// `retired` proves that traversal removal happened, and `ownership` is the +/// physical resource consumed by the callback body. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetiredOwnedObject { + object: RcuObjectId, + retired: RcuRetired, + ownership: O, +} + +impl RcuRetiredOwnedObject { + pub closed spec fn object(self) -> RcuObjectId { + self.object + } + + pub closed spec fn retired(self) -> RcuRetired { + self.retired + } + + pub closed spec fn ownership(self) -> O { + self.ownership + } + + pub closed spec fn domain(self) -> Loc { + self.object().domain() + } + + pub closed spec fn obj(self) -> nat { + self.object().obj() + } + + pub closed spec fn ptr(self) -> *mut T { + self.object().ptr() + } + + pub proof fn tracked_into_parts(tracked self) -> (tracked res: ( + RcuObjectId, + RcuRetired, + O, + )) + ensures + res.0 == self.object(), + res.1 == self.retired(), + res.2 == self.ownership(), + res.0.domain() == res.1.domain(), + res.0.obj() == res.1.obj(), + res.0.ptr() == res.1.ptr(), + { + use_type_invariant(&self); + (self.object, self.retired, self.ownership) + } + + pub closed spec fn wf(self) -> bool { + &&& self.object().domain() == self.retired().domain() + &&& self.object().obj() == self.retired().obj() + &&& self.object().ptr() == self.retired().ptr() + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +impl RcuOwnedObject { + pub closed spec fn registration(self) -> RcuRegistration { + self.registration + } + + pub closed spec fn block_info(self) -> RcuBlockInfo { + self.registration().0 + } + + pub closed spec fn retire_perm(self) -> RcuBaseRetirePerm { + self.registration().1 + } + + pub closed spec fn ownership(self) -> O { + self.ownership + } + + pub proof fn tracked_into_parts(tracked self) -> (tracked res: (RcuRegistration, O)) + ensures + res.0 == self.registration(), + res.1 == self.ownership(), + { + (self.registration, self.ownership) + } +} + +/// Agreement between one registration resource and publication metadata. +pub open spec fn registration_matches_publication( + registration: RcuRegistration, + object: RcuPublishedObject, +) -> bool { + &&& registration.0.wf() + &&& registration.1.wf() + &&& registration.0.domain() == object.domain + &&& registration.0.obj() == object.obj + &&& registration.0.addr() == object.addr + &&& registration.0.obj() == registration.1.obj() + &&& registration.0.domain() == registration.1.domain() + &&& registration.0.ptr() == registration.1.ptr() +} + +pub open spec fn current_registration_matches( + root: RcuRootGhost, + registration: Option>, +) -> bool { + match (root.current(), registration) { + (None, None) => true, + (Some(object), Some(registration)) => { + &&& registration_matches_publication(registration, object) + &&& registration.1.belongs_to(root.domain_auth()) + }, + _ => false, + } +} + /// Publication metadata paired with an RCU root's atomic message history. /// /// Entry `publications[i]` describes atomic message `i`. A null message has no @@ -83,6 +217,10 @@ pub tracked struct RcuRootGhost { } impl RcuRootGhost { + pub closed spec fn domain_auth(self) -> RcuDomainAuth { + self.domain + } + pub closed spec fn domain(self) -> Loc { self.domain.id() } @@ -136,6 +274,7 @@ impl RcuRootGhost { res.1 is Some ==> res.1->Some_0.0.domain() == res.0.domain(), res.1 is Some ==> res.0.publications()[0] == Some(res.1->Some_0.0.obj()), res.1 is Some ==> res.1->Some_0.0.wf(), + current_registration_matches(res.0, res.1), { let tracked mut domain = RcuDomainAuth::tracked_new(); if ptr.addr() == 0 { @@ -169,9 +308,13 @@ impl RcuRootGhost { ensures rcu_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), + final(self).domain_auth().retire_registry() == old( + self, + ).domain_auth().retire_registry(), (res is Some) == (msg.value.addr() != 0), res is Some ==> res->Some_0.0.ptr() == msg.value, res is Some ==> res->Some_0.0.obj() == res->Some_0.1.obj(), + current_registration_matches(*final(self), res), { let ghost ts = prev.len(); assert(self.publications().len() == ts); @@ -234,6 +377,9 @@ impl RcuRootGhost { ensures rcu_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), + final(self).domain_auth().retire_registry() == old( + self, + ).domain_auth().retire_registry(), final(self).objects() == old(self).objects(), final(self).publications() == old(self).publications().push(Some(info.obj())), { @@ -314,13 +460,297 @@ impl WeakAtomicInvariantPredicate for RcuWeakAtom } } +/// Typed ownership state paired with one executable RCU root atomic. +/// +/// `root` owns the append-only publication registry. `current` owns the unique +/// registration resources for the latest non-null root value. Historical +/// messages retain only persistent allocation metadata, so replacing the root +/// can move the old unique retire permission out exactly once. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRootOwnedGhost { + root: RcuRootGhost, + current: Option>, +} + +impl RcuRootOwnedGhost { + pub closed spec fn root(self) -> RcuRootGhost { + self.root + } + + pub closed spec fn domain(self) -> Loc { + self.root().domain() + } + + pub closed spec fn publications(self) -> Seq> { + self.root().publications() + } + + pub open spec fn published_at(self, ts: nat) -> Option + recommends + ts < self.publications().len(), + { + self.root().published_at(ts) + } + + pub closed spec fn current_registration(self) -> Option> { + match self.current { + Some(owned) => Some(owned.registration()), + None => None, + } + } + + pub closed spec fn current_owned(self) -> Option> { + self.current + } + + pub closed spec fn current_ownership(self) -> Option { + match self.current { + Some(owned) => Some(owned.ownership()), + None => None, + } + } + + pub open spec fn ownership_wf(self) -> bool { + current_registration_matches(self.root(), self.current_registration()) + } + + /// Initializes root history and retains the initial registration as the + /// current unique ownership resource. + pub proof fn tracked_initial(ptr: *mut T, tracked ownership: Option) -> (tracked res: Self) + requires + (ownership is Some) == (ptr.addr() != 0), + ensures + rcu_owned_root_history_inv(seq![Msg { value: ptr, view: WmView::empty() }], res), + (res.current_registration() is Some) == (ptr.addr() != 0), + res.current_registration() is Some ==> res.current_registration()->Some_0.0.ptr() + == ptr, + res.current_ownership() == ownership, + match res.current_owned() { + Some(owned) => { + &&& ptr.addr() != 0 + &&& equal(owned.block_info().ptr(), ptr) + &&& ownership == Some(owned.ownership()) + }, + None => { + &&& ptr.addr() == 0 + &&& ownership is None + }, + }, + { + let tracked (root, registration) = RcuRootGhost::tracked_initial(ptr); + let tracked current = match registration { + Some(registration) => { + Some(RcuOwnedObject { registration, ownership: ownership.tracked_unwrap() }) + }, + None => None, + }; + RcuRootOwnedGhost { root, current } + } + + /// Publishes a fresh allocation and retires the previously current root. + /// + /// A root replacement is also the complete traversal-removal proof for + /// the old root: this cell was its only incoming root edge. The old base + /// retire permission is therefore consumed before ownership leaves the + /// atomic invariant. + pub proof fn tracked_push_fresh( + tracked &mut self, + prev: History<*mut T>, + next: History<*mut T>, + msg: Msg<*mut T>, + tracked ownership: Option, + ) -> (tracked detached: Option>) where + OwnPred: RcuRootOwnershipPredicate, + + requires + rcu_owned_root_history_inv(prev, *old(self)), + rcu_current_ownership_inv::(*old(self)), + next == prev.push(msg), + (ownership is Some) == (msg.value.addr() != 0), + ensures + rcu_owned_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + match detached { + Some(detached) => { + &&& old(self).current_registration() is Some + &&& detached.object() == old(self).current_registration()->Some_0.0 + &&& detached.retired().domain() == detached.domain() + &&& detached.retired().obj() == detached.obj() + &&& detached.retired().ptr() == detached.ptr() + &&& old(self).current_ownership() == Some(detached.ownership()) + &&& equal(detached.ptr(), prev[(prev.len() - 1) as int].value) + &&& OwnPred::owns(detached.ptr(), detached.ownership()) + }, + None => old(self).current_registration() is None, + }, + (final(self).current_registration() is Some) == (msg.value.addr() != 0), + final(self).current_registration() is Some + ==> final(self).current_registration()->Some_0.0.ptr() == msg.value, + final(self).current_ownership() == ownership, + match final(self).current_owned() { + Some(owned) => { + &&& msg.value.addr() != 0 + &&& equal(owned.block_info().ptr(), msg.value) + &&& ownership == Some(owned.ownership()) + }, + None => { + &&& msg.value.addr() == 0 + &&& ownership is None + }, + }, + { + assert(current_registration_matches(self.root(), self.current_registration())); + let tracked old_current = if self.current is Some { + Some(self.current.tracked_take()) + } else { + None + }; + let tracked new_registration = self.root.tracked_push_fresh(prev, next, msg); + let tracked new_current = match new_registration { + Some(registration) => { + Some(RcuOwnedObject { registration, ownership: ownership.tracked_unwrap() }) + }, + None => None, + }; + let tracked detached = match old_current { + Some(owned) => { + let tracked (registration, old_ownership) = owned.tracked_into_parts(); + let tracked (object, base) = registration; + assert(base.belongs_to(self.root.domain)); + let ghost seen_removed = RcuSeenRemoved { + removed: Set::empty().insert(object.obj()), + link_view: RcuLinkView::empty(), + }; + let tracked retire = lift_retire_perm(base, seen_removed); + let tracked retired = self.root.domain.tracked_retire(retire); + Some(RcuRetiredOwnedObject { object, retired, ownership: old_ownership }) + }, + None => None, + }; + self.current = new_current; + assert(current_registration_matches(self.root(), self.current_registration())); + detached + } + + /// Re-publishes the currently owned registration without changing its AId + /// or releasing its unique retire permission. + pub proof fn tracked_republish_current( + tracked &mut self, + prev: History<*mut T>, + next: History<*mut T>, + msg: Msg<*mut T>, + ) + requires + rcu_owned_root_history_inv(prev, *old(self)), + next == prev.push(msg), + old(self).current_registration() is Some, + old(self).current_registration()->Some_0.0.ptr() == msg.value, + ensures + rcu_owned_root_history_inv(next, *final(self)), + final(self).domain() == old(self).domain(), + final(self).current_registration() == old(self).current_registration(), + { + let tracked owned = self.current.tracked_take(); + self.root.tracked_push_registered(prev, next, msg, &owned.registration.0); + self.current = Some(owned); + assert(current_registration_matches(self.root(), self.current_registration())); + } +} + +/// The current ownership resource agrees with the latest publication, while +/// older history entries need only agree with persistent registration metadata. +pub open spec fn rcu_owned_root_history_inv( + history: History<*mut T>, + ghost: RcuRootOwnedGhost, +) -> bool { + &&& rcu_root_history_inv(history, ghost.root()) + &&& ghost.ownership_wf() + &&& match ghost.current_registration() { + Some(registration) => equal( + registration.0.ptr(), + history[(history.len() - 1) as int].value, + ), + None => history[(history.len() - 1) as int].value.addr() == 0, + } +} + +/// Client relation between a pointer and its physical ownership resource. +pub trait RcuRootOwnershipPredicate { + spec fn owns(ptr: *mut T, ownership: O) -> bool; +} + +/// Trivial ownership relation used by proof-only examples carrying `()`. +pub struct UnitRcuRootOwnership; + +impl RcuRootOwnershipPredicate for UnitRcuRootOwnership { + open spec fn owns(_ptr: *mut T, _ownership: ()) -> bool { + true + } +} + +pub open spec fn rcu_current_ownership_inv( + ghost: RcuRootOwnedGhost, +) -> bool where OwnPred: RcuRootOwnershipPredicate { + match ghost.current_owned() { + Some(owned) => OwnPred::owns(owned.block_info().ptr(), owned.ownership()), + None => true, + } +} + +/// Opens the structural current-ownership relation for atomic clients. +pub proof fn lemma_current_owned_resources( + history: History<*mut T>, + tracked ghost: &RcuRootOwnedGhost, +) where OwnPred: RcuRootOwnershipPredicate + requires + rcu_owned_root_history_inv(history, *ghost), + rcu_current_ownership_inv::(*ghost), + ensures + match ghost.current_owned() { + Some(owned) => { + &&& owned.block_info().wf() + &&& equal(owned.block_info().ptr(), history[(history.len() - 1) as int].value) + &&& OwnPred::owns(owned.block_info().ptr(), owned.ownership()) + }, + None => history[(history.len() - 1) as int].value.addr() == 0, + }, +{ + match ghost.current_owned() { + Some(owned) => { + assert(ghost.current_registration() == Some(owned.registration())); + }, + None => {}, + } +} + +/// RCU weak-atomic invariant with typed ownership for the current root value. +pub struct RcuOwnedWeakAtomicInv { + _marker: PhantomData, +} + +impl WeakAtomicInvariantPredicate< + bool, + *mut T, + RcuRootOwnedGhost, +> for RcuOwnedWeakAtomicInv where OwnPred: RcuRootOwnershipPredicate { + open spec fn atomic_inv( + nullable: bool, + history: History<*mut T>, + g: RcuRootOwnedGhost, + ) -> bool { + &&& rcu_history_inv(nullable, history) + &&& rcu_owned_root_history_inv(history, g) + &&& rcu_current_ownership_inv::(g) + } +} + /// Proof-facing summary of one grace period. /// -/// The executable CPU mask is intentionally not part of this first state model: -/// the current proof cut only needs to connect pending callbacks to the monitor -/// flag. A later epoch/quiescent-state invariant should refine this view with -/// CPU progress. +/// `epoch` is assigned by the monitor, not by callback producers. Every +/// callback in a batch carries exactly this epoch, so completion of an older +/// grace period cannot authorize a callback queued for a later one. pub ghost struct GracePeriodView { + pub epoch: nat, pub callbacks: Seq, pub is_complete: bool, } @@ -329,7 +759,7 @@ impl GracePeriodView { /// The state of the grace period when the monitor is created: complete, /// with no callbacks attached. pub open spec fn initial() -> Self { - GracePeriodView { callbacks: Seq::empty(), is_complete: true } + GracePeriodView { epoch: 0, callbacks: Seq::empty(), is_complete: true } } pub open spec fn has_pending_work(self) -> bool { @@ -341,7 +771,10 @@ impl GracePeriodView { /// a critical section (between completing a grace period and taking its /// callbacks), but it must hold whenever the monitor lock is released. pub open spec fn wf(self) -> bool { - self.is_complete ==> self.callbacks.len() == 0 + &&& self.is_complete ==> self.callbacks.len() == 0 + &&& forall|i: int| + 0 <= i < self.callbacks.len() ==> (#[trigger] self.callbacks[i]).retire_epoch + == self.epoch } } @@ -378,6 +811,9 @@ impl MonitorStateView { pub open spec fn wf(self) -> bool { &&& self.current_gp.wf() &&& self.current_gp.is_complete ==> self.next_callbacks.len() == 0 + &&& forall|i: int| + 0 <= i < self.next_callbacks.len() ==> (#[trigger] self.next_callbacks[i]).retire_epoch + == self.current_gp.epoch + 1 } } @@ -668,6 +1104,7 @@ pub tracked struct RcuDomainAuth { ghost next_obj: nat, ghost next_reader: nat, ghost retired: Set, + ghost expired: Set, } impl RcuDomainAuth { @@ -689,6 +1126,15 @@ impl RcuDomainAuth { self.retired } + /// Allocations whose retirement is covered by a completed grace period. + /// + /// This is the paper's implementation-specific expired set. It is kept + /// separate from `retired`: a newly retired allocation remains protected + /// by critical sections until monitor completion moves it here. + pub closed spec fn expired(self) -> Set { + self.expired + } + pub closed spec fn reader_registry(self) -> Loc { self.readers.id() } @@ -709,6 +1155,7 @@ impl RcuDomainAuth { &&& forall|obj: nat| #[trigger] self.objects@.contains_key(obj) ==> obj < self.next_obj() &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> tid < self.next_reader() &&& self.retired().subset_of(self.objects().dom()) + &&& self.expired().subset_of(self.retired()) } /// Allocates a fresh RCU protection domain. @@ -728,6 +1175,7 @@ impl RcuDomainAuth { next_obj: 0, next_reader: 0, retired: Set::empty(), + expired: Set::empty(), } } @@ -746,8 +1194,11 @@ impl RcuDomainAuth { ensures final(self).wf(), final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), final(self).next_obj() == old(self).next_obj() + 1, final(self).retired() == old(self).retired(), + final(self).expired() == old(self).expired(), final(self).objects() == old(self).objects().insert(old(self).next_obj(), ptr.addr()), res.0.domain() == final(self).id(), res.0.obj() == old(self).next_obj(), @@ -802,8 +1253,11 @@ impl RcuDomainAuth { ensures final(self).wf(), final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), + final(self).expired() == old(self).expired(), res.domain() == final(self).id(), res.tid() == old(self).next_reader(), res.belongs_to(*final(self)), @@ -823,7 +1277,12 @@ impl RcuDomainAuth { } /// Starts a read-side critical section, snapshotting the set `X` of AIds - /// that had already been retired. + /// whose grace period had already completed. + /// + /// The paper only requires `X` to be a subset of all retired allocations. + /// Using `retired` itself here would be too strong: a reader may safely + /// observe a newly retired stale pointer while that retirement's grace + /// period is still in progress. pub proof fn tracked_guard_start( tracked &mut self, tracked mut inactive: RcuInactive, @@ -835,11 +1294,14 @@ impl RcuDomainAuth { ensures final(self).wf(), final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), + final(self).expired() == old(self).expired(), res.belongs_to(*final(self)), res.tid() == inactive.tid(), - res.expired() == old(self).retired(), + res.expired() == old(self).expired(), res.protected() == Map::::empty(), res.wf(), { @@ -858,7 +1320,7 @@ impl RcuDomainAuth { RcuBaseGuard { domain: self.id(), state: inactive.state, - expired: self.retired, + expired: self.expired, protected: Map::empty(), } } @@ -876,8 +1338,11 @@ impl RcuDomainAuth { ensures final(self).wf(), final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), + final(self).expired() == old(self).expired(), res.belongs_to(*final(self)), res.tid() == guard.tid(), res.wf(), @@ -910,8 +1375,11 @@ impl RcuDomainAuth { ensures final(self).wf(), final(self).id() == old(self).id(), + final(self).retire_registry() == old(self).retire_registry(), + final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired().insert(retire.obj()), + final(self).expired() == old(self).expired(), res.domain() == final(self).id(), res.obj() == retire.obj(), res.ptr() == retire.ptr(), @@ -920,6 +1388,7 @@ impl RcuDomainAuth { assert(self.objects().contains_key(retire.obj())); self.retired = self.retired.insert(retire.obj()); assert(self.retired().subset_of(self.objects().dom())); + assert(self.expired().subset_of(self.retired())); RcuRetired { retire } } } @@ -1122,6 +1591,41 @@ pub proof fn registered_republication_preserves_allocation_id(ptr: *mut T) -> (root, registration) } +/// Regression proof for ownership transfer on root replacement. +/// +/// The old registration leaves the atomic ownership state exactly once, while +/// the fresh registration for `next_ptr` becomes current. This is the resource +/// handoff that later feeds traversal retirement and callback construction. +pub proof fn owned_root_replacement_retires_previous_registration( + first_ptr: *mut T, + next_ptr: *mut T, +) -> (tracked res: (RcuRootOwnedGhost, RcuRetiredOwnedObject)) + requires + first_ptr.addr() != 0, + next_ptr.addr() != 0, + ensures + res.1.ptr() == first_ptr, + res.1.object().obj() == res.1.retired().obj(), + res.0.current_registration() is Some, + res.0.current_registration()->Some_0.0.ptr() == next_ptr, + res.0.current_registration()->Some_0.0.obj() + == res.0.current_registration()->Some_0.1.obj(), + res.1.domain() == res.0.domain(), +{ + let ghost initial = seq![Msg { value: first_ptr, view: WmView::empty() }]; + let tracked mut root = RcuRootOwnedGhost::tracked_initial(first_ptr, Some(())); + let ghost next_msg = Msg { value: next_ptr, view: WmView::empty() }; + let ghost next_history = initial.push(next_msg); + let tracked detached = root.tracked_push_fresh::( + initial, + next_history, + next_msg, + Some(()), + ); + let tracked detached = detached.tracked_unwrap(); + (root, detached) +} + /// Low-level base retire permission. /// /// This is the paper's unique `BaseRetirePerm(l, a)`. The embedded owning @@ -1173,6 +1677,10 @@ pub tracked struct RcuRetirePerm { } impl RcuRetirePerm { + pub closed spec fn base(self) -> RcuBaseRetirePerm { + self.base + } + pub closed spec fn domain(self) -> Loc { self.base.domain() } @@ -1209,6 +1717,7 @@ pub proof fn lift_retire_perm( requires seen_removed.removed.contains(base.obj()), ensures + perm.base() == base, perm.domain() == base.domain(), perm.ptr() == base.ptr(), perm.obj() == base.obj(), @@ -1240,30 +1749,72 @@ impl RcuRetired { } } +/// Regression proof for the distinction between retirement and expiration. +/// +/// Even a reader registered after `retire` may protect the object while no +/// grace period has expired it. This is the stale-read case required by the +/// paper's relaxed-memory base specification. +pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> (tracked res: ( + RcuBaseGuard, + RcuBlockInfo, +)) + requires + ptr.addr() != 0, + ensures + res.1.ptr() == ptr, + res.0.domain() == res.1.domain(), + res.0.expired() == Set::::empty(), + res.0.protects(res.1.addr(), res.1.obj()), +{ + let tracked mut domain = RcuDomainAuth::tracked_new(); + let tracked (info, base) = domain.tracked_register(ptr); + let ghost seen_removed = RcuSeenRemoved { + removed: Set::empty().insert(info.obj()), + link_view: RcuLinkView::empty(), + }; + let tracked retire = lift_retire_perm(base, seen_removed); + let tracked _retired = domain.tracked_retire(retire); + + let tracked inactive = domain.tracked_register_reader(); + let tracked mut guard = domain.tracked_guard_start(inactive); + assert(guard.expired() == Set::::empty()); + assert(!guard.expired().contains(info.obj())); + guard.tracked_protect(&info); + (guard, info) +} + /// Non-generic proof certificate carried across the type-erasure boundary. /// /// A certificate can only be produced from a typed traversal retire permission, /// but after that point the monitor only needs the erased callback summary. pub tracked struct RcuCallbackSafety { - ghost summary: RcuCallbackSummary, + ghost domain: Loc, + ghost obj: nat, } -impl View for RcuCallbackSafety { - type V = RcuCallbackSummary; +impl RcuCallbackSafety { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn obj(self) -> nat { + self.obj + } - closed spec fn view(&self) -> RcuCallbackSummary { - self.summary + /// The monitor may assign any future batch generation, but it cannot + /// change the retired object's domain or allocation identity. + pub open spec fn matches(self, summary: RcuCallbackSummary) -> bool { + &&& summary.domain == self.domain() + &&& summary.obj == self.obj() } } pub open spec fn callback_safety_from_traversal( cert: RcuCallbackSafety, object: RcuObjectId, - retire_epoch: nat, ) -> bool { - &&& cert@.domain == object.domain() - &&& cert@.obj == object.obj() - &&& cert@.retire_epoch == retire_epoch + &&& cert.domain() == object.domain() + &&& cert.obj() == object.obj() } /// Consume a typed traversal retire permission and compress it into the @@ -1271,19 +1822,17 @@ pub open spec fn callback_safety_from_traversal( pub proof fn certify_callback_from_retired( tracked object: &RcuObjectId, tracked retired: RcuRetired, - retire_epoch: nat, ) -> (tracked cert: RcuCallbackSafety) requires object.domain() == retired.domain(), object.obj() == retired.obj(), object.ptr() == retired.ptr(), ensures - cert@ == (RcuCallbackSummary { domain: retired.domain(), obj: object.obj(), retire_epoch }), - callback_safety_from_traversal(cert, *object, retire_epoch), + cert.domain() == retired.domain(), + cert.obj() == object.obj(), + callback_safety_from_traversal(cert, *object), { - RcuCallbackSafety { - summary: RcuCallbackSummary { domain: retired.domain(), obj: object.obj(), retire_epoch }, - } + RcuCallbackSafety { domain: retired.domain(), obj: object.obj() } } /// Read-side guard token for one critical section. diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 4b1d00f60..9bef19882 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -658,7 +658,12 @@ impl WeakAtomicPtr { } } -impl WeakAtomicPtr { +impl WeakAtomicPtr< + T, + bool, + rcu_spec::RcuRootOwnedGhost, + rcu_spec::RcuOwnedWeakAtomicInv, +> where OwnPred: rcu_spec::RcuRootOwnershipPredicate { /// Acquire-load helper for RCU root pointers. #[inline(always)] pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( @@ -716,24 +721,39 @@ impl WeakAtomicPtr) -> (res: - Tracked>>) + pub fn swap_release_rcu( + &self, + value: *mut T, + Tracked(ownership): Tracked>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (*mut T, Tracked>>)) requires self.well_formed(), self.constant() || !value.is_null(), + match ownership { + Some(ownership) => { + &&& !value.is_null() + &&& OwnPred::owns(value, ownership) + }, + None => value.is_null(), + }, ensures - (res@ is Some) == !value.is_null(), - res@ is Some ==> res@->Some_0.0.ptr() == value, - res@ is Some ==> res@->Some_0.0.obj() == res@->Some_0.1.obj(), + (res.1@ is Some) == !res.0.is_null(), + res.1@ is Some ==> res.1@->Some_0.object().wf(), + res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), + res.1@ is Some ==> res.1@->Some_0.retired().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> OwnPred::owns(res.0, res.1@->Some_0.ownership()), { + let result; proof_decl! { - let tracked registration; + let tracked retired_ownership; } proof { use_type_invariant(self); @@ -746,10 +766,14 @@ impl WeakAtomicPtr(g)); + rcu_spec::lemma_current_owned_resources::(prev, &g); if !self.constant() { assert(!value.is_null()); assert(snap@.msg().value.addr() != 0); @@ -760,41 +784,74 @@ impl WeakAtomicPtr( + prev, + next, + snap@.msg(), + ownership, + ); + assert(detached is Some ==> detached->Some_0.object().wf()); + assert(detached is Some ==> equal(detached->Some_0.ptr(), result)); + assert(detached is Some ==> OwnPred::owns( + result, + detached->Some_0.ownership(), + )); + assert(rcu_spec::rcu_current_ownership_inv::(g)) by { + match g.current_owned() { + Some(owned) => { + assert(ownership == Some(owned.ownership())); + assert(equal(owned.block_info().ptr(), value)); + }, + None => {}, + } + }; + retired_ownership = detached; pair = (hist, g); } }); - Tracked(registration) + (result, Tracked(retired_ownership)) } /// Strong AcqRel/Acquire CAS helper for a freshly introduced RCU pointer. /// - /// Registration occurs only in the successful CAS branch. A failed CAS - /// therefore returns no allocation ID or retire permission, allowing the - /// same owning pointer to be retried without acquiring a second identity. + /// Registration occurs only in the successful CAS branch. A successful + /// CAS returns the previous root registration; a failed CAS leaves the + /// ownership state unchanged and returns no detached registration. #[inline(always)] pub fn compare_exchange_acqrel_acquire_rcu( &self, current: *mut T, new: *mut T, + Tracked(new_ownership): Tracked>, Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ( Result<*mut T, *mut T>, Ghost, - Tracked>>, + Tracked<(Option>, Option)>, )) requires self.well_formed(), self.constant() || !new.is_null(), + match new_ownership { + Some(ownership) => { + &&& !new.is_null() + &&& OwnPred::owns(new, ownership) + }, + None => new.is_null(), + }, ensures - res.0 is Ok ==> ((res.2@ is Some) == !new.is_null()), - res.0 is Err ==> res.2@ is None, - res.2@ is Some ==> res.2@->Some_0.0.ptr() == new, - res.2@ is Some ==> res.2@->Some_0.0.obj() == res.2@->Some_0.1.obj(), + res.0 is Err ==> res.2@.0 is None, + res.0 is Err ==> res.2@.1 == new_ownership, + res.0 is Ok ==> res.2@.1 is None, + res.0 is Ok ==> ((res.2@.0 is Some) == !res.0->Ok_0.is_null()), + res.2@.0 is Some ==> res.2@.0->Some_0.object().wf(), + res.2@.0 is Some ==> equal(res.2@.0->Some_0.ptr(), res.0->Ok_0), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().obj() == res.2@.0->Some_0.obj(), + res.2@.0 is Some ==> OwnPred::owns(res.0->Ok_0, res.2@.0->Some_0.ownership()), { let result; proof_decl! { - let tracked registration; + let tracked retired_ownership; } proof { use_type_invariant(self); @@ -816,7 +873,9 @@ impl WeakAtomicPtr(g)); + rcu_spec::lemma_current_owned_resources::(prev, &g); match cas_result.0 { Result::Ok(_) => { let tracked snap_opt = cas_result.2.get(); @@ -832,16 +891,40 @@ impl WeakAtomicPtr( + prev, + next, + snap.msg(), + new_ownership, + ); + assert(detached is Some ==> detached->Some_0.object().wf()); + assert(detached is Some ==> equal( + detached->Some_0.ptr(), + cas_result.0->Ok_0, + )); + assert(detached is Some ==> OwnPred::owns( + cas_result.0->Ok_0, + detached->Some_0.ownership(), + )); + assert(rcu_spec::rcu_current_ownership_inv::(g)) by { + match g.current_owned() { + Some(owned) => { + assert(new_ownership == Some(owned.ownership())); + assert(equal(owned.block_info().ptr(), new)); + }, + None => {}, + } + }; + retired_ownership = (detached, None); }, Option::None => { assert(false); - registration = None; + retired_ownership = (None, None); }, } }, Result::Err(_) => { - registration = None; + retired_ownership = (None, new_ownership); assert(next == prev); assert(rcu_spec::rcu_history_inv(self.constant(), next)); }, @@ -849,7 +932,7 @@ impl WeakAtomicPtr AtomicPtrW { self.value.store(value, Ordering::Release); Tracked::assume_new() } + + /// Release swap: return the latest pointer and append a new release + /// message in the same atomic read-modify-write step. + /// + /// `Ordering::Release` does not acquire the old message's view. The caller + /// observes the new modification-order timestamp and publishes its existing + /// thread view, exactly as for a release store. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn swap_release( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: *mut T, + ) -> (res: (*mut T, Tracked>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let write_ts = old(auth).history().len(); + let old_msg = old(auth).history()[(write_ts - 1) as int]; + let write_msg = Msg { value, view: old(tv)@.observe(self.id(), write_ts) }; + &&& write_ts >= 1 + &&& equal(res.0, old_msg.value) + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), write_ts) + &&& res.1@.id() == self.id() + &&& res.1@.ts() == write_ts + &&& res.1@.msg() == write_msg + &&& res.1@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + let old = self.value.swap(value, Ordering::Release); + (old, Tracked::assume_new()) + } } impl AtomicPtrW { diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 6ec1a4b19..738b696c1 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -75,15 +75,17 @@ //! //! Delayed reclamation is still being wired into the weak-memory proof. The //! remaining boundary is concrete: executable preemption guards do not yet own -//! the domain's `Inactive/Guard` reader token. Fresh stores and successful CAS -//! operations now produce `RcuRegistration`, but `RcuInner` does not yet retain -//! that resource together with `P::Permission` or route the replaced object's -//! retire permission into the monitor. For now, `RcuDrop` preserves the +//! the domain's `Inactive/Guard` reader token. The weak atomic invariant now +//! retains the current registration together with `P::Permission`; Release +//! swap and successful CAS return the old raw pointer and matching owned +//! resource. `RcuInner` does not yet establish traversal removal or route that +//! detached ownership into the monitor. For now, `RcuDrop` preserves the //! public wrapper API. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; use vstd_extra::prelude::*; +use vstd_extra::raw_callback::{RawCallback, RawCallbackContext}; use crate::{ specs::{ @@ -93,6 +95,7 @@ use crate::{ }, task::InAtomicMode, }, + sync::Once, task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, }; @@ -105,6 +108,35 @@ verus! { broadcast use vstd_extra::external::nonnull::group_nonull_axioms; +exec static RCU_MONITOR: Once< + monitor::RcuMonitor, + monitor::RcuMonitorOwner, + monitor::RcuMonitorPred, +> + ensures + RCU_MONITOR.wf(), + RCU_MONITOR.inv() == monitor::RcuMonitorPred, +{ + Once::new(Ghost(monitor::RcuMonitorPred)) +} + +struct RcuPointerOwnership { + _marker: PhantomData

, +} + +impl rcu_spec::RcuRootOwnershipPredicate< +

::Target, +

::Permission, +> for RcuPointerOwnership

{ + open spec fn owns( + ptr: *mut

::Target, + ownership:

::Permission, + ) -> bool { + &&& P::ptr_perm_match(ptr, ownership) + &&& ownership.inv() + } +} + /// The weak-memory atomic slot used by RCU. /// /// `bool` is the constant key: `true` means the public cell may contain null @@ -113,11 +145,16 @@ broadcast use vstd_extra::external::nonnull::group_nonull_axioms; /// non-null history messages. Its publication registry also assigns every /// non-null message a domain-local allocation identity, matching the paper's /// distinction between physical addresses and allocation IDs. +type RcuAtomicGhost

= rcu_spec::RcuRootOwnedGhost< +

::Target, +

::Permission, +>; + type RcuAtomicPtr

= WeakAtomicPtr<

::Target, bool, - rcu_spec::RcuRootGhost, - rcu_spec::RcuWeakAtomicInv, + RcuAtomicGhost

, + rcu_spec::RcuOwnedWeakAtomicInv>, >; /// A Read-Copy Update cell for sharing a non-null pointer. @@ -148,6 +185,71 @@ struct RcuReadGuardInner<'a, P: NonNullPtr> { _inner_guard: DisabledPreemptGuard, } +/// Sized callback payload that retains the physical ownership of one detached +/// RCU object until the monitor executes its callback. +struct RcuDropCallbackContext { + pointer: NonNull<

::Target>, + permission: Tracked<

::Permission>, +} + +// SAFETY: the callback consumes the same owning pointer type `P` that was +// accepted by the RCU cell. The tracked permission has no runtime payload. +#[verifier::external] +unsafe impl Send for RcuDropCallbackContext

{ + +} + +impl RawCallbackContext for RcuDropCallbackContext

{ + fn run(self) { + proof { + use_type_invariant(&self); + } + proof_decl! { + let tracked permission = self.permission.get(); + } + let _pointer = unsafe { P::from_raw(self.pointer, Tracked(permission)) }; + } +} + +impl RcuDropCallbackContext

{ + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& P::ptr_perm_match(self.pointer.as_ptr(), self.permission@) + &&& self.permission@.inv() + } +} + +/// Erases a detached owned object into an executable callback payload. +/// +/// This function does not certify or enqueue the callback. Those operations +/// still require `RcuRetired` and a monitor grace-period certificate. +fn callback_from_detached( + pointer: *mut

::Target, + Tracked(owned): Tracked< + rcu_spec::RcuRetiredOwnedObject<

::Target,

::Permission>, + >, +) -> (res: (RawCallback, Tracked)) + requires + !pointer.is_null(), + equal(owned.ptr(), pointer), + P::ptr_perm_match(pointer, owned.ownership()), + owned.ownership().inv(), +{ + proof { + use_type_invariant(&owned); + } + proof_decl! { + let tracked (object, retired, permission) = owned.tracked_into_parts(); + let tracked cert = rcu_spec::certify_callback_from_retired(&object, retired); + } + let pointer = unsafe { NonNull::new_unchecked(pointer) }; + let context = RcuDropCallbackContext::

{ pointer, permission: Tracked(permission) }; + proof { + use_type_invariant(&context); + } + (RawCallback::new(context), Tracked(cert)) +} + impl RcuInner

{ closed spec fn is_nullable(self) -> bool { self.ghost_nullable@ @@ -182,9 +284,10 @@ impl RcuInner

{ res.is_nullable(), { proof_decl! { - // TODO: retain this beside the initial `P::Permission`. - let tracked (root_ghost, _registration) = rcu_spec::RcuRootGhost::tracked_initial( + let tracked root_ghost: RcuAtomicGhost

= + rcu_spec::RcuRootOwnedGhost::tracked_initial( core::ptr::null_mut::<

::Target>(), + None, ); } let ptr = WeakAtomicPtr::new(Ghost(true), core::ptr::null_mut(), Tracked(root_ghost)); @@ -204,15 +307,14 @@ impl RcuInner

{ res.is_nullable() == nullable, )] fn new(pointer: P) -> Self { - let (raw, Tracked(_perm)) = P::into_raw(pointer); + let (raw, Tracked(perm)) = P::into_raw(pointer); let raw_ptr = raw.as_ptr(); proof { assert(!raw_ptr.is_null()); } proof_decl! { - // TODO: retain this beside the initial `P::Permission`. - let tracked (root_ghost, _registration) = - rcu_spec::RcuRootGhost::tracked_initial(raw_ptr); + let tracked root_ghost = + rcu_spec::RcuRootOwnedGhost::tracked_initial(raw_ptr, Some(perm)); } let ptr = WeakAtomicPtr::new(Ghost(nullable), raw_ptr, Tracked(root_ghost)); Self { @@ -244,23 +346,45 @@ impl RcuInner

{ } #[inline(always)] - fn store_ptr_release( + fn swap_ptr_release( &self, new_ptr: *mut

::Target, + Tracked(ownership): Tracked::Permission>>, Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: Tracked::Target>>>) + ) -> (res: ( + *mut

::Target, + Tracked< + Option< + rcu_spec::RcuRetiredOwnedObject< +

::Target, +

::Permission, + >, + >, + >, + )) requires self.type_inv(), self.is_nullable() || !new_ptr.is_null(), + match ownership { + Some(ownership) => { + &&& !new_ptr.is_null() + &&& P::ptr_perm_match(new_ptr, ownership) + &&& ownership.inv() + }, + None => new_ptr.is_null(), + }, ensures - (res@ is Some) == !new_ptr.is_null(), - res@ is Some ==> res@->Some_0.0.ptr() == new_ptr, + (res.1@ is Some) == !res.0.is_null(), + res.1@ is Some ==> res.1@->Some_0.object().wf(), + res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), + res.1@ is Some ==> P::ptr_perm_match(res.0, res.1@->Some_0.ownership()), + res.1@ is Some ==> res.1@->Some_0.ownership().inv(), { proof { assert(self.ptr.constant() == self.is_nullable()); assert(self.ptr.constant() || !new_ptr.is_null()); } - self.ptr.store_release_rcu(new_ptr, Tracked(tv)) + self.ptr.swap_release_rcu(new_ptr, Tracked(ownership), Tracked(tv)) } fn update(&self, new_ptr: Option

, Tracked(session): Tracked<&mut RunningTaskContext>) @@ -278,24 +402,35 @@ impl RcuInner

{ proof_decl! { let ghost new_ptr_is_some = new_ptr is Some; } - let (raw, Tracked(_perm)) = if let Some(new_ptr) = new_ptr { + let (raw, Tracked(perm)) = if let Some(new_ptr) = new_ptr { let (ptr, Tracked(perm)) = P::into_raw(new_ptr); (ptr.as_ptr(), Tracked(Some(perm))) } else { (core::ptr::null_mut(), Tracked(None)) }; - proof_decl! { - let tracked tv = session.tracked_borrow_thread_view_mut(); - } proof { if !self.is_nullable() { assert(new_ptr_is_some); } assert(self.is_nullable() || !raw.is_null()); } - // TODO: retain the registration with `perm` until this object is detached. - let Tracked(_registration) = self.store_ptr_release(raw, Tracked(tv)); + let (old_raw, Tracked(detached)) = { + proof_decl! { + let tracked tv = session.tracked_borrow_thread_view_mut(); + } + self.swap_ptr_release(raw, Tracked(perm), Tracked(tv)) + }; + if !old_raw.is_null() { + proof_decl! { + let tracked detached = detached.tracked_unwrap(); + } + let (callback, cert) = callback_from_detached::

(old_raw, Tracked(detached)); + if let Some(monitor) = RCU_MONITOR.get() { + #[verus_spec(with Tracked(session))] + monitor.after_grace_period(callback, cert); + } + } } fn read(&self, Tracked(session): Tracked<&mut RunningTaskContext>) -> (res: RcuReadGuardInner< @@ -392,10 +527,6 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { proof_decl! { let ghost new_ptr_is_some = new_ptr is Some; - let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( - session, - &self._inner_guard, - ); } let (new_raw, Tracked(new_perm)) = if let Some(new_ptr) = new_ptr { @@ -415,16 +546,46 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { assert(rcu.ptr.constant() == rcu.is_nullable()); assert(rcu.ptr.constant() || !new_raw.is_null()); } - let cas_res = rcu.ptr.compare_exchange_acqrel_acquire_rcu(expected, new_raw, Tracked(tv)); - // A successful CAS must route this registration into the ownership map. - let Tracked(_registration) = cas_res.2; + let cas_res = { + proof_decl! { + let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( + session, + &self._inner_guard, + ); + } + rcu.ptr.compare_exchange_acqrel_acquire_rcu( + expected, + new_raw, + Tracked(new_perm), + Tracked(tv), + ) + }; + let ghost context_before_enqueue = *session; + proof { + assert(self._inner_guard.matches_context(context_before_enqueue)); + } + proof_decl! { + let tracked (detached, rejected_new_perm) = cas_res.2.get(); + } let res = match cas_res.0 { - Result::Ok(_) => Ok(()), + Result::Ok(old_raw) => { + if !old_raw.is_null() { + proof_decl! { + let tracked detached = detached.tracked_unwrap(); + } + let (callback, cert) = callback_from_detached::

(old_raw, Tracked(detached)); + if let Some(monitor) = RCU_MONITOR.get() { + #[verus_spec(with Tracked(session))] + monitor.after_grace_period(callback, cert); + } + } + Ok(()) + }, Result::Err(_) => { if let Some(new_nonnull) = NonNull::new(new_raw) { proof_decl! { - let tracked perm = new_perm.tracked_unwrap(); + let tracked perm = rejected_new_perm.tracked_unwrap(); } Err(Some(unsafe { P::from_raw(new_nonnull, Tracked(perm)) })) } else { @@ -433,6 +594,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { }, }; proof { + self._inner_guard.lemma_matches_context_preserved(context_before_enqueue, session); self._inner_guard.lemma_matches_context_depth(session); } self._inner_guard.release_to_context(Tracked(session)); @@ -478,7 +640,7 @@ impl Rcu

{ ) } - /// Replaces the current pointer with `new_ptr` using a release store. + /// Replaces the current pointer with `new_ptr` using a release swap. #[inline] #[verus_spec( with @@ -540,7 +702,7 @@ impl RcuOption

{ Self(RcuInner::new_none()) } - /// Replaces the current pointer using a release store. + /// Replaces the current pointer using a release swap. #[inline] #[verus_spec( with @@ -747,11 +909,31 @@ impl Deref for RcuDrop { /// Finishes a grace period on the current CPU. /// -/// No-op until the weak-memory monitor/reclamation path is rebuilt. +#[verus_spec( + with + Tracked(session): Tracked<&mut RunningTaskContext>, + requires + old(session).wf(), + old(session).is_quiescent(), + ensures + final(session).wf(), + final(session).is_quiescent(), + final(session).task() == old(session).task(), + final(session).session_id() == old(session).session_id(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), +)] pub unsafe fn finish_grace_period() { + if let Some(monitor) = RCU_MONITOR.get() { + unsafe { + #[verus_spec(with Tracked(session))] + monitor.finish_grace_period(); + } + } } pub fn init() { + RCU_MONITOR.init(monitor::RcuMonitor::new_data()); } } // verus! diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 8accd9a22..7581113e2 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -2,7 +2,7 @@ use alloc::collections::VecDeque; use core::sync::atomic::Ordering; -use vstd::prelude::*; +use vstd::{predicate::Predicate as DataPredicate, prelude::*}; use vstd_extra::raw_callback::RawCallback; use crate::specs::{ @@ -13,7 +13,9 @@ use crate::specs::{ weak_memory::{History, ThreadView, WeakAtomicBool}, }, }; -use crate::sync::{LocalIrqDisabled, SpinLock}; +use crate::sync::{ + AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate, +}; use crate::task::RunningTaskContext; verus! { @@ -35,6 +37,7 @@ type MonitorAtomicBool = WeakAtomicBool< pub struct RcuCallback { raw: RawCallback, summary: Ghost, + safety: Tracked, } impl View for RcuCallback { @@ -49,13 +52,25 @@ impl RcuCallback { /// Converts a raw callback into an RCU callback, given a proof that the callback is /// safe to run after a grace period. #[inline] - pub fn from_raw(raw: RawCallback, Tracked(cert): Tracked) -> (res: - Self) + fn from_raw( + raw: RawCallback, + Tracked(cert): Tracked, + Ghost(retire_epoch): Ghost, + ) -> (res: Self) ensures - res@ == cert@, + res.wf(), + res@ == (rcu_spec::RcuCallbackSummary { + domain: cert.domain(), + obj: cert.obj(), + retire_epoch, + }), { - let ghost summary = cert@; - Self { raw, summary: Ghost(summary) } + let ghost summary = rcu_spec::RcuCallbackSummary { + domain: cert.domain(), + obj: cert.obj(), + retire_epoch, + }; + Self { raw, summary: Ghost(summary), safety: Tracked(cert) } } /// Runs the underlying callback once the monitor has completed the grace @@ -64,12 +79,22 @@ impl RcuCallback { #[verifier::external_body] unsafe fn call_once(self, Tracked(completed): Tracked<&CompletedGracePeriod>) requires + self.wf(), completed.covers(self@), { unsafe { self.raw.call_once(); } } + + closed spec fn wf(self) -> bool { + self.safety@.matches(self@) + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } } /// Proof token produced by the monitor when a grace period finishes. @@ -78,20 +103,22 @@ impl RcuCallback { /// certify that a callback is safe to enqueue, but cannot manufacture the /// completion fact needed to execute the callback. tracked struct CompletedGracePeriod { + ghost epoch: nat, ghost callbacks: Seq, } -impl View for CompletedGracePeriod { - type V = Seq; - - closed spec fn view(&self) -> Seq { +impl CompletedGracePeriod { + closed spec fn callbacks(self) -> Seq { self.callbacks } -} -impl CompletedGracePeriod { + closed spec fn epoch(self) -> nat { + self.epoch + } + closed spec fn covers(self, callback: rcu_spec::RcuCallbackSummary) -> bool { - self@.contains(callback) + &&& self.callbacks().contains(callback) + &&& callback.retire_epoch == self.epoch() } } @@ -104,7 +131,11 @@ fn run_completed_callbacks( Tracked(completed): Tracked, ) requires - completed@ == callback_summaries(callbacks), + completed.callbacks() == callback_summaries(callbacks), + forall|i: int| + 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( + callbacks, + )[i]).retire_epoch == completed.epoch(), { proof { assert forall|i: int| 0 <= i < callbacks@.len() implies completed.covers( @@ -134,6 +165,9 @@ fn run_completed_callbacks( } } unsafe { + proof { + use_type_invariant(&callback); + } callback.call_once(Tracked(&completed)); } } @@ -157,6 +191,31 @@ proof fn callback_summaries_len(callbacks: Callbacks) { } +fn push_callback(callbacks: &mut Callbacks, callback: RcuCallback) + ensures + callback_summaries(*final(callbacks)) == callback_summaries(*old(callbacks)).push( + callback@, + ), +{ + let ghost before = callbacks@; + callbacks.push_back(callback); + proof { + assert(callbacks@ == before.push(callback)); + vstd::seq_lib::assert_seqs_equal!( + callback_summaries(*callbacks) + == callback_summaries(*old(callbacks)).push(callback@), + i => { + if i < before.len() { + assert(callbacks@[i] == before[i]); + } else { + assert(i == before.len()); + assert(callbacks@[i] == callback); + } + } + ); + } +} + // The proof-facing views `GracePeriodView` and `MonitorStateView` live in // `specs::sync::rcu` so that the monitor flag's weak-memory ghost state can // record a state snapshot per flag message without depending on this module. @@ -164,6 +223,7 @@ pub(super) struct GracePeriod { callbacks: Callbacks, cpu_mask: AtomicCpuSet, is_complete: bool, + ghost_epoch: Ghost, } impl View for GracePeriod { @@ -171,6 +231,7 @@ impl View for GracePeriod { closed spec fn view(&self) -> GracePeriodView { GracePeriodView { + epoch: self.ghost_epoch@, callbacks: callback_summaries(self.callbacks), is_complete: self.is_complete, } @@ -187,7 +248,7 @@ impl GracePeriod { { let callbacks = Callbacks::new(); let cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); - let res = Self { callbacks, cpu_mask, is_complete: true }; + let res = Self { callbacks, cpu_mask, is_complete: true, ghost_epoch: Ghost(0) }; proof { callback_summaries_empty(res.callbacks); } @@ -198,14 +259,22 @@ impl GracePeriod { /// after it completes. The CPU mask is reset because all CPUs must pass a /// fresh quiescent state for this new batch. Keep the same atomic object so /// later weak-memory ghost state can attach stable identity to this mask. - fn restart(&mut self, callbacks: Callbacks) + fn restart(&mut self, callbacks: Callbacks, Ghost(epoch): Ghost) + requires + forall|i: int| + 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( + callbacks, + )[i]).retire_epoch == epoch, ensures final(self).callback_summaries() == callback_summaries(callbacks), + final(self)@.epoch == epoch, !final(self).is_complete, + final(self).wf(), no_unwind { self.is_complete = false; self.callbacks = callbacks; + self.ghost_epoch = Ghost(epoch); self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); } @@ -237,11 +306,6 @@ impl GracePeriod { closed spec fn wf(self) -> bool { self@.wf() } - - #[verifier::type_invariant] - closed spec fn type_inv(self) -> bool { - self.wf() - } } pub(super) struct State { @@ -261,6 +325,10 @@ impl View for State { } impl State { + closed spec fn next_callback_epoch(self) -> nat { + self@.current_gp.epoch + 1 + } + /// Creates the lock-protected initial monitor state: there is no active /// grace period and no callbacks waiting to be attached to the next one. pub(super) fn new() -> (res: Self) @@ -288,21 +356,58 @@ impl State { /// keeps `next_callbacks` empty whenever the current grace period is /// complete, so the idle case constructs the current batch directly. fn enqueue_after_grace_period(&mut self, callback: RcuCallback) -> (started_gp: bool) + requires + callback.wf(), + callback@.retire_epoch == old(self).next_callback_epoch(), ensures final(self).wf(), final(self).has_pending_work(), started_gp ==> !final(self)@.current_gp.is_complete, !started_gp ==> !old(self)@.current_gp.is_complete, { + proof { + use_type_invariant(&*self); + } + let ghost callback_epoch = self.next_callback_epoch(); if self.current_gp.is_complete { let mut callbacks = Callbacks::new(); - callbacks.push_back(callback); - self.current_gp.restart(callbacks); + push_callback(&mut callbacks, callback); + proof { + assert(callback_summaries(callbacks).len() == 1); + assert(callback_summaries(self.next_callbacks).len() == 0); + } + self.current_gp.restart(callbacks, Ghost(callback_epoch)); true } else { let mut next_callbacks = Callbacks::new(); + let ghost existing_summaries = callback_summaries(self.next_callbacks); + proof { + assert(self.current_gp.wf()); + assert(!self.current_gp.is_complete); + assert(self.wf()); + assert(self@.wf()); + assert(self@.next_callbacks == existing_summaries); + assert forall|i: int| 0 <= i < existing_summaries.len() implies ( + #[trigger] existing_summaries[i]).retire_epoch == self.current_gp@.epoch + 1 by {}; + } core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); - next_callbacks.push_back(callback); + let ghost before_summaries = callback_summaries(next_callbacks); + proof { + assert(before_summaries == existing_summaries); + } + push_callback(&mut next_callbacks, callback); + proof { + assert forall|i: int| 0 <= i < callback_summaries(next_callbacks).len() implies ( + #[trigger] callback_summaries(next_callbacks)[i]).retire_epoch + == self.current_gp@.epoch + 1 by { + if i < before_summaries.len() { + assert(callback_summaries(next_callbacks)[i] == before_summaries[i]); + } else { + assert(i == before_summaries.len()); + assert(callback_summaries(next_callbacks)[i] == callback@); + } + }; + } self.next_callbacks = next_callbacks; false } @@ -324,10 +429,17 @@ impl State { ): (bool, Callbacks, Tracked)) ensures final(self).wf(), - completed_token@@ == callback_summaries(completed_callbacks), + completed_token@.callbacks() == callback_summaries(completed_callbacks), completed_gp ==> !old(self)@.current_gp.is_complete, - completed_gp ==> completed_token@@ == old(self)@.current_gp.callbacks, - !completed_gp ==> completed_token@@ == Seq::::empty(), + completed_gp ==> completed_token@.callbacks() == old(self)@.current_gp.callbacks, + completed_gp ==> completed_token@.epoch() == old(self)@.current_gp.epoch, + !completed_gp ==> completed_token@.callbacks() == Seq::< + rcu_spec::RcuCallbackSummary, + >::empty(), + forall|i: int| + 0 <= i < callback_summaries(completed_callbacks).len() ==> ( + #[trigger] callback_summaries(completed_callbacks)[i]).retire_epoch + == completed_token@.epoch(), (!completed_gp && !(old(self)@.current_gp.is_complete)) ==> !( final(self)@.current_gp.is_complete), { @@ -336,6 +448,21 @@ impl State { } let ghost initially_complete = self.current_gp.is_complete; let ghost initial_current_callbacks = self.current_gp@.callbacks; + let ghost initial_current_epoch = self.current_gp@.epoch; + let ghost initial_next_callbacks = callback_summaries(self.next_callbacks); + proof { + assert(self.wf()); + assert(self@.wf()); + assert(self@.next_callbacks == initial_next_callbacks); + assert forall|i: int| 0 <= i < initial_current_callbacks.len() implies ( + #[trigger] initial_current_callbacks[i]).retire_epoch == initial_current_epoch by { + assert(self.current_gp.wf()); + }; + assert forall|i: int| 0 <= i < initial_next_callbacks.len() implies ( + #[trigger] initial_next_callbacks[i]).retire_epoch == initial_current_epoch + 1 by { + assert(self.wf()); + }; + } let mut completed_callbacks = Callbacks::new(); let mut completed_gp = false; if !self.current_gp.is_complete { @@ -352,8 +479,9 @@ impl State { core::mem::swap(&mut next_callbacks, &mut self.next_callbacks); proof { callback_summaries_empty(self.next_callbacks); + assert(callback_summaries(next_callbacks) == initial_next_callbacks); } - self.current_gp.restart(next_callbacks); + self.current_gp.restart(next_callbacks, Ghost(initial_current_epoch + 1)); } else { self.current_gp.is_complete = true; proof { @@ -367,18 +495,26 @@ impl State { } proof_decl! { let tracked completed = CompletedGracePeriod { + epoch: if completed_gp { initial_current_epoch } else { 0 }, callbacks: callback_summaries(completed_callbacks), }; } proof { callback_summaries_len(self.current_gp.callbacks); callback_summaries_len(self.next_callbacks); - assert(completed@ == callback_summaries(completed_callbacks)); + assert(completed.callbacks() == callback_summaries(completed_callbacks)); if !completed_gp { callback_summaries_empty(completed_callbacks); } else { assert(!initially_complete); assert(callback_summaries(completed_callbacks) == initial_current_callbacks); + assert forall|i: int| + 0 <= i < callback_summaries(completed_callbacks).len() implies ( + #[trigger] callback_summaries(completed_callbacks)[i]).retire_epoch + == completed.epoch() by { + assert(callback_summaries(completed_callbacks)[i] + == initial_current_callbacks[i]); + }; } if initially_complete { assert(initial_current_callbacks.len() == 0); @@ -578,11 +714,20 @@ impl RcuMonitor { final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] - pub(super) fn after_grace_period(&self, callback: RcuCallback) { + pub(super) fn after_grace_period( + &self, + raw: RawCallback, + cert: Tracked, + ) { proof { use_type_invariant(self); } let mut state = self.state.lock(); + let ghost retire_epoch = state.view()@.current_gp.epoch + 1; + proof_decl! { + let tracked cert = cert.get(); + } + let callback = RcuCallback::from_raw(raw, Tracked(cert), Ghost(retire_epoch)); let started_gp = state.enqueue_after_grace_period(callback); if started_gp { proof { @@ -610,8 +755,10 @@ impl RcuMonitor { Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + old(session).is_quiescent(), ensures final(session).wf(), + final(session).is_quiescent(), final(session).task() == old(session).task(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), @@ -668,6 +815,45 @@ impl RcuMonitor { } } +/// Tracked witness stored beside the global monitor in `Once`. +pub(super) tracked struct RcuMonitorOwner {} + +impl DataPredicate for RcuMonitorOwner { + closed spec fn predicate(&self, monitor: RcuMonitor) -> bool { + monitor.wf() + } +} + +/// Invariant used by the global `Once` cell. +pub(super) struct RcuMonitorPred; + +impl OncePredicate> for RcuMonitorPred { + closed spec fn inv(self, value: AtomicDataWithOwner) -> bool { + value.permission@.predicate(value.data) + } +} + +impl RcuMonitor { + /// Packages a fresh monitor with the owner expected by the global `Once`. + pub(super) fn new_data() -> (res: AtomicDataWithOwner) + ensures + RcuMonitorPred.inv(res), + { + let data = Self::new(); + proof_decl! { + let tracked owner = RcuMonitorOwner {}; + } + proof { + use_type_invariant(&data); + } + let res = AtomicDataWithOwner { data, permission: Tracked(owner) }; + proof { + assert(res.permission@.predicate(res.data)); + } + res + } +} + } // verus! // use vstd::{ // atomic_ghost::AtomicBool, atomic_with_ghost, predicate::Predicate as DataPredicate, prelude::*, diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index ed50747f5..317975778 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -634,6 +634,30 @@ impl DisabledPreemptGuard { { } + /// Changing only the task's weak-memory view preserves this guard's + /// relation to the running context. + pub proof fn lemma_matches_context_preserved( + &self, + before: RunningTaskContext, + tracked after: &RunningTaskContext, + ) + requires + self.matches_context(before), + after.wf(), + after.task() == before.task(), + after.session_id() == before.session_id(), + after.available_fractions() == before.available_fractions(), + after.preempt_depth() == before.preempt_depth(), + ensures + self.matches_context(*after), + { + assert(before.session.session_task() == before.task()); + assert(after.session.session_task() == after.task()); + assert(self.tracked_resource@.session_token().task() == before.task()); + assert(self.tracked_resource@.session_token().task() == after.task()); + assert(after.session.token_matches(self.tracked_resource@.session_token())); + } + /// Borrows the running task's view while this guard witnesses that /// preemption is disabled. Both outermost and nested guards use the same /// context-owned view. From fd56d78b1d1f2e839cf57837dc59c367ea41bd05 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 20 Jul 2026 22:39:59 -0400 Subject: [PATCH 25/47] add thread management caps for the scheduler --- ostd/specs/sync/weak_memory.rs | 12 +- ostd/src/mm/frame/segment.rs | 2 +- ostd/src/sync/rcu/mod.rs | 15 ++ ostd/src/sync/rcu/monitor.rs | 14 +- ostd/src/task/preempt/guard.rs | 54 ++++-- ostd/src/task/scheduler/mod.rs | 320 ++++++++++++++++++++++++++++++--- 6 files changed, 364 insertions(+), 53 deletions(-) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 9bef19882..4954c9e77 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -33,7 +33,7 @@ use vstd::seq::Seq; verus! { -// The "global" memory is defined wihtin the invariant we need to preserve and, +// The "global" memory is defined within the invariant we need to preserve and, // by the definition of Iris operations, invariant can be opened by a thread // provided that the invariant holds and it can close afterwards provided that // the invariant holds as well. @@ -117,7 +117,7 @@ impl WmView { } /// Partial ordering two threads' views. - pub open spec fn le(self, other: Self) -> bool { + pub open spec fn spec_le(self, other: Self) -> bool { forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) } } @@ -1230,9 +1230,11 @@ impl ThreadView { /// through release stores, so executable code should thread one token per /// logical operation or critical section, and eventually one per task. /// - /// TODO: This API should not be exposed as a public because the only way - /// to create this is via critical section markers such as `disable_preempt`. - pub proof fn new() -> (tracked res: Self) + /// This constructor is crate-private so clients cannot discard an + /// established task view and restart from the empty view. Production code + /// creates one view when the scheduler registers a task, then moves that + /// same linear token through schedule-in and schedule-out. + pub(crate) proof fn new() -> (tracked res: Self) ensures res@ == WmView::empty(), { diff --git a/ostd/src/mm/frame/segment.rs b/ostd/src/mm/frame/segment.rs index 5082daca0..7e427a427 100644 --- a/ostd/src/mm/frame/segment.rs +++ b/ostd/src/mm/frame/segment.rs @@ -1109,7 +1109,7 @@ impl<'a, M: AnyFrameMeta + Repr + OwnerOf> SegmentIterator<'a, let item = (frame, Tracked(from_raw_obl)); proof { remaining.resolve_cons(item); - broadcast use vstd::seq::group_seq_lemmas; + broadcast use vstd::seq::group_seq_axioms; assert(remaining.seq() == old_remaining.drop_first()); assert(item == old_remaining[0]); diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 738b696c1..b10e9f380 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -395,6 +395,7 @@ impl RcuInner

{ ensures final(session).wf(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), @@ -445,6 +446,7 @@ impl RcuInner

{ res.type_inv(), res.rcu.is_nullable() == self.is_nullable(), final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -471,6 +473,7 @@ impl RcuInner

{ old(session).wf(), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), { @@ -514,6 +517,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { new_ptr is Some && res is Err ==> res->Err_0 is Some, final(session).wf(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), @@ -609,6 +613,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { ensures final(session).wf(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).view() == old(session).view(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions() + 1, @@ -649,6 +654,7 @@ impl Rcu

{ old(session).wf(), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -669,6 +675,7 @@ impl Rcu

{ old(session).available_fractions() > 1, ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -711,6 +718,7 @@ impl RcuOption

{ old(session).wf(), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -731,6 +739,7 @@ impl RcuOption

{ old(session).available_fractions() > 1, ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -750,6 +759,7 @@ impl RcuOption

{ old(session).wf(), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -774,6 +784,7 @@ impl RcuReadGuard<'_, P> { self.matches_context(*old(session)), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -800,6 +811,7 @@ impl RcuReadGuard<'_, P> { self.matches_context(*old(session)), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -824,6 +836,7 @@ impl RcuOptionReadGuard<'_, P> { self.matches_context(*old(session)), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -851,6 +864,7 @@ impl RcuOptionReadGuard<'_, P> { self.matches_context(*old(session)), ensures final(session).wf(), + final(session).scheduler() == old(session).scheduler(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -919,6 +933,7 @@ impl Deref for RcuDrop { final(session).wf(), final(session).is_quiescent(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 7581113e2..a3c00ce88 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -103,17 +103,17 @@ impl RcuCallback { /// certify that a callback is safe to enqueue, but cannot manufacture the /// completion fact needed to execute the callback. tracked struct CompletedGracePeriod { - ghost epoch: nat, - ghost callbacks: Seq, + epoch: Ghost, + callbacks: Ghost>, } impl CompletedGracePeriod { closed spec fn callbacks(self) -> Seq { - self.callbacks + self.callbacks@ } closed spec fn epoch(self) -> nat { - self.epoch + self.epoch@ } closed spec fn covers(self, callback: rcu_spec::RcuCallbackSummary) -> bool { @@ -495,8 +495,8 @@ impl State { } proof_decl! { let tracked completed = CompletedGracePeriod { - epoch: if completed_gp { initial_current_epoch } else { 0 }, - callbacks: callback_summaries(completed_callbacks), + epoch: Ghost(if completed_gp { initial_current_epoch } else { 0 }), + callbacks: Ghost(callback_summaries(completed_callbacks)), }; } proof { @@ -710,6 +710,7 @@ impl RcuMonitor { ensures final(session).wf(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), @@ -760,6 +761,7 @@ impl RcuMonitor { final(session).wf(), final(session).is_quiescent(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 317975778..a17618da3 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -19,7 +19,7 @@ pub const PREEMPT_SESSION_FRACTIONS: u64 = 1 << 31; /// guard. This token only records that the guard was created while preemption /// was already disabled. pub tracked struct NestedPreemptToken { - ghost depth_before: nat, + depth_before: Ghost, } impl NestedPreemptToken { @@ -30,11 +30,11 @@ impl NestedPreemptToken { res.depth_before() == depth_before, res.wf(), { - NestedPreemptToken { depth_before } + NestedPreemptToken { depth_before: Ghost(depth_before) } } pub closed spec fn depth_before(self) -> nat { - self.depth_before + self.depth_before@ } pub closed spec fn wf(self) -> bool { @@ -104,8 +104,8 @@ impl PreemptSessionToken { /// `ThreadView` per running task while still allowing nested RCU code to /// perform weak atomic operations. pub tracked struct PreemptThreadViewSession { - tracked task_view: TaskThreadView, - tracked tokens: CountGhostResource, + task_view: TaskThreadView, + tokens: CountGhostResource, } impl PreemptThreadViewSession { @@ -114,6 +114,7 @@ impl PreemptThreadViewSession { requires task_view.wf(sched_view), ensures + res.scheduler() == task_view.scheduler(), res.task() == task_view.task(), res.view() == task_view.view(), res.session_task() == task_view.task(), @@ -139,6 +140,10 @@ impl PreemptThreadViewSession { self.task_view.task() } + pub closed spec fn scheduler(self) -> Loc { + self.task_view.scheduler() + } + pub closed spec fn view(self) -> WmView { self.task_view.view() } @@ -183,6 +188,7 @@ impl PreemptThreadViewSession { old(self).available_fractions() > 1, ensures final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), @@ -202,6 +208,7 @@ impl PreemptThreadViewSession { old(self).token_matches(token), ensures final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), @@ -232,6 +239,7 @@ impl PreemptThreadViewSession { ensures (*tv)@ == old(self).view(), final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), final(self).available_fractions() == old(self).available_fractions(), @@ -246,12 +254,13 @@ impl PreemptThreadViewSession { /// This is the proof-side counterpart of dropping the outermost /// preemption-disable scope: the session stops owning the task view, and /// the caller can write it back with - /// `SchedulerThreadViews::tracked_put_checked_out_thread_view`. + /// `SchedulerGhostState::tracked_schedule_out`. pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) requires self.wf_session_resource(), self.available_fractions() == PREEMPT_SESSION_FRACTIONS, ensures + res.scheduler() == self.scheduler(), res.task() == self.task(), res.view() == self.view(), { @@ -267,6 +276,7 @@ impl PreemptThreadViewSession { self.wf(sched_view), self.available_fractions() == PREEMPT_SESSION_FRACTIONS, ensures + res.scheduler() == self.scheduler(), res.task() == self.task(), res.view() == self.view(), res.wf(sched_view), @@ -283,8 +293,8 @@ impl PreemptThreadViewSession { /// the inverse transition. Consequently the context can only be returned to /// the scheduler when no guard remains live. pub tracked struct RunningTaskContext { - tracked session: PreemptThreadViewSession, - ghost preempt_depth: nat, + session: PreemptThreadViewSession, + preempt_depth: Ghost, } impl RunningTaskContext { @@ -294,6 +304,7 @@ impl RunningTaskContext { requires task_view.wf(sched_view), ensures + res.scheduler() == task_view.scheduler(), res.task() == task_view.task(), res.view() == task_view.view(), res.preempt_depth() == 0, @@ -303,7 +314,7 @@ impl RunningTaskContext { res.wf_scheduler(sched_view), { let tracked session = PreemptThreadViewSession::new(task_view, sched_view); - let tracked res = RunningTaskContext { session, preempt_depth: 0 }; + let tracked res = RunningTaskContext { session, preempt_depth: Ghost(0) }; assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(res.wf()); assert(res.session.wf(sched_view)); @@ -315,6 +326,10 @@ impl RunningTaskContext { self.session.task() } + pub closed spec fn scheduler(self) -> Loc { + self.session.scheduler() + } + pub closed spec fn view(self) -> WmView { self.session.view() } @@ -328,7 +343,7 @@ impl RunningTaskContext { } pub closed spec fn preempt_depth(self) -> nat { - self.preempt_depth + self.preempt_depth@ } pub closed spec fn wf(self) -> bool { @@ -349,6 +364,7 @@ impl RunningTaskContext { requires self.wf(), sched_view.wf(), + sched_view.id == self.scheduler(), sched_view.task_view_is_checked_out(self.task()), sched_view.checked_out_views[self.task()] == self.view(), sched_view.task_views.contains_key(self.task()), @@ -372,6 +388,7 @@ impl RunningTaskContext { ensures (*tv)@ == old(self).view(), final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth(), @@ -388,6 +405,7 @@ impl RunningTaskContext { self.wf(), self.preempt_depth() == 0, ensures + res.scheduler() == self.scheduler(), res.task() == self.task(), res.view() == self.view(), { @@ -405,6 +423,7 @@ impl RunningTaskContext { self.wf_scheduler(sched_view), self.is_quiescent(), ensures + res.scheduler() == self.scheduler(), res.task() == self.task(), res.view() == self.view(), res.wf(sched_view), @@ -496,6 +515,7 @@ impl PreemptGuardResource { ensures final(session).wf_session_resource(), final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), final(session).view() == old(session).view(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions() + 1, @@ -522,6 +542,7 @@ impl RunningTaskContext { ensures final(self).wf(), final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() + 1 == old(self).available_fractions(), @@ -530,7 +551,7 @@ impl RunningTaskContext { resource.is_outermost() <==> old(self).preempt_depth() == 0, resource.is_nested() <==> old(self).preempt_depth() > 0, { - let ghost depth_before = self.preempt_depth; + let ghost depth_before = self.preempt_depth@; let tracked token = self.session.tracked_split_guard_token(); let tracked resource = if depth_before == 0 { PreemptGuardResource::Outermost(token) @@ -538,7 +559,7 @@ impl RunningTaskContext { let tracked nested = NestedPreemptToken::new(depth_before); PreemptGuardResource::Nested { session: token, nested } }; - self.preempt_depth = depth_before + 1; + self.preempt_depth = Ghost(depth_before + 1); assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(self.wf()); resource @@ -554,14 +575,15 @@ impl RunningTaskContext { ensures final(self).wf(), final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions() + 1, final(self).preempt_depth() + 1 == old(self).preempt_depth(), { - let ghost old_depth = self.preempt_depth; + let ghost old_depth = self.preempt_depth@; resource.tracked_return_to_session(&mut self.session); - self.preempt_depth = (old_depth - 1) as nat; + self.preempt_depth = Ghost((old_depth - 1) as nat); assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(self.wf()); } @@ -645,6 +667,7 @@ impl DisabledPreemptGuard { self.matches_context(before), after.wf(), after.task() == before.task(), + after.scheduler() == before.scheduler(), after.session_id() == before.session_id(), after.available_fractions() == before.available_fractions(), after.preempt_depth() == before.preempt_depth(), @@ -671,6 +694,7 @@ impl DisabledPreemptGuard { ensures (*tv)@ == old(context).view(), final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), final(context).session_id() == old(context).session_id(), final(context).available_fractions() == old(context).available_fractions(), final(context).preempt_depth() == old(context).preempt_depth(), @@ -691,6 +715,7 @@ impl DisabledPreemptGuard { ensures final(context).wf(), final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), final(context).view() == old(context).view(), final(context).session_id() == old(context).session_id(), final(context).available_fractions() == old(context).available_fractions() + 1, @@ -743,6 +768,7 @@ pub(crate) fn disable_preempt_in_context( ensures final(context).wf(), final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), final(context).view() == old(context).view(), final(context).session_id() == old(context).session_id(), final(context).available_fractions() + 1 == old(context).available_fractions(), diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 45be5ed1d..ad0afd761 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -63,6 +63,7 @@ //! Violating this invariant—e.g., running the same task on two CPUs concurrently— //! can have catastrophic consequences, //! as the task's stack and internal state may be corrupted by concurrent modifications. +use vstd::resource::map::GhostMapAuth; use vstd::{map::Map, prelude::*, resource::Loc}; use super::{Task, preempt::RunningTaskContext}; @@ -98,12 +99,12 @@ pub ghost enum TaskSchedState { /// /// The scheduler proof state has two layers. `SchedulerView` is the copyable /// ghost snapshot used in specifications: it records scheduling state and the -/// current weak-memory view for each live task. `SchedulerThreadViews` is the -/// tracked owner that stores the actual linear `ThreadView` resources for -/// tasks whose views are still owned by the scheduler. +/// current weak-memory view for each live task. `SchedulerGhostState` is the +/// authoritative tracked root that stores both that snapshot and the actual +/// linear `ThreadView` resources. /// /// When a task is scheduled in, its `ThreadView` is checked out of -/// `SchedulerThreadViews` and moved into a `RunningTaskContext`. While it is +/// `SchedulerGhostState` and moved into a `RunningTaskContext`. While it is /// checked out, /// `SchedulerView::task_views` remains the logical source of truth, but the /// ownership partition records that the view is in `checked_out_views` rather @@ -114,6 +115,9 @@ pub ghost enum TaskSchedState { /// /// In short, the resource flow is: /// `stored_views -> RunningTaskContext -> checked_out_views update -> stored_views`. +/// Scheduler-policy transitions may change runqueues, current tasks, and task +/// states only through `same_thread_view_ownership`, which frames all three +/// weak-memory ownership maps. /// /// Abstract proof view of the scheduler state. /// @@ -123,6 +127,7 @@ pub ghost enum TaskSchedState { /// weak-memory view. `stored_views` records views still owned by the scheduler /// resource; `checked_out_views` records views temporarily held by guards. pub ghost struct SchedulerView { + pub id: Loc, pub runqueues: Map>, pub current: Map>, pub state: Map, @@ -132,6 +137,25 @@ pub ghost struct SchedulerView { } impl SchedulerView { + /// Initial scheduler state before any task has been registered. + pub open spec fn empty(id: Loc) -> Self { + SchedulerView { + id, + runqueues: Map::empty(), + current: Map::empty(), + state: Map::empty(), + task_views: Map::empty(), + stored_views: Map::empty(), + checked_out_views: Map::empty(), + } + } + + pub proof fn lemma_empty_wf(id: Loc) + ensures + Self::empty(id).wf(), + { + } + pub open spec fn task_is_known(self, task: Loc) -> bool { self.state.contains_key(task) } @@ -155,6 +179,19 @@ impl SchedulerView { self.checked_out_views.contains_key(task) } + /// The scheduling policy changed no weak-memory ownership state. + /// + /// This relation deliberately ignores runqueues, current tasks, and task + /// scheduling states. A concrete scheduler may update those fields while + /// choosing where a task runs, but it must leave all three view maps to the + /// authoritative weak-memory transitions below. + pub open spec fn same_thread_view_ownership(self, other: Self) -> bool { + &&& self.id == other.id + &&& self.task_views == other.task_views + &&& self.stored_views == other.stored_views + &&& self.checked_out_views == other.checked_out_views + } + pub open spec fn task_in_runqueue(self, task: Loc) -> bool { exists|cpu: CpuId, idx: int| self.runqueues.contains_key(cpu) && 0 <= idx && idx < self.runqueues[cpu].len() && ( @@ -196,6 +233,38 @@ impl SchedulerView { &&& !(self.task_in_runqueue(task) && self.task_is_current(task)) } + /// Registers a task and installs its initial weak-memory view. + /// + /// Registration is the only transition that introduces a task view. A new + /// task starts with the empty view and remains in `New` until the scheduler + /// enqueues it. In particular, a caller cannot transfer observations from + /// another task into a newly-created task. + pub open spec fn register_task(self, task: Loc) -> SchedulerView + recommends + !self.state.contains_key(task), + { + SchedulerView { + state: self.state.insert(task, TaskSchedState::New), + task_views: self.task_views.insert(task, WmView::empty()), + stored_views: self.stored_views.insert(task, WmView::empty()), + ..self + } + } + + /// Task registration preserves the scheduler ownership partition. + pub proof fn lemma_register_task_preserves_wf(self, task: Loc) + requires + self.wf(), + !self.state.contains_key(task), + ensures + self.register_task(task).wf(), + self.register_task(task).state[task] is New, + self.register_task(task).task_thread_view(task) == WmView::empty(), + self.register_task(task).task_view_is_stored(task), + !self.register_task(task).task_view_is_checked_out(task), + { + } + /// Moves the current task's weak-memory view out of scheduler storage. /// /// This is the logical transition for the outermost preemption-disable @@ -326,12 +395,13 @@ impl SchedulerView { /// Tracked owner of per-task weak-memory views. /// -/// This is the resource that should eventually back `disable_preempt()`: +/// This is the internal resource backing `SchedulerGhostState`: /// a guard borrows or moves out the current task's `ThreadView`, atomic /// operations update it, and schedule-out writes it back to the same task /// entry after all preemption guards have been released. -pub tracked struct SchedulerThreadViews { - tracked views: Map, +tracked struct SchedulerThreadViews { + scheduler: Ghost, + views: Map, } /// A checked-out per-task `ThreadView`. @@ -340,21 +410,27 @@ pub tracked struct SchedulerThreadViews { /// back. This prevents proofs from taking one task's view and re-inserting it /// under another task id. pub tracked struct TaskThreadView { - ghost task: Loc, - tracked thread_view: ThreadView, + scheduler: Ghost, + task: Ghost, + thread_view: ThreadView, } impl TaskThreadView { - pub proof fn new(task: Loc, tracked thread_view: ThreadView) -> (tracked res: Self) + proof fn new(scheduler: Loc, task: Loc, tracked thread_view: ThreadView) -> (tracked res: Self) ensures + res.scheduler() == scheduler, res.task() == task, res.view() == thread_view@, { - TaskThreadView { task, thread_view } + TaskThreadView { scheduler: Ghost(scheduler), task: Ghost(task), thread_view } + } + + pub closed spec fn scheduler(self) -> Loc { + self.scheduler@ } pub closed spec fn task(self) -> Loc { - self.task + self.task@ } pub closed spec fn view(self) -> WmView { @@ -367,6 +443,7 @@ impl TaskThreadView { /// partition and the total logical `task_views` snapshot. pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { &&& sched_view.wf() + &&& sched_view.id == self.scheduler() &&& sched_view.checked_out_views.contains_key(self.task()) &&& sched_view.checked_out_views[self.task()] == self.view() &&& sched_view.task_views.contains_key(self.task()) @@ -378,6 +455,7 @@ impl TaskThreadView { pub proof fn lemma_wf(tracked &self, sched_view: SchedulerView) requires sched_view.wf(), + sched_view.id == self.scheduler(), sched_view.task_view_is_checked_out(self.task()), sched_view.checked_out_views[self.task()] == self.view(), sched_view.task_views.contains_key(self.task()), @@ -396,6 +474,7 @@ impl TaskThreadView { ensures (*tv)@ == old(self).view(), final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), final(self).view() == (*final(tv))@, { &mut self.thread_view @@ -403,12 +482,17 @@ impl TaskThreadView { } impl SchedulerThreadViews { - pub proof fn empty() -> (tracked res: Self) + proof fn empty(scheduler: Loc) -> (tracked res: Self) ensures + res.scheduler() == scheduler, res.view() == Map::::empty(), { let tracked views = Map::::tracked_empty(); - SchedulerThreadViews { views } + SchedulerThreadViews { scheduler: Ghost(scheduler), views } + } + + closed spec fn scheduler(self) -> Loc { + self.scheduler@ } pub closed spec fn view(self) -> Map { @@ -430,31 +514,56 @@ impl SchedulerThreadViews { /// state. Checked-out views are represented by `TaskThreadView` tokens /// instead, so they are intentionally absent here. pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { - self.view() == sched_view.stored_views + &&& self.scheduler() == sched_view.id + &&& self.view() == sched_view.stored_views } /// Inserts a task view created during task registration. /// /// This is separate from check-in: initial insertion creates a new stored /// entry, while check-in returns an existing checked-out view. - pub proof fn tracked_insert_initial_thread_view( - tracked &mut self, - tracked token: TaskThreadView, - ) + proof fn tracked_insert_initial_thread_view(tracked &mut self, tracked token: TaskThreadView) requires !old(self).contains(token.task()), + token.scheduler() == old(self).scheduler(), ensures + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(token.task(), token.view()), { - let tracked TaskThreadView { task, thread_view } = token; + let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; self.views.tracked_insert(task, thread_view); } + /// Registers a task with its unique initial weak-memory view. + /// + /// This is the linear counterpart of [`SchedulerView::register_task`]. It + /// mints the empty `ThreadView` internally, so callers cannot initialize a + /// task using a view detached from another scheduler entry. + proof fn tracked_register_task(tracked &mut self, sched_view: SchedulerView, task: Loc) + requires + old(self).wf(sched_view), + sched_view.wf(), + !sched_view.state.contains_key(task), + ensures + final(self).scheduler() == old(self).scheduler(), + final(self).view() == sched_view.register_task(task).stored_views, + final(self).wf(sched_view.register_task(task)), + final(self).contains(task), + final(self).thread_view(task) == WmView::empty(), + { + sched_view.lemma_register_task_preserves_wf(task); + let tracked thread_view = ThreadView::new(); + let tracked token = TaskThreadView::new(self.scheduler(), task, thread_view); + self.tracked_insert_initial_thread_view(token); + assert(final(self).view() == sched_view.register_task(task).stored_views); + assert(final(self).wf(sched_view.register_task(task))); + } + /// Checks out the current CPU's task view from the tracked owner. /// /// The proof-side owner loses this task entry and returns the linear token /// that a guard will carry until check-in. - pub proof fn tracked_take_current_thread_view( + proof fn tracked_take_current_thread_view( tracked &mut self, sched_view: SchedulerView, cpu: CpuId, @@ -468,7 +577,9 @@ impl SchedulerThreadViews { old(self).contains(sched_view.current[cpu]->0), ensures token.task() == sched_view.current[cpu]->0, + token.scheduler() == sched_view.id, token.view() == old(self).thread_view(sched_view.current[cpu]->0), + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), final(self).view() == sched_view.checkout_task_view(cpu).stored_views, final(self).wf(sched_view.checkout_task_view(cpu)), @@ -476,7 +587,11 @@ impl SchedulerThreadViews { { let task = sched_view.current[cpu]->0; let tracked thread_view = self.views.tracked_remove(task); - let tracked token = TaskThreadView { task, thread_view }; + let tracked token = TaskThreadView { + scheduler: Ghost(self.scheduler()), + task: Ghost(task), + thread_view, + }; let next = sched_view.checkout_task_view(cpu); assert(final(self).view() == next.stored_views); assert(final(self).wf(next)); @@ -486,7 +601,7 @@ impl SchedulerThreadViews { /// Checks out the current task's weak-memory view and starts its running /// context. - pub proof fn tracked_take_current_running_context( + proof fn tracked_take_current_running_context( tracked &mut self, sched_view: SchedulerView, cpu: CpuId, @@ -500,10 +615,12 @@ impl SchedulerThreadViews { old(self).contains(sched_view.current[cpu]->0), ensures context.task() == sched_view.current[cpu]->0, + context.scheduler() == sched_view.id, context.view() == old(self).thread_view(sched_view.current[cpu]->0), context.is_quiescent(), context.wf_scheduler(sched_view.checkout_task_view(cpu)), final(self).view() == sched_view.checkout_task_view(cpu).stored_views, + final(self).scheduler() == old(self).scheduler(), final(self).wf(sched_view.checkout_task_view(cpu)), { let ghost next = sched_view.checkout_task_view(cpu); @@ -517,7 +634,7 @@ impl SchedulerThreadViews { /// The `token.wf(sched_view)` precondition ties this write-back to the /// scheduler's checked-out partition, so the task id and view cannot be /// swapped with another task. - pub proof fn tracked_put_checked_out_thread_view( + proof fn tracked_put_checked_out_thread_view( tracked &mut self, sched_view: SchedulerView, tracked token: TaskThreadView, @@ -527,6 +644,7 @@ impl SchedulerThreadViews { token.wf(sched_view), !old(self).contains(token.task()), ensures + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(token.task(), token.view()), final(self).view() == sched_view.checkin_task_view( token.task(), @@ -535,7 +653,7 @@ impl SchedulerThreadViews { final(self).wf(sched_view.checkin_task_view(token.task(), token.view())), { let ghost view = token.view(); - let tracked TaskThreadView { task, thread_view } = token; + let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; self.views.tracked_insert(task, thread_view); let next = sched_view.checkin_task_view(task, view); assert(final(self).view() == next.stored_views); @@ -544,7 +662,7 @@ impl SchedulerThreadViews { /// Ends a quiescent running interval and checks its updated task view back /// into scheduler ownership. - pub proof fn tracked_put_running_context( + proof fn tracked_put_running_context( tracked &mut self, sched_view: SchedulerView, tracked context: RunningTaskContext, @@ -553,10 +671,12 @@ impl SchedulerThreadViews { old(self).wf(sched_view), sched_view.wf(), sched_view.task_view_is_checked_out(context.task()), + context.scheduler() == sched_view.id, context.wf(), context.is_quiescent(), !old(self).contains(context.task()), ensures + final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(context.task(), context.view()), final(self).view() == sched_view.update_checked_out_task_view( context.task(), @@ -579,6 +699,147 @@ impl SchedulerThreadViews { } } +/// Authoritative scheduler proof state. +/// +/// This linear root keeps the copyable scheduler snapshot and the owned +/// per-task `ThreadView` resources in one object. Its transitions update both +/// layers together, preventing a proof from changing `SchedulerView` without +/// moving the corresponding linear token (or vice versa). +pub tracked struct SchedulerGhostState { + identity: GhostMapAuth, + view: Ghost, + thread_views: SchedulerThreadViews, +} + +impl SchedulerGhostState { + /// Creates an empty authority for one scheduler instance. + pub proof fn new() -> (tracked res: Self) + ensures + res.wf(), + res.view() == SchedulerView::empty(res.id()), + { + let tracked (identity, _entries) = GhostMapAuth::new(Map::::empty()); + let ghost id = identity.id(); + SchedulerView::lemma_empty_wf(id); + let tracked thread_views = SchedulerThreadViews::empty(id); + let tracked res = SchedulerGhostState { + identity, + view: Ghost(SchedulerView::empty(id)), + thread_views, + }; + assert(res.wf()); + res + } + + pub closed spec fn view(self) -> SchedulerView { + self.view@ + } + + pub closed spec fn id(self) -> Loc { + self.identity.id() + } + + pub closed spec fn wf(self) -> bool { + &&& self.view().wf() + &&& self.view().id == self.id() + &&& self.thread_views.wf(self.view()) + } + + /// Applies a scheduler-policy transition without changing thread views. + /// + /// The caller proves the concrete runqueue/current/state update is a valid + /// `SchedulerView`. Equality of the ownership maps prevents this glue step + /// from minting a view, discarding observations, or moving a checked-out + /// token behind the linear root's back. + pub proof fn tracked_apply_scheduling_transition(tracked &mut self, next: SchedulerView) + requires + old(self).wf(), + next.wf(), + old(self).view().same_thread_view_ownership(next), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == next, + { + self.view = Ghost(next); + assert(self.thread_views.wf(self.view())); + assert(self.wf()); + } + + /// Registers a new task with one empty weak-memory view. + pub proof fn tracked_register_task(tracked &mut self, task: Loc) + requires + old(self).wf(), + !old(self).view().state.contains_key(task), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().register_task(task), + final(self).view().state[task] is New, + final(self).view().task_view_is_stored(task), + final(self).view().task_thread_view(task) == WmView::empty(), + { + let ghost old_view = self.view@; + self.thread_views.tracked_register_task(old_view, task); + self.view = Ghost(old_view.register_task(task)); + assert(self.wf()); + } + + /// Checks the current task's view out for one running interval. + pub proof fn tracked_schedule_in(tracked &mut self, cpu: CpuId) -> (tracked context: + RunningTaskContext) + requires + old(self).wf(), + old(self).view().current.contains_key(cpu), + old(self).view().current[cpu] is Some, + old(self).view().task_view_is_stored(old(self).view().current[cpu]->0), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().checkout_task_view(cpu), + context.task() == old(self).view().current[cpu]->0, + context.scheduler() == old(self).id(), + context.view() == old(self).view().task_thread_view(context.task()), + context.is_quiescent(), + context.wf_scheduler(final(self).view()), + { + let ghost old_view = self.view@; + let tracked context = self.thread_views.tracked_take_current_running_context(old_view, cpu); + self.view = Ghost(old_view.checkout_task_view(cpu)); + assert(self.wf()); + context + } + + /// Ends a quiescent running interval and stores its updated view. + pub proof fn tracked_schedule_out(tracked &mut self, tracked context: RunningTaskContext) + requires + old(self).wf(), + old(self).view().task_view_is_checked_out(context.task()), + context.scheduler() == old(self).id(), + context.wf(), + context.is_quiescent(), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().update_checked_out_task_view( + context.task(), + context.view(), + ).checkin_task_view(context.task(), context.view()), + final(self).view().task_view_is_stored(context.task()), + final(self).view().task_thread_view(context.task()) == context.view(), + !final(self).view().task_view_is_checked_out(context.task()), + { + let ghost old_view = self.view@; + let ghost next = old_view.update_checked_out_task_view( + context.task(), + context.view(), + ).checkin_task_view(context.task(), context.view()); + self.thread_views.tracked_put_running_context(old_view, context); + self.view = Ghost(next); + assert(self.wf()); + } +} + /// Logical identity of a runnable task handle. /// /// `RoArc` does not yet expose a proof-level view of its pointee. Keep this @@ -592,7 +853,12 @@ pub open spec fn valid_cpu(_cpu: CpuId) -> bool { pub open spec fn can_enqueue(view: SchedulerView, task: Loc, flags: EnqueueFlags) -> bool { match flags { - EnqueueFlags::Spawn => !view.state.contains_key(task) || view.state[task] is New, + EnqueueFlags::Spawn => { + &&& view.state.contains_key(task) + &&& view.state[task] is New + &&& view.task_view_is_stored(task) + &&& view.task_thread_view(task) == WmView::empty() + }, EnqueueFlags::Wake => view.state.contains_key(task) && !(view.state[task] is Exited), } } From 5170c403b0d72bae0369bbdf92f42f41426e325b Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 23 Jul 2026 03:09:16 -0400 Subject: [PATCH 26/47] Connect weak-memory RCU proofs --- ostd/specs/mm/cpu.rs | 151 +++++++- ostd/specs/sync/rcu.rs | 644 ++++++++++++++++++++++++++++++--- ostd/specs/sync/weak_memory.rs | 249 ++++++++++++- ostd/src/sync/rcu/mod.rs | 210 +++++++++-- ostd/src/sync/rcu/monitor.rs | 532 +++++++++++++++++++++++++-- ostd/src/task/preempt/guard.rs | 27 +- ostd/src/task/scheduler/mod.rs | 230 +++++++++++- 7 files changed, 1882 insertions(+), 161 deletions(-) diff --git a/ostd/specs/mm/cpu.rs b/ostd/specs/mm/cpu.rs index 9398a1b56..ac5c0634c 100644 --- a/ostd/specs/mm/cpu.rs +++ b/ostd/specs/mm/cpu.rs @@ -1,26 +1,134 @@ use core::sync::atomic::Ordering; use vstd::prelude::*; +use vstd::resource::Loc; +use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; + +use crate::task::RunningTaskContext; verus! { pub struct CpuId(u32); +/// The fixed set of CPUs participating in system-wide protocols. +/// +/// OSTD does not support CPU hotplug, so the executable `num_cpus()` value is +/// stable after boot. Keeping that set abstract here avoids exposing the +/// executable bitset representation to clients of this specification. +pub uninterp spec fn online_cpus() -> Set; + impl CpuId { #[verifier::external_body] - pub fn current() -> Self { + pub fn current(Tracked(context): Tracked<&RunningTaskContext>) -> (res: Self) + requires + context.wf(), + ensures + res == context.cpu(), + online_cpus().contains(res), + { unimplemented!() } } -pub struct AtomicCpuSet; +/// Linear shadow state for an [`AtomicCpuSet`]. +/// +/// The token is deliberately separate from the executable bitset. Clients +/// that serialize all operations (the RCU monitor does so with its state +/// lock) move this token through `store`, `add`, and `load`, obtaining an exact +/// logical view despite the executable type using atomic words internally. +pub tracked struct AtomicCpuSetToken { + identity: GhostVarAuth<()>, + ghost cpus: Set, +} + +impl AtomicCpuSetToken { + pub closed spec fn id(self) -> Loc { + self.identity.id() + } + + pub closed spec fn cpus(self) -> Set { + self.cpus + } + + pub closed spec fn wf(self) -> bool { + self.cpus().subset_of(online_cpus()) + } +} + +pub struct AtomicCpuSet { + tracked_identity: Tracked>>, + tracked_peer: Tracked>, + ghost_initial: Ghost>, +} impl AtomicCpuSet { - #[verifier::external_body] - pub fn new(_initial: CpuSet) -> Self { - unimplemented!() + pub closed spec fn id(self) -> Loc { + self.tracked_peer@.id() + } + + pub closed spec fn token_available(self) -> bool { + self.tracked_identity@ is Some + } + + pub closed spec fn initial_cpus(self) -> Set { + self.ghost_initial@ + } + + pub closed spec fn wf(self) -> bool { + &&& self.initial_cpus().subset_of(online_cpus()) + &&& self.token_available() ==> self.tracked_identity@->Some_0.id() == self.id() + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } + + pub fn new(_initial: CpuSet) -> (res: Self) + requires + _initial.cpus.subset_of(online_cpus()), + ensures + res.token_available(), + res.initial_cpus() == _initial.cpus, + res.wf(), + { + let tracked (identity, peer) = GhostVarAuth::new(()); + Self { + tracked_identity: Tracked(Some(identity)), + tracked_peer: Tracked(peer), + ghost_initial: Ghost(_initial.cpus), + } + } + + /// Extracts the unique logical state used to verify serialized operations + /// on this atomic set. + pub proof fn tracked_take_token(tracked &mut self) -> (tracked res: AtomicCpuSetToken) + requires + old(self).token_available(), + ensures + !final(self).token_available(), + final(self).id() == old(self).id(), + final(self).initial_cpus() == old(self).initial_cpus(), + res.id() == final(self).id(), + res.cpus() == final(self).initial_cpus(), + res.wf(), + { + use_type_invariant(&*self); + let tracked identity = self.tracked_identity.borrow_mut().tracked_take(); + let tracked res = AtomicCpuSetToken { identity, cpus: self.ghost_initial@ }; + assert(res.id() == self.id()); + res } + #[verus_spec(res => + with + Tracked(token): Tracked<&AtomicCpuSetToken>, + requires + token.id() == self.id(), + token.wf(), + ensures + res.cpus == token.cpus(), + )] #[verifier::external_body] pub fn load(&self, _ordering: Ordering) -> CpuSet no_unwind @@ -28,14 +136,43 @@ impl AtomicCpuSet { unimplemented!() } + #[verus_spec( + with + Tracked(token): Tracked<&mut AtomicCpuSetToken>, + requires + old(token).id() == self.id(), + _value.cpus.subset_of(online_cpus()), + ensures + final(token).id() == old(token).id(), + final(token).cpus() == _value.cpus, + final(token).wf(), + )] pub fn store(&self, _value: &CpuSet, _ordering: Ordering) no_unwind { + proof { + token.cpus = _value.cpus; + } } + #[verus_spec( + with + Tracked(token): Tracked<&mut AtomicCpuSetToken>, + requires + old(token).id() == self.id(), + old(token).wf(), + online_cpus().contains(_cpu), + ensures + final(token).id() == old(token).id(), + final(token).cpus() == old(token).cpus().insert(_cpu), + final(token).wf(), + )] pub fn add(&self, _cpu: CpuId, _ordering: Ordering) no_unwind { + proof { + token.cpus = token.cpus.insert(_cpu); + } } } @@ -59,7 +196,9 @@ impl CpuSet { } #[verifier::external_body] - pub fn is_full(&self) -> bool + pub fn is_full(&self) -> (res: bool) + ensures + res == (self.cpus == online_cpus()), no_unwind { unimplemented!() diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index bdfcdd200..c1ecac57e 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -15,12 +15,17 @@ //! `BlockInfo` and the guard's `address -> AId` protection map. This distinction //! is required to handle stale weak-memory messages after address reuse. //! -//! The module remains proof-only. The executable RCU must still connect its -//! preemption guard to the same domain's reader token and route the retire -//! permission released by pointer replacement into the callback monitor. +//! The module remains proof-only. Pointer replacement turns the detached +//! traversal-retire permission and its root-history removal observation into +//! the callback monitor's erased safety certificate. The remaining executable +//! connection is the paper's `Guard-seen-retired` rule: reader guards must +//! retain persistent removal observations for their start snapshot so a +//! readable atomic timestamp can be proved inconsistent with an already +//! observed removal. Grace-period synchronization must then make those +//! observations available to later readers before physical reclamation. use core::marker::PhantomData; -use super::weak_memory::{History, Msg, WeakAtomicInvariantPredicate, WmView}; +use super::weak_memory::{History, Msg, Timestamp, WeakAtomicInvariantPredicate, WmView}; use vstd::prelude::*; use vstd::resource::Loc; use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; @@ -37,14 +42,36 @@ pub type LinkEdge = (nat, LinkIndex); /// proof only needs to know which logical object it will reclaim and which /// grace-period generation retired that object. `domain` identifies the RCU /// protection domain, and `obj` identifies the reclaimed allocation/object -/// inside that domain. +/// inside that domain. `retire_view` is the retiring task's weak-memory view +/// after unlink and before the callback is enqueued; completion must eventually +/// prove that every CPU report has advanced beyond this view. pub ghost struct RcuCallbackSummary { /// The RCU protection domain whose grace period governs this callback. pub domain: Loc, /// Logical identity of the retired object inside `domain`. pub obj: nat, + /// Root-atomic removal observation retained from `Retired(a, Q)`. + pub removal: RcuRemovalObservation, /// The domain-local epoch in which `obj` was retired. pub retire_epoch: nat, + /// Weak-memory observations that must precede safe reclamation. + pub retire_view: WmView, +} + +/// The paper's detachment observation `Q` for a root publication. +/// +/// A view observes this fact once it has advanced to at least `timestamp` in +/// the root atomic's modification history. The message at `timestamp` is the +/// first publication after the retired object ceased to be the root. +pub ghost struct RcuRemovalObservation { + pub root: Loc, + pub timestamp: nat, +} + +impl RcuRemovalObservation { + pub open spec fn observed_by(self, view: WmView) -> bool { + self.timestamp <= view.seen_at(self.root) + } } /// Logical identity attached to one non-null publication in an RCU root. @@ -274,6 +301,13 @@ impl RcuRootGhost { res.1 is Some ==> res.1->Some_0.0.domain() == res.0.domain(), res.1 is Some ==> res.0.publications()[0] == Some(res.1->Some_0.0.obj()), res.1 is Some ==> res.1->Some_0.0.wf(), + match res.1 { + Some(registration) => res.0.objects() == Map::empty().insert( + registration.0.obj(), + ptr.addr(), + ), + None => res.0.objects() == Map::empty(), + }, current_registration_matches(res.0, res.1), { let tracked mut domain = RcuDomainAuth::tracked_new(); @@ -311,9 +345,26 @@ impl RcuRootGhost { final(self).domain_auth().retire_registry() == old( self, ).domain_auth().retire_registry(), + final(self).domain_auth().reader_registry() == old( + self, + ).domain_auth().reader_registry(), (res is Some) == (msg.value.addr() != 0), res is Some ==> res->Some_0.0.ptr() == msg.value, res is Some ==> res->Some_0.0.obj() == res->Some_0.1.obj(), + res is Some ==> !old(self).objects().contains_key(res->Some_0.0.obj()), + final(self).publications() == old(self).publications().push( + match res { + Some(registration) => Some(registration.0.obj()), + None => None, + }, + ), + match res { + Some(registration) => final(self).objects() == old(self).objects().insert( + registration.0.obj(), + msg.value.addr(), + ), + None => final(self).objects() == old(self).objects(), + }, current_registration_matches(*final(self), res), { let ghost ts = prev.len(); @@ -380,6 +431,9 @@ impl RcuRootGhost { final(self).domain_auth().retire_registry() == old( self, ).domain_auth().retire_registry(), + final(self).domain_auth().reader_registry() == old( + self, + ).domain_auth().reader_registry(), final(self).objects() == old(self).objects(), final(self).publications() == old(self).publications().push(Some(info.obj())), { @@ -460,6 +514,17 @@ impl WeakAtomicInvariantPredicate for RcuWeakAtom } } +/// Immutable identity carried by an executable RCU root atomic. +/// +/// Besides nullability, the key records the two resource locations needed to +/// associate read-side guard tokens with the same root invariant after the +/// invariant has been closed. +pub ghost struct RcuRootKey { + pub nullable: bool, + pub domain: Loc, + pub reader_registry: Loc, +} + /// Typed ownership state paired with one executable RCU root atomic. /// /// `root` owns the append-only publication registry. `current` owns the unique @@ -470,6 +535,8 @@ impl WeakAtomicInvariantPredicate for RcuWeakAtom pub tracked struct RcuRootOwnedGhost { root: RcuRootGhost, current: Option>, + infos: Map>, + ghost removals: Map, } impl RcuRootOwnedGhost { @@ -485,6 +552,10 @@ impl RcuRootOwnedGhost { self.root().publications() } + pub closed spec fn reader_registry(self) -> Loc { + self.root().domain_auth().reader_registry() + } + pub open spec fn published_at(self, ts: nat) -> Option recommends ts < self.publications().len(), @@ -510,10 +581,169 @@ impl RcuRootOwnedGhost { } } + pub closed spec fn infos(self) -> Map> { + self.infos + } + + pub closed spec fn removals(self) -> Map { + self.removals + } + pub open spec fn ownership_wf(self) -> bool { current_registration_matches(self.root(), self.current_registration()) } + /// Every registered allocation retains a persistent typed identity token. + /// + /// Entries are append-only. Retiring an object moves its unique ownership + /// and retire permission out of `current`, but leaves this persistent + /// `BlockInfo` available to justify stale weak-memory history reads. + pub open spec fn infos_wf(self) -> bool { + &&& self.infos().dom() == self.root().objects().dom() + &&& forall|obj: nat| + self.infos().contains_key(obj) ==> { + let info = #[trigger] self.infos()[obj]; + &&& info.wf() + &&& info.domain() == self.domain() + &&& info.obj() == obj + &&& self.root().objects().contains_pair(obj, info.addr()) + } + } + + /// Root-history interpretation of the paper's detachment observations. + /// + /// Once `removals[obj] = ts`, no message at or after `ts` may publish that + /// allocation ID again. The currently owned registration is therefore + /// never in the removed domain. + pub open spec fn removals_wf(self, history: History<*mut T>) -> bool { + &&& self.removals().dom().subset_of(self.infos().dom()) + &&& match self.current_registration() { + Some(registration) => !self.removals().contains_key(registration.0.obj()), + None => true, + } + &&& forall|obj: nat| + self.removals().contains_key(obj) ==> { + let ts = #[trigger] self.removals()[obj]; + &&& 0 < ts < history.len() + &&& forall|i: int| + ts <= i < history.len() ==> #[trigger] self.publications()[i] != Some(obj) + } + } + + /// Copies the persistent identity corresponding to one published message. + pub proof fn tracked_info_for(tracked &self, object: RcuPublishedObject) -> (tracked res: + RcuBlockInfo) + requires + self.infos_wf(), + object.domain == self.domain(), + self.root().objects().contains_pair(object.obj, object.addr), + ensures + res.wf(), + res.domain() == object.domain, + res.obj() == object.obj, + res.addr() == object.addr, + equal(res.ptr(), self.infos()[object.obj].ptr()), + { + let tracked info = self.infos.tracked_borrow(object.obj); + info.tracked_duplicate() + } + + /// Resolves one atomic-history timestamp to its persistent typed identity. + /// + /// This is the proof interface used by weak atomic loads. It keeps the + /// root's internal publication and identity maps opaque to the atomic + /// wrapper while exporting exact pointer provenance, not just an address. + pub proof fn tracked_info_at(tracked &self, history: History<*mut T>, ts: nat) -> (tracked res: + Option>) + requires + rcu_owned_root_history_inv(history, *self), + ts < history.len(), + ensures + ts < self.publications().len(), + match (self.published_at(ts), res) { + (None, None) => history[ts as int].value.addr() == 0, + (Some(object), Some(info)) => { + &&& object.domain == self.domain() + &&& object.addr == history[ts as int].value.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), history[ts as int].value) + }, + _ => false, + }, + { + assert(ts < self.publications().len()); + match self.publications()[ts as int] { + Some(obj) => { + let ghost object = RcuPublishedObject { + domain: self.domain(), + obj, + addr: self.root().objects()[obj], + }; + assert(self.published_at(ts) == Some(object)); + let tracked info = self.tracked_info_for(object); + assert(equal(info.ptr(), history[ts as int].value)); + Some(info) + }, + None => { + assert(self.published_at(ts) is None); + None + }, + } + } + + /// Registers and starts one fresh paper reader while preserving root state. + /// + /// Reader slots are proof-only and currently allocated per critical + /// section. `tracked_stop_reader` consumes the live slot again; no runtime + /// reader counter is introduced. + pub proof fn tracked_start_reader( + tracked &mut self, + history: History<*mut T>, + ) -> (tracked res: RcuBaseGuard) + requires + rcu_owned_root_history_inv(history, *old(self)), + ensures + rcu_owned_root_history_inv(history, *final(self)), + final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).current_owned() == old(self).current_owned(), + res.wf(), + res.domain() == final(self).domain(), + res.reader_registry() == final(self).reader_registry(), + { + let tracked inactive = self.root.domain.tracked_register_reader(); + let tracked guard = self.root.domain.tracked_guard_start(inactive); + assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.infos_wf()); + guard + } + + /// Ends a reader started by `tracked_start_reader`. + pub proof fn tracked_stop_reader( + tracked &mut self, + history: History<*mut T>, + tracked guard: RcuBaseGuard, + ) + requires + rcu_owned_root_history_inv(history, *old(self)), + guard.wf(), + guard.domain() == old(self).domain(), + guard.reader_registry() == old(self).reader_registry(), + ensures + rcu_owned_root_history_inv(history, *final(self)), + final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).current_owned() == old(self).current_owned(), + { + assert(guard.belongs_to(self.root.domain)); + let tracked _inactive = self.root.domain.tracked_guard_stop(guard); + assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.infos_wf()); + } + /// Initializes root history and retains the initial registration as the /// current unique ownership resource. pub proof fn tracked_initial(ptr: *mut T, tracked ownership: Option) -> (tracked res: Self) @@ -538,13 +768,38 @@ impl RcuRootOwnedGhost { }, { let tracked (root, registration) = RcuRootGhost::tracked_initial(ptr); + let tracked mut infos = Map::>::tracked_empty(); let tracked current = match registration { Some(registration) => { + let ghost obj = registration.0.obj(); + let tracked info = registration.0.tracked_duplicate(); + infos.tracked_insert(obj, info); + assert(infos.dom() == root.objects().dom()); + assert forall|registered: nat| infos.contains_key(registered) implies { + let saved = #[trigger] infos[registered]; + &&& saved.wf() + &&& saved.domain() == root.domain() + &&& saved.obj() == registered + &&& root.objects().contains_pair(registered, saved.addr()) + } by { + assert(registered == obj); + }; Some(RcuOwnedObject { registration, ownership: ownership.tracked_unwrap() }) }, - None => None, + None => { + assert(infos.dom() == root.objects().dom()); + None + }, }; - RcuRootOwnedGhost { root, current } + let tracked res = RcuRootOwnedGhost { + root, + current, + infos, + removals: Map::empty(), + }; + assert(res.infos_wf()); + assert(res.removals_wf(seq![Msg { value: ptr, view: WmView::empty() }])); + res } /// Publishes a fresh allocation and retires the previously current root. @@ -558,6 +813,7 @@ impl RcuRootOwnedGhost { prev: History<*mut T>, next: History<*mut T>, msg: Msg<*mut T>, + root: Loc, tracked ownership: Option, ) -> (tracked detached: Option>) where OwnPred: RcuRootOwnershipPredicate, @@ -570,6 +826,7 @@ impl RcuRootOwnedGhost { ensures rcu_owned_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), match detached { Some(detached) => { &&& old(self).current_registration() is Some @@ -577,6 +834,8 @@ impl RcuRootOwnedGhost { &&& detached.retired().domain() == detached.domain() &&& detached.retired().obj() == detached.obj() &&& detached.retired().ptr() == detached.ptr() + &&& detached.retired().removal() + == (RcuRemovalObservation { root, timestamp: prev.len() }) &&& old(self).current_ownership() == Some(detached.ownership()) &&& equal(detached.ptr(), prev[(prev.len() - 1) as int].value) &&& OwnPred::owns(detached.ptr(), detached.ownership()) @@ -600,6 +859,10 @@ impl RcuRootOwnedGhost { }, { assert(current_registration_matches(self.root(), self.current_registration())); + let ghost removed_obj = match self.current_registration() { + Some(registration) => Some(registration.0.obj()), + None => None, + }; let tracked old_current = if self.current is Some { Some(self.current.tracked_take()) } else { @@ -608,9 +871,32 @@ impl RcuRootOwnedGhost { let tracked new_registration = self.root.tracked_push_fresh(prev, next, msg); let tracked new_current = match new_registration { Some(registration) => { + let ghost obj = registration.0.obj(); + let tracked info = registration.0.tracked_duplicate(); + self.infos.tracked_insert(obj, info); + assert(self.infos.dom() == self.root.objects().dom()); + assert forall|registered: nat| self.infos.contains_key(registered) implies { + let saved = #[trigger] self.infos[registered]; + &&& saved.wf() + &&& saved.domain() == self.root.domain() + &&& saved.obj() == registered + &&& self.root.objects().contains_pair(registered, saved.addr()) + } by { + if registered != obj { + assert(old(self).infos().contains_key(registered)); + assert(self.infos[registered] == old(self).infos()[registered]); + assert(old(self).root().objects().contains_pair( + registered, + self.infos[registered].addr(), + )); + } + }; Some(RcuOwnedObject { registration, ownership: ownership.tracked_unwrap() }) }, - None => None, + None => { + assert(self.infos_wf()); + None + }, }; let tracked detached = match old_current { Some(owned) => { @@ -622,13 +908,61 @@ impl RcuRootOwnedGhost { link_view: RcuLinkView::empty(), }; let tracked retire = lift_retire_perm(base, seen_removed); - let tracked retired = self.root.domain.tracked_retire(retire); + let ghost removal = RcuRemovalObservation { root, timestamp: prev.len() }; + let tracked retired = self.root.domain.tracked_retire(retire, removal); Some(RcuRetiredOwnedObject { object, retired, ownership: old_ownership }) }, None => None, }; self.current = new_current; + self.removals = match removed_obj { + Some(obj) => self.removals.insert(obj, prev.len()), + None => self.removals, + }; assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.infos_wf()); + assert(self.removals_wf(next)) by { + assert forall|obj: nat| self.removals().contains_key(obj) implies { + let ts = #[trigger] self.removals()[obj]; + &&& 0 < ts < next.len() + &&& forall|i: int| + ts <= i < next.len() ==> #[trigger] self.publications()[i] != Some(obj) + } by { + if removed_obj == Some(obj) { + assert(self.removals()[obj] == prev.len()); + assert(next.len() == prev.len() + 1); + assert(self.publications()[prev.len() as int] + == match new_registration { + Some(registration) => Some(registration.0.obj()), + None => None, + }); + if new_registration is Some { + assert(!old(self).root().objects().contains_key( + new_registration->Some_0.0.obj(), + )); + assert(old(self).infos().contains_key(obj)); + } + } else { + assert(old(self).removals().contains_key(obj)); + assert(self.removals()[obj] == old(self).removals()[obj]); + assert forall|i: int| + self.removals()[obj] <= i < next.len() implies + #[trigger] self.publications()[i] != Some(obj) by { + if i < prev.len() { + assert(self.publications()[i] == old(self).publications()[i]); + } else { + assert(i == prev.len()); + if new_registration is Some { + assert(!old(self).root().objects().contains_key( + new_registration->Some_0.0.obj(), + )); + assert(old(self).infos().contains_key(obj)); + } + } + }; + } + }; + } detached } @@ -648,12 +982,34 @@ impl RcuRootOwnedGhost { ensures rcu_owned_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), + final(self).reader_registry() == old(self).reader_registry(), final(self).current_registration() == old(self).current_registration(), { let tracked owned = self.current.tracked_take(); self.root.tracked_push_registered(prev, next, msg, &owned.registration.0); self.current = Some(owned); assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.removals_wf(next)) by { + assert forall|obj: nat| self.removals().contains_key(obj) implies { + let ts = #[trigger] self.removals()[obj]; + &&& 0 < ts < next.len() + &&& forall|i: int| + ts <= i < next.len() ==> #[trigger] self.publications()[i] != Some(obj) + } by { + assert(old(self).removals().contains_key(obj)); + assert(!old(self).removals().contains_key(owned.registration.0.obj())); + assert(obj != owned.registration.0.obj()); + assert forall|i: int| + self.removals()[obj] <= i < next.len() implies + #[trigger] self.publications()[i] != Some(obj) by { + if i < prev.len() { + assert(self.publications()[i] == old(self).publications()[i]); + } else { + assert(i == prev.len()); + } + }; + }; + } } } @@ -665,6 +1021,15 @@ pub open spec fn rcu_owned_root_history_inv( ) -> bool { &&& rcu_root_history_inv(history, ghost.root()) &&& ghost.ownership_wf() + &&& ghost.infos_wf() + &&& ghost.removals_wf(history) + &&& forall|i: int| + 0 <= i < history.len() ==> { + match #[trigger] ghost.publications()[i] { + Some(obj) => equal(ghost.infos()[obj].ptr(), history[i].value), + None => true, + } + } &&& match ghost.current_registration() { Some(registration) => equal( registration.0.ptr(), @@ -729,16 +1094,18 @@ pub struct RcuOwnedWeakAtomicInv { } impl WeakAtomicInvariantPredicate< - bool, + RcuRootKey, *mut T, RcuRootOwnedGhost, > for RcuOwnedWeakAtomicInv where OwnPred: RcuRootOwnershipPredicate { open spec fn atomic_inv( - nullable: bool, + key: RcuRootKey, history: History<*mut T>, g: RcuRootOwnedGhost, ) -> bool { - &&& rcu_history_inv(nullable, history) + &&& key.domain == g.domain() + &&& key.reader_registry == g.reader_registry() + &&& rcu_history_inv(key.nullable, history) &&& rcu_owned_root_history_inv(history, g) &&& rcu_current_ownership_inv::(g) } @@ -1367,10 +1734,12 @@ impl RcuDomainAuth { pub proof fn tracked_retire( tracked &mut self, tracked retire: RcuRetirePerm, + removal: RcuRemovalObservation, ) -> (tracked res: RcuRetired) requires old(self).wf(), retire.belongs_to(*old(self)), + retire.wf(), retire.ready_to_retire(), ensures final(self).wf(), @@ -1383,13 +1752,19 @@ impl RcuDomainAuth { res.domain() == final(self).id(), res.obj() == retire.obj(), res.ptr() == retire.ptr(), + res.removal() == removal, + res.wf(), { + let ghost domain = retire.domain(); + let ghost obj = retire.obj(); + let ghost ptr = retire.ptr(); retire.base.perm.agree(&self.retire_perms); - assert(self.objects().contains_key(retire.obj())); - self.retired = self.retired.insert(retire.obj()); + assert(self.objects().contains_key(obj)); + self.retired = self.retired.insert(obj); assert(self.retired().subset_of(self.objects().dom())); assert(self.expired().subset_of(self.retired())); - RcuRetired { retire } + let tracked fact = retire.base.perm.persist(); + RcuRetired { fact: RcuRetiredFact { domain, fact, removal }, ptr } } } @@ -1438,6 +1813,10 @@ impl RcuBaseGuard { self.state.key() } + pub closed spec fn reader_registry(self) -> Loc { + self.state.id() + } + pub closed spec fn expired(self) -> Set { self.expired } @@ -1620,6 +1999,7 @@ pub proof fn owned_root_replacement_retires_previous_registration( initial, next_history, next_msg, + root.domain(), Some(()), ); let tracked detached = detached.tracked_unwrap(); @@ -1697,6 +2077,10 @@ impl RcuRetirePerm { self.seen_removed } + pub open spec fn wf(self) -> bool { + self.base().wf() + } + /// The traversal layer has established that this object may be retired. /// Reclamation still requires a completed base-RCU grace period. pub open spec fn ready_to_retire(self) -> bool { @@ -1715,6 +2099,7 @@ pub proof fn lift_retire_perm( seen_removed: RcuSeenRemoved, ) -> (tracked perm: RcuRetirePerm) requires + base.wf(), seen_removed.removed.contains(base.obj()), ensures perm.base() == base, @@ -1722,30 +2107,111 @@ pub proof fn lift_retire_perm( perm.ptr() == base.ptr(), perm.obj() == base.obj(), perm.seen_removed() == seen_removed, + perm.wf(), perm.ready_to_retire(), { RcuRetirePerm { base, seen_removed } } -/// Objective record that an allocation has passed the base `rcu-retire` -/// transition. It is safe to enqueue its callback, but not yet safe to execute -/// it; execution additionally needs monitor grace-period completion. +/// Persistent, type-erased evidence that one allocation passed the base +/// `rcu-retire` transition. +/// +/// The points-to fact comes from consuming the allocation's unique +/// `BaseRetirePerm`. Its fields are private, so clients cannot manufacture a +/// retirement fact from a `(domain, obj)` pair. The fact remains duplicable +/// after callback type erasure and can therefore be retained in the final +/// reclaim permit. +pub tracked struct RcuRetiredFact { + ghost domain: Loc, + fact: GhostPersistentPointsTo, + ghost removal: RcuRemovalObservation, +} + +impl RcuRetiredFact { + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn obj(self) -> nat { + self.fact.key() + } + + pub closed spec fn addr(self) -> usize { + self.fact.value() + } + + pub closed spec fn removal(self) -> RcuRemovalObservation { + self.removal + } + + pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { + &&& summary.domain == self.domain() + &&& summary.obj == self.obj() + &&& summary.removal == self.removal() + } + + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.domain() == self.domain(), + res.obj() == self.obj(), + res.addr() == self.addr(), + res.removal() == self.removal(), + { + let tracked fact = self.fact.duplicate(); + RcuRetiredFact { domain: self.domain, fact, removal: self.removal } + } +} + +/// Typed objective record that an allocation has passed the base +/// `rcu-retire` transition. It is safe to enqueue its callback, but not yet +/// safe to execute it; execution additionally needs monitor grace-period +/// completion. #[verifier::reject_recursive_types(T)] pub tracked struct RcuRetired { - retire: RcuRetirePerm, + fact: RcuRetiredFact, + ghost ptr: *mut T, } impl RcuRetired { pub closed spec fn domain(self) -> Loc { - self.retire.domain() + self.fact.domain() } pub closed spec fn obj(self) -> nat { - self.retire.obj() + self.fact.obj() } pub closed spec fn ptr(self) -> *mut T { - self.retire.ptr() + self.ptr + } + + pub closed spec fn addr(self) -> usize { + self.fact.addr() + } + + pub closed spec fn removal(self) -> RcuRemovalObservation { + self.fact.removal() + } + + pub open spec fn wf(self) -> bool { + self.addr() == self.ptr().addr() + } + + proof fn tracked_into_fact(tracked self) -> (tracked res: RcuRetiredFact) + requires + self.wf(), + ensures + res.domain() == self.domain(), + res.obj() == self.obj(), + res.addr() == self.ptr().addr(), + res.removal() == self.removal(), + { + self.fact + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() } } @@ -1773,7 +2239,8 @@ pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> link_view: RcuLinkView::empty(), }; let tracked retire = lift_retire_perm(base, seen_removed); - let tracked _retired = domain.tracked_retire(retire); + let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 0 }; + let tracked _retired = domain.tracked_retire(retire, removal); let tracked inactive = domain.tracked_register_reader(); let tracked mut guard = domain.tracked_guard_start(inactive); @@ -1788,24 +2255,53 @@ pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> /// A certificate can only be produced from a typed traversal retire permission, /// but after that point the monitor only needs the erased callback summary. pub tracked struct RcuCallbackSafety { - ghost domain: Loc, - ghost obj: nat, + retired: RcuRetiredFact, } impl RcuCallbackSafety { pub closed spec fn domain(self) -> Loc { - self.domain + self.retired.domain() } pub closed spec fn obj(self) -> nat { - self.obj + self.retired.obj() + } + + pub closed spec fn removal(self) -> RcuRemovalObservation { + self.retired.removal() } /// The monitor may assign any future batch generation, but it cannot /// change the retired object's domain or allocation identity. - pub open spec fn matches(self, summary: RcuCallbackSummary) -> bool { - &&& summary.domain == self.domain() - &&& summary.obj == self.obj() + pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { + self.retired.matches(summary) + } + + /// Establishes the abstract matching predicate without exposing the + /// persistent retirement resource to callback-monitor clients. + pub proof fn lemma_matches(tracked &self, summary: RcuCallbackSummary) + requires + summary.domain == self.domain(), + summary.obj == self.obj(), + summary.removal == self.removal(), + ensures + self.matches(summary), + { + } + + /// Duplicates the persistent base-retirement fact for an object-level + /// reclaim permit. + pub proof fn tracked_retired_fact(tracked &self, summary: RcuCallbackSummary) -> (tracked res: + RcuRetiredFact) + requires + self.matches(summary), + ensures + res.domain() == self.domain(), + res.obj() == self.obj(), + res.removal() == self.removal(), + res.matches(summary), + { + self.retired.tracked_duplicate() } } @@ -1830,9 +2326,12 @@ pub proof fn certify_callback_from_retired( ensures cert.domain() == retired.domain(), cert.obj() == object.obj(), + cert.removal() == retired.removal(), callback_safety_from_traversal(cert, *object), { - RcuCallbackSafety { domain: retired.domain(), obj: object.obj() } + use_type_invariant(&retired); + let tracked fact = retired.tracked_into_fact(); + RcuCallbackSafety { retired: fact } } /// Read-side guard token for one critical section. @@ -1854,6 +2353,10 @@ impl RcuReadGuardToken { self.base.tid() } + pub closed spec fn reader_registry(self) -> Loc { + self.base.reader_registry() + } + pub closed spec fn expired(self) -> Set { self.base.expired() } @@ -1887,11 +2390,23 @@ impl RcuReadGuardToken { self.base.belongs_to(domain) } - pub open spec fn can_protect(self, info: RcuBlockInfo) -> bool { + /// Preconditions of the paper's base `Guard-protect` rule. + /// + /// This deliberately says only that the allocation was not already + /// expired when the critical section started. Whether a stale traversal + /// may still reach the allocation is proved separately from + /// `SeenRemoved` and the link history. + pub open spec fn can_base_protect(self, info: RcuBlockInfo) -> bool { &&& self.wf() &&& info.wf() &&& info.domain() == self.domain() &&& !self.expired().contains(info.obj()) + } + + /// A base-protectable allocation that traversal has also proved is not in + /// the guard's observed removed set. + pub open spec fn can_protect(self, info: RcuBlockInfo) -> bool { + &&& self.can_base_protect(info) &&& !self.seen_removed().removed.contains(info.obj()) } @@ -1907,11 +2422,53 @@ impl RcuReadGuardToken { res.wf(), res.domain() == base.domain(), res.tid() == base.tid(), + res.reader_registry() == base.reader_registry(), + res.expired() == base.expired(), + res.protected() == base.protected(), res.seen_removed() == seen_removed, { RcuReadGuardToken { base, seen_removed } } + /// Lift a base guard using its start-time expired set as the initial + /// traversal observation. + pub proof fn tracked_from_base( + tracked base: RcuBaseGuard, + ) -> (tracked res: Self) + requires + base.wf(), + ensures + res.wf(), + res.domain() == base.domain(), + res.tid() == base.tid(), + res.reader_registry() == base.reader_registry(), + res.expired() == base.expired(), + res.seen_removed().removed == base.expired(), + res.link_view() == RcuLinkView::::empty(), + { + let ghost seen_removed = RcuSeenRemoved { + removed: base.expired(), + link_view: RcuLinkView::empty(), + }; + RcuReadGuardToken::tracked_new(base, seen_removed) + } + + /// Consume the traversal wrapper when ending the read-side critical + /// section. + pub proof fn tracked_into_base(tracked self) -> (tracked res: RcuBaseGuard) + requires + self.wf(), + ensures + res.wf(), + res.domain() == self.domain(), + res.tid() == self.tid(), + res.reader_registry() == self.reader_registry(), + res.expired() == self.expired(), + res.protected() == self.protected(), + { + self.base + } + /// Records one successful base `Guard-protect` operation in `G`. pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) requires @@ -2023,21 +2580,19 @@ pub trait RcuTraversalSafety: Sized { /// Protect a freshly acquired root pointer. pub proof fn protect_root( - tracked domain: &RcuDomainAuth, tracked guard: &mut RcuReadGuardToken, tracked info: &RcuBlockInfo, p: *mut S::Node, g: S::Ghost, ) -> (tracked root: RcuProtectedPtr) requires - old(guard).is_for(*domain), old(guard).can_protect(*info), info.ptr() == p, S::root_inv(p, info.obj(), g), ensures root.ptr() == p, root.obj() == info.obj(), - root.domain() == domain.id(), + root.domain() == old(guard).domain(), root.protected_by(*final(guard)), final(guard).wf(), final(guard).domain() == old(guard).domain(), @@ -2048,7 +2603,7 @@ pub proof fn protect_root( S::root_is_node_inv(p, info.obj(), g); guard.tracked_protect(info); RcuProtectedPtr { - domain: domain.id(), + domain: guard.domain(), obj: info.obj(), ptr: p, seen_removed: guard.seen_removed(), @@ -2066,7 +2621,7 @@ pub proof fn protect_link( ) -> (tracked to_protected: RcuProtectedPtr) requires from.protected_by(*old(guard)), - old(guard).can_protect(*to_info), + old(guard).can_base_protect(*to_info), to_info.ptr() == to, S::node_inv(from.ptr(), from.obj(), g), S::link_inv(from.ptr(), from.obj(), n, to, to_info.obj(), g), @@ -2092,6 +2647,7 @@ pub proof fn protect_link( guard.seen_removed(), g, ); + assert(old(guard).can_protect(*to_info)); guard.tracked_protect(to_info); RcuProtectedPtr { domain: from.domain(), @@ -2198,7 +2754,6 @@ impl RcuTraversalSafety for LinkedListTraversalSpec { /// Example: after protecting the root, following a non-stale successor-history /// event protects the next node under the same guard. pub proof fn linked_list_protect_next_example( - tracked domain: &RcuDomainAuth, tracked guard: &mut RcuReadGuardToken, tracked root_info: &RcuBlockInfo, tracked next_info: &RcuBlockInfo, @@ -2208,9 +2763,8 @@ pub proof fn linked_list_protect_next_example( g: LinkedListGhost, ) -> (tracked next_protected: RcuProtectedPtr) requires - old(guard).is_for(*domain), old(guard).can_protect(*root_info), - old(guard).can_protect(*next_info), + old(guard).can_base_protect(*next_info), root_info.ptr() == root, next_info.ptr() == next, LinkedListTraversalSpec::root_inv(root, root_info.obj(), g), @@ -2220,18 +2774,12 @@ pub proof fn linked_list_protect_next_example( ensures next_protected.ptr() == next, next_protected.obj() == next_info.obj(), - next_protected.domain() == domain.id(), + next_protected.domain() == old(guard).domain(), next_protected.protected_by(*final(guard)), final(guard).wf(), LinkedListTraversalSpec::node_inv(next, next_info.obj(), g), { - let tracked root_protected = protect_root::( - domain, - guard, - root_info, - root, - g, - ); + let tracked root_protected = protect_root::(guard, root_info, root, g); protect_link::(guard, &root_protected, next_info, n, next, g) } diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 4954c9e77..67efa3566 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -120,6 +120,27 @@ impl WmView { pub open spec fn spec_le(self, other: Self) -> bool { forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) } + + pub proof fn lemma_join_left(self, other: Self) + ensures + self.spec_le(self.join(other)), + { + } + + pub proof fn lemma_join_right(self, other: Self) + ensures + other.spec_le(self.join(other)), + { + } + + pub proof fn lemma_spec_le_transitive(self, middle: Self, upper: Self) + requires + self.spec_le(middle), + middle.spec_le(upper), + ensures + self.spec_le(upper), + { + } } /// One message in an atomic object's modification history. @@ -514,8 +535,13 @@ impl WeakAtomicPtr { self.atomic_inv@.constant().0 } + /// Logical modification-history identity of this atomic pointer. + pub closed spec fn id(&self) -> AtomicId { + self.atomic_inv@.constant().1 + } + pub closed spec fn well_formed(&self) -> bool { - self.atomic_inv@.constant().1 == self.atomic.id() + self.id() == self.atomic.id() } #[verifier::type_invariant] @@ -660,7 +686,7 @@ impl WeakAtomicPtr { impl WeakAtomicPtr< T, - bool, + rcu_spec::RcuRootKey, rcu_spec::RcuRootOwnedGhost, rcu_spec::RcuOwnedWeakAtomicInv, > where OwnPred: rcu_spec::RcuRootOwnershipPredicate { @@ -670,17 +696,24 @@ impl WeakAtomicPtr< *mut T, Ghost, Ghost>, + Tracked>>, )) requires self.well_formed(), ensures - !self.constant() ==> !res.0.is_null(), - match res.2@ { - None => res.0.addr() == 0, - Some(object) => { + !self.constant().nullable ==> !res.0.is_null(), + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { &&& res.0.addr() != 0 &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) }, + _ => false, }, { let result; @@ -695,25 +728,141 @@ impl WeakAtomicPtr< assert(hist.id() == self.atomic.id()); } let loaded = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + assert(hist.valid_ts(loaded.1@)); + assert(loaded.1@ < hist.history().len()); + assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); + } proof_decl! { let ghost published = g.published_at(loaded.1@); + let tracked loaded_info; } proof { - assert(rcu_spec::rcu_history_inv(self.constant(), hist.history())); - match published { - Some(object) => { - assert(object.addr == loaded.0.addr()); + assert(rcu_spec::rcu_history_inv(self.constant().nullable, hist.history())); + assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); + loaded_info = g.tracked_info_at(hist.history(), loaded.1@); + match (published, &loaded_info) { + (Some(object), Some(info)) => { + assert(equal(hist.history()[loaded.1@ as int].value, loaded.0)); + assert(equal(info.ptr(), loaded.0)); }, - None => { + (None, None) => { assert(loaded.0.addr() == 0); }, + _ => assert(false), + }; + if !self.constant().nullable { + rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), loaded.1@); + assert(!loaded.0.is_null()); } - if !self.constant() { + } + result = (loaded.0, loaded.1, Ghost(published), Tracked(loaded_info)); + proof { + pair = (hist, g); + } + }); + result + } + + /// Acquire-load an RCU root while starting a paper read-side guard. + /// + /// The ghost reader transition occurs in the same invariant opening as the + /// real acquire load. Executably this is identical to `load_acquire_rcu`. + #[inline(always)] + pub fn load_acquire_rcu_guarded(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + Tracked>, + )) + requires + self.well_formed(), + ensures + !self.constant().nullable ==> !res.0.is_null(), + res.4@.wf(), + res.4@.domain() == self.constant().domain, + res.4@.reader_registry() == self.constant().reader_registry, + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == res.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + }, + _ => false, + }, + { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, mut g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + proof_decl! { + let tracked base_guard = g.tracked_start_reader(hist.history()); + } + let loaded = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + assert(hist.valid_ts(loaded.1@)); + assert(loaded.1@ < hist.history().len()); + assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); + } + proof_decl! { + let tracked loaded_info; + } + proof { + assert(rcu_spec::rcu_history_inv( + self.constant().nullable, + hist.history(), + )); + loaded_info = g.tracked_info_at(hist.history(), loaded.1@); + assert(loaded.1@ < g.publications().len()); + } + proof_decl! { + let ghost published = g.published_at(loaded.1@); + } + proof { + match (published, &loaded_info) { + (Some(object), Some(info)) => { + assert(equal(hist.history()[loaded.1@ as int].value, loaded.0)); + assert(equal(info.ptr(), loaded.0)); + assert(info.domain() == g.domain()); + assert(info.domain() == base_guard.domain()); + }, + (None, None) => { + assert(loaded.0.addr() == 0); + }, + _ => assert(false), + }; + if !self.constant().nullable { rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), loaded.1@); assert(!loaded.0.is_null()); } + assert(base_guard.domain() == self.constant().domain); + assert(base_guard.reader_registry() == self.constant().reader_registry); + assert(rcu_spec::rcu_current_ownership_inv::(g)); + } + proof_decl! { + let tracked guard = rcu_spec::RcuReadGuardToken::tracked_from_base(base_guard); } - result = (loaded.0, loaded.1, Ghost(published)); + result = ( + loaded.0, + loaded.1, + Ghost(published), + Tracked(loaded_info), + Tracked(guard), + ); proof { pair = (hist, g); } @@ -721,6 +870,34 @@ impl WeakAtomicPtr< result } + /// End a paper read-side guard without executing another atomic operation. + #[inline(always)] + pub fn stop_rcu_reader(&self, Tracked(guard): Tracked>) + requires + self.well_formed(), + guard.wf(), + guard.domain() == self.constant().domain, + guard.reader_registry() == self.constant().reader_registry, + { + proof_decl! { + let tracked base_guard = guard.tracked_into_base(); + } + let credit = vstd::invariant::create_open_invariant_credit(); + proof { + use_type_invariant(self); + vstd::invariant::open_atomic_invariant_in_proof!( + credit.get() => self.atomic_inv.borrow() => pair => { + let tracked (hist, mut g) = pair; + assert(g.domain() == self.constant().domain); + assert(g.reader_registry() == self.constant().reader_registry); + g.tracked_stop_reader(hist.history(), base_guard); + assert(rcu_spec::rcu_current_ownership_inv::(g)); + pair = (hist, g); + } + ); + } + } + /// Release-swap helper for a freshly introduced RCU root pointer. /// /// The new registration remains owned by the atomic invariant. The return @@ -736,7 +913,7 @@ impl WeakAtomicPtr< ) -> (res: (*mut T, Tracked>>)) requires self.well_formed(), - self.constant() || !value.is_null(), + self.constant().nullable || !value.is_null(), match ownership { Some(ownership) => { &&& !value.is_null() @@ -749,6 +926,8 @@ impl WeakAtomicPtr< res.1@ is Some ==> res.1@->Some_0.object().wf(), res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), res.1@ is Some ==> res.1@->Some_0.retired().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> res.1@->Some_0.retired().removal().root == self.id(), + res.1@ is Some ==> res.1@->Some_0.retired().removal().observed_by(final(tv)@), res.1@ is Some ==> OwnPred::owns(res.0, res.1@->Some_0.ownership()), { let result; @@ -774,12 +953,12 @@ impl WeakAtomicPtr< assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); assert(rcu_spec::rcu_current_ownership_inv::(g)); rcu_spec::lemma_current_owned_resources::(prev, &g); - if !self.constant() { + if !self.constant().nullable { assert(!value.is_null()); assert(snap@.msg().value.addr() != 0); } rcu_spec::preserve_rcu_history_inv_on_push( - self.constant(), + self.constant().nullable, prev, next, snap@.msg(), @@ -788,6 +967,7 @@ impl WeakAtomicPtr< prev, next, snap@.msg(), + self.atomic.id(), ownership, ); assert(detached is Some ==> detached->Some_0.object().wf()); @@ -796,6 +976,13 @@ impl WeakAtomicPtr< result, detached->Some_0.ownership(), )); + assert(detached is Some ==> detached->Some_0.retired().removal().root + == self.atomic.id()); + assert(detached is Some ==> detached->Some_0.retired().removal().timestamp + == prev.len()); + assert(detached is Some ==> detached->Some_0.retired().removal().observed_by( + tv@, + )); assert(rcu_spec::rcu_current_ownership_inv::(g)) by { match g.current_owned() { Some(owned) => { @@ -831,7 +1018,7 @@ impl WeakAtomicPtr< )) requires self.well_formed(), - self.constant() || !new.is_null(), + self.constant().nullable || !new.is_null(), match new_ownership { Some(ownership) => { &&& !new.is_null() @@ -847,6 +1034,8 @@ impl WeakAtomicPtr< res.2@.0 is Some ==> res.2@.0->Some_0.object().wf(), res.2@.0 is Some ==> equal(res.2@.0->Some_0.ptr(), res.0->Ok_0), res.2@.0 is Some ==> res.2@.0->Some_0.retired().obj() == res.2@.0->Some_0.obj(), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().root == self.id(), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().observed_by(final(tv)@), res.2@.0 is Some ==> OwnPred::owns(res.0->Ok_0, res.2@.0->Some_0.ownership()), { let result; @@ -881,12 +1070,12 @@ impl WeakAtomicPtr< let tracked snap_opt = cas_result.2.get(); match snap_opt { Option::Some(snap) => { - if !self.constant() { + if !self.constant().nullable { assert(!new.is_null()); assert(snap.msg().value.addr() != 0); } rcu_spec::preserve_rcu_history_inv_on_push( - self.constant(), + self.constant().nullable, prev, next, snap.msg(), @@ -895,6 +1084,7 @@ impl WeakAtomicPtr< prev, next, snap.msg(), + self.atomic.id(), new_ownership, ); assert(detached is Some ==> detached->Some_0.object().wf()); @@ -906,6 +1096,12 @@ impl WeakAtomicPtr< cas_result.0->Ok_0, detached->Some_0.ownership(), )); + assert(detached is Some ==> detached->Some_0.retired().removal().root + == self.atomic.id()); + assert(detached is Some ==> + detached->Some_0.retired().removal().timestamp == prev.len()); + assert(detached is Some ==> + detached->Some_0.retired().removal().observed_by(tv@)); assert(rcu_spec::rcu_current_ownership_inv::(g)) by { match g.current_owned() { Some(owned) => { @@ -926,7 +1122,7 @@ impl WeakAtomicPtr< Result::Err(_) => { retired_ownership = (None, new_ownership); assert(next == prev); - assert(rcu_spec::rcu_history_inv(self.constant(), next)); + assert(rcu_spec::rcu_history_inv(self.constant().nullable, next)); }, } pair = (hist, g); @@ -1240,6 +1436,19 @@ impl ThreadView { { ThreadView { view: WmView::empty() } } + + /// Imports observations from another genuine thread/CPU view. + /// + /// Unlike a raw ghost mutator, this operation cannot introduce an + /// unwritten timestamp: both operands are tracked `ThreadView` values that + /// originated from the weak-memory TCB. Scheduler context switches use it + /// to transfer observations between a CPU view and a task view. + pub(crate) proof fn tracked_join(tracked &mut self, tracked other: &Self) + ensures + final(self)@ == old(self)@.join(other@), + { + self.view = self.view.join(other.view); + } } #[repr(transparent)] diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index b10e9f380..989aeb18b 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -21,10 +21,14 @@ //! stale messages remain distinguishable even if a physical address is later //! reused. Multiple messages may refer to one registration and therefore carry //! the same allocation ID; an atomic timestamp is never used as an allocation -//! identity. `Rcu` roots are non-null in every message, while `RcuOption` roots -//! may contain null messages without allocation IDs. Physical `P::Permission`, -//! reader permissions, traversal snapshots, and reclamation are modeled -//! separately in [`specs::sync::rcu`] and are being connected incrementally. +//! identity. The owned root invariant retains a persistent typed `BlockInfo` +//! for every registered AId, including retired historical entries. An acquire +//! load therefore returns proof of the exact typed pointer and AId it observed, +//! rather than reconstructing identity from the address. `Rcu` roots are +//! non-null in every message, while `RcuOption` roots may contain null messages +//! without allocation IDs. Physical `P::Permission`, reader permissions, +//! traversal snapshots, and reclamation are modeled separately in +//! [`specs::sync::rcu`] and are being connected incrementally. //! //! The traversal layer follows the paper's shape: //! @@ -37,8 +41,13 @@ //! removed set. The domain's base `rcu-retire` transition then records it in //! `RcuState.R` as `RcuRetired`. //! - `RcuCallbackSafety` compresses that recorded retire proof into an erased -//! `RcuCallbackSummary { domain, obj, retire_epoch }`, which is what the -//! monitor stores next to a type-erased executable callback. +//! `RcuCallbackSummary { domain, obj, removal, retire_epoch, retire_view }`, +//! which is what the monitor stores next to a type-erased executable +//! callback. `removal` is the paper's `Retired(a, Q)` detachment observation: +//! it records the root atomic and the first timestamp after the object was +//! removed. The retire view observes that timestamp and records the +//! observations that every quiescent report must cover before physical +//! reclamation. //! //! # Callback boundary //! @@ -54,8 +63,14 @@ //! lock-protected monitor state (`specs::sync::rcu::MonitorStateView`), and a //! `false` message certifies that its snapshot has no pending callbacks and no //! incomplete grace period. `finish_grace_period` removes the completed batch -//! under the monitor lock, produces a `CompletedGracePeriod` certificate, and -//! executes exactly that batch outside the lock. +//! under the monitor lock and produces a private `CompletedGracePeriod` +//! certificate. For each callback, the monitor combines that certificate with +//! the callback's traversal-retire safety token to produce a linear +//! object-level reclaim permit, then executes exactly that batch outside the +//! lock. The monitor lock carries a linear release view: enqueue publishes the +//! callback's `retire_view`, and each CPU report is created only after an +//! acquire imports that view. A completed certificate therefore proves that +//! every online CPU's report view covers every callback in the batch. //! //! # Usage outline //! @@ -74,13 +89,34 @@ //! updated view back in only after the context is quiescent. //! //! Delayed reclamation is still being wired into the weak-memory proof. The -//! remaining boundary is concrete: executable preemption guards do not yet own -//! the domain's `Inactive/Guard` reader token. The weak atomic invariant now -//! retains the current registration together with `P::Permission`; Release -//! swap and successful CAS return the old raw pointer and matching owned -//! resource. `RcuInner` does not yet establish traversal removal or route that -//! detached ownership into the monitor. For now, `RcuDrop` preserves the -//! public wrapper API. +//! weak atomic invariant retains the current registration together with +//! `P::Permission`; release swap and successful CAS establish root removal, +//! return the old raw pointer and matching ownership, and route a certified +//! callback into the monitor. Scheduler handoff now preserves a per-CPU +//! `ThreadView`: schedule-out joins the departing task's observations into the +//! CPU view, and schedule-in imports that view into the incoming task. +//! `RunningTaskContext` retains the CPU identity, so a quiescent report is tied +//! to the CPU whose persistent view it reports. +//! +//! An executable `read()` now performs the paper's `Inactive -> Guard` +//! transition while opening the root weak-atomic invariant. The resulting +//! `RcuReadGuardToken` and exact historical `BlockInfo` remain in the +//! executable read guard until destruction or consuming CAS performs +//! `Guard -> Inactive`. +//! +//! Two boundaries remain. First, the loaded root's `BlockInfo` is not yet +//! installed in the guard's protection map. That step requires connecting +//! expired-object completion to the root atomic's timestamp and the task's +//! `ThreadView`; consequently `assume_shared_ref` still stands in for the final +//! traversal argument that grants the client pointer's reference permission. +//! Second, the unsafe +//! quiescent-report entrypoint is not yet owned by the executable scheduler's +//! context-switch path. The scheduler ghost API proves the per-CPU view +//! handoff, but the real hook still has to carry that ghost state, perform the +//! domain expired-set transition, and connect it to monitor grace-period +//! completion. Until both boundaries are closed, the reclaim permit is a +//! monitor-level authorization rather than the final end-to-end memory-safety +//! authority. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; @@ -152,7 +188,7 @@ type RcuAtomicGhost

= rcu_spec::RcuRootOwnedGhost< type RcuAtomicPtr

= WeakAtomicPtr<

::Target, - bool, + rcu_spec::RcuRootKey, RcuAtomicGhost

, rcu_spec::RcuOwnedWeakAtomicInv>, >; @@ -183,6 +219,8 @@ struct RcuReadGuardInner<'a, P: NonNullPtr> { obj_ptr: *mut

::Target, rcu: &'a RcuInner

, _inner_guard: DisabledPreemptGuard, + tracked_info: Tracked::Target>>>, + tracked_guard: Tracked::Target>>, } /// Sized callback payload that retains the physical ownership of one detached @@ -234,6 +272,8 @@ fn callback_from_detached( equal(owned.ptr(), pointer), P::ptr_perm_match(pointer, owned.ownership()), owned.ownership().inv(), + ensures + res.1@.removal() == owned.retired().removal(), { proof { use_type_invariant(&owned); @@ -257,7 +297,7 @@ impl RcuInner

{ closed spec fn wf(self) -> bool { &&& self.ptr.well_formed() - &&& self.ptr.constant() == self.ghost_nullable@ + &&& self.ptr.constant().nullable == self.ghost_nullable@ } } @@ -286,11 +326,16 @@ impl RcuInner

{ proof_decl! { let tracked root_ghost: RcuAtomicGhost

= rcu_spec::RcuRootOwnedGhost::tracked_initial( - core::ptr::null_mut::<

::Target>(), - None, - ); + core::ptr::null_mut::<

::Target>(), + None, + ); + let ghost key = rcu_spec::RcuRootKey { + nullable: true, + domain: root_ghost.domain(), + reader_registry: root_ghost.reader_registry(), + }; } - let ptr = WeakAtomicPtr::new(Ghost(true), core::ptr::null_mut(), Tracked(root_ghost)); + let ptr = WeakAtomicPtr::new(Ghost(key), core::ptr::null_mut(), Tracked(root_ghost)); Self { ptr, ghost_nullable: Ghost(true), @@ -315,8 +360,13 @@ impl RcuInner

{ proof_decl! { let tracked root_ghost = rcu_spec::RcuRootOwnedGhost::tracked_initial(raw_ptr, Some(perm)); + let ghost key = rcu_spec::RcuRootKey { + nullable, + domain: root_ghost.domain(), + reader_registry: root_ghost.reader_registry(), + }; } - let ptr = WeakAtomicPtr::new(Ghost(nullable), raw_ptr, Tracked(root_ghost)); + let ptr = WeakAtomicPtr::new(Ghost(key), raw_ptr, Tracked(root_ghost)); Self { ptr, ghost_nullable: Ghost(nullable), @@ -325,24 +375,70 @@ impl RcuInner

{ } #[inline(always)] - fn load_ptr_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: - *mut

::Target) + fn load_ptr_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + *mut

::Target, + Tracked::Target>>>, + )) requires self.type_inv(), ensures - !self.is_nullable() ==> !res.is_null(), + !self.is_nullable() ==> !res.0.is_null(), + match res.1@ { + None => res.0.is_null(), + Some(info) => { + &&& !res.0.is_null() + &&& info.wf() + &&& equal(info.ptr(), res.0) + }, + }, { proof { - assert(self.ptr.constant() == self.is_nullable()); + assert(self.ptr.constant().nullable == self.is_nullable()); } let res = self.ptr.load_acquire_rcu(Tracked(tv)); proof { if !self.is_nullable() { - assert(!self.ptr.constant()); + assert(!self.ptr.constant().nullable); assert(!res.0.is_null()); } } - res.0 + (res.0, res.3) + } + + #[inline(always)] + fn load_ptr_acquire_guarded(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + *mut

::Target, + Tracked::Target>>>, + Tracked::Target>>, + )) + requires + self.type_inv(), + ensures + !self.is_nullable() ==> !res.0.is_null(), + res.2@.wf(), + res.2@.domain() == self.ptr.constant().domain, + res.2@.reader_registry() == self.ptr.constant().reader_registry, + match res.1@ { + None => res.0.is_null(), + Some(info) => { + &&& !res.0.is_null() + &&& info.wf() + &&& info.domain() == res.2@.domain() + &&& equal(info.ptr(), res.0) + }, + }, + { + proof { + assert(self.ptr.constant().nullable == self.is_nullable()); + } + let res = self.ptr.load_acquire_rcu_guarded(Tracked(tv)); + proof { + if !self.is_nullable() { + assert(!self.ptr.constant().nullable); + assert(!res.0.is_null()); + } + } + (res.0, res.3, res.4) } #[inline(always)] @@ -377,12 +473,13 @@ impl RcuInner

{ (res.1@ is Some) == !res.0.is_null(), res.1@ is Some ==> res.1@->Some_0.object().wf(), res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), + res.1@ is Some ==> res.1@->Some_0.retired().removal().observed_by(final(tv)@), res.1@ is Some ==> P::ptr_perm_match(res.0, res.1@->Some_0.ownership()), res.1@ is Some ==> res.1@->Some_0.ownership().inv(), { proof { - assert(self.ptr.constant() == self.is_nullable()); - assert(self.ptr.constant() || !new_ptr.is_null()); + assert(self.ptr.constant().nullable == self.is_nullable()); + assert(self.ptr.constant().nullable || !new_ptr.is_null()); } self.ptr.swap_release_rcu(new_ptr, Tracked(ownership), Tracked(tv)) } @@ -396,6 +493,7 @@ impl RcuInner

{ final(session).wf(), final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), @@ -447,6 +545,7 @@ impl RcuInner

{ res.rcu.is_nullable() == self.is_nullable(), final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -458,8 +557,14 @@ impl RcuInner

{ &inner_guard, ); } - let obj_ptr = self.load_ptr_acquire(Tracked(tv)); - RcuReadGuardInner { obj_ptr, rcu: self, _inner_guard: inner_guard } + let (obj_ptr, tracked_info, tracked_guard) = self.load_ptr_acquire_guarded(Tracked(tv)); + RcuReadGuardInner { + obj_ptr, + rcu: self, + _inner_guard: inner_guard, + tracked_info, + tracked_guard, + } } #[inline] @@ -474,13 +579,14 @@ impl RcuInner

{ ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), { proof_decl! { let tracked tv = session.tracked_borrow_thread_view_mut(); } - let obj_ptr = self.load_ptr_acquire(Tracked(tv)); + let (obj_ptr, _tracked_info) = self.load_ptr_acquire(Tracked(tv)); NonNull::new(obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }) } } @@ -518,6 +624,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { final(session).wf(), final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), @@ -547,8 +654,8 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { } proof { - assert(rcu.ptr.constant() == rcu.is_nullable()); - assert(rcu.ptr.constant() || !new_raw.is_null()); + assert(rcu.ptr.constant().nullable == rcu.is_nullable()); + assert(rcu.ptr.constant().nullable || !new_raw.is_null()); } let cas_res = { proof_decl! { @@ -601,6 +708,10 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { self._inner_guard.lemma_matches_context_preserved(context_before_enqueue, session); self._inner_guard.lemma_matches_context_depth(session); } + proof_decl! { + let tracked guard = self.tracked_guard.get(); + } + rcu.ptr.stop_rcu_reader(Tracked(guard)); self._inner_guard.release_to_context(Tracked(session)); res } @@ -614,14 +725,20 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { final(session).wf(), final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).view() == old(session).view(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), { proof { + use_type_invariant(&self); self._inner_guard.lemma_matches_context_depth(session); } + proof_decl! { + let tracked guard = self.tracked_guard.get(); + } + self.rcu.ptr.stop_rcu_reader(Tracked(guard)); self._inner_guard.release_to_context(Tracked(session)); } } @@ -655,6 +772,7 @@ impl Rcu

{ ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -676,6 +794,7 @@ impl Rcu

{ ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -719,6 +838,7 @@ impl RcuOption

{ ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -740,6 +860,7 @@ impl RcuOption

{ ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -760,6 +881,7 @@ impl RcuOption

{ ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -785,6 +907,7 @@ impl RcuReadGuard<'_, P> { ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -812,6 +935,7 @@ impl RcuReadGuard<'_, P> { ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -837,6 +961,7 @@ impl RcuOptionReadGuard<'_, P> { ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -865,6 +990,7 @@ impl RcuOptionReadGuard<'_, P> { ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -934,6 +1060,7 @@ impl Deref for RcuDrop { final(session).is_quiescent(), final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), @@ -1013,7 +1140,20 @@ impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.rcu.type_inv() + &&& self.tracked_guard@.wf() + &&& self.tracked_guard@.domain() == self.rcu.ptr.constant().domain + &&& self.tracked_guard@.reader_registry() + == self.rcu.ptr.constant().reader_registry &&& !self.rcu.is_nullable() ==> !self.obj_ptr.is_null() + &&& match self.tracked_info@ { + None => self.obj_ptr.is_null(), + Some(info) => { + &&& !self.obj_ptr.is_null() + &&& info.wf() + &&& info.domain() == self.tracked_guard@.domain() + &&& equal(info.ptr(), self.obj_ptr) + }, + } } } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index a3c00ce88..643f7be45 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -2,15 +2,15 @@ use alloc::collections::VecDeque; use core::sync::atomic::Ordering; -use vstd::{predicate::Predicate as DataPredicate, prelude::*}; +use vstd::{predicate::Predicate as DataPredicate, prelude::*, resource::Loc}; use vstd_extra::raw_callback::RawCallback; use crate::specs::{ - mm::cpu::{AtomicCpuSet, CpuId, CpuSet}, + mm::cpu::{AtomicCpuSet, AtomicCpuSetToken, CpuId, CpuSet, online_cpus}, sync::{ rcu as rcu_spec, rcu::{GracePeriodView, MonitorStateView}, - weak_memory::{History, ThreadView, WeakAtomicBool}, + weak_memory::{History, ThreadView, WeakAtomicBool, WmView}, }, }; use crate::sync::{ @@ -28,6 +28,53 @@ type MonitorAtomicBool = WeakAtomicBool< rcu_spec::RcuMonitorFlagInv, >; +/// Evidence captured at a call site where the current task is quiescent. +/// +/// This token deliberately does not claim that a complete RCU grace period has +/// elapsed. It records the task and weak-memory view at one CPU-local +/// observation; the monitor binds it to the currently active generation below. +/// A later scheduler proof must additionally show that this unsafe entrypoint +/// is reached by the required context-switch path. +tracked struct RcuQuiescentContext { + ghost cpu: CpuId, + ghost task: Loc, + ghost scheduler: Loc, + ghost view: WmView, +} + +impl RcuQuiescentContext { + proof fn tracked_from_running_context( + tracked context: &RunningTaskContext, + cpu: CpuId, + ) -> (tracked res: Self) + requires + context.wf(), + context.is_quiescent(), + cpu == context.cpu(), + ensures + res.cpu == cpu, + res.task == context.task(), + res.scheduler == context.scheduler(), + res.view == context.view(), + { + RcuQuiescentContext { + cpu, + task: context.task(), + scheduler: context.scheduler(), + view: context.view(), + } + } +} + +/// Historical record of one quiescent context bound to a monitor generation. +ghost struct RcuCpuQuiescentReport { + cpu: CpuId, + task: Loc, + scheduler: Loc, + view: WmView, + epoch: nat, +} + /// RCU-specific wrapper around a type-erased executable callback. /// /// `RawCallback` is intentionally proof-opaque. The summary records the object @@ -56,20 +103,30 @@ impl RcuCallback { raw: RawCallback, Tracked(cert): Tracked, Ghost(retire_epoch): Ghost, + Ghost(retire_view): Ghost, ) -> (res: Self) + requires + cert.removal().observed_by(retire_view), ensures res.wf(), res@ == (rcu_spec::RcuCallbackSummary { domain: cert.domain(), obj: cert.obj(), + removal: cert.removal(), retire_epoch, + retire_view, }), { let ghost summary = rcu_spec::RcuCallbackSummary { domain: cert.domain(), obj: cert.obj(), + removal: cert.removal(), retire_epoch, + retire_view, }; + proof { + cert.lemma_matches(summary); + } Self { raw, summary: Ghost(summary), safety: Tracked(cert) } } @@ -77,10 +134,10 @@ impl RcuCallback { /// period that contained this callback's retire summary. #[inline] #[verifier::external_body] - unsafe fn call_once(self, Tracked(completed): Tracked<&CompletedGracePeriod>) + unsafe fn call_once(self, Tracked(permit): Tracked) requires self.wf(), - completed.covers(self@), + permit.authorizes(self@), { unsafe { self.raw.call_once(); @@ -105,6 +162,35 @@ impl RcuCallback { tracked struct CompletedGracePeriod { epoch: Ghost, callbacks: Ghost>, + reported_cpus: Ghost>, + reports: Ghost>, +} + +/// Object-level authorization to execute one reclamation callback. +/// +/// Unlike [`CompletedGracePeriod`], this token is specific to one callback. +/// It combines the callback's traversal-retirement certificate with monitor +/// completion of the batch containing that callback. Keeping its constructor +/// private prevents executable callback code from treating batch membership +/// alone as proof that an arbitrary object was retired safely. +tracked struct RcuReclaimPermit { + summary: Ghost, + retired: rcu_spec::RcuRetiredFact, + reports: Ghost>, +} + +impl RcuReclaimPermit { + closed spec fn authorizes(self, callback: rcu_spec::RcuCallbackSummary) -> bool { + &&& self.summary@ == callback + &&& self.retired.matches(callback) + &&& self.reports@.dom() == online_cpus() + &&& forall|cpu: CpuId| #[trigger] + self.reports@.contains_key(cpu) ==> { + &&& self.reports@[cpu].cpu == cpu + &&& self.reports@[cpu].epoch == callback.retire_epoch + &&& callback.retire_view.spec_le(self.reports@[cpu].view) + } + } } impl CompletedGracePeriod { @@ -116,9 +202,54 @@ impl CompletedGracePeriod { self.epoch@ } + closed spec fn reported_cpus(self) -> Set { + self.reported_cpus@ + } + + closed spec fn reports(self) -> Map { + self.reports@ + } + + closed spec fn reports_wf(self) -> bool { + &&& self.reports().dom() == self.reported_cpus() + &&& forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) ==> { + &&& self.reports()[cpu].cpu == cpu + &&& self.reports()[cpu].epoch == self.epoch() + } + } + + closed spec fn callbacks_covered(self) -> bool { + forall|i: int, cpu: CpuId| + 0 <= i < self.callbacks().len() && #[trigger] self.reports().contains_key(cpu) ==> ( + #[trigger] self.callbacks()[i]).retire_view.spec_le(self.reports()[cpu].view) + } + closed spec fn covers(self, callback: rcu_spec::RcuCallbackSummary) -> bool { &&& self.callbacks().contains(callback) &&& callback.retire_epoch == self.epoch() + &&& self.reported_cpus() == online_cpus() + &&& self.reports_wf() + &&& forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) ==> callback.retire_view.spec_le( + self.reports()[cpu].view, + ) + } + + /// Combines traversal retirement with monitor completion for one callback. + proof fn tracked_authorize_callback( + tracked &self, + tracked safety: &rcu_spec::RcuCallbackSafety, + callback: rcu_spec::RcuCallbackSummary, + ) -> (tracked permit: RcuReclaimPermit) + requires + safety.matches(callback), + self.covers(callback), + ensures + permit.authorizes(callback), + { + let tracked retired = safety.tracked_retired_fact(callback); + RcuReclaimPermit { summary: Ghost(callback), retired, reports: Ghost(self.reports()) } } } @@ -132,6 +263,9 @@ fn run_completed_callbacks( ) requires completed.callbacks() == callback_summaries(callbacks), + completed.reported_cpus() == online_cpus(), + completed.reports_wf(), + completed.callbacks_covered(), forall|i: int| 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( callbacks, @@ -142,6 +276,12 @@ fn run_completed_callbacks( (#[trigger] callbacks@[i])@, ) by { callback_summaries(callbacks).lemma_index_contains(i); + assert forall|cpu: CpuId| #[trigger] + completed.reports().contains_key(cpu) implies callbacks@[i]@.retire_view.spec_le( + completed.reports()[cpu].view, + ) by { + assert(callback_summaries(callbacks)[i] == callbacks@[i]@); + }; } } @@ -168,7 +308,13 @@ fn run_completed_callbacks( proof { use_type_invariant(&callback); } - callback.call_once(Tracked(&completed)); + proof_decl! { + let tracked permit = completed.tracked_authorize_callback( + callback.safety.borrow(), + callback@, + ); + } + callback.call_once(Tracked(permit)); } } } @@ -222,6 +368,8 @@ fn push_callback(callbacks: &mut Callbacks, callback: RcuCallback) pub(super) struct GracePeriod { callbacks: Callbacks, cpu_mask: AtomicCpuSet, + tracked_cpu_mask: Tracked, + ghost_reports: Ghost>, is_complete: bool, ghost_epoch: Ghost, } @@ -245,12 +393,30 @@ impl GracePeriod { pub(super) fn new() -> (res: Self) ensures res@ == GracePeriodView::initial(), + res.wf(), { let callbacks = Callbacks::new(); - let cpu_mask = AtomicCpuSet::new(CpuSet::new_empty()); - let res = Self { callbacks, cpu_mask, is_complete: true, ghost_epoch: Ghost(0) }; + let empty_cpu_set = CpuSet::new_empty(); + proof { + assert(empty_cpu_set.cpus == Set::::empty()); + } + let mut cpu_mask = AtomicCpuSet::new(empty_cpu_set); + proof_decl! { + let tracked cpu_mask_token = cpu_mask.tracked_take_token(); + } + let res = Self { + callbacks, + cpu_mask, + tracked_cpu_mask: Tracked(cpu_mask_token), + ghost_reports: Ghost(Map::empty()), + is_complete: true, + ghost_epoch: Ghost(0), + }; proof { callback_summaries_empty(res.callbacks); + assert(res.cpu_mask.initial_cpus() == Set::::empty()); + assert(res.ghost_reports@.dom() == Set::::empty()); + assert(res.tracked_cpu_mask@.cpus() == Set::::empty()); } res } @@ -261,6 +427,7 @@ impl GracePeriod { /// later weak-memory ghost state can attach stable identity to this mask. fn restart(&mut self, callbacks: Callbacks, Ghost(epoch): Ghost) requires + old(self).wf(), forall|i: int| 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( callbacks, @@ -275,20 +442,86 @@ impl GracePeriod { self.is_complete = false; self.callbacks = callbacks; self.ghost_epoch = Ghost(epoch); + self.ghost_reports = Ghost(Map::empty()); + proof_decl! { + let tracked cpu_mask_token = self.tracked_cpu_mask.borrow_mut(); + } + #[verus_spec(with Tracked(cpu_mask_token))] self.cpu_mask.store(&CpuSet::new_empty(), Ordering::Relaxed); } - /// Records that `this_cpu` has passed a quiescent state for this grace - /// period and returns whether the executable CPU mask now covers all CPUs. - /// - /// The CPU-mask contents are not part of the current proof view, so this - /// method refines only the executable monitor protocol. The higher-level - /// proof still treats the returned completion bit abstractly. - fn record_quiescent_state(&self, this_cpu: CpuId) -> (complete: bool) + /// Records one generation-bound quiescent context and updates the + /// executable CPU mask in the same invariant-preserving transition. + fn record_quiescent_state( + &mut self, + this_cpu: CpuId, + Tracked(context): Tracked, + ) -> (complete: bool) + requires + old(self).wf(), + online_cpus().contains(this_cpu), + context.cpu == this_cpu, + forall|i: int| + 0 <= i < old(self).callback_summaries().len() ==> (#[trigger] old( + self, + ).callback_summaries()[i]).retire_view.spec_le(context.view), + ensures + final(self).wf(), + final(self)@ == old(self)@, + complete == (final(self).tracked_cpu_mask@.cpus() == online_cpus()), no_unwind { + let ghost old_reports = self.ghost_reports@; + let ghost report = RcuCpuQuiescentReport { + cpu: this_cpu, + task: context.task, + scheduler: context.scheduler, + view: context.view, + epoch: self@.epoch, + }; + proof { + self.ghost_reports = Ghost(self.ghost_reports@.insert(this_cpu, report)); + } + #[verus_spec(with Tracked(self.tracked_cpu_mask.borrow_mut()))] self.cpu_mask.add(this_cpu, Ordering::Relaxed); - self.cpu_mask.load(Ordering::Relaxed).is_full() + let cpu_mask = #[verus_spec(with Tracked(self.tracked_cpu_mask.borrow()))] + self.cpu_mask.load(Ordering::Relaxed); + let complete = cpu_mask.is_full(); + proof { + assert(self.ghost_reports@ == old_reports.insert(this_cpu, report)); + assert(self.callback_summaries() == old(self).callback_summaries()); + assert forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) implies { + &&& self.ghost_reports@[cpu].cpu == cpu + &&& self.ghost_reports@[cpu].epoch == self@.epoch + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) + } by { + if cpu == this_cpu { + assert(self.ghost_reports@[cpu] == report); + assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + } else { + assert(self.ghost_reports@[cpu] == old_reports[cpu]); + assert(old(self).ghost_reports@.contains_key(cpu)); + assert(self.ghost_reports@[cpu] == old(self).ghost_reports@[cpu]); + assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; + } + }; + assert(self.wf()); + } + complete } closed spec fn callback_summaries(self) -> Seq { @@ -303,14 +536,33 @@ impl GracePeriod { /// callbacks taken. Monitor methods may break this transiently inside a /// critical section (between completing a grace period and taking its /// callbacks), but must restore it before releasing the monitor lock. - closed spec fn wf(self) -> bool { - self@.wf() + pub(super) closed spec fn wf(self) -> bool { + &&& self@.wf() + &&& self.tracked_cpu_mask@.id() == self.cpu_mask.id() + &&& self.tracked_cpu_mask@.wf() + &&& self.ghost_reports@.dom() == self.tracked_cpu_mask@.cpus() + &&& forall|cpu: CpuId| #[trigger] + self.ghost_reports@.contains_key(cpu) ==> { + &&& self.ghost_reports@[cpu].cpu == cpu + &&& self.ghost_reports@[cpu].epoch == self@.epoch + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).retire_view.spec_le( + self.ghost_reports@[cpu].view, + ) + } } } pub(super) struct State { current_gp: GracePeriod, next_callbacks: Callbacks, + /// Release view of the monitor lock. + /// + /// This proof-only token is updated before unlocking and imported after + /// locking. It gives the existing executable spin lock the release/acquire + /// semantics needed by the RCU proof without changing its runtime layout. + tracked_lock_view: Tracked, } impl View for State { @@ -325,10 +577,82 @@ impl View for State { } impl State { + closed spec fn lock_view(self) -> WmView { + self.tracked_lock_view@@ + } + closed spec fn next_callback_epoch(self) -> nat { self@.current_gp.epoch + 1 } + /// Imports the view published by the previous monitor-lock holder. + fn tracked_acquire_lock_view(&self, Tracked(thread_view): Tracked<&mut ThreadView>) + ensures + self.wf(), + final(thread_view)@ == old(thread_view)@.join(self.lock_view()), + old(thread_view)@.spec_le(final(thread_view)@), + self.lock_view().spec_le(final(thread_view)@), + { + proof { + use_type_invariant(self); + } + proof_decl! { + let tracked published_view = self.tracked_lock_view.borrow(); + } + proof { + let ghost before = thread_view@; + let ghost lock_view = self.lock_view(); + thread_view.tracked_join(published_view); + before.lemma_join_left(lock_view); + before.lemma_join_right(lock_view); + } + } + + /// Publishes the current holder's observations to the next lock acquirer. + fn tracked_publish_lock_view(&mut self, Tracked(thread_view): Tracked<&ThreadView>) + requires + old(self).wf(), + ensures + final(self).wf(), + final(self)@ == old(self)@, + final(self).lock_view() == old(self).lock_view().join(thread_view@), + old(self).lock_view().spec_le(final(self).lock_view()), + thread_view@.spec_le(final(self).lock_view()), + { + proof { + let ghost old_lock_view = old(self).lock_view(); + let ghost holder_view = thread_view@; + self.tracked_lock_view.borrow_mut().tracked_join(thread_view); + old_lock_view.lemma_join_left(holder_view); + old_lock_view.lemma_join_right(holder_view); + assert forall|i: int| + 0 <= i < final(self).current_gp.callback_summaries().len() implies ( + #[trigger] final(self).current_gp.callback_summaries()[i]).retire_view.spec_le( + final(self).lock_view(), + ) by { + assert(final(self).current_gp.callback_summaries()[i] == old( + self, + ).current_gp.callback_summaries()[i]); + old(self).current_gp.callback_summaries()[i].retire_view.lemma_spec_le_transitive( + old_lock_view, + final(self).lock_view(), + ); + }; + assert forall|i: int| + 0 <= i < callback_summaries(final(self).next_callbacks).len() implies ( + #[trigger] callback_summaries(final(self).next_callbacks)[i]).retire_view.spec_le( + final(self).lock_view(), + ) by { + assert(callback_summaries(final(self).next_callbacks)[i] == callback_summaries( + old(self).next_callbacks, + )[i]); + callback_summaries( + old(self).next_callbacks, + )[i].retire_view.lemma_spec_le_transitive(old_lock_view, final(self).lock_view()); + }; + } + } + /// Creates the lock-protected initial monitor state: there is no active /// grace period and no callbacks waiting to be attached to the next one. pub(super) fn new() -> (res: Self) @@ -338,7 +662,10 @@ impl State { { let current_gp = GracePeriod::new(); let next_callbacks = Callbacks::new(); - let res = Self { current_gp, next_callbacks }; + proof_decl! { + let tracked lock_view = ThreadView::new(); + } + let res = Self { current_gp, next_callbacks, tracked_lock_view: Tracked(lock_view) }; proof { callback_summaries_empty(res.next_callbacks); } @@ -359,6 +686,7 @@ impl State { requires callback.wf(), callback@.retire_epoch == old(self).next_callback_epoch(), + callback@.retire_view.spec_le(old(self).lock_view()), ensures final(self).wf(), final(self).has_pending_work(), @@ -413,6 +741,31 @@ impl State { } } + /// Records one CPU report while preserving the lock-protected callback + /// state. The linear mask token makes the executable `is_full` result + /// equivalent to coverage of the fixed online-CPU set. + fn record_quiescent_state( + &mut self, + this_cpu: CpuId, + Tracked(context): Tracked, + ) -> (complete: bool) + requires + old(self).wf(), + online_cpus().contains(this_cpu), + context.cpu == this_cpu, + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( + self, + ).current_gp.callback_summaries()[i]).retire_view.spec_le(context.view), + ensures + final(self).wf(), + final(self)@ == old(self)@, + complete == (final(self).current_gp.tracked_cpu_mask@.cpus() == online_cpus()), + no_unwind + { + self.current_gp.record_quiescent_state(this_cpu, Tracked(context)) + } + /// Records a quiescent state for the current CPU, returns the callbacks /// that become reclaimable if this completes the grace period, and /// immediately starts the next grace period if callbacks accumulated while @@ -420,19 +773,33 @@ impl State { /// /// This mirrors the upstream state machine: an incomplete CPU mask leaves /// the current grace period running and returns no completed callbacks. - /// The exact CPU-mask contents are still outside the proof view; the proof - /// treats `record_quiescent_state`'s boolean result as the completion cut. - fn finish_grace_period(&mut self, this_cpu: CpuId) -> (( - completed_gp, - completed_callbacks, - completed_token, - ): (bool, Callbacks, Tracked)) + /// The mask's linear shadow token records the exact reported-CPU set, so a + /// completed batch also carries proof that every online CPU reported. + fn finish_grace_period( + &mut self, + this_cpu: CpuId, + Tracked(context): Tracked, + ) -> ((completed_gp, completed_callbacks, completed_token): ( + bool, + Callbacks, + Tracked, + )) + requires + online_cpus().contains(this_cpu), + context.cpu == this_cpu, + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( + self, + ).current_gp.callback_summaries()[i]).retire_view.spec_le(context.view), ensures final(self).wf(), completed_token@.callbacks() == callback_summaries(completed_callbacks), completed_gp ==> !old(self)@.current_gp.is_complete, completed_gp ==> completed_token@.callbacks() == old(self)@.current_gp.callbacks, completed_gp ==> completed_token@.epoch() == old(self)@.current_gp.epoch, + completed_gp ==> completed_token@.reported_cpus() == online_cpus(), + completed_gp ==> completed_token@.reports_wf(), + completed_token@.callbacks_covered(), !completed_gp ==> completed_token@.callbacks() == Seq::< rcu_spec::RcuCallbackSummary, >::empty(), @@ -465,10 +832,26 @@ impl State { } let mut completed_callbacks = Callbacks::new(); let mut completed_gp = false; + let ghost mut completed_cpu_mask = Set::::empty(); + let ghost mut completed_reports = Map::::empty(); if !self.current_gp.is_complete { - let is_complete = self.current_gp.record_quiescent_state(this_cpu); + let is_complete = self.record_quiescent_state(this_cpu, Tracked(context)); if is_complete { completed_gp = true; + proof { + completed_cpu_mask = self.current_gp.tracked_cpu_mask@.cpus(); + completed_reports = self.current_gp.ghost_reports@; + assert(completed_cpu_mask == online_cpus()); + assert(self.current_gp.callback_summaries() == initial_current_callbacks); + assert forall|i: int, cpu: CpuId| + 0 <= i < initial_current_callbacks.len() + && #[trigger] completed_reports.contains_key(cpu) implies ( + #[trigger] initial_current_callbacks[i]).retire_view.spec_le( + completed_reports[cpu].view, + ) by { + assert(self.current_gp.wf()); + }; + } core::mem::swap(&mut completed_callbacks, &mut self.current_gp.callbacks); proof { assert(callback_summaries(completed_callbacks) == initial_current_callbacks); @@ -497,6 +880,8 @@ impl State { let tracked completed = CompletedGracePeriod { epoch: Ghost(if completed_gp { initial_current_epoch } else { 0 }), callbacks: Ghost(callback_summaries(completed_callbacks)), + reported_cpus: Ghost(completed_cpu_mask), + reports: Ghost(completed_reports), }; } proof { @@ -505,9 +890,23 @@ impl State { assert(completed.callbacks() == callback_summaries(completed_callbacks)); if !completed_gp { callback_summaries_empty(completed_callbacks); + assert(completed.callbacks_covered()); } else { assert(!initially_complete); + assert(completed.reported_cpus() == online_cpus()); + assert(completed.reports_wf()); assert(callback_summaries(completed_callbacks) == initial_current_callbacks); + assert(completed.callbacks_covered()) by { + assert forall|i: int, cpu: CpuId| + 0 <= i < completed.callbacks().len() + && #[trigger] completed.reports().contains_key(cpu) implies ( + #[trigger] completed.callbacks()[i]).retire_view.spec_le( + completed.reports()[cpu].view, + ) by { + assert(completed.callbacks()[i] == initial_current_callbacks[i]); + assert(completed.reports() == completed_reports); + }; + }; assert forall|i: int| 0 <= i < callback_summaries(completed_callbacks).len() implies ( #[trigger] callback_summaries(completed_callbacks)[i]).retire_epoch @@ -554,7 +953,18 @@ impl State { /// the queued callbacks or stopped monitoring). Holds whenever the monitor /// lock is free. closed spec fn wf(self) -> bool { - self@.wf() + &&& self@.wf() + &&& self.current_gp.wf() + &&& forall|i: int| + 0 <= i < self.current_gp.callback_summaries().len() ==> ( + #[trigger] self.current_gp.callback_summaries()[i]).retire_view.spec_le( + self.lock_view(), + ) + &&& forall|i: int| + 0 <= i < callback_summaries(self.next_callbacks).len() ==> ( + #[trigger] callback_summaries(self.next_callbacks)[i]).retire_view.spec_le( + self.lock_view(), + ) } #[verifier::type_invariant] @@ -707,10 +1117,12 @@ impl RcuMonitor { Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + cert@.removal().observed_by(old(session).view()), ensures final(session).wf(), final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), @@ -723,12 +1135,33 @@ impl RcuMonitor { proof { use_type_invariant(self); } + let ghost retire_view = session.view(); let mut state = self.state.lock(); + let ghost before_acquire = session.view(); + proof_decl! { + let tracked acquire_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_acquire_lock_view(Tracked(acquire_view)); + proof { + retire_view.lemma_spec_le_transitive(before_acquire, session.view()); + } + proof_decl! { + let tracked publish_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); + proof { + retire_view.lemma_spec_le_transitive(session.view(), state.value().lock_view()); + } let ghost retire_epoch = state.view()@.current_gp.epoch + 1; proof_decl! { let tracked cert = cert.get(); } - let callback = RcuCallback::from_raw(raw, Tracked(cert), Ghost(retire_epoch)); + let callback = RcuCallback::from_raw( + raw, + Tracked(cert), + Ghost(retire_epoch), + Ghost(retire_view), + ); let started_gp = state.enqueue_after_grace_period(callback); if started_gp { proof { @@ -741,6 +1174,10 @@ impl RcuMonitor { } self.set_monitoring(true, Ghost(state.view()@), Tracked(tv)); } + proof_decl! { + let tracked publish_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); } @@ -762,6 +1199,7 @@ impl RcuMonitor { final(session).is_quiescent(), final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), @@ -778,15 +1216,45 @@ impl RcuMonitor { return; } let mut state = self.state.lock(); + proof_decl! { + let tracked acquire_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_acquire_lock_view(Tracked(acquire_view)); if state.current_gp.is_complete { + proof_decl! { + let tracked publish_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); return; } - let this_cpu = CpuId::current(); + let this_cpu = CpuId::current(Tracked(&*session)); + proof_decl! { + let tracked quiescent_context = + RcuQuiescentContext::tracked_from_running_context(session, this_cpu); + } + proof { + assert forall|i: int| + 0 <= i < state.value().current_gp.callback_summaries().len() implies ( + #[trigger] state.value().current_gp.callback_summaries()[i]).retire_view.spec_le( + quiescent_context.view, + ) by { + let ghost callback = state.value().current_gp.callback_summaries()[i]; + callback.retire_view.lemma_spec_le_transitive( + state.value().lock_view(), + session.view(), + ); + }; + } let (completed_gp, completed_callbacks, Tracked(completed)) = state.finish_grace_period( this_cpu, + Tracked(quiescent_context), ); if !completed_gp { + proof_decl! { + let tracked publish_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); return; } @@ -802,6 +1270,10 @@ impl RcuMonitor { } self.set_monitoring(false, Ghost(state.view()@), Tracked(tv)); } + proof_decl! { + let tracked publish_view = session.tracked_borrow_thread_view_mut(); + } + state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); run_completed_callbacks(completed_callbacks, Tracked(completed)); } diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index a17618da3..e766443a8 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -295,18 +295,25 @@ impl PreemptThreadViewSession { pub tracked struct RunningTaskContext { session: PreemptThreadViewSession, preempt_depth: Ghost, + cpu: Ghost, } impl RunningTaskContext { /// Starts a running interval for a checked-out task view. - pub proof fn new(tracked task_view: TaskThreadView, sched_view: SchedulerView) -> (tracked res: - Self) + pub proof fn new( + tracked task_view: TaskThreadView, + sched_view: SchedulerView, + cpu: crate::specs::mm::cpu::CpuId, + ) -> (tracked res: Self) requires task_view.wf(sched_view), + sched_view.current.contains_key(cpu), + sched_view.current[cpu] == Some(task_view.task()), ensures res.scheduler() == task_view.scheduler(), res.task() == task_view.task(), res.view() == task_view.view(), + res.cpu() == cpu, res.preempt_depth() == 0, res.available_fractions() == PREEMPT_SESSION_FRACTIONS, res.wf(), @@ -314,7 +321,7 @@ impl RunningTaskContext { res.wf_scheduler(sched_view), { let tracked session = PreemptThreadViewSession::new(task_view, sched_view); - let tracked res = RunningTaskContext { session, preempt_depth: Ghost(0) }; + let tracked res = RunningTaskContext { session, preempt_depth: Ghost(0), cpu: Ghost(cpu) }; assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(res.wf()); assert(res.session.wf(sched_view)); @@ -334,6 +341,10 @@ impl RunningTaskContext { self.session.view() } + pub closed spec fn cpu(self) -> crate::specs::mm::cpu::CpuId { + self.cpu@ + } + pub closed spec fn session_id(self) -> Loc { self.session.session_id() } @@ -356,6 +367,8 @@ impl RunningTaskContext { pub closed spec fn wf_scheduler(self, sched_view: SchedulerView) -> bool { &&& self.wf() &&& self.session.wf(sched_view) + &&& sched_view.current.contains_key(self.cpu()) + &&& sched_view.current[self.cpu()] == Some(self.task()) } /// Re-establishes the scheduler relation after the checked-out task view @@ -369,6 +382,8 @@ impl RunningTaskContext { sched_view.checked_out_views[self.task()] == self.view(), sched_view.task_views.contains_key(self.task()), sched_view.task_views[self.task()] == self.view(), + sched_view.current.contains_key(self.cpu()), + sched_view.current[self.cpu()] == Some(self.task()), ensures self.wf_scheduler(sched_view), { @@ -389,6 +404,7 @@ impl RunningTaskContext { (*tv)@ == old(self).view(), final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth(), @@ -543,6 +559,7 @@ impl RunningTaskContext { final(self).wf(), final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() + 1 == old(self).available_fractions(), @@ -576,6 +593,7 @@ impl RunningTaskContext { final(self).wf(), final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions() + 1, @@ -695,6 +713,7 @@ impl DisabledPreemptGuard { (*tv)@ == old(context).view(), final(context).task() == old(context).task(), final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), final(context).session_id() == old(context).session_id(), final(context).available_fractions() == old(context).available_fractions(), final(context).preempt_depth() == old(context).preempt_depth(), @@ -716,6 +735,7 @@ impl DisabledPreemptGuard { final(context).wf(), final(context).task() == old(context).task(), final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), final(context).view() == old(context).view(), final(context).session_id() == old(context).session_id(), final(context).available_fractions() == old(context).available_fractions() + 1, @@ -769,6 +789,7 @@ pub(crate) fn disable_preempt_in_context( final(context).wf(), final(context).task() == old(context).task(), final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), final(context).view() == old(context).view(), final(context).session_id() == old(context).session_id(), final(context).available_fractions() + 1 == old(context).available_fractions(), diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index ad0afd761..e7529bb6f 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -112,9 +112,13 @@ pub ghost enum TaskSchedState { /// `ThreadView`; the scheduler view must be updated with /// `update_checked_out_task_view` to keep the logical snapshot synchronized. /// A quiescent schedule-out checks the updated token back into `stored_views`. +/// It also joins the outgoing task view into the persistent view of that CPU. +/// The next task scheduled on the CPU joins that CPU view into its checked-out +/// task view. Thus observations survive both task migration and context +/// switches without minting a fresh `ThreadView`. /// /// In short, the resource flow is: -/// `stored_views -> RunningTaskContext -> checked_out_views update -> stored_views`. +/// `stored task view + CPU view -> RunningTaskContext -> stored task view + CPU view`. /// Scheduler-policy transitions may change runqueues, current tasks, and task /// states only through `same_thread_view_ownership`, which frames all three /// weak-memory ownership maps. @@ -126,6 +130,7 @@ pub ghost enum TaskSchedState { /// state for every known task. `task_views` is the logical per-task /// weak-memory view. `stored_views` records views still owned by the scheduler /// resource; `checked_out_views` records views temporarily held by guards. +/// `cpu_views` persists observations across context switches on each CPU. pub ghost struct SchedulerView { pub id: Loc, pub runqueues: Map>, @@ -134,6 +139,7 @@ pub ghost struct SchedulerView { pub task_views: Map, pub stored_views: Map, pub checked_out_views: Map, + pub cpu_views: Map, } impl SchedulerView { @@ -147,6 +153,7 @@ impl SchedulerView { task_views: Map::empty(), stored_views: Map::empty(), checked_out_views: Map::empty(), + cpu_views: Map::empty(), } } @@ -179,6 +186,17 @@ impl SchedulerView { self.checked_out_views.contains_key(task) } + pub open spec fn cpu_has_thread_view(self, cpu: CpuId) -> bool { + self.cpu_views.contains_key(cpu) + } + + pub open spec fn cpu_thread_view(self, cpu: CpuId) -> WmView + recommends + self.cpu_has_thread_view(cpu), + { + self.cpu_views[cpu] + } + /// The scheduling policy changed no weak-memory ownership state. /// /// This relation deliberately ignores runqueues, current tasks, and task @@ -190,6 +208,7 @@ impl SchedulerView { &&& self.task_views == other.task_views &&& self.stored_views == other.stored_views &&& self.checked_out_views == other.checked_out_views + &&& self.cpu_views == other.cpu_views } pub open spec fn task_in_runqueue(self, task: Loc) -> bool { @@ -251,6 +270,30 @@ impl SchedulerView { } } + /// Registers the persistent weak-memory view for one CPU. + /// + /// Scheduler policy may populate that CPU's runqueue/current slot only + /// after this transition. The initial empty view carries no observations. + pub open spec fn register_cpu(self, cpu: CpuId) -> SchedulerView + recommends + !self.cpu_views.contains_key(cpu), + valid_cpu(cpu), + { + SchedulerView { cpu_views: self.cpu_views.insert(cpu, WmView::empty()), ..self } + } + + pub proof fn lemma_register_cpu_preserves_wf(self, cpu: CpuId) + requires + self.wf(), + !self.cpu_views.contains_key(cpu), + valid_cpu(cpu), + ensures + self.register_cpu(cpu).wf(), + self.register_cpu(cpu).cpu_has_thread_view(cpu), + self.register_cpu(cpu).cpu_thread_view(cpu) == WmView::empty(), + { + } + /// Task registration preserves the scheduler ownership partition. pub proof fn lemma_register_task_preserves_wf(self, task: Loc) requires @@ -274,13 +317,16 @@ impl SchedulerView { recommends self.current.contains_key(cpu), self.current[cpu] is Some, + self.cpu_has_thread_view(cpu), self.task_view_is_stored(self.current[cpu]->0), !self.task_view_is_checked_out(self.current[cpu]->0), { let task = self.current[cpu]->0; + let joined = self.stored_views[task].join(self.cpu_views[cpu]); SchedulerView { + task_views: self.task_views.insert(task, joined), stored_views: self.stored_views.remove(task), - checked_out_views: self.checked_out_views.insert(task, self.stored_views[task]), + checked_out_views: self.checked_out_views.insert(task, joined), ..self } } @@ -335,11 +381,46 @@ impl SchedulerView { } } + /// Publishes the outgoing task's observations into the persistent CPU + /// view. A subsequent task scheduled on this CPU imports the result in + /// `checkout_task_view`. + pub open spec fn publish_cpu_view(self, cpu: CpuId, view: WmView) -> SchedulerView + recommends + self.cpu_has_thread_view(cpu), + { + SchedulerView { + cpu_views: self.cpu_views.insert(cpu, self.cpu_views[cpu].join(view)), + ..self + } + } + + pub proof fn lemma_publish_cpu_view_preserves_wf(self, cpu: CpuId, view: WmView) + requires + self.wf(), + self.cpu_has_thread_view(cpu), + ensures + self.publish_cpu_view(cpu, view).wf(), + self.publish_cpu_view(cpu, view).cpu_thread_view(cpu) == self.cpu_thread_view(cpu).join( + view, + ), + self.cpu_thread_view(cpu).spec_le( + self.publish_cpu_view(cpu, view).cpu_thread_view(cpu), + ), + view.spec_le(self.publish_cpu_view(cpu, view).cpu_thread_view(cpu)), + { + self.cpu_thread_view(cpu).lemma_join_left(view); + self.cpu_thread_view(cpu).lemma_join_right(view); + } + pub open spec fn wf(self) -> bool { // CPU-indexed maps may only mention valid CPUs. &&& forall|cpu: CpuId| #[trigger] self.runqueues.contains_key(cpu) ==> valid_cpu(cpu) &&& forall|cpu: CpuId| #[trigger] - self.current.contains_key(cpu) ==> valid_cpu( + self.current.contains_key(cpu) ==> valid_cpu(cpu) && self.cpu_views.contains_key(cpu) + &&& forall|cpu: CpuId| #[trigger] + self.runqueues.contains_key(cpu) ==> self.cpu_views.contains_key(cpu) + &&& forall|cpu: CpuId| #[trigger] + self.cpu_views.contains_key(cpu) ==> valid_cpu( cpu, ) // Runqueues contain exactly runnable tasks; current slots contain @@ -402,6 +483,7 @@ impl SchedulerView { tracked struct SchedulerThreadViews { scheduler: Ghost, views: Map, + cpu_views: Map, } /// A checked-out per-task `ThreadView`. @@ -486,9 +568,11 @@ impl SchedulerThreadViews { ensures res.scheduler() == scheduler, res.view() == Map::::empty(), + res.cpu_view_map() == Map::::empty(), { let tracked views = Map::::tracked_empty(); - SchedulerThreadViews { scheduler: Ghost(scheduler), views } + let tracked cpu_views = Map::::tracked_empty(); + SchedulerThreadViews { scheduler: Ghost(scheduler), views, cpu_views } } closed spec fn scheduler(self) -> Loc { @@ -499,6 +583,10 @@ impl SchedulerThreadViews { Map::new(self.views.dom(), |task: Loc| self.views[task]@) } + pub closed spec fn cpu_view_map(self) -> Map { + Map::new(self.cpu_views.dom(), |cpu: CpuId| self.cpu_views[cpu]@) + } + pub closed spec fn contains(self, task: Loc) -> bool { self.views.contains_key(task) } @@ -510,12 +598,45 @@ impl SchedulerThreadViews { self.views[task]@ } + pub closed spec fn contains_cpu(self, cpu: CpuId) -> bool { + self.cpu_views.contains_key(cpu) + } + + pub closed spec fn cpu_thread_view(self, cpu: CpuId) -> WmView + recommends + self.contains_cpu(cpu), + { + self.cpu_views[cpu]@ + } + /// The tracked owner contains exactly the views still stored in scheduler /// state. Checked-out views are represented by `TaskThreadView` tokens /// instead, so they are intentionally absent here. pub closed spec fn wf(self, sched_view: SchedulerView) -> bool { &&& self.scheduler() == sched_view.id &&& self.view() == sched_view.stored_views + &&& self.cpu_view_map() == sched_view.cpu_views + } + + proof fn tracked_register_cpu(tracked &mut self, sched_view: SchedulerView, cpu: CpuId) + requires + old(self).wf(sched_view), + sched_view.wf(), + !sched_view.cpu_has_thread_view(cpu), + valid_cpu(cpu), + ensures + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view(), + final(self).cpu_view_map() == sched_view.register_cpu(cpu).cpu_views, + final(self).wf(sched_view.register_cpu(cpu)), + final(self).contains_cpu(cpu), + final(self).cpu_thread_view(cpu) == WmView::empty(), + { + sched_view.lemma_register_cpu_preserves_wf(cpu); + let tracked cpu_view = ThreadView::new(); + self.cpu_views.tracked_insert(cpu, cpu_view); + assert(final(self).cpu_view_map() == sched_view.register_cpu(cpu).cpu_views); + assert(final(self).wf(sched_view.register_cpu(cpu))); } /// Inserts a task view created during task registration. @@ -529,6 +650,7 @@ impl SchedulerThreadViews { ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(token.task(), token.view()), + final(self).cpu_view_map() == old(self).cpu_view_map(), { let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; self.views.tracked_insert(task, thread_view); @@ -547,6 +669,7 @@ impl SchedulerThreadViews { ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == sched_view.register_task(task).stored_views, + final(self).cpu_view_map() == old(self).cpu_view_map(), final(self).wf(sched_view.register_task(task)), final(self).contains(task), final(self).thread_view(task) == WmView::empty(), @@ -573,20 +696,27 @@ impl SchedulerThreadViews { sched_view.wf(), sched_view.current.contains_key(cpu), sched_view.current[cpu] is Some, + sched_view.cpu_has_thread_view(cpu), sched_view.task_view_is_stored(sched_view.current[cpu]->0), old(self).contains(sched_view.current[cpu]->0), + old(self).contains_cpu(cpu), ensures token.task() == sched_view.current[cpu]->0, token.scheduler() == sched_view.id, - token.view() == old(self).thread_view(sched_view.current[cpu]->0), + token.view() == old(self).thread_view(sched_view.current[cpu]->0).join( + old(self).cpu_thread_view(cpu), + ), final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), + final(self).cpu_view_map() == old(self).cpu_view_map(), final(self).view() == sched_view.checkout_task_view(cpu).stored_views, final(self).wf(sched_view.checkout_task_view(cpu)), token.wf(sched_view.checkout_task_view(cpu)), { let task = sched_view.current[cpu]->0; - let tracked thread_view = self.views.tracked_remove(task); + let tracked mut thread_view = self.views.tracked_remove(task); + let tracked cpu_view = self.cpu_views.tracked_borrow(cpu); + thread_view.tracked_join(cpu_view); let tracked token = TaskThreadView { scheduler: Ghost(self.scheduler()), task: Ghost(task), @@ -611,12 +741,17 @@ impl SchedulerThreadViews { sched_view.wf(), sched_view.current.contains_key(cpu), sched_view.current[cpu] is Some, + sched_view.cpu_has_thread_view(cpu), sched_view.task_view_is_stored(sched_view.current[cpu]->0), old(self).contains(sched_view.current[cpu]->0), + old(self).contains_cpu(cpu), ensures context.task() == sched_view.current[cpu]->0, context.scheduler() == sched_view.id, - context.view() == old(self).thread_view(sched_view.current[cpu]->0), + context.view() == old(self).thread_view(sched_view.current[cpu]->0).join( + old(self).cpu_thread_view(cpu), + ), + context.cpu() == cpu, context.is_quiescent(), context.wf_scheduler(sched_view.checkout_task_view(cpu)), final(self).view() == sched_view.checkout_task_view(cpu).stored_views, @@ -625,7 +760,7 @@ impl SchedulerThreadViews { { let ghost next = sched_view.checkout_task_view(cpu); let tracked task_view = self.tracked_take_current_thread_view(sched_view, cpu); - let tracked context = RunningTaskContext::new(task_view, next); + let tracked context = RunningTaskContext::new(task_view, next, cpu); context } @@ -646,6 +781,7 @@ impl SchedulerThreadViews { ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(token.task(), token.view()), + final(self).cpu_view_map() == old(self).cpu_view_map(), final(self).view() == sched_view.checkin_task_view( token.task(), token.view(), @@ -672,30 +808,47 @@ impl SchedulerThreadViews { sched_view.wf(), sched_view.task_view_is_checked_out(context.task()), context.scheduler() == sched_view.id, + sched_view.current.contains_key(context.cpu()), + sched_view.current[context.cpu()] == Some(context.task()), + sched_view.cpu_has_thread_view(context.cpu()), context.wf(), context.is_quiescent(), !old(self).contains(context.task()), + old(self).contains_cpu(context.cpu()), ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(context.task(), context.view()), - final(self).view() == sched_view.update_checked_out_task_view( - context.task(), - context.view(), - ).checkin_task_view(context.task(), context.view()).stored_views, + final(self).cpu_view_map() == old(self).cpu_view_map().insert( + context.cpu(), + old(self).cpu_thread_view(context.cpu()).join(context.view()), + ), final(self).wf( sched_view.update_checked_out_task_view( context.task(), context.view(), - ).checkin_task_view(context.task(), context.view()), + ).checkin_task_view(context.task(), context.view()).publish_cpu_view( + context.cpu(), + context.view(), + ), ), { let ghost task = context.task(); let ghost view = context.view(); + let ghost cpu = context.cpu(); sched_view.lemma_update_checked_out_task_view_preserves_wf(task, view); let ghost updated = sched_view.update_checked_out_task_view(task, view); context.lemma_wf_scheduler(updated); let tracked task_view = context.tracked_into_task_view_for_scheduler(updated); - self.tracked_put_checked_out_thread_view(updated, task_view); + let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = task_view; + let tracked cpu_view = self.cpu_views.tracked_borrow_mut(cpu); + cpu_view.tracked_join(&thread_view); + self.views.tracked_insert(task, thread_view); + let ghost checked = updated.checkin_task_view(task, view); + checked.lemma_publish_cpu_view_preserves_wf(cpu, view); + let ghost next = checked.publish_cpu_view(cpu, view); + assert(final(self).view() == next.stored_views); + assert(final(self).cpu_view_map() == next.cpu_views); + assert(final(self).wf(next)); } } @@ -766,6 +919,26 @@ impl SchedulerGhostState { assert(self.wf()); } + /// Registers one persistent CPU view before scheduler policy installs a + /// runqueue or current-task slot for that CPU. + pub proof fn tracked_register_cpu(tracked &mut self, cpu: CpuId) + requires + old(self).wf(), + !old(self).view().cpu_has_thread_view(cpu), + valid_cpu(cpu), + ensures + final(self).wf(), + final(self).id() == old(self).id(), + final(self).view() == old(self).view().register_cpu(cpu), + final(self).view().cpu_has_thread_view(cpu), + final(self).view().cpu_thread_view(cpu) == WmView::empty(), + { + let ghost old_view = self.view@; + self.thread_views.tracked_register_cpu(old_view, cpu); + self.view = Ghost(old_view.register_cpu(cpu)); + assert(self.wf()); + } + /// Registers a new task with one empty weak-memory view. pub proof fn tracked_register_task(tracked &mut self, task: Loc) requires @@ -792,6 +965,7 @@ impl SchedulerGhostState { old(self).wf(), old(self).view().current.contains_key(cpu), old(self).view().current[cpu] is Some, + old(self).view().cpu_has_thread_view(cpu), old(self).view().task_view_is_stored(old(self).view().current[cpu]->0), ensures final(self).wf(), @@ -799,12 +973,19 @@ impl SchedulerGhostState { final(self).view() == old(self).view().checkout_task_view(cpu), context.task() == old(self).view().current[cpu]->0, context.scheduler() == old(self).id(), - context.view() == old(self).view().task_thread_view(context.task()), + context.cpu() == cpu, + context.view() == old(self).view().task_thread_view(context.task()).join( + old(self).view().cpu_thread_view(cpu), + ), + old(self).view().task_thread_view(context.task()).spec_le(context.view()), + old(self).view().cpu_thread_view(cpu).spec_le(context.view()), context.is_quiescent(), context.wf_scheduler(final(self).view()), { let ghost old_view = self.view@; let tracked context = self.thread_views.tracked_take_current_running_context(old_view, cpu); + old_view.task_thread_view(context.task()).lemma_join_left(old_view.cpu_thread_view(cpu)); + old_view.task_thread_view(context.task()).lemma_join_right(old_view.cpu_thread_view(cpu)); self.view = Ghost(old_view.checkout_task_view(cpu)); assert(self.wf()); context @@ -816,6 +997,9 @@ impl SchedulerGhostState { old(self).wf(), old(self).view().task_view_is_checked_out(context.task()), context.scheduler() == old(self).id(), + old(self).view().current.contains_key(context.cpu()), + old(self).view().current[context.cpu()] == Some(context.task()), + old(self).view().cpu_has_thread_view(context.cpu()), context.wf(), context.is_quiescent(), ensures @@ -824,18 +1008,26 @@ impl SchedulerGhostState { final(self).view() == old(self).view().update_checked_out_task_view( context.task(), context.view(), - ).checkin_task_view(context.task(), context.view()), + ).checkin_task_view(context.task(), context.view()).publish_cpu_view( + context.cpu(), + context.view(), + ), final(self).view().task_view_is_stored(context.task()), final(self).view().task_thread_view(context.task()) == context.view(), !final(self).view().task_view_is_checked_out(context.task()), + context.view().spec_le(final(self).view().cpu_thread_view(context.cpu())), { let ghost old_view = self.view@; + let ghost cpu = context.cpu(); + let ghost task = context.task(); + let ghost context_view = context.view(); let ghost next = old_view.update_checked_out_task_view( - context.task(), - context.view(), - ).checkin_task_view(context.task(), context.view()); + task, + context_view, + ).checkin_task_view(task, context_view).publish_cpu_view(cpu, context_view); self.thread_views.tracked_put_running_context(old_view, context); self.view = Ghost(next); + old_view.cpu_thread_view(cpu).lemma_join_right(context_view); assert(self.wf()); } } From 88bc28740f0ea2f4fe8da9a16f0290effcf3e904 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 23 Jul 2026 03:10:07 -0400 Subject: [PATCH 27/47] format --- ostd/specs/sync/rcu.rs | 40 ++++++++++++++++------------------------ ostd/src/sync/rcu/mod.rs | 3 +-- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index c1ecac57e..55259f415 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -699,10 +699,8 @@ impl RcuRootOwnedGhost { /// Reader slots are proof-only and currently allocated per critical /// section. `tracked_stop_reader` consumes the live slot again; no runtime /// reader counter is introduced. - pub proof fn tracked_start_reader( - tracked &mut self, - history: History<*mut T>, - ) -> (tracked res: RcuBaseGuard) + pub proof fn tracked_start_reader(tracked &mut self, history: History<*mut T>) -> (tracked res: + RcuBaseGuard) requires rcu_owned_root_history_inv(history, *old(self)), ensures @@ -791,12 +789,7 @@ impl RcuRootOwnedGhost { None }, }; - let tracked res = RcuRootOwnedGhost { - root, - current, - infos, - removals: Map::empty(), - }; + let tracked res = RcuRootOwnedGhost { root, current, infos, removals: Map::empty() }; assert(res.infos_wf()); assert(res.removals_wf(seq![Msg { value: ptr, view: WmView::empty() }])); res @@ -834,8 +827,10 @@ impl RcuRootOwnedGhost { &&& detached.retired().domain() == detached.domain() &&& detached.retired().obj() == detached.obj() &&& detached.retired().ptr() == detached.ptr() - &&& detached.retired().removal() - == (RcuRemovalObservation { root, timestamp: prev.len() }) + &&& detached.retired().removal() == (RcuRemovalObservation { + root, + timestamp: prev.len(), + }) &&& old(self).current_ownership() == Some(detached.ownership()) &&& equal(detached.ptr(), prev[(prev.len() - 1) as int].value) &&& OwnPred::owns(detached.ptr(), detached.ownership()) @@ -931,11 +926,10 @@ impl RcuRootOwnedGhost { if removed_obj == Some(obj) { assert(self.removals()[obj] == prev.len()); assert(next.len() == prev.len() + 1); - assert(self.publications()[prev.len() as int] - == match new_registration { - Some(registration) => Some(registration.0.obj()), - None => None, - }); + assert(self.publications()[prev.len() as int] == match new_registration { + Some(registration) => Some(registration.0.obj()), + None => None, + }); if new_registration is Some { assert(!old(self).root().objects().contains_key( new_registration->Some_0.0.obj(), @@ -946,8 +940,8 @@ impl RcuRootOwnedGhost { assert(old(self).removals().contains_key(obj)); assert(self.removals()[obj] == old(self).removals()[obj]); assert forall|i: int| - self.removals()[obj] <= i < next.len() implies - #[trigger] self.publications()[i] != Some(obj) by { + self.removals()[obj] <= i + < next.len() implies #[trigger] self.publications()[i] != Some(obj) by { if i < prev.len() { assert(self.publications()[i] == old(self).publications()[i]); } else { @@ -1000,8 +994,8 @@ impl RcuRootOwnedGhost { assert(!old(self).removals().contains_key(owned.registration.0.obj())); assert(obj != owned.registration.0.obj()); assert forall|i: int| - self.removals()[obj] <= i < next.len() implies - #[trigger] self.publications()[i] != Some(obj) by { + self.removals()[obj] <= i < next.len() implies #[trigger] self.publications()[i] + != Some(obj) by { if i < prev.len() { assert(self.publications()[i] == old(self).publications()[i]); } else { @@ -2432,9 +2426,7 @@ impl RcuReadGuardToken { /// Lift a base guard using its start-time expired set as the initial /// traversal observation. - pub proof fn tracked_from_base( - tracked base: RcuBaseGuard, - ) -> (tracked res: Self) + pub proof fn tracked_from_base(tracked base: RcuBaseGuard) -> (tracked res: Self) requires base.wf(), ensures diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 989aeb18b..42b2c0537 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -1142,8 +1142,7 @@ impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { &&& self.rcu.type_inv() &&& self.tracked_guard@.wf() &&& self.tracked_guard@.domain() == self.rcu.ptr.constant().domain - &&& self.tracked_guard@.reader_registry() - == self.rcu.ptr.constant().reader_registry + &&& self.tracked_guard@.reader_registry() == self.rcu.ptr.constant().reader_registry &&& !self.rcu.is_nullable() ==> !self.obj_ptr.is_null() &&& match self.tracked_info@ { None => self.obj_ptr.is_null(), From 9e83e47eef6eb48e5690d88e20e6a6799e519a86 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Sun, 26 Jul 2026 22:40:16 -0400 Subject: [PATCH 28/47] Move weak memory atomics to vstd_extra --- ostd/specs/sync/weak_memory.rs | 2471 ++----------------- ostd/src/sync/rcu/mod.rs | 13 +- ostd/src/sync/rcu/monitor.rs | 8 +- verified_libs/vstd_extra/src/atomic_weak.rs | 2376 ++++++++++++++++++ verified_libs/vstd_extra/src/lib.rs | 1 + 5 files changed, 2547 insertions(+), 2322 deletions(-) create mode 100644 verified_libs/vstd_extra/src/atomic_weak.rs diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 67efa3566..e0d93a846 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -1,547 +1,43 @@ -//! Weak-memory atomic wrappers used by the verification layer. +// SPDX-License-Identifier: MPL-2.0 +//! OSTD-specific adapters for the generic weak-memory atomic library. //! -//! This module is a TCB boundary: executable atomic operations are connected to -//! Rust atomics with `external_body`, while proofs rely only on the ghost specs -//! below. Concrete wrappers currently cover Rust integer atomics, `AtomicBoolW`, -//! and `AtomicPtrW`, all using the same view/history model. -//! -//! We focus on the repaired C11/RC11-style memory model, where relaxed behavior -//! is modeled as reading from previously written messages in a location’s modi- -//! fication history, subject to coherence. In particular, relaxed reads may ob- -//! serve stale writes, but a thread’s view prevents it from going backwards, -//! and reads do not observe future writes that have not been added to the history. -//! -//! # References -//! -//! - [RCU Verification](https://dl.acm.org/doi/pdf/10.1145/3729246) -use core::sync::atomic::{ - AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, - AtomicU32, AtomicUsize, Ordering, -}; +//! The reusable view, history, resource algebra, atomic wrappers, and +//! invariant-opening macro live in [`vstd_extra::atomic_weak`]. This module +//! re-exports that API for existing OSTD callers and keeps only transitions +//! coupled to the RCU root and monitor ghost state. +pub use vstd_extra::atomic_weak::*; +pub use vstd_extra::weak_atomic_with_ghost; use super::rcu as rcu_spec; - -#[cfg(target_has_atomic = "64")] -use core::sync::atomic::{AtomicI64, AtomicU64}; - -use vstd::assert_sets_equal; -use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::prelude::*; -use vstd::resource::Loc; -use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; -use vstd::seq::Seq; - -verus! { - -// The "global" memory is defined within the invariant we need to preserve and, -// by the definition of Iris operations, invariant can be opened by a thread -// provided that the invariant holds and it can close afterwards provided that -// the invariant holds as well. -// -// Thanks to Verus' native support for the semantics, we only need to define -// what means for `atomic` and we can freely open the invariant and provide -// customized macros for doing ergonomic updates on both the physical resources -// and the ghost tokens like message histories, views, etc. -/// An `AtomicId` is just an abstract identifier (memory location) of one atomic object. -pub type AtomicId = Loc; - -/// Logical timestamp into one atomic object's message history. -/// Timestamp 0 is always the initial message installed by `new`. -pub type Timestamp = nat; - -/// A thread-local weak-memory view. -/// -/// `seen[id] = ts` means this thread has advanced past all messages for `id` -/// older than `ts`; future reads from that atomic must not go backwards. -/// -/// Typically, if another thread has published a message with timestamp `ts` for `id`, -/// and the reader reads the message via some atomic operations, then the reader's -/// thread view will advance to at least `ts` for `id`. -pub ghost struct WmView { - pub seen: IMap, -} - -impl WmView { - /// Creates an empty view. - pub open spec fn empty() -> Self { - WmView { seen: IMap::empty() } - } - - pub open spec fn seen_at(self, id: AtomicId) -> Timestamp { - if self.seen.contains_key(id) { - self.seen[id] - } else { - // Missing entries are equivalent to only seeing the initial write. - 0nat - } - } - - /// Monotonically advance the view for one atomic object. - /// - /// Just as the name indicates, `observe` means that the current thread has observed - /// a message written by another thread with a specific timestamp; because the atomic - /// operation never "goes back", the thread's view for that atomic must advance to at - /// least that timestamp. - /// - /// "During a read from `l`, a thread can observe any message `m` from `M(l)` where - /// `m.time >= V(l)`, and updates its view to incorporate `m.time`." - pub open spec fn observe(self, id: AtomicId, ts: Timestamp) -> Self { - WmView { - seen: self.seen.insert( - id, - if self.seen_at(id) <= ts { - ts - } else { - self.seen_at(id) - }, - ), - } - } - - /// Pointwise maximum of two views. - /// - /// This is the ghost effect of an acquire read: the reader imports the - /// release view carried by the message it read. - pub open spec fn join(self, other: Self) -> Self { - WmView { - seen: IMap::new( - |id: AtomicId| self.seen.contains_key(id) || other.seen.contains_key(id), - |id: AtomicId| - if self.seen_at(id) <= other.seen_at(id) { - other.seen_at(id) - } else { - self.seen_at(id) - }, - ), - } - } - - /// Partial ordering two threads' views. - pub open spec fn spec_le(self, other: Self) -> bool { - forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) - } - - pub proof fn lemma_join_left(self, other: Self) - ensures - self.spec_le(self.join(other)), - { - } - - pub proof fn lemma_join_right(self, other: Self) - ensures - other.spec_le(self.join(other)), - { - } - - pub proof fn lemma_spec_le_transitive(self, middle: Self, upper: Self) - requires - self.spec_le(middle), - middle.spec_le(upper), - ensures - self.spec_le(upper), - { - } -} - -/// One message in an atomic object's modification history. -/// -/// `view` is the release view published with this value. Relaxed stores publish -/// only their own timestamp; release stores publish the writer's current view. -pub ghost struct Msg { - pub value: V, - pub view: WmView, -} - -pub type History = Seq>; - -/// User-supplied invariant predicate for a weak-memory atomic. -/// -/// This mirrors `vstd::atomic_ghost::AtomicInvariantPredicate`, except the -/// predicate is over the whole message history rather than one current value. -pub trait WeakAtomicInvariantPredicate { - spec fn atomic_inv(k: K, history: History, g: G) -> bool; -} - -/// Authoritative ghost state for one atomic object's history. -/// -/// This is intentionally a thin wrapper around vstd's map resource algebra: -/// [`GhostMapAuth`] owns the authoritative timestamp-to-message map, while `len` -/// records that the domain is the contiguous range `0..len`. -/// -/// The proof-facing atomic wrapper below stores this token inside an -/// `AtomicInvariant` next to the executable atomic. -pub tracked struct HistAuth { - auth: GhostMapAuth>, - // Private: code outside this TCB module must not forge the history length, - // which would desynchronize `len` from the authoritative map domain. - ghost len: nat, -} - -proof fn lemma_timestamp_range_insert_last(hi: Timestamp) - ensures - Set::range(0nat, hi).insert(hi) == Set::range(0nat, hi + 1), -{ - broadcast use vstd::set_lib::range_set_properties; - - assert_sets_equal!(Set::range(0nat, hi).insert(hi), Set::range(0nat, hi + 1), ts: Timestamp => { - if Set::range(0nat, hi).insert(hi).contains(ts) { - if ts != hi { - assert(Set::range(0nat, hi).contains(ts)); - assert(ts < hi); - } - assert(ts < hi + 1); - assert(Set::range(0nat, hi + 1).contains(ts)); - } - - if Set::range(0nat, hi + 1).contains(ts) { - assert(ts < hi + 1); - if ts == hi { - assert(Set::range(0nat, hi).insert(hi).contains(ts)); - } else { - assert(ts < hi); - assert(Set::range(0nat, hi).contains(ts)); - assert(Set::range(0nat, hi).insert(hi).contains(ts)); - } - } - }); -} - -impl HistAuth { - pub closed spec fn id(self) -> AtomicId { - self.auth.id() - } - - pub closed spec fn map(self) -> Map> { - self.auth@ - } - - pub closed spec fn len(self) -> nat { - self.len - } - - pub closed spec fn history(self) -> History - recommends - self.wf(), - { - Seq::new(self.len(), |i: int| self.map()[i as nat]) - } - - pub open spec fn wf(self) -> bool { - &&& self.len() > 0 - &&& self.map().dom() == Set::range(0nat, self.len()) - } - - pub open spec fn valid_ts(self, ts: Timestamp) -> bool { - ts < self.len() - } - - pub open spec fn msg_at(self, ts: Timestamp) -> Msg - recommends - self.valid_ts(ts), - { - self.map()[ts] - } - - /// RC11-style relaxed readability: a read may choose any message that is - /// not older than the thread's current view for this location. - pub open spec fn readable(self, view: WmView, ts: Timestamp) -> bool { - &&& self.valid_ts(ts) - &&& view.seen_at(self.id()) <= ts - } - - /// Append one message to the authoritative history and return a persistent - /// snapshot for the newly allocated timestamp. - pub proof fn append_msg(tracked &mut self, msg: Msg) -> (tracked snap: MsgSnap) - requires - old(self).wf(), - ensures - final(self).id() == old(self).id(), - final(self).history() == old(self).history().push(msg), - final(self).wf(), - snap.id() == final(self).id(), - snap.ts() == old(self).history().len(), - snap.msg() == msg, - snap.agrees_with(*final(self)), - { - let ghost ts = self.len(); - let ghost old_dom = self.map().dom(); - - let tracked pt = self.auth.insert(ts, msg); - self.len = self.len + 1; - - // Full-crate verification does not reliably rediscover this range/domain - // fact after the ghost-map insert, so keep the append step explicit. - lemma_timestamp_range_insert_last(ts); - assert(old_dom == Set::range(0nat, ts)); - assert(self.map().dom() == old_dom.insert(ts)); - assert(self.map().dom() == Set::range(0nat, ts + 1)); - assert(ts + 1 == self.len()); - assert(self.map().dom() == Set::range(0nat, self.len())); - - let tracked psnap = pt.persist(); - MsgSnap { snap: psnap } - } -} - -/// A stable proof handle for one message; a snapshot of the message. -/// -/// The underlying vstd token is persistent/duplicable, so a message snapshot -/// can be copied through proofs without granting permission to mutate history. -/// -/// Stores return snapshots so higher layers can connect a concrete write to -/// later ownership-transfer predicates without exposing the whole history. -pub tracked struct MsgSnap { - snap: GhostPersistentPointsTo>, -} - -impl MsgSnap { - pub closed spec fn id(self) -> AtomicId { - self.snap.id() - } - - /// Fetch the timestamp of this specific message. - pub closed spec fn ts(self) -> Timestamp { - self.snap.key() - } - - /// Fetch the ghost message value of this specific message. - pub closed spec fn msg(self) -> Msg { - self.snap.value() - } - - pub open spec fn agrees_with(self, auth: HistAuth) -> bool { - &&& self.id() == auth.id() - &&& auth.valid_ts(self.ts()) - &&& self.msg() == auth.msg_at(self.ts()) - } - - pub proof fn duplicate(tracked &self) -> (tracked snap: MsgSnap) - ensures - snap.id() == self.id(), - snap.ts() == self.ts(), - snap.msg() == self.msg(), - { - let tracked psnap = self.snap.duplicate(); - MsgSnap { snap: psnap } - } - - pub proof fn agree(tracked &self, tracked auth: &HistAuth) - requires - self.id() == auth.id(), - auth.wf(), - ensures - self.agrees_with(*auth), - { - self.snap.agree(&auth.auth); - assert(auth.map().contains_pair(self.ts(), self.msg())); - } -} - -} // verus! -/// Generate the proof-facing wrapper for one concrete weak-memory atomic type. -/// -/// The generated type keeps the executable TCB wrapper separate from the -/// invariant protocol. Adding `AtomicU32W` or `AtomicBoolW` later should require -/// a new executable wrapper plus one macro invocation, not another copy of the -/// invariant glue. -macro_rules! declare_weak_atomic_type { - ($weak_atomic:ident, $pred_adapter:ident, $raw_atomic:ident, $value_ty:ty) => { - verus! { - /// Predicate adapter stored inside `AtomicInvariant`. - /// - /// The invariant contains the authoritative history and user ghost - /// state. The constant pairs the user key `K` with the logical atomic id. - pub struct $pred_adapter { - p: Pred, - } - - impl InvariantPredicate<(K, AtomicId), (HistAuth<$value_ty>, G)> for $pred_adapter< - Pred, - > where Pred: WeakAtomicInvariantPredicate { - open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<$value_ty>, G)) -> bool { - let (k, id) = k_id; - let (hist, g) = hist_g; - &&& hist.id() == id - &&& hist.wf() - &&& Pred::atomic_inv(k, hist.history(), g) - } - } - - /// A weak-memory atomic with an `atomic_ghost`-style invariant. - /// - /// The executable atomic remains the TCB wrapper. This proof-facing - /// wrapper stores the authoritative history in an `AtomicInvariant` and - /// exposes `well_formed`/`type_inv` predicates tying that history to the - /// executable atomic id. As in `vstd::atomic_ghost`, outer data - /// structures put this predicate in their own - /// `#[verifier::type_invariant]`. - pub struct $weak_atomic { - #[doc(hidden)] - atomic: $raw_atomic, - #[doc(hidden)] - atomic_inv: Tracked< - AtomicInvariant<(K, AtomicId), (HistAuth<$value_ty>, G), $pred_adapter>, - >, - } - - impl $weak_atomic { - pub closed spec fn constant(&self) -> K { - self.atomic_inv@.constant().0 - } - - pub closed spec fn well_formed(&self) -> bool { - self.atomic_inv@.constant().1 == self.atomic.id() - } - - #[verifier::type_invariant] - pub closed spec fn type_inv(&self) -> bool { - self.well_formed() - } - } - - impl $weak_atomic where - Pred: WeakAtomicInvariantPredicate, - { - #[inline(always)] - pub const fn new( - Ghost(k): Ghost, - init: $value_ty, - Tracked(g): Tracked, - ) -> (res: Self) - requires - Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), - ensures - res.well_formed(), - res.constant() == k, - { - let (atomic, Tracked(hist)) = $raw_atomic::new(init); - let tracked pair = (hist, g); - assert($pred_adapter::::inv((k, atomic.id()), pair)); - let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); - $weak_atomic { atomic, atomic_inv: Tracked(atomic_inv) } - } - - #[inline(always)] - pub fn load_relaxed( - &self, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } - - #[inline(always)] - pub fn load_acquire( - &self, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } - } - } - }; -} - -declare_weak_atomic_type!(WeakAtomicU8, WeakAtomicPredU8, AtomicU8W, u8); -declare_weak_atomic_type!(WeakAtomicU16, WeakAtomicPredU16, AtomicU16W, u16); -declare_weak_atomic_type!(WeakAtomicU32, WeakAtomicPredU32, AtomicU32W, u32); -declare_weak_atomic_type!(WeakAtomicUsize, WeakAtomicPredUsize, AtomicUsizeW, usize); -declare_weak_atomic_type!(WeakAtomicBool, WeakAtomicPredBool, AtomicBoolW, bool); - -#[cfg(target_has_atomic = "64")] -declare_weak_atomic_type!(WeakAtomicU64, WeakAtomicPredU64, AtomicU64W, u64); - -declare_weak_atomic_type!(WeakAtomicI8, WeakAtomicPredI8, AtomicI8W, i8); -declare_weak_atomic_type!(WeakAtomicI16, WeakAtomicPredI16, AtomicI16W, i16); -declare_weak_atomic_type!(WeakAtomicI32, WeakAtomicPredI32, AtomicI32W, i32); -declare_weak_atomic_type!(WeakAtomicIsize, WeakAtomicPredIsize, AtomicIsizeW, isize); - -#[cfg(target_has_atomic = "64")] -declare_weak_atomic_type!(WeakAtomicI64, WeakAtomicPredI64, AtomicI64W, i64); verus! { -/// Predicate adapter for weak-memory pointer atomics. -/// -/// The history stores raw pointer values. This tracks the atomic pointer value -/// itself; ownership of the pointee must be modeled by the user ghost state `G`. -pub struct WeakAtomicPredPtr { - t: T, - p: Pred, -} - -impl InvariantPredicate<(K, AtomicId), (HistAuth<*mut T>, G)> for WeakAtomicPredPtr< - T, - Pred, -> where Pred: WeakAtomicInvariantPredicate { - open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<*mut T>, G)) -> bool { - let (k, id) = k_id; - let (hist, g) = hist_g; - &&& hist.id() == id - &&& hist.wf() - &&& Pred::atomic_inv(k, hist.history(), g) - } -} - -/// Weak-memory atomic pointer with an `atomic_ghost`-style invariant. +/// OSTD's RCU-specific specialization of the generic weak pointer atomic. /// -/// This is the pointer analogue of [`WeakAtomicUsize`]. It deliberately models -/// only the atomic pointer value and its release/acquire synchronization history; -/// any ownership or validity claim about the pointed-to allocation belongs in -/// the user-supplied ghost state `G` and invariant predicate. -#[verifier::accept_recursive_types(T)] -pub struct WeakAtomicPtr { - #[doc(hidden)] - atomic: AtomicPtrW, - #[doc(hidden)] - atomic_inv: Tracked< - AtomicInvariant<(K, AtomicId), (HistAuth<*mut T>, G), WeakAtomicPredPtr>, +/// The inner atomic and its history protocol are reusable; this wrapper adds +/// transitions that manipulate `RcuRootOwnedGhost`. +#[verifier::reject_recursive_types(T)] +pub struct RcuWeakAtomicPtr { + inner: WeakAtomicPtr< + T, + rcu_spec::RcuRootKey, + rcu_spec::RcuRootOwnedGhost, + rcu_spec::RcuOwnedWeakAtomicInv, >, } -impl WeakAtomicPtr { - pub closed spec fn constant(&self) -> K { - self.atomic_inv@.constant().0 +impl RcuWeakAtomicPtr { + pub closed spec fn constant(&self) -> rcu_spec::RcuRootKey { + self.inner.constant() } - /// Logical modification-history identity of this atomic pointer. pub closed spec fn id(&self) -> AtomicId { - self.atomic_inv@.constant().1 + self.inner.id() } pub closed spec fn well_formed(&self) -> bool { - self.id() == self.atomic.id() + self.inner.well_formed() } #[verifier::type_invariant] @@ -550,146 +46,50 @@ impl WeakAtomicPtr { } } -impl WeakAtomicPtr where - Pred: WeakAtomicInvariantPredicate, +impl RcuWeakAtomicPtr where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, { - #[inline(always)] - pub const fn new(Ghost(k): Ghost, init: *mut T, Tracked(g): Tracked) -> (res: Self) + pub const fn new( + Ghost(k): Ghost, + init: *mut T, + Tracked(g): Tracked>, + ) -> (res: Self) requires - Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), + rcu_spec::RcuOwnedWeakAtomicInv::::atomic_inv( + k, + seq![Msg { value: init, view: WmView::empty() }], + g, + ), ensures res.well_formed(), res.constant() == k, { - let (atomic, Tracked(hist)) = AtomicPtrW::::new(init); - let tracked pair = (hist, g); - assert(WeakAtomicPredPtr::::inv((k, atomic.id()), pair)); - let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); - WeakAtomicPtr { atomic, atomic_inv: Tracked(atomic_inv) } - } - - #[inline(always)] - pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( - *mut T, - Ghost, - )) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } - - #[inline(always)] - pub fn load_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( - *mut T, - Ghost, - )) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } -} - -pub struct TrueWeakAtomicInv; - -impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { - open spec fn atomic_inv(k: K, history: History, g: G) -> bool { - true + let inner = WeakAtomicPtr::new(Ghost(k), init, Tracked(g)); + Self { inner } } -} -impl WeakAtomicPtr { - // TODO: Move exec code into, `vstd_extra`? - /// Release-store helper for users with the trivial atomic invariant. - /// - /// This keeps early weak-memory clients from depending on the macro while - /// we are still shaping the client-specific ghost state. - #[inline(always)] - pub fn store_release_simple(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) { - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - let _snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); - proof { - pair = (hist, g); - } - }); + fn raw_atomic(&self) -> (res: &AtomicPtrW) + requires + self.well_formed(), + ensures + res.id() == self.id(), + { + self.inner.raw_atomic() } - /// Strong AcqRel/Acquire CAS helper for users with the trivial invariant. - #[inline(always)] - pub fn compare_exchange_acqrel_acquire_simple( - &self, - current: *mut T, - new: *mut T, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (Result<*mut T, *mut T>, Ghost)) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - let cas_result = self.atomic.compare_exchange_acqrel_acquire( - Tracked(&mut hist), - Tracked(tv), - current, - new, - ); - result = (cas_result.0, cas_result.1); - proof { - pair = (hist, g); - } - }); - result + proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &vstd::invariant::AtomicInvariant< + (rcu_spec::RcuRootKey, AtomicId), + (HistAuth<*mut T>, rcu_spec::RcuRootOwnedGhost), + WeakAtomicPredPtr>, + >) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.id()), + { + self.inner.tracked_atomic_inv() } -} -impl WeakAtomicPtr< - T, - rcu_spec::RcuRootKey, - rcu_spec::RcuRootOwnedGhost, - rcu_spec::RcuOwnedWeakAtomicInv, -> where OwnPred: rcu_spec::RcuRootOwnershipPredicate { /// Acquire-load helper for RCU root pointers. #[inline(always)] pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( @@ -720,14 +120,15 @@ impl WeakAtomicPtr< proof { use_type_invariant(self); } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { let tracked (hist, g) = pair; proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); + assert(hist.id() == self.id()); + assert(raw_atomic.id() == self.id()); + assert(hist.id() == raw_atomic.id()); } - let loaded = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { assert(hist.valid_ts(loaded.1@)); assert(loaded.1@ < hist.history().len()); @@ -802,17 +203,18 @@ impl WeakAtomicPtr< proof { use_type_invariant(self); } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { let tracked (hist, mut g) = pair; proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); + assert(hist.id() == self.id()); + assert(raw_atomic.id() == self.id()); + assert(hist.id() == raw_atomic.id()); } proof_decl! { let tracked base_guard = g.tracked_start_reader(hist.history()); } - let loaded = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { assert(hist.valid_ts(loaded.1@)); assert(loaded.1@ < hist.history().len()); @@ -886,7 +288,7 @@ impl WeakAtomicPtr< proof { use_type_invariant(self); vstd::invariant::open_atomic_invariant_in_proof!( - credit.get() => self.atomic_inv.borrow() => pair => { + credit.get() => self.tracked_atomic_inv() => pair => { let tracked (hist, mut g) = pair; assert(g.domain() == self.constant().domain); assert(g.reader_registry() == self.constant().reader_registry); @@ -937,15 +339,16 @@ impl WeakAtomicPtr< proof { use_type_invariant(self); } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { let tracked (mut hist, mut g) = pair; proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); + assert(hist.id() == self.id()); + assert(raw_atomic.id() == self.id()); + assert(hist.id() == raw_atomic.id()); } let ghost prev = hist.history(); - let swap = self.atomic.swap_release(Tracked(&mut hist), Tracked(tv), value); + let swap = raw_atomic.swap_release(Tracked(&mut hist), Tracked(tv), value); result = swap.0; let snap = swap.1; let ghost next = hist.history(); @@ -967,7 +370,7 @@ impl WeakAtomicPtr< prev, next, snap@.msg(), - self.atomic.id(), + self.id(), ownership, ); assert(detached is Some ==> detached->Some_0.object().wf()); @@ -977,7 +380,7 @@ impl WeakAtomicPtr< detached->Some_0.ownership(), )); assert(detached is Some ==> detached->Some_0.retired().removal().root - == self.atomic.id()); + == self.id()); assert(detached is Some ==> detached->Some_0.retired().removal().timestamp == prev.len()); assert(detached is Some ==> detached->Some_0.retired().removal().observed_by( @@ -1045,15 +448,16 @@ impl WeakAtomicPtr< proof { use_type_invariant(self); } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { let tracked (mut hist, mut g) = pair; proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); + assert(hist.id() == self.id()); + assert(raw_atomic.id() == self.id()); + assert(hist.id() == raw_atomic.id()); } let ghost prev = hist.history(); - let cas_result = self.atomic.compare_exchange_acqrel_acquire( + let cas_result = raw_atomic.compare_exchange_acqrel_acquire( Tracked(&mut hist), Tracked(tv), current, @@ -1084,7 +488,7 @@ impl WeakAtomicPtr< prev, next, snap.msg(), - self.atomic.id(), + self.id(), new_ownership, ); assert(detached is Some ==> detached->Some_0.object().wf()); @@ -1097,7 +501,7 @@ impl WeakAtomicPtr< detached->Some_0.ownership(), )); assert(detached is Some ==> detached->Some_0.retired().removal().root - == self.atomic.id()); + == self.id()); assert(detached is Some ==> detached->Some_0.retired().removal().timestamp == prev.len()); assert(detached is Some ==> @@ -1132,1673 +536,122 @@ impl WeakAtomicPtr< } } -impl WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv> { - /// Relaxed-store helper for the RCU monitor flag. - /// - /// The executable flag remains a relaxed atomic flag, matching the old - /// monitor protocol. The proof-side effect is stronger: each stored flag - /// message appends the lock-protected monitor-state snapshot supplied by - /// the writer. - #[inline(always)] - pub fn store_relaxed_rcu_monitor( - &self, - value: bool, - Ghost(state): Ghost, - Tracked(tv): Tracked<&mut ThreadView>, - ) - requires - self.well_formed(), - state.wf(), - !value ==> state.no_pending_work(), - { - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, mut g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - let ghost prev = hist.history(); - let snap = self.atomic.store_relaxed(Tracked(&mut hist), Tracked(tv), value); - let ghost next = hist.history(); - proof { - assert(snap@.msg().value == value); - rcu_spec::preserve_rcu_monitor_flag_inv_on_push( - prev, - next, - snap@.msg(), - g, - g.push(state), - state, - ); - g = g.tracked_push(state); - pair = (hist, g); - } - }); - } -} - -/// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, -/// but for weak-memory atomics with per-thread view tokens and message histories. -/// -/// The macro opens the atomic invariant, performs the specified operation, and -/// provides the previous history, new history, and operation snapshot to the user- -/// provided proof block. The user can then write proofs about the effects of the -/// operation on the history and thread view, using the snapshot to connect to the -/// authoritative history. -#[macro_export] -macro_rules! weak_atomic_with_ghost { - ( - $atomic:expr => compare_exchange_acqrel_acquire($current:expr, $new:expr, $tv:expr); - update $prev:ident -> $next:ident; - returning $ret:ident; - timestamp $ts:ident; - message $msg:ident; - snapshot $snap:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let result; - let atomic = &($atomic); - let current = $current; - let new = $new; - proof { - use_type_invariant(atomic); - } - ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { - #[allow(unused_mut)] - let tracked (mut hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.atomic_inv@.constant().1); - assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); - assert(hist.id() == atomic.atomic.id()); - } - let ghost $prev = hist.history(); - let cas_result = atomic.atomic.compare_exchange_acqrel_acquire( - Tracked(&mut hist), - $tv, - current, - new, - ); - result = (cas_result.0, cas_result.1); - let ghost $next = hist.history(); - let ghost $ret = cas_result.0; - let ghost $ts = cas_result.1@; - let ghost $msg = $prev[$ts as int]; - - proof { - let tracked $snap = cas_result.2.get(); - $b - } - - proof { - pair = (hist, $g); - } - }); - result - }} - }; - ( - $atomic:expr => load_acquire($tv:expr); - returning $ret:ident; - timestamp $ts:ident; - message $msg:ident; - history $history:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let result; - let atomic = &($atomic); - proof { - use_type_invariant(atomic); - } - ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { - #[allow(unused_mut)] - let tracked (hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.atomic_inv@.constant().1); - assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); - assert(hist.id() == atomic.atomic.id()); - } - let ghost $history = hist.history(); - result = atomic.atomic.load_acquire(Tracked(&hist), $tv); - let ghost $ret = result.0; - let ghost $ts = result.1@; - let ghost $msg = hist.msg_at($ts); - - proof { $b } - - proof { - pair = (hist, $g); - } - }); - result - }} - }; - ( - $atomic:expr => load_relaxed($tv:expr); - returning $ret:ident; - timestamp $ts:ident; - message $msg:ident; - history $history:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let result; - let atomic = &($atomic); - proof { - use_type_invariant(atomic); - } - ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { - #[allow(unused_mut)] - let tracked (hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.atomic_inv@.constant().1); - assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); - assert(hist.id() == atomic.atomic.id()); - } - let ghost $history = hist.history(); - result = atomic.atomic.load_relaxed(Tracked(&hist), $tv); - let ghost $ret = result.0; - let ghost $ts = result.1@; - let ghost $msg = hist.msg_at($ts); - - proof { $b } - - proof { - pair = (hist, $g); - } - }); - result - }} - }; - ( - $atomic:expr => store_release($value:expr, $tv:expr); - update $prev:ident -> $next:ident; - snapshot $snap:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let atomic = &($atomic); - let value = $value; - proof { - use_type_invariant(atomic); - } - ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { - #[allow(unused_mut)] - let tracked (mut hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.atomic_inv@.constant().1); - assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); - assert(hist.id() == atomic.atomic.id()); - } - let ghost $prev = hist.history(); - let snap_tracked = atomic.atomic.store_release(Tracked(&mut hist), $tv, value); - let ghost $next = hist.history(); - - proof { - let tracked $snap = snap_tracked.get(); - $b - } - - proof { - pair = (hist, $g); - } - }); - }} - }; - ( - $atomic:expr => store_relaxed($value:expr, $tv:expr); - update $prev:ident -> $next:ident; - snapshot $snap:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let atomic = &($atomic); - let value = $value; - proof { - use_type_invariant(atomic); - } - ::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => { - #[allow(unused_mut)] - let tracked (mut hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.atomic_inv@.constant().1); - assert(atomic.atomic_inv@.constant().1 == atomic.atomic.id()); - assert(hist.id() == atomic.atomic.id()); - } - let ghost $prev = hist.history(); - let snap_tracked = atomic.atomic.store_relaxed(Tracked(&mut hist), $tv, value); - let ghost $next = hist.history(); - - proof { - let tracked $snap = snap_tracked.get(); - $b - } - - proof { - pair = (hist, $g); - } - }); - }} - }; -} - -/// Explicit per-thread view token. -/// -/// Passing this token through atomic operations makes the weak-memory effects -/// visible in specs instead of hiding them in global or thread-local state. -/// -/// # Soundness -/// -/// The wrapped view is private and can only evolve through the TCB atomic -/// operations in this module. Those operations maintain the invariant that a -/// view never claims a timestamp at or beyond the length of that location's -/// history: loads observe an existing message, stores observe the message they -/// just appended, and acquire joins only import message views that were built -/// from existing timestamps. This keeps the `readable`-based postconditions of -/// loads satisfiable. Do not add raw mutators (e.g. an unconditional -/// `observe`/`join` proof fn): a forged view claiming an unwritten timestamp -/// would make the next load's postcondition vacuously false. -pub tracked struct ThreadView { - ghost view: WmView, -} - -impl View for ThreadView { - type V = WmView; - - closed spec fn view(&self) -> WmView { - self.view - } +/// RCU monitor specialization of the generic weak boolean atomic. +pub struct RcuMonitorWeakAtomicBool { + inner: WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv>, } -impl ThreadView { - /// Creates a fresh token holding the empty view. - /// - /// The empty view is the weakest token: it lower-bounds every location at - /// timestamp 0, so minting one is always sound — the holder merely - /// forfeits all ordering knowledge. Note that minting a fresh view - /// mid-thread over-approximates real executions (it forgets per-location - /// coherence the thread has already observed) and publishes nothing useful - /// through release stores, so executable code should thread one token per - /// logical operation or critical section, and eventually one per task. - /// - /// This constructor is crate-private so clients cannot discard an - /// established task view and restart from the empty view. Production code - /// creates one view when the scheduler registers a task, then moves that - /// same linear token through schedule-in and schedule-out. - pub(crate) proof fn new() -> (tracked res: Self) - ensures - res@ == WmView::empty(), - { - ThreadView { view: WmView::empty() } - } - - /// Imports observations from another genuine thread/CPU view. - /// - /// Unlike a raw ghost mutator, this operation cannot introduce an - /// unwritten timestamp: both operands are tracked `ThreadView` values that - /// originated from the weak-memory TCB. Scheduler context switches use it - /// to transfer observations between a CPU view and a task view. - pub(crate) proof fn tracked_join(tracked &mut self, tracked other: &Self) - ensures - final(self)@ == old(self)@.join(other@), - { - self.view = self.view.join(other.view); - } -} - -#[repr(transparent)] -#[verifier::external_body] -/// TCB wrapper around Rust's `AtomicUsize`. -/// -/// The executable field is the real atomic object. The proof layer sees only -/// the specs below, plus the uninterpreted logical identity `id`. -pub struct AtomicUsizeW { - value: AtomicUsize, -} - -impl AtomicUsizeW { - /// Logical identity of this atomic object. - /// - /// `id` has no runtime representation; it indexes ghost histories and - /// thread views. The `new` spec ties the fresh history to this identity. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: usize) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = AtomicUsizeW { value: AtomicUsize::new(init) }; - (atomic, Tracked::assume_new()) +impl RcuMonitorWeakAtomicBool { + pub closed spec fn id(&self) -> AtomicId { + self.inner.id() } - /// Relaxed load: choose a readable message and advance only this location's - /// timestamp in the caller's thread view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (usize, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) + pub closed spec fn well_formed(&self) -> bool { + self.inner.well_formed() } - /// Acquire load: same readable message choice as relaxed, plus import the - /// release view carried by the selected message. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (usize, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() } - /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` - /// failure ordering. - /// - /// On success, RMW atomicity forces the read to be the latest message in - /// the modification history, and the new release message is appended - /// immediately after it. On failure, the operation is only an acquire - /// load: it may read *any* readable message whose value differs from - /// `current`, not necessarily the latest one. A strong CAS merely never - /// fails after reading a value equal to `current`. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - current: usize, - new: usize, - ) -> (res: (Result, Ghost, Tracked>>)) + pub const fn new( + Ghost(k): Ghost<()>, + init: bool, + Tracked(g): Tracked, + ) -> (res: Self) requires - old(auth).id() == self.id(), - old(auth).wf(), + rcu_spec::RcuMonitorFlagInv::atomic_inv( + k, + seq![Msg { value: init, view: WmView::empty() }], + g, + ), ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& v == current - &&& read_msg.value == current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& v == read_msg.value - &&& read_msg.value != current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind + res.well_formed(), { - let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); - (result, Ghost::assume_new(), Tracked::assume_new()) + let inner = WeakAtomicBool::new(Ghost(k), init, Tracked(g)); + Self { inner } } - /// Relaxed store: append a new message whose published view contains only - /// this store's own timestamp. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: usize, - ) -> (snap: Tracked>) + pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + bool, + Ghost, + )) requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind + self.well_formed(), { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() + self.inner.load_relaxed(Tracked(tv)) } - /// Release store: append a new message carrying the writer's current view, - /// then advance the writer's view for this location. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: usize, - ) -> (snap: Tracked>) + fn raw_atomic(&self) -> (res: &AtomicBoolW) requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } -} - -} // verus! -/// Generate a TCB executable wrapper around one Rust integer atomic type. -/// -/// All integer atomics share the same weak-memory history shape: load chooses a -/// readable message, stores append a message, and CAS either reads the latest -/// message and appends its write right after it (success), or acts as an -/// acquire read of any readable message with a different value (failure). -macro_rules! declare_integer_atomic_wrapper { - ($wrapper:ident, $rust_atomic:ident, $value_ty:ty) => { - verus! { - #[repr(transparent)] - #[verifier::external_body] - /// TCB wrapper around a Rust integer atomic. - pub struct $wrapper { - value: $rust_atomic, - } - - impl $wrapper { - /// Logical identity of this atomic object. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: $value_ty) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = $wrapper { value: $rust_atomic::new(init) }; - (atomic, Tracked::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - current: $value_ty, - new: $value_ty, - ) -> (res: ( - Result<$value_ty, $value_ty>, - Ghost, - Tracked>>, - )) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& v == current - &&& read_msg.value == current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& v == read_msg.value - &&& read_msg.value != current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind - { - let result = self.value.compare_exchange( - current, - new, - Ordering::AcqRel, - Ordering::Acquire, - ); - (result, Ghost::assume_new(), Tracked::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: $value_ty, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: $value_ty, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } - } - } - }; -} - -declare_integer_atomic_wrapper!(AtomicU8W, AtomicU8, u8); - -declare_integer_atomic_wrapper!(AtomicU16W, AtomicU16, u16); - -declare_integer_atomic_wrapper!(AtomicU32W, AtomicU32, u32); - -declare_integer_atomic_wrapper!(AtomicIsizeW, AtomicIsize, isize); - -declare_integer_atomic_wrapper!(AtomicI8W, AtomicI8, i8); - -declare_integer_atomic_wrapper!(AtomicI16W, AtomicI16, i16); - -declare_integer_atomic_wrapper!(AtomicI32W, AtomicI32, i32); - -#[cfg(target_has_atomic = "64")] -declare_integer_atomic_wrapper!(AtomicU64W, AtomicU64, u64); - -#[cfg(target_has_atomic = "64")] -declare_integer_atomic_wrapper!(AtomicI64W, AtomicI64, i64); - -verus! { - -#[repr(transparent)] -#[verifier::external_body] -/// TCB wrapper around Rust's `AtomicBool`. -/// -/// Bool atomics share the load/store/CAS weak-memory protocol with integer -/// atomics, but they are not numeric atomics: this wrapper intentionally exposes -/// no arithmetic or bitwise fetch operations. -pub struct AtomicBoolW { - value: AtomicBool, -} - -impl AtomicBoolW { - /// Logical identity of this atomic object. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: bool) -> (res: (Self, Tracked>)) + self.well_formed(), ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), + res.id() == self.id(), { - let atomic = AtomicBoolW { value: AtomicBool::new(init) }; - (atomic, Tracked::assume_new()) + self.inner.raw_atomic() } - /// Relaxed load: choose a readable bool message and advance only this - /// location's timestamp in the caller's thread view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (bool, Ghost)) + proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &vstd::invariant::AtomicInvariant< + ((), AtomicId), + (HistAuth, rcu_spec::RcuMonitorFlagGhost), + WeakAtomicPredBool, + >) requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - /// Acquire load: same bool choice as relaxed, plus import the release view - /// carried by the selected message. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (bool, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), + self.well_formed(), ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind + res.constant() == ((), self.id()), { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) + self.inner.tracked_atomic_inv() } - /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` - /// failure ordering. + /// Relaxed-store helper for the RCU monitor flag. /// - /// On success it reads the latest message and appends `new` immediately - /// after it; on failure it acts as an acquire load that may read any - /// readable message with a different value, importing that message's view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - current: bool, - new: bool, - ) -> (res: (Result, Ghost, Tracked>>)) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& v == current - &&& read_msg.value == current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& v == read_msg.value - &&& read_msg.value != current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind - { - let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); - (result, Ghost::assume_new(), Tracked::assume_new()) - } - - /// Relaxed store: append a bool-valued message whose published view contains - /// only this store's own timestamp. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: bool, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - /// Release store: append a bool-valued message carrying the writer's current - /// view, then advance the writer's view for this location. + /// The executable flag remains a relaxed atomic flag, matching the old + /// monitor protocol. The proof-side effect is stronger: each stored flag + /// message appends the lock-protected monitor-state snapshot supplied by + /// the writer. #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( + pub fn store_relaxed_rcu_monitor( &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, value: bool, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } -} - -#[repr(transparent)] -#[verifier::accept_recursive_types(T)] -#[verifier::external_body] -/// TCB wrapper around Rust's `AtomicPtr`. -/// -/// This wrapper tracks the pointer value in the weak-memory history, but it -/// does not claim ownership of, or permission to dereference, the pointee. A -/// higher-level invariant must connect pointer values to `PointsTo`, refcount, -/// hazard-pointer, RCU, or other ownership ghost state when dereference safety -/// matters. -pub struct AtomicPtrW { - value: AtomicPtr, -} - -impl AtomicPtrW { - /// Logical identity of this atomic pointer object. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: *mut T) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = AtomicPtrW { value: AtomicPtr::new(init) }; - (atomic, Tracked::assume_new()) - } - - /// Relaxed load: choose a readable pointer message and advance only this - /// location's timestamp in the caller's thread view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (*mut T, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& equal(res.0, auth.msg_at(ts).value) - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - /// Acquire load: same pointer choice as relaxed, plus import the release - /// view carried by the selected message. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (*mut T, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& equal(res.0, auth.msg_at(ts).value) - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) - } - - /// Relaxed store: append a pointer-valued message whose published view - /// contains only this store's own timestamp. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: *mut T, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - /// Release store: append a pointer-valued message carrying the writer's - /// current view, then advance the writer's view for this location. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: *mut T, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } - - /// Release swap: return the latest pointer and append a new release - /// message in the same atomic read-modify-write step. - /// - /// `Ordering::Release` does not acquire the old message's view. The caller - /// observes the new modification-order timestamp and publishes its existing - /// thread view, exactly as for a release store. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn swap_release( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: *mut T, - ) -> (res: (*mut T, Tracked>)) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let write_ts = old(auth).history().len(); - let old_msg = old(auth).history()[(write_ts - 1) as int]; - let write_msg = Msg { value, view: old(tv)@.observe(self.id(), write_ts) }; - &&& write_ts >= 1 - &&& equal(res.0, old_msg.value) - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), write_ts) - &&& res.1@.id() == self.id() - &&& res.1@.ts() == write_ts - &&& res.1@.msg() == write_msg - &&& res.1@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - let old = self.value.swap(value, Ordering::Release); - (old, Tracked::assume_new()) - } -} - -impl AtomicPtrW { - /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` - /// failure ordering. - /// - /// Pointer CAS compares runtime pointer identity, which Verus models as - /// address equality for sized pointers. The returned pointer and written - /// message still carry the full pointer value, including provenance. - /// - /// On success the read is the latest message and the write is appended - /// immediately after it; on failure the operation is an acquire load that - /// may read any readable message whose address differs from `current`. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Ghost(state): Ghost, Tracked(tv): Tracked<&mut ThreadView>, - current: *mut T, - new: *mut T, - ) -> (res: (Result<*mut T, *mut T>, Ghost, Tracked>>)) + ) requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& current.addr() == read_msg.value.addr() - &&& equal(v, read_msg.value) - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& current.addr() != read_msg.value.addr() - &&& equal(v, read_msg.value) - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind + self.well_formed(), + state.wf(), + !value ==> state.no_pending_work(), { - let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); - (result, Ghost::assume_new(), Tracked::assume_new()) - } -} - -#[cfg(verus_keep_ghost)] -fn smoke_test_weak_atomic_with_ghost() { - let atomic = WeakAtomicUsize::<(), (), TrueWeakAtomicInv>::new(Ghost(()), 0, Tracked(())); - let tracked mut tv = ThreadView::new(); - weak_atomic_with_ghost! { - atomic => store_release(1, Tracked(&mut tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(snap.msg().value == 1); - } - } - weak_atomic_with_ghost! { - atomic => store_relaxed(2, Tracked(&mut tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(snap.msg().value == 2); - } - } - let _ = - weak_atomic_with_ghost! { - atomic => load_acquire(Tracked(&mut tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(ret == msg.value); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - atomic => load_relaxed(Tracked(&mut tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(ret == msg.value); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - atomic => compare_exchange_acqrel_acquire(2, 3, Tracked(&mut tv)); - update prev -> next; - returning ret; - timestamp ts; - message msg; - snapshot snap; - ghost g => { - assert(ts < prev.len()); - match ret { - Result::Ok(v) => { - assert(ts + 1 == prev.len()); - assert(v == 2); - assert(msg.value == 2); - match snap { - Option::Some(s) => { - assert(next == prev.push(s.msg())); - assert(s.msg().value == 3); - }, - Option::None => { - assert(false); - }, - } - }, - Result::Err(v) => { - assert(v == msg.value); - assert(msg.value != 2); - match snap { - Option::Some(_) => { - assert(false); - }, - Option::None => {}, - } - assert(next == prev); - }, - } - } - }; - - let bool_atomic = WeakAtomicBool::<(), (), TrueWeakAtomicInv>::new( - Ghost(()), - false, - Tracked(()), - ); - let tracked mut bool_tv = ThreadView::new(); - weak_atomic_with_ghost! { - bool_atomic => store_release(true, Tracked(&mut bool_tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(snap.msg().value == true); - } - } - let _ = - weak_atomic_with_ghost! { - bool_atomic => load_acquire(Tracked(&mut bool_tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(ret == msg.value); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - bool_atomic => compare_exchange_acqrel_acquire(true, false, Tracked(&mut bool_tv)); - update prev -> next; - returning ret; - timestamp ts; - message msg; - snapshot snap; - ghost g => { - assert(ts < prev.len()); - match ret { - Result::Ok(v) => { - assert(ts + 1 == prev.len()); - assert(v == true); - assert(msg.value == true); - match snap { - Option::Some(s) => { - assert(next == prev.push(s.msg())); - assert(s.msg().value == false); - }, - Option::None => { - assert(false); - }, - } - }, - Result::Err(v) => { - assert(v == msg.value); - assert(msg.value != true); - match snap { - Option::Some(_) => { - assert(false); - }, - Option::None => {}, - } - assert(next == prev); - }, - } - } - }; - - let null = core::ptr::null_mut::(); - let ptr_atomic = WeakAtomicPtr::::new( - Ghost(()), - null, - Tracked(()), - ); - let tracked mut ptr_tv = ThreadView::new(); - weak_atomic_with_ghost! { - ptr_atomic => store_release(null, Tracked(&mut ptr_tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(equal(snap.msg().value, null)); - } - } - let _ = - weak_atomic_with_ghost! { - ptr_atomic => load_acquire(Tracked(&mut ptr_tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(equal(ret, msg.value)); - assert(ts < history.len()); + proof { + use_type_invariant(self); } - }; - let _ = - weak_atomic_with_ghost! { - ptr_atomic => compare_exchange_acqrel_acquire(null, null, Tracked(&mut ptr_tv)); - update prev -> next; - returning ret; - timestamp ts; - message msg; - snapshot snap; - ghost g => { - assert(ts < prev.len()); - match ret { - Result::Ok(v) => { - assert(ts + 1 == prev.len()); - assert(equal(v, msg.value)); - assert(msg.value.addr() == null.addr()); - match snap { - Option::Some(s) => { - assert(next == prev.push(s.msg())); - assert(equal(s.msg().value, null)); - }, - Option::None => { - assert(false); - }, - } - }, - Result::Err(v) => { - assert(equal(v, msg.value)); - assert(msg.value.addr() != null.addr()); - match snap { - Option::Some(_) => { - assert(false); - }, - Option::None => {}, - } - assert(next == prev); - }, + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { + let tracked (mut hist, mut g) = pair; + proof { + assert(hist.id() == self.id()); + assert(raw_atomic.id() == self.id()); + assert(hist.id() == raw_atomic.id()); } - } - }; -} - -#[cfg(verus_keep_ghost)] -pub struct MessagePassingDataInv; - -#[cfg(verus_keep_ghost)] -impl WeakAtomicInvariantPredicate<(), usize, ()> for MessagePassingDataInv { - open spec fn atomic_inv(k: (), history: History, g: ()) -> bool { - &&& history.len() >= 1 - &&& history[0].value == 0 - &&& forall|i: int| 1 <= i < history.len() ==> #[trigger] history[i].value == 1 - } -} - -#[cfg(verus_keep_ghost)] -pub struct MessagePassingFlagInv; - -#[cfg(verus_keep_ghost)] -impl WeakAtomicInvariantPredicate for MessagePassingFlagInv { - open spec fn atomic_inv(data_id: AtomicId, history: History, g: ()) -> bool { - &&& history.len() >= 1 - &&& history[0].value == 0 - &&& forall|i: int| - 1 <= i < history.len() ==> { - &&& #[trigger] history[i].value == 1 - &&& history[i].view.seen_at(data_id) >= 1 + let ghost prev = hist.history(); + let snap = raw_atomic.store_relaxed(Tracked(&mut hist), Tracked(tv), value); + let ghost next = hist.history(); + proof { + assert(snap@.msg().value == value); + rcu_spec::preserve_rcu_monitor_flag_inv_on_push( + prev, + next, + snap@.msg(), + g, + g.push(state), + state, + ); + g = g.tracked_push(state); + pair = (hist, g); } + }); } } -#[cfg(verus_keep_ghost)] -proof fn preserve_message_passing_data_inv_on_push( - prev: History, - next: History, - msg: Msg, -) - requires - MessagePassingDataInv::atomic_inv((), prev, ()), - next == prev.push(msg), - msg.value == 1, - ensures - MessagePassingDataInv::atomic_inv((), next, ()), -{ - assert(next.len() >= 1); - assert(next[0].value == 0); - assert forall|i: int| 1 <= i < next.len() implies #[trigger] next[i].value == 1 by { - if i == prev.len() { - assert(next[i] == msg); - } else { - assert(i < prev.len()); - } - }; -} - -#[cfg(verus_keep_ghost)] -proof fn preserve_message_passing_flag_inv_on_push( - data_id: AtomicId, - prev: History, - next: History, - msg: Msg, -) - requires - MessagePassingFlagInv::atomic_inv(data_id, prev, ()), - next == prev.push(msg), - msg.value == 1, - msg.view.seen_at(data_id) >= 1, - ensures - MessagePassingFlagInv::atomic_inv(data_id, next, ()), -{ - assert(next.len() >= 1); - assert(next[0].value == 0); - assert forall|i: int| 1 <= i < next.len() implies { - &&& #[trigger] next[i].value == 1 - &&& next[i].view.seen_at(data_id) >= 1 - } by { - if i == prev.len() { - assert(next[i] == msg); - } else { - assert(i < prev.len()); - } - }; -} - -#[cfg(verus_keep_ghost)] -proof fn prove_message_passing_data_read( - data_id: AtomicId, - history: History, - ret: usize, - ts: Timestamp, - msg: Msg, - tv: WmView, -) - requires - MessagePassingDataInv::atomic_inv((), history, ()), - ts < history.len(), - ret == msg.value, - msg == history[ts as int], - tv.seen_at(data_id) >= 1, - tv.seen_at(data_id) <= ts, - ensures - ret == 1, -{ - assert(ts >= 1); - assert(history[ts as int].value == 1); -} - -// #[cfg(verus_keep_ghost)] -// fn message_passing_release_acquire_threads_can_prove() { -// let data = std::sync::Arc::new( -// WeakAtomicUsize::<(), (), MessagePassingDataInv>::new(Ghost(()), 0, Tracked(())), -// ); -// let ghost data_id = data.atomic.id(); -// let flag = std::sync::Arc::new( -// WeakAtomicUsize::::new(Ghost(data_id), 0, Tracked(())), -// ); -// let data_writer = data.clone(); -// let flag_writer = flag.clone(); -// let data_reader = data.clone(); -// let flag_reader = flag.clone(); -// let writer = vstd::thread::spawn( -// move || -// { -// let tracked mut writer_tv = ThreadView::new(); -// weak_atomic_with_ghost! { -// *data_writer => store_release(1, Tracked(&mut writer_tv)); -// update prev -> next; -// snapshot snap; -// ghost g => { -// preserve_message_passing_data_inv_on_push(prev, next, snap.msg()); -// assert(writer_tv.view.seen_at(data_id) >= 1); -// } -// } -// let ghost before_flag = writer_tv.view; -// assert(before_flag.seen_at(data_id) >= 1); -// weak_atomic_with_ghost! { -// *flag_writer => store_release(1, Tracked(&mut writer_tv)); -// update prev -> next; -// snapshot snap; -// ghost g => { -// assert(snap.msg().view == before_flag.observe(flag_writer.atomic.id(), snap.ts())); -// assert(snap.msg().view.seen_at(data_id) >= 1); -// preserve_message_passing_flag_inv_on_push(data_id, prev, next, snap.msg()); -// } -// } -// }, -// ); -// let reader = vstd::thread::spawn( -// move || -// { -// let tracked mut reader_tv = ThreadView::new(); -// let flag_result = -// weak_atomic_with_ghost! { -// *flag_reader => load_acquire(Tracked(&mut reader_tv)); -// returning ret; -// timestamp ts; -// message msg; -// history history; -// ghost g => { -// if ret == 1 { -// assert(msg.value == 1); -// if ts == 0 { -// assert(history[0].value == 0); -// assert(false); -// } -// assert(ts >= 1); -// assert(history[ts as int].value == 1); -// assert(history[ts as int].view.seen_at(data_id) >= 1); -// assert(msg == history[ts as int]); -// assert(msg.view.seen_at(data_id) >= 1); -// assert(reader_tv.view.seen_at(data_id) >= 1); -// } -// } -// }; -// if flag_result.0 == 1 { -// assert(reader_tv.view.seen_at(data_id) >= 1); -// let data_result = -// weak_atomic_with_ghost! { -// *data_reader => load_relaxed(Tracked(&mut reader_tv)); -// returning ret; -// timestamp ts; -// message msg; -// history history; -// ghost g => { -// prove_message_passing_data_read(data_id, history, ret, ts, msg, reader_tv.view); -// } -// }; -// assert(data_result.0 == 1); -// } -// }, -// ); -// let _ = writer.join(); -// let _ = reader.join(); -// } } // verus! diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 42b2c0537..805f5ad75 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -127,7 +127,7 @@ use crate::{ specs::{ sync::{ rcu as rcu_spec, - weak_memory::{ThreadView, WeakAtomicPtr}, + weak_memory::{RcuWeakAtomicPtr, ThreadView}, }, task::InAtomicMode, }, @@ -186,11 +186,10 @@ type RcuAtomicGhost

= rcu_spec::RcuRootOwnedGhost<

::Permission, >; -type RcuAtomicPtr

= WeakAtomicPtr< +type RcuAtomicPtr

= RcuWeakAtomicPtr<

::Target, - rcu_spec::RcuRootKey, - RcuAtomicGhost

, - rcu_spec::RcuOwnedWeakAtomicInv>, +

::Permission, + RcuPointerOwnership

, >; /// A Read-Copy Update cell for sharing a non-null pointer. @@ -335,7 +334,7 @@ impl RcuInner

{ reader_registry: root_ghost.reader_registry(), }; } - let ptr = WeakAtomicPtr::new(Ghost(key), core::ptr::null_mut(), Tracked(root_ghost)); + let ptr = RcuAtomicPtr::

::new(Ghost(key), core::ptr::null_mut(), Tracked(root_ghost)); Self { ptr, ghost_nullable: Ghost(true), @@ -366,7 +365,7 @@ impl RcuInner

{ reader_registry: root_ghost.reader_registry(), }; } - let ptr = WeakAtomicPtr::new(Ghost(key), raw_ptr, Tracked(root_ghost)); + let ptr = RcuAtomicPtr::

::new(Ghost(key), raw_ptr, Tracked(root_ghost)); Self { ptr, ghost_nullable: Ghost(nullable), diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 643f7be45..cf4bbbe03 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -10,7 +10,7 @@ use crate::specs::{ sync::{ rcu as rcu_spec, rcu::{GracePeriodView, MonitorStateView}, - weak_memory::{History, ThreadView, WeakAtomicBool, WmView}, + weak_memory::{History, RcuMonitorWeakAtomicBool, ThreadView, WmView}, }, }; use crate::sync::{ @@ -22,11 +22,7 @@ verus! { pub type Callbacks = VecDeque; -type MonitorAtomicBool = WeakAtomicBool< - (), - rcu_spec::RcuMonitorFlagGhost, - rcu_spec::RcuMonitorFlagInv, ->; +type MonitorAtomicBool = RcuMonitorWeakAtomicBool; /// Evidence captured at a call site where the current task is quiescent. /// diff --git a/verified_libs/vstd_extra/src/atomic_weak.rs b/verified_libs/vstd_extra/src/atomic_weak.rs new file mode 100644 index 000000000..8c5a5da05 --- /dev/null +++ b/verified_libs/vstd_extra/src/atomic_weak.rs @@ -0,0 +1,2376 @@ +//! Weak-memory atomic wrappers used by the verification layer. +//! +//! This module is a TCB boundary: executable atomic operations are connected to +//! Rust atomics with `external_body`, while proofs rely only on the ghost specs +//! below. Concrete wrappers currently cover Rust integer atomics, `AtomicBoolW`, +//! and `AtomicPtrW`, all using the same view/history model. +//! +//! We focus on the repaired C11/RC11-style memory model, where relaxed behavior +//! is modeled as reading from previously written messages in a location’s modi- +//! fication history, subject to coherence. In particular, relaxed reads may ob- +//! serve stale writes, but a thread’s view prevents it from going backwards, +//! and reads do not observe future writes that have not been added to the history. +//! +//! # References +//! +//! - [RCU Verification](https://dl.acm.org/doi/pdf/10.1145/3729246) +use core::sync::atomic::{ + AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, + AtomicU32, AtomicUsize, Ordering, +}; + +#[cfg(target_has_atomic = "64")] +use core::sync::atomic::{AtomicI64, AtomicU64}; + +use vstd::assert_sets_equal; +use vstd::invariant::{AtomicInvariant, InvariantPredicate}; +use vstd::prelude::*; +use vstd::resource::Loc; +use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; +use vstd::seq::Seq; + +verus! { + +// The "global" memory is defined within the invariant we need to preserve and, +// by the definition of Iris operations, invariant can be opened by a thread +// provided that the invariant holds and it can close afterwards provided that +// the invariant holds as well. +// +// Thanks to Verus' native support for the semantics, we only need to define +// what means for `atomic` and we can freely open the invariant and provide +// customized macros for doing ergonomic updates on both the physical resources +// and the ghost tokens like message histories, views, etc. +/// An `AtomicId` is just an abstract identifier (memory location) of one atomic object. +pub type AtomicId = Loc; + +/// Logical timestamp into one atomic object's message history. +/// Timestamp 0 is always the initial message installed by `new`. +pub type Timestamp = nat; + +/// A thread-local weak-memory view. +/// +/// `seen[id] = ts` means this thread has advanced past all messages for `id` +/// older than `ts`; future reads from that atomic must not go backwards. +/// +/// Typically, if another thread has published a message with timestamp `ts` for `id`, +/// and the reader reads the message via some atomic operations, then the reader's +/// thread view will advance to at least `ts` for `id`. +pub ghost struct WmView { + pub seen: IMap, +} + +impl WmView { + /// Creates an empty view. + pub open spec fn empty() -> Self { + WmView { seen: IMap::empty() } + } + + pub open spec fn seen_at(self, id: AtomicId) -> Timestamp { + if self.seen.contains_key(id) { + self.seen[id] + } else { + // Missing entries are equivalent to only seeing the initial write. + 0nat + } + } + + /// Monotonically advance the view for one atomic object. + /// + /// Just as the name indicates, `observe` means that the current thread has observed + /// a message written by another thread with a specific timestamp; because the atomic + /// operation never "goes back", the thread's view for that atomic must advance to at + /// least that timestamp. + /// + /// "During a read from `l`, a thread can observe any message `m` from `M(l)` where + /// `m.time >= V(l)`, and updates its view to incorporate `m.time`." + pub open spec fn observe(self, id: AtomicId, ts: Timestamp) -> Self { + WmView { + seen: self.seen.insert( + id, + if self.seen_at(id) <= ts { + ts + } else { + self.seen_at(id) + }, + ), + } + } + + /// Pointwise maximum of two views. + /// + /// This is the ghost effect of an acquire read: the reader imports the + /// release view carried by the message it read. + pub open spec fn join(self, other: Self) -> Self { + WmView { + seen: IMap::new( + |id: AtomicId| self.seen.contains_key(id) || other.seen.contains_key(id), + |id: AtomicId| + if self.seen_at(id) <= other.seen_at(id) { + other.seen_at(id) + } else { + self.seen_at(id) + }, + ), + } + } + + /// Partial ordering two threads' views. + pub open spec fn spec_le(self, other: Self) -> bool { + forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) + } + + pub proof fn lemma_join_left(self, other: Self) + ensures + self.spec_le(self.join(other)), + { + } + + pub proof fn lemma_join_right(self, other: Self) + ensures + other.spec_le(self.join(other)), + { + } + + pub proof fn lemma_spec_le_transitive(self, middle: Self, upper: Self) + requires + self.spec_le(middle), + middle.spec_le(upper), + ensures + self.spec_le(upper), + { + } +} + +/// One message in an atomic object's modification history. +/// +/// `view` is the release view published with this value. Relaxed stores publish +/// only their own timestamp; release stores publish the writer's current view. +pub ghost struct Msg { + pub value: V, + pub view: WmView, +} + +pub type History = Seq>; + +/// User-supplied invariant predicate for a weak-memory atomic. +/// +/// This mirrors `vstd::atomic_ghost::AtomicInvariantPredicate`, except the +/// predicate is over the whole message history rather than one current value. +pub trait WeakAtomicInvariantPredicate { + spec fn atomic_inv(k: K, history: History, g: G) -> bool; +} + +/// Authoritative ghost state for one atomic object's history. +/// +/// This is intentionally a thin wrapper around vstd's map resource algebra: +/// [`GhostMapAuth`] owns the authoritative timestamp-to-message map, while `len` +/// records that the domain is the contiguous range `0..len`. +/// +/// The proof-facing atomic wrapper below stores this token inside an +/// `AtomicInvariant` next to the executable atomic. +pub tracked struct HistAuth { + auth: GhostMapAuth>, + // Private: code outside this TCB module must not forge the history length, + // which would desynchronize `len` from the authoritative map domain. + ghost len: nat, +} + +proof fn lemma_timestamp_range_insert_last(hi: Timestamp) + ensures + Set::range(0nat, hi).insert(hi) == Set::range(0nat, hi + 1), +{ + broadcast use vstd::set_lib::range_set_properties; + + assert_sets_equal!(Set::range(0nat, hi).insert(hi), Set::range(0nat, hi + 1), ts: Timestamp => { + if Set::range(0nat, hi).insert(hi).contains(ts) { + if ts != hi { + assert(Set::range(0nat, hi).contains(ts)); + assert(ts < hi); + } + assert(ts < hi + 1); + assert(Set::range(0nat, hi + 1).contains(ts)); + } + + if Set::range(0nat, hi + 1).contains(ts) { + assert(ts < hi + 1); + if ts == hi { + assert(Set::range(0nat, hi).insert(hi).contains(ts)); + } else { + assert(ts < hi); + assert(Set::range(0nat, hi).contains(ts)); + assert(Set::range(0nat, hi).insert(hi).contains(ts)); + } + } + }); +} + +impl HistAuth { + pub closed spec fn id(self) -> AtomicId { + self.auth.id() + } + + pub closed spec fn map(self) -> Map> { + self.auth@ + } + + pub closed spec fn len(self) -> nat { + self.len + } + + pub open spec fn history(self) -> History + recommends + self.wf(), + { + Seq::new(self.len(), |i: int| self.map()[i as nat]) + } + + pub open spec fn wf(self) -> bool { + &&& self.len() > 0 + &&& self.map().dom() == Set::range(0nat, self.len()) + } + + pub open spec fn valid_ts(self, ts: Timestamp) -> bool { + ts < self.len() + } + + pub open spec fn msg_at(self, ts: Timestamp) -> Msg + recommends + self.valid_ts(ts), + { + self.map()[ts] + } + + /// RC11-style relaxed readability: a read may choose any message that is + /// not older than the thread's current view for this location. + pub open spec fn readable(self, view: WmView, ts: Timestamp) -> bool { + &&& self.valid_ts(ts) + &&& view.seen_at(self.id()) <= ts + } + + /// Append one message to the authoritative history and return a persistent + /// snapshot for the newly allocated timestamp. + pub proof fn append_msg(tracked &mut self, msg: Msg) -> (tracked snap: MsgSnap) + requires + old(self).wf(), + ensures + final(self).id() == old(self).id(), + final(self).history() == old(self).history().push(msg), + final(self).wf(), + snap.id() == final(self).id(), + snap.ts() == old(self).history().len(), + snap.msg() == msg, + snap.agrees_with(*final(self)), + { + let ghost ts = self.len(); + let ghost old_dom = self.map().dom(); + + let tracked pt = self.auth.insert(ts, msg); + self.len = self.len + 1; + + // Full-crate verification does not reliably rediscover this range/domain + // fact after the ghost-map insert, so keep the append step explicit. + lemma_timestamp_range_insert_last(ts); + assert(old_dom == Set::range(0nat, ts)); + assert(self.map().dom() == old_dom.insert(ts)); + assert(self.map().dom() == Set::range(0nat, ts + 1)); + assert(ts + 1 == self.len()); + assert(self.map().dom() == Set::range(0nat, self.len())); + + let tracked psnap = pt.persist(); + MsgSnap { snap: psnap } + } +} + +/// A stable proof handle for one message; a snapshot of the message. +/// +/// The underlying vstd token is persistent/duplicable, so a message snapshot +/// can be copied through proofs without granting permission to mutate history. +/// +/// Stores return snapshots so higher layers can connect a concrete write to +/// later ownership-transfer predicates without exposing the whole history. +pub tracked struct MsgSnap { + snap: GhostPersistentPointsTo>, +} + +impl MsgSnap { + pub closed spec fn id(self) -> AtomicId { + self.snap.id() + } + + /// Fetch the timestamp of this specific message. + pub closed spec fn ts(self) -> Timestamp { + self.snap.key() + } + + /// Fetch the ghost message value of this specific message. + pub closed spec fn msg(self) -> Msg { + self.snap.value() + } + + pub open spec fn agrees_with(self, auth: HistAuth) -> bool { + &&& self.id() == auth.id() + &&& auth.valid_ts(self.ts()) + &&& self.msg() == auth.msg_at(self.ts()) + } + + pub proof fn duplicate(tracked &self) -> (tracked snap: MsgSnap) + ensures + snap.id() == self.id(), + snap.ts() == self.ts(), + snap.msg() == self.msg(), + { + let tracked psnap = self.snap.duplicate(); + MsgSnap { snap: psnap } + } + + pub proof fn agree(tracked &self, tracked auth: &HistAuth) + requires + self.id() == auth.id(), + auth.wf(), + ensures + self.agrees_with(*auth), + { + self.snap.agree(&auth.auth); + assert(auth.map().contains_pair(self.ts(), self.msg())); + } +} + +} // verus! +/// Generate the proof-facing wrapper for one concrete weak-memory atomic type. +/// +/// The generated type keeps the executable TCB wrapper separate from the +/// invariant protocol. Adding `AtomicU32W` or `AtomicBoolW` later should require +/// a new executable wrapper plus one macro invocation, not another copy of the +/// invariant glue. +macro_rules! declare_weak_atomic_type { + ($weak_atomic:ident, $pred_adapter:ident, $raw_atomic:ident, $value_ty:ty) => { + verus! { + /// Predicate adapter stored inside `AtomicInvariant`. + /// + /// The invariant contains the authoritative history and user ghost + /// state. The constant pairs the user key `K` with the logical atomic id. + pub struct $pred_adapter { + p: Pred, + } + + impl InvariantPredicate<(K, AtomicId), (HistAuth<$value_ty>, G)> for $pred_adapter< + Pred, + > where Pred: WeakAtomicInvariantPredicate { + open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<$value_ty>, G)) -> bool { + let (k, id) = k_id; + let (hist, g) = hist_g; + &&& hist.id() == id + &&& hist.wf() + &&& Pred::atomic_inv(k, hist.history(), g) + } + } + + /// A weak-memory atomic with an `atomic_ghost`-style invariant. + /// + /// The executable atomic remains the TCB wrapper. This proof-facing + /// wrapper stores the authoritative history in an `AtomicInvariant` and + /// exposes `well_formed`/`type_inv` predicates tying that history to the + /// executable atomic id. As in `vstd::atomic_ghost`, outer data + /// structures put this predicate in their own + /// `#[verifier::type_invariant]`. + pub struct $weak_atomic { + #[doc(hidden)] + atomic: $raw_atomic, + #[doc(hidden)] + atomic_inv: Tracked< + AtomicInvariant<(K, AtomicId), (HistAuth<$value_ty>, G), $pred_adapter>, + >, + } + + impl $weak_atomic { + pub closed spec fn constant(&self) -> K { + self.atomic_inv@.constant().0 + } + + /// Logical modification-history identity of this atomic. + pub closed spec fn id(&self) -> AtomicId { + self.atomic_inv@.constant().1 + } + + pub closed spec fn well_formed(&self) -> bool { + self.id() == self.atomic.id() + } + + /// Borrows the executable atomic for a client-specific invariant + /// transition. + #[doc(hidden)] + pub fn raw_atomic(&self) -> (res: &$raw_atomic) + requires + self.well_formed(), + ensures + res.id() == self.id(), + { + &self.atomic + } + + /// Borrows the invariant for a client-specific atomic operation. + /// + /// The fields remain private so the type invariant cannot be + /// bypassed by construction or replacement. + #[doc(hidden)] + pub proof fn tracked_atomic_inv( + tracked &self, + ) -> (tracked res: &AtomicInvariant< + (K, AtomicId), + (HistAuth<$value_ty>, G), + $pred_adapter, + >) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.id()), + { + self.atomic_inv.borrow() + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } + } + + impl $weak_atomic where + Pred: WeakAtomicInvariantPredicate, + { + #[inline(always)] + pub const fn new( + Ghost(k): Ghost, + init: $value_ty, + Tracked(g): Tracked, + ) -> (res: Self) + requires + Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), + ensures + res.well_formed(), + res.constant() == k, + { + let (atomic, Tracked(hist)) = $raw_atomic::new(init); + let tracked pair = (hist, g); + assert($pred_adapter::::inv((k, atomic.id()), pair)); + let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); + $weak_atomic { atomic, atomic_inv: Tracked(atomic_inv) } + } + + #[inline(always)] + pub fn load_relaxed( + &self, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } + + #[inline(always)] + pub fn load_acquire( + &self, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } + } + } + }; +} + +declare_weak_atomic_type!(WeakAtomicU8, WeakAtomicPredU8, AtomicU8W, u8); +declare_weak_atomic_type!(WeakAtomicU16, WeakAtomicPredU16, AtomicU16W, u16); +declare_weak_atomic_type!(WeakAtomicU32, WeakAtomicPredU32, AtomicU32W, u32); +declare_weak_atomic_type!(WeakAtomicUsize, WeakAtomicPredUsize, AtomicUsizeW, usize); +declare_weak_atomic_type!(WeakAtomicBool, WeakAtomicPredBool, AtomicBoolW, bool); + +#[cfg(target_has_atomic = "64")] +declare_weak_atomic_type!(WeakAtomicU64, WeakAtomicPredU64, AtomicU64W, u64); + +declare_weak_atomic_type!(WeakAtomicI8, WeakAtomicPredI8, AtomicI8W, i8); +declare_weak_atomic_type!(WeakAtomicI16, WeakAtomicPredI16, AtomicI16W, i16); +declare_weak_atomic_type!(WeakAtomicI32, WeakAtomicPredI32, AtomicI32W, i32); +declare_weak_atomic_type!(WeakAtomicIsize, WeakAtomicPredIsize, AtomicIsizeW, isize); + +#[cfg(target_has_atomic = "64")] +declare_weak_atomic_type!(WeakAtomicI64, WeakAtomicPredI64, AtomicI64W, i64); + +verus! { + +/// Predicate adapter for weak-memory pointer atomics. +/// +/// The history stores raw pointer values. This tracks the atomic pointer value +/// itself; ownership of the pointee must be modeled by the user ghost state `G`. +pub struct WeakAtomicPredPtr { + t: T, + p: Pred, +} + +impl InvariantPredicate<(K, AtomicId), (HistAuth<*mut T>, G)> for WeakAtomicPredPtr< + T, + Pred, +> where Pred: WeakAtomicInvariantPredicate { + open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<*mut T>, G)) -> bool { + let (k, id) = k_id; + let (hist, g) = hist_g; + &&& hist.id() == id + &&& hist.wf() + &&& Pred::atomic_inv(k, hist.history(), g) + } +} + +/// Weak-memory atomic pointer with an `atomic_ghost`-style invariant. +/// +/// This is the pointer analogue of [`WeakAtomicUsize`]. It deliberately models +/// only the atomic pointer value and its release/acquire synchronization history; +/// any ownership or validity claim about the pointed-to allocation belongs in +/// the user-supplied ghost state `G` and invariant predicate. +#[verifier::accept_recursive_types(T)] +pub struct WeakAtomicPtr { + #[doc(hidden)] + atomic: AtomicPtrW, + #[doc(hidden)] + atomic_inv: Tracked< + AtomicInvariant<(K, AtomicId), (HistAuth<*mut T>, G), WeakAtomicPredPtr>, + >, +} + +impl WeakAtomicPtr { + pub closed spec fn constant(&self) -> K { + self.atomic_inv@.constant().0 + } + + /// Logical modification-history identity of this atomic pointer. + pub closed spec fn id(&self) -> AtomicId { + self.atomic_inv@.constant().1 + } + + pub closed spec fn well_formed(&self) -> bool { + self.id() == self.atomic.id() + } + + /// Borrows the executable pointer atomic for a client-specific invariant + /// transition. + #[doc(hidden)] + pub fn raw_atomic(&self) -> (res: &AtomicPtrW) + requires + self.well_formed(), + ensures + res.id() == self.id(), + { + &self.atomic + } + + /// Borrows the invariant for a client-specific atomic operation. + #[doc(hidden)] + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &AtomicInvariant< + (K, AtomicId), + (HistAuth<*mut T>, G), + WeakAtomicPredPtr, + >) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.id()), + { + self.atomic_inv.borrow() + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } +} + +impl WeakAtomicPtr where + Pred: WeakAtomicInvariantPredicate, + { + #[inline(always)] + pub const fn new(Ghost(k): Ghost, init: *mut T, Tracked(g): Tracked) -> (res: Self) + requires + Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), + ensures + res.well_formed(), + res.constant() == k, + { + let (atomic, Tracked(hist)) = AtomicPtrW::::new(init); + let tracked pair = (hist, g); + assert(WeakAtomicPredPtr::::inv((k, atomic.id()), pair)); + let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); + WeakAtomicPtr { atomic, atomic_inv: Tracked(atomic_inv) } + } + + #[inline(always)] + pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + *mut T, + Ghost, + )) { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } + + #[inline(always)] + pub fn load_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + *mut T, + Ghost, + )) { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); + proof { + pair = (hist, g); + } + }); + result + } +} + +pub struct TrueWeakAtomicInv; + +impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { + open spec fn atomic_inv(k: K, history: History, g: G) -> bool { + true + } +} + +impl WeakAtomicPtr { + /// Release-store helper for users with the trivial atomic invariant. + /// + /// This keeps early weak-memory clients from depending on the macro while + /// we are still shaping the client-specific ghost state. + #[inline(always)] + pub fn store_release_simple(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) { + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + let _snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); + proof { + pair = (hist, g); + } + }); + } + + /// Strong AcqRel/Acquire CAS helper for users with the trivial invariant. + #[inline(always)] + pub fn compare_exchange_acqrel_acquire_simple( + &self, + current: *mut T, + new: *mut T, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (Result<*mut T, *mut T>, Ghost)) { + let result; + proof { + use_type_invariant(self); + } + vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { + let tracked (mut hist, g) = pair; + proof { + assert(hist.id() == self.atomic_inv@.constant().1); + assert(self.atomic_inv@.constant().1 == self.atomic.id()); + assert(hist.id() == self.atomic.id()); + } + let cas_result = self.atomic.compare_exchange_acqrel_acquire( + Tracked(&mut hist), + Tracked(tv), + current, + new, + ); + result = (cas_result.0, cas_result.1); + proof { + pair = (hist, g); + } + }); + result + } +} + +/// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, +/// but for weak-memory atomics with per-thread view tokens and message histories. +/// +/// The macro opens the atomic invariant, performs the specified operation, and +/// provides the previous history, new history, and operation snapshot to the user- +/// provided proof block. The user can then write proofs about the effects of the +/// operation on the history and thread view, using the snapshot to connect to the +/// authoritative history. +#[macro_export] +macro_rules! weak_atomic_with_ghost { + ( + $atomic:expr => compare_exchange_acqrel_acquire($current:expr, $new:expr, $tv:expr); + update $prev:ident -> $next:ident; + returning $ret:ident; + timestamp $ts:ident; + message $msg:ident; + snapshot $snap:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let result; + let atomic = &($atomic); + let current = $current; + let new = $new; + proof { + use_type_invariant(atomic); + } + let raw_atomic = atomic.raw_atomic(); + ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { + #[allow(unused_mut)] + let tracked (mut hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.id()); + assert(raw_atomic.id() == atomic.id()); + assert(hist.id() == raw_atomic.id()); + } + let ghost $prev = hist.history(); + let cas_result = raw_atomic.compare_exchange_acqrel_acquire( + Tracked(&mut hist), + $tv, + current, + new, + ); + result = (cas_result.0, cas_result.1); + let ghost $next = hist.history(); + let ghost $ret = cas_result.0; + let ghost $ts = cas_result.1@; + let ghost $msg = $prev[$ts as int]; + + proof { + let tracked $snap = cas_result.2.get(); + $b + } + + proof { + pair = (hist, $g); + } + }); + result + }} + }; + ( + $atomic:expr => load_acquire($tv:expr); + returning $ret:ident; + timestamp $ts:ident; + message $msg:ident; + history $history:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let result; + let atomic = &($atomic); + proof { + use_type_invariant(atomic); + } + let raw_atomic = atomic.raw_atomic(); + ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { + #[allow(unused_mut)] + let tracked (hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.id()); + assert(raw_atomic.id() == atomic.id()); + assert(hist.id() == raw_atomic.id()); + } + let ghost $history = hist.history(); + result = raw_atomic.load_acquire(Tracked(&hist), $tv); + let ghost $ret = result.0; + let ghost $ts = result.1@; + let ghost $msg = hist.msg_at($ts); + + proof { $b } + + proof { + pair = (hist, $g); + } + }); + result + }} + }; + ( + $atomic:expr => load_relaxed($tv:expr); + returning $ret:ident; + timestamp $ts:ident; + message $msg:ident; + history $history:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let result; + let atomic = &($atomic); + proof { + use_type_invariant(atomic); + } + let raw_atomic = atomic.raw_atomic(); + ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { + #[allow(unused_mut)] + let tracked (hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.id()); + assert(raw_atomic.id() == atomic.id()); + assert(hist.id() == raw_atomic.id()); + } + let ghost $history = hist.history(); + result = raw_atomic.load_relaxed(Tracked(&hist), $tv); + let ghost $ret = result.0; + let ghost $ts = result.1@; + let ghost $msg = hist.msg_at($ts); + + proof { $b } + + proof { + pair = (hist, $g); + } + }); + result + }} + }; + ( + $atomic:expr => store_release($value:expr, $tv:expr); + update $prev:ident -> $next:ident; + snapshot $snap:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let atomic = &($atomic); + let value = $value; + proof { + use_type_invariant(atomic); + } + let raw_atomic = atomic.raw_atomic(); + ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { + #[allow(unused_mut)] + let tracked (mut hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.id()); + assert(raw_atomic.id() == atomic.id()); + assert(hist.id() == raw_atomic.id()); + } + let ghost $prev = hist.history(); + let snap_tracked = raw_atomic.store_release(Tracked(&mut hist), $tv, value); + let ghost $next = hist.history(); + + proof { + let tracked $snap = snap_tracked.get(); + $b + } + + proof { + pair = (hist, $g); + } + }); + }} + }; + ( + $atomic:expr => store_relaxed($value:expr, $tv:expr); + update $prev:ident -> $next:ident; + snapshot $snap:ident; + ghost $g:ident => $b:block + ) => { + ::vstd::prelude::verus_exec_expr! {{ + let atomic = &($atomic); + let value = $value; + proof { + use_type_invariant(atomic); + } + let raw_atomic = atomic.raw_atomic(); + ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { + #[allow(unused_mut)] + let tracked (mut hist, mut $g) = pair; + proof { + assert(hist.id() == atomic.id()); + assert(raw_atomic.id() == atomic.id()); + assert(hist.id() == raw_atomic.id()); + } + let ghost $prev = hist.history(); + let snap_tracked = raw_atomic.store_relaxed(Tracked(&mut hist), $tv, value); + let ghost $next = hist.history(); + + proof { + let tracked $snap = snap_tracked.get(); + $b + } + + proof { + pair = (hist, $g); + } + }); + }} + }; +} + +pub use weak_atomic_with_ghost; + +/// Explicit per-thread view token. +/// +/// Passing this token through atomic operations makes the weak-memory effects +/// visible in specs instead of hiding them in global or thread-local state. +/// +/// # Soundness +/// +/// The wrapped view is private and can only evolve through the TCB atomic +/// operations in this module. Those operations maintain the invariant that a +/// view never claims a timestamp at or beyond the length of that location's +/// history: loads observe an existing message, stores observe the message they +/// just appended, and acquire joins only import message views that were built +/// from existing timestamps. This keeps the `readable`-based postconditions of +/// loads satisfiable. Do not add raw mutators (e.g. an unconditional +/// `observe`/`join` proof fn): a forged view claiming an unwritten timestamp +/// would make the next load's postcondition vacuously false. +pub tracked struct ThreadView { + ghost view: WmView, +} + +impl View for ThreadView { + type V = WmView; + + closed spec fn view(&self) -> WmView { + self.view + } +} + +impl ThreadView { + /// Creates a fresh token holding the empty view. + /// + /// The empty view is the weakest token: it lower-bounds every location at + /// timestamp 0, so minting one is always sound — the holder merely + /// forfeits all ordering knowledge. Note that minting a fresh view + /// mid-thread over-approximates real executions (it forgets per-location + /// coherence the thread has already observed) and publishes nothing useful + /// through release stores, so executable code should thread one token per + /// logical operation or critical section, and eventually one per task. + /// + /// This generic constructor is public because schedulers live outside + /// `vstd_extra`. Minting a fresh empty view is sound but loses ordering + /// knowledge. Production OSTD code therefore calls it only when registering + /// a task or CPU, then moves that same linear token through schedule-in and + /// schedule-out. + pub proof fn new() -> (tracked res: Self) + ensures + res@ == WmView::empty(), + { + ThreadView { view: WmView::empty() } + } + + /// Imports observations from another genuine thread/CPU view. + /// + /// Unlike a raw ghost mutator, this operation cannot introduce an + /// unwritten timestamp: both operands are tracked `ThreadView` values that + /// originated from the weak-memory TCB. Scheduler context switches use it + /// to transfer observations between a CPU view and a task view. + pub proof fn tracked_join(tracked &mut self, tracked other: &Self) + ensures + final(self)@ == old(self)@.join(other@), + { + self.view = self.view.join(other.view); + } +} + +#[repr(transparent)] +#[verifier::external_body] +/// TCB wrapper around Rust's `AtomicUsize`. +/// +/// The executable field is the real atomic object. The proof layer sees only +/// the specs below, plus the uninterpreted logical identity `id`. +pub struct AtomicUsizeW { + value: AtomicUsize, +} + +impl AtomicUsizeW { + /// Logical identity of this atomic object. + /// + /// `id` has no runtime representation; it indexes ghost histories and + /// thread views. The `new` spec ties the fresh history to this identity. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: usize) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = AtomicUsizeW { value: AtomicUsize::new(init) }; + (atomic, Tracked::assume_new()) + } + + /// Relaxed load: choose a readable message and advance only this location's + /// timestamp in the caller's thread view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (usize, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + /// Acquire load: same readable message choice as relaxed, plus import the + /// release view carried by the selected message. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (usize, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` + /// failure ordering. + /// + /// On success, RMW atomicity forces the read to be the latest message in + /// the modification history, and the new release message is appended + /// immediately after it. On failure, the operation is only an acquire + /// load: it may read *any* readable message whose value differs from + /// `current`, not necessarily the latest one. A strong CAS merely never + /// fails after reading a value equal to `current`. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + current: usize, + new: usize, + ) -> (res: (Result, Ghost, Tracked>>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& read_ts + 1 == old(auth).history().len() + &&& v == current + &&& read_msg.value == current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& v == read_msg.value + &&& read_msg.value != current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); + (result, Ghost::assume_new(), Tracked::assume_new()) + } + + /// Relaxed store: append a new message whose published view contains only + /// this store's own timestamp. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: usize, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + /// Release store: append a new message carrying the writer's current view, + /// then advance the writer's view for this location. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: usize, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } +} + +} // verus! +/// Generate a TCB executable wrapper around one Rust integer atomic type. +/// +/// All integer atomics share the same weak-memory history shape: load chooses a +/// readable message, stores append a message, and CAS either reads the latest +/// message and appends its write right after it (success), or acts as an +/// acquire read of any readable message with a different value (failure). +macro_rules! declare_integer_atomic_wrapper { + ($wrapper:ident, $rust_atomic:ident, $value_ty:ty) => { + verus! { + #[repr(transparent)] + #[verifier::external_body] + /// TCB wrapper around a Rust integer atomic. + pub struct $wrapper { + value: $rust_atomic, + } + + impl $wrapper { + /// Logical identity of this atomic object. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: $value_ty) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = $wrapper { value: $rust_atomic::new(init) }; + (atomic, Tracked::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ($value_ty, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + current: $value_ty, + new: $value_ty, + ) -> (res: ( + Result<$value_ty, $value_ty>, + Ghost, + Tracked>>, + )) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& read_ts + 1 == old(auth).history().len() + &&& v == current + &&& read_msg.value == current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& v == read_msg.value + &&& read_msg.value != current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange( + current, + new, + Ordering::AcqRel, + Ordering::Acquire, + ); + (result, Ghost::assume_new(), Tracked::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: $value_ty, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: $value_ty, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } + } + } + }; +} + +declare_integer_atomic_wrapper!(AtomicU8W, AtomicU8, u8); + +declare_integer_atomic_wrapper!(AtomicU16W, AtomicU16, u16); + +declare_integer_atomic_wrapper!(AtomicU32W, AtomicU32, u32); + +declare_integer_atomic_wrapper!(AtomicIsizeW, AtomicIsize, isize); + +declare_integer_atomic_wrapper!(AtomicI8W, AtomicI8, i8); + +declare_integer_atomic_wrapper!(AtomicI16W, AtomicI16, i16); + +declare_integer_atomic_wrapper!(AtomicI32W, AtomicI32, i32); + +#[cfg(target_has_atomic = "64")] +declare_integer_atomic_wrapper!(AtomicU64W, AtomicU64, u64); + +#[cfg(target_has_atomic = "64")] +declare_integer_atomic_wrapper!(AtomicI64W, AtomicI64, i64); + +verus! { + +#[repr(transparent)] +#[verifier::external_body] +/// TCB wrapper around Rust's `AtomicBool`. +/// +/// Bool atomics share the load/store/CAS weak-memory protocol with integer +/// atomics, but they are not numeric atomics: this wrapper intentionally exposes +/// no arithmetic or bitwise fetch operations. +pub struct AtomicBoolW { + value: AtomicBool, +} + +impl AtomicBoolW { + /// Logical identity of this atomic object. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: bool) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = AtomicBoolW { value: AtomicBool::new(init) }; + (atomic, Tracked::assume_new()) + } + + /// Relaxed load: choose a readable bool message and advance only this + /// location's timestamp in the caller's thread view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (bool, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + /// Acquire load: same bool choice as relaxed, plus import the release view + /// carried by the selected message. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (bool, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& res.0 == auth.msg_at(ts).value + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` + /// failure ordering. + /// + /// On success it reads the latest message and appends `new` immediately + /// after it; on failure it acts as an acquire load that may read any + /// readable message with a different value, importing that message's view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + current: bool, + new: bool, + ) -> (res: (Result, Ghost, Tracked>>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& read_ts + 1 == old(auth).history().len() + &&& v == current + &&& read_msg.value == current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& v == read_msg.value + &&& read_msg.value != current + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); + (result, Ghost::assume_new(), Tracked::assume_new()) + } + + /// Relaxed store: append a bool-valued message whose published view contains + /// only this store's own timestamp. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: bool, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + /// Release store: append a bool-valued message carrying the writer's current + /// view, then advance the writer's view for this location. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth>, + Tracked(tv): Tracked<&mut ThreadView>, + value: bool, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } +} + +#[repr(transparent)] +#[verifier::accept_recursive_types(T)] +#[verifier::external_body] +/// TCB wrapper around Rust's `AtomicPtr`. +/// +/// This wrapper tracks the pointer value in the weak-memory history, but it +/// does not claim ownership of, or permission to dereference, the pointee. A +/// higher-level invariant must connect pointer values to `PointsTo`, refcount, +/// hazard-pointer, RCU, or other ownership ghost state when dereference safety +/// matters. +pub struct AtomicPtrW { + value: AtomicPtr, +} + +impl AtomicPtrW { + /// Logical identity of this atomic pointer object. + pub uninterp spec fn id(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(init: *mut T) -> (res: (Self, Tracked>)) + ensures + res.1@.id() == res.0.id(), + res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], + res.1@.wf(), + { + let atomic = AtomicPtrW { value: AtomicPtr::new(init) }; + (atomic, Tracked::assume_new()) + } + + /// Relaxed load: choose a readable pointer message and advance only this + /// location's timestamp in the caller's thread view. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_relaxed( + &self, + Tracked(auth): Tracked<&HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (*mut T, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& equal(res.0, auth.msg_at(ts).value) + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Relaxed); + (value, Ghost::assume_new()) + } + + /// Acquire load: same pointer choice as relaxed, plus import the release + /// view carried by the selected message. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load_acquire( + &self, + Tracked(auth): Tracked<&HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: (*mut T, Ghost)) + requires + auth.id() == self.id(), + auth.wf(), + ensures + ({ + let ts = res.1@; + &&& auth.readable(old(tv)@, ts) + &&& equal(res.0, auth.msg_at(ts).value) + &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) + }), + opens_invariants none + no_unwind + { + let value = self.value.load(Ordering::Acquire); + (value, Ghost::assume_new()) + } + + /// Relaxed store: append a pointer-valued message whose published view + /// contains only this store's own timestamp. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_relaxed( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: *mut T, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Relaxed); + Tracked::assume_new() + } + + /// Release store: append a pointer-valued message carrying the writer's + /// current view, then advance the writer's view for this location. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store_release( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: *mut T, + ) -> (snap: Tracked>) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let ts = old(auth).history().len(); + let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), ts) + &&& snap@.id() == self.id() + &&& snap@.ts() == ts + &&& snap@.msg() == msg + &&& snap@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + self.value.store(value, Ordering::Release); + Tracked::assume_new() + } + + /// Release swap: return the latest pointer and append a new release + /// message in the same atomic read-modify-write step. + /// + /// `Ordering::Release` does not acquire the old message's view. The caller + /// observes the new modification-order timestamp and publishes its existing + /// thread view, exactly as for a release store. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn swap_release( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + value: *mut T, + ) -> (res: (*mut T, Tracked>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let write_ts = old(auth).history().len(); + let old_msg = old(auth).history()[(write_ts - 1) as int]; + let write_msg = Msg { value, view: old(tv)@.observe(self.id(), write_ts) }; + &&& write_ts >= 1 + &&& equal(res.0, old_msg.value) + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == old(tv)@.observe(self.id(), write_ts) + &&& res.1@.id() == self.id() + &&& res.1@.ts() == write_ts + &&& res.1@.msg() == write_msg + &&& res.1@.agrees_with(*final(auth)) + }), + opens_invariants none + no_unwind + { + let old = self.value.swap(value, Ordering::Release); + (old, Tracked::assume_new()) + } +} + +impl AtomicPtrW { + /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` + /// failure ordering. + /// + /// Pointer CAS compares runtime pointer identity, which Verus models as + /// address equality for sized pointers. The returned pointer and written + /// message still carry the full pointer value, including provenance. + /// + /// On success the read is the latest message and the write is appended + /// immediately after it; on failure the operation is an acquire load that + /// may read any readable message whose address differs from `current`. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange_acqrel_acquire( + &self, + Tracked(auth): Tracked<&mut HistAuth<*mut T>>, + Tracked(tv): Tracked<&mut ThreadView>, + current: *mut T, + new: *mut T, + ) -> (res: (Result<*mut T, *mut T>, Ghost, Tracked>>)) + requires + old(auth).id() == self.id(), + old(auth).wf(), + ensures + ({ + let read_ts = res.1@; + let read_msg = old(auth).msg_at(read_ts); + let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); + &&& old(auth).readable(old(tv)@, read_ts) + &&& match res.0 { + Ok(v) => { + let write_ts = old(auth).history().len(); + let write_msg = Msg { + value: new, + view: after_read.observe(self.id(), write_ts), + }; + &&& read_ts + 1 == old(auth).history().len() + &&& current.addr() == read_msg.value.addr() + &&& equal(v, read_msg.value) + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history().push(write_msg) + &&& final(auth).wf() + &&& final(tv)@ == after_read.observe(self.id(), write_ts) + &&& res.2@ is Some + &&& res.2@->Some_0.id() == self.id() + &&& res.2@->Some_0.ts() == write_ts + &&& res.2@->Some_0.msg() == write_msg + &&& res.2@->Some_0.agrees_with(*final(auth)) + }, + Err(v) => { + &&& current.addr() != read_msg.value.addr() + &&& equal(v, read_msg.value) + &&& final(auth).id() == old(auth).id() + &&& final(auth).history() == old(auth).history() + &&& final(auth).wf() + &&& final(tv)@ == after_read + &&& res.2@ is None + }, + } + }), + opens_invariants none + no_unwind + { + let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); + (result, Ghost::assume_new(), Tracked::assume_new()) + } +} + +#[cfg(verus_keep_ghost)] +fn smoke_test_weak_atomic_with_ghost() { + let atomic = WeakAtomicUsize::<(), (), TrueWeakAtomicInv>::new(Ghost(()), 0, Tracked(())); + let tracked mut tv = ThreadView::new(); + weak_atomic_with_ghost! { + atomic => store_release(1, Tracked(&mut tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(snap.msg().value == 1); + } + } + weak_atomic_with_ghost! { + atomic => store_relaxed(2, Tracked(&mut tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(snap.msg().value == 2); + } + } + let _ = + weak_atomic_with_ghost! { + atomic => load_acquire(Tracked(&mut tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(ret == msg.value); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + atomic => load_relaxed(Tracked(&mut tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(ret == msg.value); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + atomic => compare_exchange_acqrel_acquire(2, 3, Tracked(&mut tv)); + update prev -> next; + returning ret; + timestamp ts; + message msg; + snapshot snap; + ghost g => { + assert(ts < prev.len()); + match ret { + Result::Ok(v) => { + assert(ts + 1 == prev.len()); + assert(v == 2); + assert(msg.value == 2); + match snap { + Option::Some(s) => { + assert(next == prev.push(s.msg())); + assert(s.msg().value == 3); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(v) => { + assert(v == msg.value); + assert(msg.value != 2); + match snap { + Option::Some(_) => { + assert(false); + }, + Option::None => {}, + } + assert(next == prev); + }, + } + } + }; + + let bool_atomic = WeakAtomicBool::<(), (), TrueWeakAtomicInv>::new( + Ghost(()), + false, + Tracked(()), + ); + let tracked mut bool_tv = ThreadView::new(); + weak_atomic_with_ghost! { + bool_atomic => store_release(true, Tracked(&mut bool_tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(snap.msg().value == true); + } + } + let _ = + weak_atomic_with_ghost! { + bool_atomic => load_acquire(Tracked(&mut bool_tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(ret == msg.value); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + bool_atomic => compare_exchange_acqrel_acquire(true, false, Tracked(&mut bool_tv)); + update prev -> next; + returning ret; + timestamp ts; + message msg; + snapshot snap; + ghost g => { + assert(ts < prev.len()); + match ret { + Result::Ok(v) => { + assert(ts + 1 == prev.len()); + assert(v == true); + assert(msg.value == true); + match snap { + Option::Some(s) => { + assert(next == prev.push(s.msg())); + assert(s.msg().value == false); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(v) => { + assert(v == msg.value); + assert(msg.value != true); + match snap { + Option::Some(_) => { + assert(false); + }, + Option::None => {}, + } + assert(next == prev); + }, + } + } + }; + + let null = core::ptr::null_mut::(); + let ptr_atomic = WeakAtomicPtr::::new( + Ghost(()), + null, + Tracked(()), + ); + let tracked mut ptr_tv = ThreadView::new(); + weak_atomic_with_ghost! { + ptr_atomic => store_release(null, Tracked(&mut ptr_tv)); + update prev -> next; + snapshot snap; + ghost g => { + assert(next == prev.push(snap.msg())); + assert(snap.ts() == prev.len()); + assert(equal(snap.msg().value, null)); + } + } + let _ = + weak_atomic_with_ghost! { + ptr_atomic => load_acquire(Tracked(&mut ptr_tv)); + returning ret; + timestamp ts; + message msg; + history history; + ghost g => { + assert(equal(ret, msg.value)); + assert(ts < history.len()); + } + }; + let _ = + weak_atomic_with_ghost! { + ptr_atomic => compare_exchange_acqrel_acquire(null, null, Tracked(&mut ptr_tv)); + update prev -> next; + returning ret; + timestamp ts; + message msg; + snapshot snap; + ghost g => { + assert(ts < prev.len()); + match ret { + Result::Ok(v) => { + assert(ts + 1 == prev.len()); + assert(equal(v, msg.value)); + assert(msg.value.addr() == null.addr()); + match snap { + Option::Some(s) => { + assert(next == prev.push(s.msg())); + assert(equal(s.msg().value, null)); + }, + Option::None => { + assert(false); + }, + } + }, + Result::Err(v) => { + assert(equal(v, msg.value)); + assert(msg.value.addr() != null.addr()); + match snap { + Option::Some(_) => { + assert(false); + }, + Option::None => {}, + } + assert(next == prev); + }, + } + } + }; +} + +#[cfg(verus_keep_ghost)] +pub struct MessagePassingDataInv; + +#[cfg(verus_keep_ghost)] +impl WeakAtomicInvariantPredicate<(), usize, ()> for MessagePassingDataInv { + open spec fn atomic_inv(k: (), history: History, g: ()) -> bool { + &&& history.len() >= 1 + &&& history[0].value == 0 + &&& forall|i: int| 1 <= i < history.len() ==> #[trigger] history[i].value == 1 + } +} + +#[cfg(verus_keep_ghost)] +pub struct MessagePassingFlagInv; + +#[cfg(verus_keep_ghost)] +impl WeakAtomicInvariantPredicate for MessagePassingFlagInv { + open spec fn atomic_inv(data_id: AtomicId, history: History, g: ()) -> bool { + &&& history.len() >= 1 + &&& history[0].value == 0 + &&& forall|i: int| + 1 <= i < history.len() ==> { + &&& #[trigger] history[i].value == 1 + &&& history[i].view.seen_at(data_id) >= 1 + } + } +} + +#[cfg(verus_keep_ghost)] +proof fn preserve_message_passing_data_inv_on_push( + prev: History, + next: History, + msg: Msg, +) + requires + MessagePassingDataInv::atomic_inv((), prev, ()), + next == prev.push(msg), + msg.value == 1, + ensures + MessagePassingDataInv::atomic_inv((), next, ()), +{ + assert(next.len() >= 1); + assert(next[0].value == 0); + assert forall|i: int| 1 <= i < next.len() implies #[trigger] next[i].value == 1 by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + } + }; +} + +#[cfg(verus_keep_ghost)] +proof fn preserve_message_passing_flag_inv_on_push( + data_id: AtomicId, + prev: History, + next: History, + msg: Msg, +) + requires + MessagePassingFlagInv::atomic_inv(data_id, prev, ()), + next == prev.push(msg), + msg.value == 1, + msg.view.seen_at(data_id) >= 1, + ensures + MessagePassingFlagInv::atomic_inv(data_id, next, ()), +{ + assert(next.len() >= 1); + assert(next[0].value == 0); + assert forall|i: int| 1 <= i < next.len() implies { + &&& #[trigger] next[i].value == 1 + &&& next[i].view.seen_at(data_id) >= 1 + } by { + if i == prev.len() { + assert(next[i] == msg); + } else { + assert(i < prev.len()); + } + }; +} + +#[cfg(verus_keep_ghost)] +proof fn prove_message_passing_data_read( + data_id: AtomicId, + history: History, + ret: usize, + ts: Timestamp, + msg: Msg, + tv: WmView, +) + requires + MessagePassingDataInv::atomic_inv((), history, ()), + ts < history.len(), + ret == msg.value, + msg == history[ts as int], + tv.seen_at(data_id) >= 1, + tv.seen_at(data_id) <= ts, + ensures + ret == 1, +{ + assert(ts >= 1); + assert(history[ts as int].value == 1); +} + +// #[cfg(verus_keep_ghost)] +// fn message_passing_release_acquire_threads_can_prove() { +// let data = std::sync::Arc::new( +// WeakAtomicUsize::<(), (), MessagePassingDataInv>::new(Ghost(()), 0, Tracked(())), +// ); +// let ghost data_id = data.atomic.id(); +// let flag = std::sync::Arc::new( +// WeakAtomicUsize::::new(Ghost(data_id), 0, Tracked(())), +// ); +// let data_writer = data.clone(); +// let flag_writer = flag.clone(); +// let data_reader = data.clone(); +// let flag_reader = flag.clone(); +// let writer = vstd::thread::spawn( +// move || +// { +// let tracked mut writer_tv = ThreadView::new(); +// weak_atomic_with_ghost! { +// *data_writer => store_release(1, Tracked(&mut writer_tv)); +// update prev -> next; +// snapshot snap; +// ghost g => { +// preserve_message_passing_data_inv_on_push(prev, next, snap.msg()); +// assert(writer_tv.view.seen_at(data_id) >= 1); +// } +// } +// let ghost before_flag = writer_tv.view; +// assert(before_flag.seen_at(data_id) >= 1); +// weak_atomic_with_ghost! { +// *flag_writer => store_release(1, Tracked(&mut writer_tv)); +// update prev -> next; +// snapshot snap; +// ghost g => { +// assert(snap.msg().view == before_flag.observe(flag_writer.atomic.id(), snap.ts())); +// assert(snap.msg().view.seen_at(data_id) >= 1); +// preserve_message_passing_flag_inv_on_push(data_id, prev, next, snap.msg()); +// } +// } +// }, +// ); +// let reader = vstd::thread::spawn( +// move || +// { +// let tracked mut reader_tv = ThreadView::new(); +// let flag_result = +// weak_atomic_with_ghost! { +// *flag_reader => load_acquire(Tracked(&mut reader_tv)); +// returning ret; +// timestamp ts; +// message msg; +// history history; +// ghost g => { +// if ret == 1 { +// assert(msg.value == 1); +// if ts == 0 { +// assert(history[0].value == 0); +// assert(false); +// } +// assert(ts >= 1); +// assert(history[ts as int].value == 1); +// assert(history[ts as int].view.seen_at(data_id) >= 1); +// assert(msg == history[ts as int]); +// assert(msg.view.seen_at(data_id) >= 1); +// assert(reader_tv.view.seen_at(data_id) >= 1); +// } +// } +// }; +// if flag_result.0 == 1 { +// assert(reader_tv.view.seen_at(data_id) >= 1); +// let data_result = +// weak_atomic_with_ghost! { +// *data_reader => load_relaxed(Tracked(&mut reader_tv)); +// returning ret; +// timestamp ts; +// message msg; +// history history; +// ghost g => { +// prove_message_passing_data_read(data_id, history, ret, ts, msg, reader_tv.view); +// } +// }; +// assert(data_result.0 == 1); +// } +// }, +// ); +// let _ = writer.join(); +// let _ = reader.join(); +// } +} // verus! diff --git a/verified_libs/vstd_extra/src/lib.rs b/verified_libs/vstd_extra/src/lib.rs index 99752d854..8569da81b 100644 --- a/verified_libs/vstd_extra/src/lib.rs +++ b/verified_libs/vstd_extra/src/lib.rs @@ -16,6 +16,7 @@ extern crate alloc; pub mod arithmetic; pub mod array_ptr; +pub mod atomic_weak; pub mod auxiliary; pub mod cast_ptr; pub mod drop_tracking; From 906c657ba71ad0104a37868b75bf0d50633ea2b4 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 27 Jul 2026 02:16:14 -0400 Subject: [PATCH 29/47] Add specs for expiration and loaded pointer into guard protection map --- ostd/specs/sync/rcu.rs | 285 +++++++++++++++--- ostd/specs/sync/weak_memory.rs | 63 +++- ostd/src/sync/rcu/mod.rs | 107 +++++-- ostd/src/sync/rcu/monitor.rs | 71 ++++- ostd/src/task/preempt/guard.rs | 172 +++++++++-- ostd/src/task/preempt/mod.rs | 3 +- verified_libs/vstd_extra/src/lib.rs | 1 + verified_libs/vstd_extra/src/rcu_read_pool.rs | 257 ++++++++++++++++ 8 files changed, 866 insertions(+), 93 deletions(-) create mode 100644 verified_libs/vstd_extra/src/rcu_read_pool.rs diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 55259f415..15c8fc16f 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -25,6 +25,8 @@ //! observations available to later readers before physical reclamation. use core::marker::PhantomData; +use crate::specs::mm::cpu::CpuId; + use super::weak_memory::{History, Msg, Timestamp, WeakAtomicInvariantPredicate, WmView}; use vstd::prelude::*; use vstd::resource::Loc; @@ -36,6 +38,21 @@ pub type LinkIndex = nat; pub type LinkEdge = (nat, LinkIndex); +/// Scheduler identity of the execution context that owns one RCU reader. +/// +/// `session` is the fresh preemption-session resource identity created when +/// the scheduler checks a task in on `cpu`. Recording the full tuple prevents +/// an RCU guard from being detached, in the proof, from the preemption guard +/// that keeps its task on that CPU. +pub ghost struct RcuReaderContext { + pub scheduler: Loc, + pub task: Loc, + pub session: Loc, + pub cpu: CpuId, + /// Quiescent interval in which this reader started. + pub generation: nat, +} + /// Proof summary for a type-erased RCU callback. /// /// The executable callback may close over any sized Rust value, but the RCU @@ -309,6 +326,8 @@ impl RcuRootGhost { None => res.0.objects() == Map::empty(), }, current_registration_matches(res.0, res.1), + res.0.domain_auth().retired() == Set::::empty(), + res.0.domain_auth().retire_observations() == Map::::empty(), { let tracked mut domain = RcuDomainAuth::tracked_new(); if ptr.addr() == 0 { @@ -348,6 +367,10 @@ impl RcuRootGhost { final(self).domain_auth().reader_registry() == old( self, ).domain_auth().reader_registry(), + final(self).domain_auth().retired() == old(self).domain_auth().retired(), + final(self).domain_auth().retire_observations() == old( + self, + ).domain_auth().retire_observations(), (res is Some) == (msg.value.addr() != 0), res is Some ==> res->Some_0.0.ptr() == msg.value, res is Some ==> res->Some_0.0.obj() == res->Some_0.1.obj(), @@ -434,6 +457,10 @@ impl RcuRootGhost { final(self).domain_auth().reader_registry() == old( self, ).domain_auth().reader_registry(), + final(self).domain_auth().retired() == old(self).domain_auth().retired(), + final(self).domain_auth().retire_observations() == old( + self, + ).domain_auth().retire_observations(), final(self).objects() == old(self).objects(), final(self).publications() == old(self).publications().push(Some(info.obj())), { @@ -536,7 +563,7 @@ pub tracked struct RcuRootOwnedGhost { root: RcuRootGhost, current: Option>, infos: Map>, - ghost removals: Map, + ghost removals: Map, } impl RcuRootOwnedGhost { @@ -585,7 +612,7 @@ impl RcuRootOwnedGhost { self.infos } - pub closed spec fn removals(self) -> Map { + pub closed spec fn removals(self) -> Map { self.removals } @@ -623,7 +650,7 @@ impl RcuRootOwnedGhost { } &&& forall|obj: nat| self.removals().contains_key(obj) ==> { - let ts = #[trigger] self.removals()[obj]; + let ts = (#[trigger] self.removals()[obj]).timestamp; &&& 0 < ts < history.len() &&& forall|i: int| ts <= i < history.len() ==> #[trigger] self.publications()[i] != Some(obj) @@ -694,13 +721,74 @@ impl RcuRootOwnedGhost { } } + /// Extracts the allocation ID stored in a non-null root publication. + pub proof fn lemma_published_object_id( + tracked &self, + history: History<*mut T>, + ts: nat, + object: RcuPublishedObject, + ) + requires + rcu_owned_root_history_inv(history, *self), + ts < history.len(), + self.published_at(ts) == Some(object), + ensures + self.publications()[ts as int] == Some(object.obj), + { + match self.publications()[ts as int] { + Some(obj) => { + assert(self.root().objects().contains_pair(obj, history[ts as int].value.addr())); + assert(self.published_at(ts) == Some( + RcuPublishedObject { + domain: self.domain(), + obj, + addr: self.root().objects()[obj], + }, + )); + }, + None => { + assert(self.published_at(ts) is None); + }, + } + } + + /// Opens the paper's entry-time expired-set membership into the recorded + /// root-removal observation for that allocation. + pub proof fn lemma_observed_retired( + tracked &self, + history: History<*mut T>, + root: Loc, + view: WmView, + obj: nat, + ) + requires + rcu_owned_root_history_inv(history, *self), + self.root().domain_auth().observed_retired(root, view).contains(obj), + ensures + self.removals().contains_key(obj), + self.removals()[obj].root == root, + self.removals()[obj].observed_by(view), + { + assert(self.root().domain_auth().wf()); + assert(self.root().domain_auth().retired().contains(obj)); + assert(self.root().domain_auth().retire_observations().dom() + == self.root().domain_auth().retired()); + assert(self.root().domain_auth().retire_observations().dom().contains(obj)); + assert(self.root().domain_auth().retire_observations().contains_key(obj)); + } + /// Registers and starts one fresh paper reader while preserving root state. /// /// Reader slots are proof-only and currently allocated per critical /// section. `tracked_stop_reader` consumes the live slot again; no runtime /// reader counter is introduced. - pub proof fn tracked_start_reader(tracked &mut self, history: History<*mut T>) -> (tracked res: - RcuBaseGuard) + pub proof fn tracked_start_reader( + tracked &mut self, + history: History<*mut T>, + root: Loc, + start_view: WmView, + reader: RcuReaderContext, + ) -> (tracked res: RcuBaseGuard) requires rcu_owned_root_history_inv(history, *old(self)), ensures @@ -708,12 +796,21 @@ impl RcuRootOwnedGhost { final(self).domain() == old(self).domain(), final(self).reader_registry() == old(self).reader_registry(), final(self).current_owned() == old(self).current_owned(), + final(self).publications() == old(self).publications(), + final(self).infos() == old(self).infos(), + final(self).removals() == old(self).removals(), + final(self).root().domain_auth().retired() == old(self).root().domain_auth().retired(), + final(self).root().domain_auth().retire_observations() == old( + self, + ).root().domain_auth().retire_observations(), res.wf(), res.domain() == final(self).domain(), res.reader_registry() == final(self).reader_registry(), + res.reader() == reader, + res.expired() == final(self).root().domain_auth().observed_retired(root, start_view), { - let tracked inactive = self.root.domain.tracked_register_reader(); - let tracked guard = self.root.domain.tracked_guard_start(inactive); + let tracked inactive = self.root.domain.tracked_register_reader(reader); + let tracked guard = self.root.domain.tracked_guard_start(inactive, root, start_view); assert(current_registration_matches(self.root(), self.current_registration())); assert(self.infos_wf()); guard @@ -792,6 +889,7 @@ impl RcuRootOwnedGhost { let tracked res = RcuRootOwnedGhost { root, current, infos, removals: Map::empty() }; assert(res.infos_wf()); assert(res.removals_wf(seq![Msg { value: ptr, view: WmView::empty() }])); + assert(res.removals() == res.root().domain_auth().retire_observations()); res } @@ -831,8 +929,8 @@ impl RcuRootOwnedGhost { root, timestamp: prev.len(), }) - &&& old(self).current_ownership() == Some(detached.ownership()) &&& equal(detached.ptr(), prev[(prev.len() - 1) as int].value) + &&& old(self).current_ownership() == Some(detached.ownership()) &&& OwnPred::owns(detached.ptr(), detached.ownership()) }, None => old(self).current_registration() is None, @@ -893,6 +991,7 @@ impl RcuRootOwnedGhost { None }, }; + let ghost removal = RcuRemovalObservation { root, timestamp: prev.len() }; let tracked detached = match old_current { Some(owned) => { let tracked (registration, old_ownership) = owned.tracked_into_parts(); @@ -903,7 +1002,6 @@ impl RcuRootOwnedGhost { link_view: RcuLinkView::empty(), }; let tracked retire = lift_retire_perm(base, seen_removed); - let ghost removal = RcuRemovalObservation { root, timestamp: prev.len() }; let tracked retired = self.root.domain.tracked_retire(retire, removal); Some(RcuRetiredOwnedObject { object, retired, ownership: old_ownership }) }, @@ -911,20 +1009,43 @@ impl RcuRootOwnedGhost { }; self.current = new_current; self.removals = match removed_obj { - Some(obj) => self.removals.insert(obj, prev.len()), + Some(obj) => self.removals.insert(obj, removal), None => self.removals, }; + assert(self.removals() == self.root().domain_auth().retire_observations()) by { + match removed_obj { + Some(obj) => { + assert(old(self).removals() == old( + self, + ).root().domain_auth().retire_observations()); + assert(self.removals() == old(self).removals().insert(obj, removal)); + assert(self.root().domain_auth().retire_observations() == old( + self, + ).root().domain_auth().retire_observations().insert(obj, removal)); + }, + None => { + assert(old(self).removals() == old( + self, + ).root().domain_auth().retire_observations()); + assert(self.removals() == old(self).removals()); + assert(self.root().domain_auth().retire_observations() == old( + self, + ).root().domain_auth().retire_observations()); + }, + } + }; assert(current_registration_matches(self.root(), self.current_registration())); assert(self.infos_wf()); assert(self.removals_wf(next)) by { assert forall|obj: nat| self.removals().contains_key(obj) implies { - let ts = #[trigger] self.removals()[obj]; + let ts = (#[trigger] self.removals()[obj]).timestamp; &&& 0 < ts < next.len() &&& forall|i: int| ts <= i < next.len() ==> #[trigger] self.publications()[i] != Some(obj) } by { if removed_obj == Some(obj) { - assert(self.removals()[obj] == prev.len()); + assert(self.removals()[obj] == removal); + assert(self.removals()[obj].timestamp == prev.len()); assert(next.len() == prev.len() + 1); assert(self.publications()[prev.len() as int] == match new_registration { Some(registration) => Some(registration.0.obj()), @@ -940,7 +1061,7 @@ impl RcuRootOwnedGhost { assert(old(self).removals().contains_key(obj)); assert(self.removals()[obj] == old(self).removals()[obj]); assert forall|i: int| - self.removals()[obj] <= i + self.removals()[obj].timestamp <= i < next.len() implies #[trigger] self.publications()[i] != Some(obj) by { if i < prev.len() { assert(self.publications()[i] == old(self).publications()[i]); @@ -983,9 +1104,10 @@ impl RcuRootOwnedGhost { self.root.tracked_push_registered(prev, next, msg, &owned.registration.0); self.current = Some(owned); assert(current_registration_matches(self.root(), self.current_registration())); + assert(self.removals() == self.root().domain_auth().retire_observations()); assert(self.removals_wf(next)) by { assert forall|obj: nat| self.removals().contains_key(obj) implies { - let ts = #[trigger] self.removals()[obj]; + let ts = (#[trigger] self.removals()[obj]).timestamp; &&& 0 < ts < next.len() &&& forall|i: int| ts <= i < next.len() ==> #[trigger] self.publications()[i] != Some(obj) @@ -994,8 +1116,8 @@ impl RcuRootOwnedGhost { assert(!old(self).removals().contains_key(owned.registration.0.obj())); assert(obj != owned.registration.0.obj()); assert forall|i: int| - self.removals()[obj] <= i < next.len() implies #[trigger] self.publications()[i] - != Some(obj) by { + self.removals()[obj].timestamp <= i + < next.len() implies #[trigger] self.publications()[i] != Some(obj) by { if i < prev.len() { assert(self.publications()[i] == old(self).publications()[i]); } else { @@ -1017,6 +1139,7 @@ pub open spec fn rcu_owned_root_history_inv( &&& ghost.ownership_wf() &&& ghost.infos_wf() &&& ghost.removals_wf(history) + &&& ghost.removals() == ghost.root().domain_auth().retire_observations() &&& forall|i: int| 0 <= i < history.len() ==> { match #[trigger] ghost.publications()[i] { @@ -1465,7 +1588,7 @@ pub tracked struct RcuDomainAuth { ghost next_obj: nat, ghost next_reader: nat, ghost retired: Set, - ghost expired: Set, + ghost retire_observations: Map, } impl RcuDomainAuth { @@ -1487,13 +1610,22 @@ impl RcuDomainAuth { self.retired } - /// Allocations whose retirement is covered by a completed grace period. + /// Physical detachment observation recorded for each retired allocation. /// - /// This is the paper's implementation-specific expired set. It is kept - /// separate from `retired`: a newly retired allocation remains protected - /// by critical sections until monitor completion moves it here. - pub closed spec fn expired(self) -> Set { - self.expired + /// A guard's implementation-specific expired set `X` is derived from this + /// map at critical-section entry: it contains exactly the retired + /// allocations whose detachment observation is already covered by the + /// entering thread's weak-memory view. + pub closed spec fn retire_observations(self) -> Map { + self.retire_observations + } + + pub open spec fn observed_retired(self, root: Loc, view: WmView) -> Set { + self.retired().filter( + |obj: nat| + self.retire_observations()[obj].root == root + && self.retire_observations()[obj].observed_by(view), + ) } pub closed spec fn reader_registry(self) -> Loc { @@ -1516,7 +1648,7 @@ impl RcuDomainAuth { &&& forall|obj: nat| #[trigger] self.objects@.contains_key(obj) ==> obj < self.next_obj() &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> tid < self.next_reader() &&& self.retired().subset_of(self.objects().dom()) - &&& self.expired().subset_of(self.retired()) + &&& self.retire_observations().dom() == self.retired() } /// Allocates a fresh RCU protection domain. @@ -1525,6 +1657,8 @@ impl RcuDomainAuth { res.wf(), res.objects() == Map::::empty(), res.next_obj() == 0, + res.retired() == Set::::empty(), + res.retire_observations() == Map::::empty(), { let tracked (objects, _objects_entries) = GhostMapAuth::new(Map::empty()); let tracked (retire_perms, _retire_entries) = GhostMapAuth::new(Map::empty()); @@ -1536,7 +1670,7 @@ impl RcuDomainAuth { next_obj: 0, next_reader: 0, retired: Set::empty(), - expired: Set::empty(), + retire_observations: Map::empty(), } } @@ -1559,7 +1693,7 @@ impl RcuDomainAuth { final(self).reader_registry() == old(self).reader_registry(), final(self).next_obj() == old(self).next_obj() + 1, final(self).retired() == old(self).retired(), - final(self).expired() == old(self).expired(), + final(self).retire_observations() == old(self).retire_observations(), final(self).objects() == old(self).objects().insert(old(self).next_obj(), ptr.addr()), res.0.domain() == final(self).id(), res.0.obj() == old(self).next_obj(), @@ -1608,7 +1742,10 @@ impl RcuDomainAuth { /// Registers one reader slot and returns the paper's `Inactive(tid)` /// resource for it. - pub proof fn tracked_register_reader(tracked &mut self) -> (tracked res: RcuInactive) + pub proof fn tracked_register_reader( + tracked &mut self, + reader: RcuReaderContext, + ) -> (tracked res: RcuInactive) requires old(self).wf(), ensures @@ -1618,9 +1755,10 @@ impl RcuDomainAuth { final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), - final(self).expired() == old(self).expired(), + final(self).retire_observations() == old(self).retire_observations(), res.domain() == final(self).id(), res.tid() == old(self).next_reader(), + res.reader() == reader, res.belongs_to(*final(self)), res.wf(), { @@ -1634,11 +1772,12 @@ impl RcuDomainAuth { assert(old(self).readers@.contains_key(registered)); } }; - RcuInactive { domain: self.id(), state } + RcuInactive { domain: self.id(), state, reader } } /// Starts a read-side critical section, snapshotting the set `X` of AIds - /// whose grace period had already completed. + /// whose root-removal observation is covered by the entering thread's + /// weak-memory view. /// /// The paper only requires `X` to be a subset of all retired allocations. /// Using `retired` itself here would be too strong: a reader may safely @@ -1647,6 +1786,8 @@ impl RcuDomainAuth { pub proof fn tracked_guard_start( tracked &mut self, tracked mut inactive: RcuInactive, + root: Loc, + start_view: WmView, ) -> (tracked res: RcuBaseGuard) requires old(self).wf(), @@ -1659,10 +1800,11 @@ impl RcuDomainAuth { final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), - final(self).expired() == old(self).expired(), + final(self).retire_observations() == old(self).retire_observations(), res.belongs_to(*final(self)), res.tid() == inactive.tid(), - res.expired() == old(self).expired(), + res.reader() == inactive.reader(), + res.expired() == old(self).observed_retired(root, start_view), res.protected() == Map::::empty(), res.wf(), { @@ -1681,7 +1823,8 @@ impl RcuDomainAuth { RcuBaseGuard { domain: self.id(), state: inactive.state, - expired: self.expired, + reader: inactive.reader, + expired: self.observed_retired(root, start_view), protected: Map::empty(), } } @@ -1703,9 +1846,10 @@ impl RcuDomainAuth { final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), - final(self).expired() == old(self).expired(), + final(self).retire_observations() == old(self).retire_observations(), res.belongs_to(*final(self)), res.tid() == guard.tid(), + res.reader() == guard.reader(), res.wf(), { let ghost tid = guard.tid(); @@ -1720,7 +1864,7 @@ impl RcuDomainAuth { assert(old(self).readers@.contains_key(registered)); } }; - RcuInactive { domain: self.id(), state: guard.state } + RcuInactive { domain: self.id(), state: guard.state, reader: guard.reader } } /// Implements the base `rcu-retire` transition by adding the detached AId @@ -1742,7 +1886,10 @@ impl RcuDomainAuth { final(self).reader_registry() == old(self).reader_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired().insert(retire.obj()), - final(self).expired() == old(self).expired(), + final(self).retire_observations() == old(self).retire_observations().insert( + retire.obj(), + removal, + ), res.domain() == final(self).id(), res.obj() == retire.obj(), res.ptr() == retire.ptr(), @@ -1755,8 +1902,9 @@ impl RcuDomainAuth { retire.base.perm.agree(&self.retire_perms); assert(self.objects().contains_key(obj)); self.retired = self.retired.insert(obj); + self.retire_observations = self.retire_observations.insert(obj, removal); assert(self.retired().subset_of(self.objects().dom())); - assert(self.expired().subset_of(self.retired())); + assert(self.retire_observations().dom() == self.retired()); let tracked fact = retire.base.perm.persist(); RcuRetired { fact: RcuRetiredFact { domain, fact, removal }, ptr } } @@ -1766,6 +1914,7 @@ impl RcuDomainAuth { pub tracked struct RcuInactive { ghost domain: Loc, state: GhostPointsTo, + ghost reader: RcuReaderContext, } impl RcuInactive { @@ -1777,6 +1926,10 @@ impl RcuInactive { self.state.key() } + pub closed spec fn reader(self) -> RcuReaderContext { + self.reader + } + pub closed spec fn wf(self) -> bool { !self.state.value() } @@ -1794,6 +1947,7 @@ impl RcuInactive { pub tracked struct RcuBaseGuard { ghost domain: Loc, state: GhostPointsTo, + ghost reader: RcuReaderContext, ghost expired: Set, ghost protected: Map, } @@ -1811,6 +1965,10 @@ impl RcuBaseGuard { self.state.id() } + pub closed spec fn reader(self) -> RcuReaderContext { + self.reader + } + pub closed spec fn expired(self) -> Set { self.expired } @@ -1844,6 +2002,8 @@ impl RcuBaseGuard { final(self).wf(), final(self).domain() == old(self).domain(), final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), final(self).expired() == old(self).expired(), final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), final(self).protects(info.addr(), info.obj()), @@ -2233,17 +2393,53 @@ pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> link_view: RcuLinkView::empty(), }; let tracked retire = lift_retire_perm(base, seen_removed); - let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 0 }; + let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 1 }; let tracked _retired = domain.tracked_retire(retire, removal); - let tracked inactive = domain.tracked_register_reader(); - let tracked mut guard = domain.tracked_guard_start(inactive); + let ghost reader = arbitrary(); + let tracked inactive = domain.tracked_register_reader(reader); + let tracked mut guard = domain.tracked_guard_start(inactive, domain.id(), WmView::empty()); assert(guard.expired() == Set::::empty()); assert(!guard.expired().contains(info.obj())); guard.tracked_protect(&info); (guard, info) } +/// Regression proof that observing a retirement makes it expired for a new +/// guard. +/// +/// Timestamp zero is covered by an empty weak-memory view. Consequently the +/// retired allocation enters the new guard's `X` snapshot and cannot be added +/// to its protection map. +pub proof fn observed_retired_object_enters_guard_expired(ptr: *mut T) -> (tracked res: ( + RcuBaseGuard, + RcuBlockInfo, +)) + requires + ptr.addr() != 0, + ensures + res.1.ptr() == ptr, + res.0.domain() == res.1.domain(), + res.0.expired().contains(res.1.obj()), +{ + let tracked mut domain = RcuDomainAuth::tracked_new(); + let tracked (info, base) = domain.tracked_register(ptr); + let ghost seen_removed = RcuSeenRemoved { + removed: Set::empty().insert(info.obj()), + link_view: RcuLinkView::empty(), + }; + let tracked retire = lift_retire_perm(base, seen_removed); + let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 0 }; + let tracked _retired = domain.tracked_retire(retire, removal); + + let ghost reader = arbitrary(); + let tracked inactive = domain.tracked_register_reader(reader); + let tracked guard = domain.tracked_guard_start(inactive, domain.id(), WmView::empty()); + assert(removal.observed_by(WmView::empty())); + assert(guard.expired().contains(info.obj())); + (guard, info) +} + /// Non-generic proof certificate carried across the type-erasure boundary. /// /// A certificate can only be produced from a typed traversal retire permission, @@ -2351,6 +2547,10 @@ impl RcuReadGuardToken { self.base.reader_registry() } + pub closed spec fn reader(self) -> RcuReaderContext { + self.base.reader() + } + pub closed spec fn expired(self) -> Set { self.base.expired() } @@ -2417,6 +2617,7 @@ impl RcuReadGuardToken { res.domain() == base.domain(), res.tid() == base.tid(), res.reader_registry() == base.reader_registry(), + res.reader() == base.reader(), res.expired() == base.expired(), res.protected() == base.protected(), res.seen_removed() == seen_removed, @@ -2434,6 +2635,7 @@ impl RcuReadGuardToken { res.domain() == base.domain(), res.tid() == base.tid(), res.reader_registry() == base.reader_registry(), + res.reader() == base.reader(), res.expired() == base.expired(), res.seen_removed().removed == base.expired(), res.link_view() == RcuLinkView::::empty(), @@ -2455,6 +2657,7 @@ impl RcuReadGuardToken { res.domain() == self.domain(), res.tid() == self.tid(), res.reader_registry() == self.reader_registry(), + res.reader() == self.reader(), res.expired() == self.expired(), res.protected() == self.protected(), { @@ -2469,6 +2672,8 @@ impl RcuReadGuardToken { final(self).wf(), final(self).domain() == old(self).domain(), final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), final(self).expired() == old(self).expired(), final(self).seen_removed() == old(self).seen_removed(), final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index e0d93a846..95255d062 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -170,7 +170,11 @@ impl RcuWeakAtomicPtr where /// The ghost reader transition occurs in the same invariant opening as the /// real acquire load. Executably this is identical to `load_acquire_rcu`. #[inline(always)] - pub fn load_acquire_rcu_guarded(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + pub fn load_acquire_rcu_guarded( + &self, + Ghost(reader): Ghost, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ( *mut T, Ghost, Ghost>, @@ -184,6 +188,7 @@ impl RcuWeakAtomicPtr where res.4@.wf(), res.4@.domain() == self.constant().domain, res.4@.reader_registry() == self.constant().reader_registry, + res.4@.reader() == reader, match (res.2@, res.3@) { (None, None) => res.0.addr() == 0, (Some(object), Some(info)) => { @@ -195,6 +200,8 @@ impl RcuWeakAtomicPtr where &&& info.obj() == object.obj &&& info.addr() == object.addr &&& equal(info.ptr(), res.0) + &&& !res.4@.expired().contains(info.obj()) + &&& res.4@.protects(info.addr(), info.obj()) }, _ => false, }, @@ -203,6 +210,7 @@ impl RcuWeakAtomicPtr where proof { use_type_invariant(self); } + let ghost start_view = tv@; let raw_atomic = self.raw_atomic(); vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { let tracked (hist, mut g) = pair; @@ -212,7 +220,8 @@ impl RcuWeakAtomicPtr where assert(hist.id() == raw_atomic.id()); } proof_decl! { - let tracked base_guard = g.tracked_start_reader(hist.history()); + let tracked base_guard = + g.tracked_start_reader(hist.history(), self.id(), start_view, reader); } let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { @@ -256,7 +265,55 @@ impl RcuWeakAtomicPtr where assert(rcu_spec::rcu_current_ownership_inv::(g)); } proof_decl! { - let tracked guard = rcu_spec::RcuReadGuardToken::tracked_from_base(base_guard); + let tracked mut guard = + rcu_spec::RcuReadGuardToken::tracked_from_base(base_guard); + } + proof { + assert(guard.expired() + == g.root().domain_auth().observed_retired(self.id(), start_view)); + match &loaded_info { + Some(info) => { + if guard.expired().contains(info.obj()) { + assert(g.root().domain_auth().observed_retired( + self.id(), + start_view, + ).contains(info.obj())); + g.lemma_observed_retired( + hist.history(), + self.id(), + start_view, + info.obj(), + ); + let ghost removal = g.removals()[info.obj()]; + assert(removal.root == self.id()); + assert(removal.observed_by(start_view)); + assert(start_view.seen_at(self.id()) <= loaded.1@); + assert(removal.timestamp <= loaded.1@); + assert(g.removals_wf(hist.history())); + assert(removal.timestamp < hist.history().len()); + assert(g.publications()[loaded.1@ as int] != Some(info.obj())); + assert(published == Some(rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + })); + g.lemma_published_object_id( + hist.history(), + loaded.1@, + rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + }, + ); + assert(g.publications()[loaded.1@ as int] == Some(info.obj())); + assert(false); + } + assert(guard.can_protect(*info)); + guard.tracked_protect(info); + }, + None => {}, + } } result = ( loaded.0, diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 805f5ad75..d38d0002c 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -104,24 +104,36 @@ //! executable read guard until destruction or consuming CAS performs //! `Guard -> Inactive`. //! -//! Two boundaries remain. First, the loaded root's `BlockInfo` is not yet -//! installed in the guard's protection map. That step requires connecting -//! expired-object completion to the root atomic's timestamp and the task's -//! `ThreadView`; consequently `assume_shared_ref` still stands in for the final -//! traversal argument that grants the client pointer's reference permission. -//! Second, the unsafe -//! quiescent-report entrypoint is not yet owned by the executable scheduler's -//! context-switch path. The scheduler ghost API proves the per-CPU view -//! handoff, but the real hook still has to carry that ghost state, perform the -//! domain expired-set transition, and connect it to monitor grace-period -//! completion. Until both boundaries are closed, the reclaim permit is a -//! monitor-level authorization rather than the final end-to-end memory-safety -//! authority. +//! A guarded weak load now installs the loaded root's exact `BlockInfo` in the +//! guard's protection map. The proof derives the guard's expired set from the +//! entering task's view and the recorded root-removal observations. If the +//! loaded AId were expired, weak-memory coherence and the root history's +//! removal invariant would contradict the load timestamp. The remaining +//! traversal boundary is converting that abstract protection into the client +//! pointer's physical reference permission; `assume_shared_ref` still stands +//! in for that final argument. +//! +//! Each reader token now records its scheduler, task, CPU, preemption-session +//! identity, and quiescent generation. The generation is part of the +//! fractional preemption resource. A monitor report advances it only while the +//! running context owns the full resource, so a live preemption-disabled +//! reader from the reported generation makes that transition impossible. New +//! readers in the same session carry the next generation and retain the +//! report's weak-memory view; schedule-out similarly requires full ownership +//! and publishes that view to the CPU view imported by the next session. +//! +//! The remaining scheduler boundary is to verify the executable +//! `processor::switch_to_task` call site against this tracked transition and +//! package the per-session argument into a global theorem over every CPU +//! report. Until that and the physical-reference boundary are closed, the +//! reclaim permit remains a monitor-level authorization rather than the final +//! end-to-end memory-safety authority. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; use vstd_extra::prelude::*; use vstd_extra::raw_callback::{RawCallback, RawCallbackContext}; +use vstd_extra::rcu_read_pool::RcuReadLease; use crate::{ specs::{ @@ -132,7 +144,7 @@ use crate::{ task::InAtomicMode, }, sync::Once, - task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, + task::{disable_preempt_in_context, DisabledPreemptGuard, RunningTaskContext}, }; use non_null::{NonNullPtr, NonNullPtrRef}; @@ -405,7 +417,11 @@ impl RcuInner

{ } #[inline(always)] - fn load_ptr_acquire_guarded(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + fn load_ptr_acquire_guarded( + &self, + Ghost(reader): Ghost, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ( *mut

::Target, Tracked::Target>>>, Tracked::Target>>, @@ -417,6 +433,7 @@ impl RcuInner

{ res.2@.wf(), res.2@.domain() == self.ptr.constant().domain, res.2@.reader_registry() == self.ptr.constant().reader_registry, + res.2@.reader() == reader, match res.1@ { None => res.0.is_null(), Some(info) => { @@ -424,13 +441,15 @@ impl RcuInner

{ &&& info.wf() &&& info.domain() == res.2@.domain() &&& equal(info.ptr(), res.0) + &&& !res.2@.expired().contains(info.obj()) + &&& res.2@.protects(info.addr(), info.obj()) }, }, { proof { assert(self.ptr.constant().nullable == self.is_nullable()); } - let res = self.ptr.load_acquire_rcu_guarded(Tracked(tv)); + let res = self.ptr.load_acquire_rcu_guarded(Ghost(reader), Tracked(tv)); proof { if !self.is_nullable() { assert(!self.ptr.constant().nullable); @@ -494,6 +513,7 @@ impl RcuInner

{ final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), { @@ -545,18 +565,29 @@ impl RcuInner

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), { let inner_guard = disable_preempt_in_context(Tracked(session)); + let ghost reader = rcu_spec::RcuReaderContext { + scheduler: session.scheduler(), + task: session.task(), + session: session.session_id(), + cpu: session.cpu(), + generation: session.quiescent_generation(), + }; proof_decl! { let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( session, &inner_guard, ); } - let (obj_ptr, tracked_info, tracked_guard) = self.load_ptr_acquire_guarded(Tracked(tv)); + let (obj_ptr, tracked_info, tracked_guard) = self.load_ptr_acquire_guarded( + Ghost(reader), + Tracked(tv), + ); RcuReadGuardInner { obj_ptr, rcu: self, @@ -579,6 +610,7 @@ impl RcuInner

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), { @@ -625,6 +657,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), { @@ -727,6 +760,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { final(session).cpu() == old(session).cpu(), final(session).view() == old(session).view(), final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), { @@ -750,6 +784,23 @@ unsafe fn assume_shared_ref<'a, P: NonNullPtrRef<'a>>(ptr: NonNull) - unsafe { P::raw_as_ref(ptr, Tracked(perm)) } } +/// Converts an RCU storage-protocol lease into the pointer implementation's +/// reusable shared-reference permission. +/// +/// Once guarded loads carry this lease, `RcuReadGuardInner::get` can pass the +/// result directly to `P::raw_as_ref` without manufacturing a permission. +proof fn borrow_lease_as_ref_permission<'a, P: NonNullPtrRef<'a>>( + tracked lease: &'a RcuReadLease, +) -> (tracked res: P::RefPermission) + requires + lease.resource().inv(), + ensures + res.inv(), + P::ref_perm_view_permission(res) == lease.resource(), +{ + P::borrow_perm_as_ref_perm(lease.borrow()) +} + #[verus_verify] impl Rcu

{ /// Creates a new RCU primitive with the given pointer. @@ -772,6 +823,7 @@ impl Rcu

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -794,6 +846,7 @@ impl Rcu

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -838,6 +891,7 @@ impl RcuOption

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -860,6 +914,7 @@ impl RcuOption

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() + 1 == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth() + 1, res.matches_context(*final(session)), @@ -881,6 +936,7 @@ impl RcuOption

{ final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -907,6 +963,7 @@ impl RcuReadGuard<'_, P> { final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -935,6 +992,7 @@ impl RcuReadGuard<'_, P> { final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -961,6 +1019,7 @@ impl RcuOptionReadGuard<'_, P> { final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -990,6 +1049,7 @@ impl RcuOptionReadGuard<'_, P> { final(session).wf(), final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, final(session).preempt_depth() + 1 == old(session).preempt_depth(), )] @@ -1061,6 +1121,8 @@ impl Deref for RcuDrop { final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), + old(session).quiescent_generation() <= final(session).quiescent_generation(), + final(session).quiescent_generation() <= old(session).quiescent_generation() + 1, final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -1133,7 +1195,14 @@ impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { closed spec fn matches_context(self, session: RunningTaskContext) -> bool { - self._inner_guard.matches_context(session) + &&& self._inner_guard.matches_context(session) + &&& self.tracked_guard@.reader() == (rcu_spec::RcuReaderContext { + scheduler: session.scheduler(), + task: session.task(), + session: session.session_id(), + cpu: session.cpu(), + generation: session.quiescent_generation(), + }) } #[verifier::type_invariant] @@ -1150,6 +1219,8 @@ impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { &&& info.wf() &&& info.domain() == self.tracked_guard@.domain() &&& equal(info.ptr(), self.obj_ptr) + &&& !self.tracked_guard@.expired().contains(info.obj()) + &&& self.tracked_guard@.protects(info.addr(), info.obj()) }, } } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index cf4bbbe03..f3508a2f1 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -35,30 +35,42 @@ tracked struct RcuQuiescentContext { ghost cpu: CpuId, ghost task: Loc, ghost scheduler: Loc, + ghost session: Loc, + ghost generation: nat, ghost view: WmView, } impl RcuQuiescentContext { proof fn tracked_from_running_context( - tracked context: &RunningTaskContext, + tracked context: &mut RunningTaskContext, cpu: CpuId, ) -> (tracked res: Self) requires - context.wf(), - context.is_quiescent(), - cpu == context.cpu(), + old(context).wf(), + old(context).is_quiescent(), + cpu == old(context).cpu(), ensures res.cpu == cpu, - res.task == context.task(), - res.scheduler == context.scheduler(), - res.view == context.view(), + res.task == old(context).task(), + res.scheduler == old(context).scheduler(), + res.session == old(context).session_id(), + res.generation == old(context).quiescent_generation(), + res.view == old(context).view(), + final(context).wf(), + final(context).is_quiescent(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == res.generation + 1, + final(context).view() == old(context).view(), { - RcuQuiescentContext { - cpu, - task: context.task(), - scheduler: context.scheduler(), - view: context.view(), - } + let ghost task = context.task(); + let ghost scheduler = context.scheduler(); + let ghost session = context.session_id(); + let ghost view = context.view(); + let ghost generation = context.tracked_record_quiescent(); + RcuQuiescentContext { cpu, task, scheduler, session, generation, view } } } @@ -67,10 +79,27 @@ ghost struct RcuCpuQuiescentReport { cpu: CpuId, task: Loc, scheduler: Loc, + session: Loc, + /// Last reader generation closed by this quiescent transition. + generation: nat, view: WmView, epoch: nat, } +impl RcuCpuQuiescentReport { + /// Whether this report is the quiescent boundary immediately following a + /// reader generation in the same scheduler session. + closed spec fn closes_same_session_generation( + self, + reader: rcu_spec::RcuReaderContext, + ) -> bool { + &&& self.cpu == reader.cpu + &&& self.scheduler == reader.scheduler + &&& self.session == reader.session + &&& reader.generation <= self.generation + } +} + /// RCU-specific wrapper around a type-erased executable callback. /// /// `RawCallback` is intentionally proof-opaque. The summary records the object @@ -141,7 +170,8 @@ impl RcuCallback { } closed spec fn wf(self) -> bool { - self.safety@.matches(self@) + &&& self.safety@.matches(self@) + &&& self@.removal.observed_by(self@.retire_view) } #[verifier::type_invariant] @@ -185,6 +215,7 @@ impl RcuReclaimPermit { &&& self.reports@[cpu].cpu == cpu &&& self.reports@[cpu].epoch == callback.retire_epoch &&& callback.retire_view.spec_le(self.reports@[cpu].view) + &&& callback.removal.observed_by(self.reports@[cpu].view) } } } @@ -241,10 +272,17 @@ impl CompletedGracePeriod { requires safety.matches(callback), self.covers(callback), + callback.removal.observed_by(callback.retire_view), ensures permit.authorizes(callback), { let tracked retired = safety.tracked_retired_fact(callback); + assert forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) implies callback.removal.observed_by( + self.reports()[cpu].view, + ) by { + assert(callback.retire_view.spec_le(self.reports()[cpu].view)); + }; RcuReclaimPermit { summary: Ghost(callback), retired, reports: Ghost(self.reports()) } } } @@ -472,6 +510,8 @@ impl GracePeriod { cpu: this_cpu, task: context.task, scheduler: context.scheduler, + session: context.session, + generation: context.generation, view: context.view, epoch: self@.epoch, }; @@ -1120,6 +1160,7 @@ impl RcuMonitor { final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] @@ -1197,6 +1238,8 @@ impl RcuMonitor { final(session).scheduler() == old(session).scheduler(), final(session).cpu() == old(session).cpu(), final(session).session_id() == old(session).session_id(), + old(session).quiescent_generation() <= final(session).quiescent_generation(), + final(session).quiescent_generation() <= old(session).quiescent_generation() + 1, final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), )] diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index e766443a8..c533d0ab3 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -44,12 +44,16 @@ impl NestedPreemptToken { /// A shareable proof token tying a guard to the active preemption session. /// -/// The token is a fractional resource-algebra fragment. It records only stable -/// session identity, currently the running task id. The mutable weak-memory -/// view is intentionally not stored here because weak atomic operations update -/// that view while guard fragments may be outstanding. +/// The token is a fractional resource-algebra fragment. Its generation changes +/// only while the session owns the full fraction, which is exactly the +/// quiescent state in which no preemption-disabled reader can remain live. +pub ghost struct PreemptSessionState { + task: Loc, + quiescent_generation: nat, +} + pub tracked struct PreemptSessionToken { - token: CountGhost, + token: CountGhost, } impl PreemptSessionToken { @@ -59,9 +63,10 @@ impl PreemptSessionToken { { assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); - let tracked mut tokens = CountGhostResource::::alloc( - arbitrary(), - ); + let tracked mut tokens = CountGhostResource::< + PreemptSessionState, + PREEMPT_SESSION_FRACTIONS, + >::alloc(arbitrary()); let tracked token = tokens.split_one(); assert(token.frac() == 1); let tracked res = PreemptSessionToken { token }; @@ -74,7 +79,11 @@ impl PreemptSessionToken { } pub closed spec fn task(self) -> Loc { - self.token@ + self.token@.task + } + + pub closed spec fn quiescent_generation(self) -> nat { + self.token@.quiescent_generation } pub closed spec fn frac(self) -> int { @@ -91,6 +100,7 @@ impl PreemptSessionToken { self.id() == other.id(), ensures self.task() == other.task(), + self.quiescent_generation() == other.quiescent_generation(), { self.token.agree(&other.token); } @@ -105,7 +115,7 @@ impl PreemptSessionToken { /// perform weak atomic operations. pub tracked struct PreemptThreadViewSession { task_view: TaskThreadView, - tokens: CountGhostResource, + tokens: CountGhostResource, } impl PreemptThreadViewSession { @@ -118,6 +128,7 @@ impl PreemptThreadViewSession { res.task() == task_view.task(), res.view() == task_view.view(), res.session_task() == task_view.task(), + res.quiescent_generation() == 0, res.available_fractions() == PREEMPT_SESSION_FRACTIONS, res.wf_session_resource(), res.wf(sched_view), @@ -125,7 +136,11 @@ impl PreemptThreadViewSession { assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); let task = task_view.task(); - let tracked tokens = CountGhostResource::::alloc(task); + let ghost state = PreemptSessionState { task, quiescent_generation: 0 }; + let tracked tokens = CountGhostResource::< + PreemptSessionState, + PREEMPT_SESSION_FRACTIONS, + >::alloc(state); assert(tokens.is_full()); tokens.validate_full(); assert(tokens.frac() == PREEMPT_SESSION_FRACTIONS); @@ -153,13 +168,21 @@ impl PreemptThreadViewSession { } pub closed spec fn session_task(self) -> Loc { - self.tokens@ + self.tokens@.task + } + + pub closed spec fn quiescent_generation(self) -> nat { + self.tokens@.quiescent_generation } pub closed spec fn available_fractions(self) -> int { self.tokens.frac() } + pub closed spec fn has_full_authority(self) -> bool { + self.tokens.is_full() + } + pub closed spec fn wf_session_resource(self) -> bool { &&& self.tokens.wf() &&& self.session_task() == self.task() @@ -175,6 +198,7 @@ impl PreemptThreadViewSession { &&& token.wf() &&& token.id() == self.session_id() &&& token.task() == self.session_task() + &&& token.quiescent_generation() == self.quiescent_generation() } /// Splits one guard fragment from the active session. @@ -192,6 +216,7 @@ impl PreemptThreadViewSession { final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() + 1 == old(self).available_fractions(), final(self).wf_session_resource(), token.wf(), @@ -212,6 +237,7 @@ impl PreemptThreadViewSession { final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions() + token.frac(), final(self).wf_session_resource(), { @@ -226,7 +252,7 @@ impl PreemptThreadViewSession { self.tokens.validate(); assert(self.tokens.frac() == old_frac + returned_frac); assert(0 < self.tokens.frac() <= PREEMPT_SESSION_FRACTIONS); - assert(self.tokens@ == self.task_view.task()); + assert(self.tokens@.task == self.task_view.task()); assert(self.wf_session_resource()); } @@ -242,13 +268,47 @@ impl PreemptThreadViewSession { final(self).scheduler() == old(self).scheduler(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), final(self).wf_session_resource() == old(self).wf_session_resource(), final(self).view() == (*final(tv))@, { self.task_view.tracked_borrow_thread_view_mut() } + /// Advances the session's quiescent boundary. + /// + /// Updating the fractional resource requires full ownership. Therefore no + /// `PreemptSessionToken` from the previous generation can coexist with + /// this transition. Tokens split afterwards carry the new generation. + proof fn tracked_advance_quiescent_generation(tracked &mut self) -> (generation: nat) + requires + old(self).wf_session_resource(), + old(self).available_fractions() == PREEMPT_SESSION_FRACTIONS, + old(self).has_full_authority(), + ensures + generation == old(self).quiescent_generation(), + final(self).quiescent_generation() == generation + 1, + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).wf_session_resource(), + { + let ghost generation = self.quiescent_generation(); + let ghost state = PreemptSessionState { + task: self.task(), + quiescent_generation: generation + 1, + }; + self.tokens.update(state); + self.tokens.validate_full(); + assert(self.tokens.frac() == PREEMPT_SESSION_FRACTIONS); + assert(self.tokens@.task == self.task_view.task()); + generation + } + /// Returns the checked-out view to the caller for scheduler check-in. /// /// This is the proof-side counterpart of dropping the outermost @@ -315,6 +375,7 @@ impl RunningTaskContext { res.view() == task_view.view(), res.cpu() == cpu, res.preempt_depth() == 0, + res.quiescent_generation() == 0, res.available_fractions() == PREEMPT_SESSION_FRACTIONS, res.wf(), res.is_quiescent(), @@ -349,10 +410,18 @@ impl RunningTaskContext { self.session.session_id() } + pub closed spec fn quiescent_generation(self) -> nat { + self.session.quiescent_generation() + } + pub closed spec fn available_fractions(self) -> int { self.session.available_fractions() } + pub closed spec fn has_full_authority(self) -> bool { + self.session.has_full_authority() + } + pub closed spec fn preempt_depth(self) -> nat { self.preempt_depth@ } @@ -394,6 +463,7 @@ impl RunningTaskContext { pub open spec fn is_quiescent(self) -> bool { &&& self.preempt_depth() == 0 &&& self.available_fractions() == PREEMPT_SESSION_FRACTIONS + &&& self.has_full_authority() } /// Borrows the running task's persistent weak-memory view. @@ -406,7 +476,9 @@ impl RunningTaskContext { final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), final(self).preempt_depth() == old(self).preempt_depth(), final(self).wf(), final(self).view() == (*final(tv))@, @@ -414,6 +486,31 @@ impl RunningTaskContext { self.session.tracked_borrow_thread_view_mut() } + /// Records one quiescent boundary for this running session. + /// + /// The returned generation names the interval that has just ended. The + /// context advances to the next generation before another RCU reader can + /// split a preemption-session fragment. + pub proof fn tracked_record_quiescent(tracked &mut self) -> (generation: nat) + requires + old(self).wf(), + old(self).is_quiescent(), + ensures + generation == old(self).quiescent_generation(), + final(self).quiescent_generation() == generation + 1, + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).wf(), + final(self).is_quiescent(), + { + self.session.tracked_advance_quiescent_generation() + } + /// Ends a running interval and returns the updated task view to scheduler /// ownership. The full-fraction requirement rules out live preempt guards. pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) @@ -499,6 +596,10 @@ impl PreemptGuardResource { self.session_token().task() } + pub closed spec fn quiescent_generation(self) -> nat { + self.session_token().quiescent_generation() + } + pub closed spec fn wf(self, _sched_view: SchedulerView) -> bool { match self { PreemptGuardResource::Outermost(token) => token.wf(), @@ -534,6 +635,7 @@ impl PreemptGuardResource { final(session).scheduler() == old(session).scheduler(), final(session).view() == old(session).view(), final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, { match self { @@ -562,6 +664,7 @@ impl RunningTaskContext { final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() + 1 == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth() + 1, resource.matches_context(*final(self)), @@ -596,6 +699,7 @@ impl RunningTaskContext { final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions() + 1, final(self).preempt_depth() + 1 == old(self).preempt_depth(), { @@ -636,10 +740,11 @@ impl DisabledPreemptGuard { res.wf(arbitrary()), res.tracked_resource@ == tracked_resource, { - // The current verification slice does not include the CPU-local - // runtime preemption counter backend. This body verifies construction - // of the guard resource; wiring the real counter increment back in - // should happen when that backend is part of this dependency closure. + // The CPU-local backend is outside the current Verus dependency + // closure, but executable builds must still perform the real + // preemption-disable transition. + #[cfg(not(verus_keep_ghost))] + super::cpu_local::inc_guard_count(); Self { _private: (), tracked_resource: Tracked(tracked_resource) } } } @@ -665,15 +770,32 @@ impl DisabledPreemptGuard { self.tracked_resource@.matches_context(context) } + pub closed spec fn quiescent_generation(&self) -> nat { + self.tracked_resource@.quiescent_generation() + } + /// Extracts the positive preemption depth witnessed by this guard. pub proof fn lemma_matches_context_depth(&self, tracked context: &RunningTaskContext) requires self.matches_context(*context), ensures context.preempt_depth() > 0, + self.quiescent_generation() == context.quiescent_generation(), { } + /// A live preemption guard rules out a quiescent report from the same + /// running context. Returning every session fraction is therefore a + /// necessary proof step before the monitor can close this generation. + pub proof fn lemma_blocks_quiescent_report(&self, tracked context: &RunningTaskContext) + requires + self.matches_context(*context), + ensures + !context.is_quiescent(), + { + self.lemma_matches_context_depth(context); + } + /// Changing only the task's weak-memory view preserves this guard's /// relation to the running context. pub proof fn lemma_matches_context_preserved( @@ -687,6 +809,7 @@ impl DisabledPreemptGuard { after.task() == before.task(), after.scheduler() == before.scheduler(), after.session_id() == before.session_id(), + after.quiescent_generation() == before.quiescent_generation(), after.available_fractions() == before.available_fractions(), after.preempt_depth() == before.preempt_depth(), ensures @@ -696,6 +819,10 @@ impl DisabledPreemptGuard { assert(after.session.session_task() == after.task()); assert(self.tracked_resource@.session_token().task() == before.task()); assert(self.tracked_resource@.session_token().task() == after.task()); + assert(self.tracked_resource@.session_token().quiescent_generation() + == before.quiescent_generation()); + assert(self.tracked_resource@.session_token().quiescent_generation() + == after.quiescent_generation()); assert(after.session.token_matches(self.tracked_resource@.session_token())); } @@ -715,7 +842,9 @@ impl DisabledPreemptGuard { final(context).scheduler() == old(context).scheduler(), final(context).cpu() == old(context).cpu(), final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), final(context).available_fractions() == old(context).available_fractions(), + final(context).has_full_authority() == old(context).has_full_authority(), final(context).preempt_depth() == old(context).preempt_depth(), final(context).wf(), final(context).view() == (*final(tv))@, @@ -738,6 +867,7 @@ impl DisabledPreemptGuard { final(context).cpu() == old(context).cpu(), final(context).view() == old(context).view(), final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), final(context).available_fractions() == old(context).available_fractions() + 1, final(context).preempt_depth() + 1 == old(context).preempt_depth(), { @@ -751,6 +881,13 @@ impl DisabledPreemptGuard { } } // verus! +#[cfg(not(verus_keep_ghost))] +impl Drop for DisabledPreemptGuard { + fn drop(&mut self) { + super::cpu_local::dec_guard_count(); + } +} + #[verus_verify] impl GuardTransfer for DisabledPreemptGuard { #[verifier::external_body] @@ -792,6 +929,7 @@ pub(crate) fn disable_preempt_in_context( final(context).cpu() == old(context).cpu(), final(context).view() == old(context).view(), final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), final(context).available_fractions() + 1 == old(context).available_fractions(), final(context).preempt_depth() == old(context).preempt_depth() + 1, res.is_outermost() <==> old(context).preempt_depth() == 0, diff --git a/ostd/src/task/preempt/mod.rs b/ostd/src/task/preempt/mod.rs index 9ff63decc..561d6e5e0 100644 --- a/ostd/src/task/preempt/mod.rs +++ b/ostd/src/task/preempt/mod.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 -// pub(super) mod cpu_local; +#[cfg(not(verus_keep_ghost))] +pub(super) mod cpu_local; mod guard; pub(crate) use self::guard::disable_preempt_in_context; diff --git a/verified_libs/vstd_extra/src/lib.rs b/verified_libs/vstd_extra/src/lib.rs index 8569da81b..1f8100ca0 100644 --- a/verified_libs/vstd_extra/src/lib.rs +++ b/verified_libs/vstd_extra/src/lib.rs @@ -34,6 +34,7 @@ pub mod map_extra; pub mod prelude; pub mod raw_callback; pub mod raw_ptr_extra; +pub mod rcu_read_pool; pub mod seq_extra; pub mod set_extra; pub mod spec_operators; diff --git a/verified_libs/vstd_extra/src/rcu_read_pool.rs b/verified_libs/vstd_extra/src/rcu_read_pool.rs new file mode 100644 index 000000000..416991ae8 --- /dev/null +++ b/verified_libs/vstd_extra/src/rcu_read_pool.rs @@ -0,0 +1,257 @@ +//! Unbounded fractional read leases for delayed reclamation. +//! +//! The pool stores one linear resource in Verus' Leaf-style storage protocol. +//! Each lease receives half of the pool's current rational fraction, so the +//! number of outstanding leases has no fixed integer bound. A lease guards a +//! shared borrow of the stored resource. Reclamation can recover the resource +//! only after all leases have been returned and the pool fraction is whole. +use vstd::{ + prelude::*, + resource::{frac_opt::Frac, Loc}, +}; + +verus! { + +/// Owner-side accumulator for one delayed-reclamation resource. +pub tracked struct RcuReadPool { + frac: Frac, +} + +/// One read-side fraction split from an [`RcuReadPool`]. +pub tracked struct RcuReadLease { + frac: Frac, +} + +/// Allocation-indexed ownership pools retained across publication changes. +/// +/// An RCU root needs this indirection because a weak load may select an older +/// publication after a newer pointer has already been installed. +pub tracked struct RcuReadPoolRegistry { + pools: Map>, +} + +impl RcuReadPool { + /// Stores `resource` and creates a whole read pool. + pub proof fn new(tracked resource: T) -> (tracked res: Self) + ensures + res.resource() == resource, + res.fraction() == 1real, + { + let tracked frac = Frac::new(resource); + RcuReadPool { frac } + } + + /// Storage-protocol identity shared by this pool and all of its leases. + pub closed spec fn id(self) -> Loc { + self.frac.id() + } + + /// The resource retained in storage while read leases exist. + pub closed spec fn resource(self) -> T { + self.frac.resource() + } + + /// Rational fraction currently accumulated by the owner. + pub closed spec fn fraction(self) -> real { + self.frac.frac() + } + + /// Splits a fresh lease without imposing a fixed reader capacity. + pub proof fn split_lease(tracked &mut self) -> (tracked lease: RcuReadLease) + ensures + final(self).id() == old(self).id(), + final(self).resource() == old(self).resource(), + lease.id() == old(self).id(), + lease.resource() == old(self).resource(), + final(self).fraction() == old(self).fraction() / 2real, + lease.fraction() == old(self).fraction() / 2real, + { + let tracked frac = self.frac.split(); + RcuReadLease { frac } + } + + /// Returns one lease to its originating pool. + pub proof fn return_lease(tracked &mut self, tracked lease: RcuReadLease) + requires + old(self).id() == lease.id(), + ensures + final(self).id() == old(self).id(), + final(self).resource() == old(self).resource(), + final(self).resource() == lease.resource(), + final(self).fraction() == old(self).fraction() + lease.fraction(), + { + self.frac.combine(lease.frac); + } + + /// Recovers the stored resource after every lease has returned. + pub proof fn reclaim(tracked self) -> (tracked resource: T) + requires + self.fraction() == 1real, + ensures + resource == self.resource(), + { + let tracked (resource, _empty) = self.frac.take_resource(); + resource + } + + /// Establishes the valid range of the accumulated rational fraction. + pub proof fn lemma_fraction_bounded(tracked &self) + ensures + 0real < self.fraction() <= 1real, + { + self.frac.bounded(); + } +} + +impl RcuReadLease { + /// Storage-protocol identity of the originating pool. + pub closed spec fn id(self) -> Loc { + self.frac.id() + } + + /// The resource protected by this lease. + pub closed spec fn resource(self) -> T { + self.frac.resource() + } + + /// Rational fraction carried by this lease. + pub closed spec fn fraction(self) -> real { + self.frac.frac() + } + + /// Borrows the protected resource for the lifetime of this lease borrow. + pub proof fn borrow(tracked &self) -> (tracked resource: &T) + ensures + *resource == self.resource(), + { + self.frac.borrow() + } + + /// Establishes that every lease carries a positive rational fraction. + pub proof fn lemma_fraction_bounded(tracked &self) + ensures + 0real < self.fraction() <= 1real, + { + self.frac.bounded(); + } +} + +impl RcuReadPoolRegistry { + /// Creates an empty pool registry. + pub proof fn empty() -> (tracked res: Self) + ensures + res.keys() == Set::::empty(), + { + RcuReadPoolRegistry { pools: Map::tracked_empty() } + } + + /// Registered allocation identities. + pub closed spec fn keys(self) -> Set { + self.pools.dom() + } + + pub closed spec fn contains(self, key: K) -> bool { + self.pools.contains_key(key) + } + + pub closed spec fn pool(self, key: K) -> RcuReadPool + recommends + self.contains(key), + { + self.pools[key] + } + + /// Registers a fresh allocation and stores its linear permission. + pub proof fn insert(tracked &mut self, key: K, tracked resource: T) + requires + !old(self).contains(key), + ensures + final(self).keys() == old(self).keys().insert(key), + final(self).contains(key), + final(self).pool(key).resource() == resource, + final(self).pool(key).fraction() == 1real, + forall|other: K| + old(self).contains(other) ==> final(self).pool(other) == old(self).pool(other), + { + let tracked pool = RcuReadPool::new(resource); + self.pools.tracked_insert(key, pool); + } + + /// Splits a lease from the allocation selected by `key`. + pub proof fn split_lease(tracked &mut self, key: K) -> (tracked lease: RcuReadLease) + requires + old(self).contains(key), + ensures + final(self).keys() == old(self).keys(), + final(self).contains(key), + final(self).pool(key).id() == old(self).pool(key).id(), + final(self).pool(key).resource() == old(self).pool(key).resource(), + lease.id() == old(self).pool(key).id(), + lease.resource() == old(self).pool(key).resource(), + final(self).pool(key).fraction() == old(self).pool(key).fraction() / 2real, + lease.fraction() == old(self).pool(key).fraction() / 2real, + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + let tracked pool = self.pools.tracked_borrow_mut(key); + pool.split_lease() + } + + /// Returns a lease to the pool identified by `key`. + pub proof fn return_lease(tracked &mut self, key: K, tracked lease: RcuReadLease) + requires + old(self).contains(key), + old(self).pool(key).id() == lease.id(), + ensures + final(self).keys() == old(self).keys(), + final(self).contains(key), + final(self).pool(key).id() == old(self).pool(key).id(), + final(self).pool(key).resource() == old(self).pool(key).resource(), + final(self).pool(key).resource() == lease.resource(), + final(self).pool(key).fraction() == old(self).pool(key).fraction() + lease.fraction(), + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + let tracked pool = self.pools.tracked_borrow_mut(key); + pool.return_lease(lease); + } + + /// Removes a whole pool and recovers its stored ownership resource. + pub proof fn reclaim(tracked &mut self, key: K) -> (tracked resource: T) + requires + old(self).contains(key), + old(self).pool(key).fraction() == 1real, + ensures + final(self).keys() == old(self).keys().remove(key), + !final(self).contains(key), + resource == old(self).pool(key).resource(), + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + let tracked pool = self.pools.tracked_remove(key); + pool.reclaim() + } +} + +/// Regression proof: recursively splitting leases does not require a capacity +/// assumption, and returning them restores the whole resource. +pub proof fn split_return_reclaims(tracked resource: T) -> (tracked res: T) + ensures + res == resource, +{ + let tracked mut pool = RcuReadPool::new(resource); + let tracked first = pool.split_lease(); + let tracked second = pool.split_lease(); + pool.return_lease(first); + pool.return_lease(second); + assert(pool.fraction() == 1real); + pool.reclaim() +} + +} // verus! From e453ca6a5069dc443e92a7200d52ac8d60219e82 Mon Sep 17 00:00:00 2001 From: Hiroki Chen Date: Tue, 28 Jul 2026 12:50:20 +0800 Subject: [PATCH 30/47] Cpu local model (#672) * preliminary CPU core and local models * formatting * update names --- ostd/specs/task/cpu_core.rs | 299 +++++++++++++++++++++++++++++++++++ ostd/specs/task/cpu_local.rs | 286 +++++++++++++++++++++++++++++++++ ostd/specs/task/mod.rs | 2 + 3 files changed, 587 insertions(+) create mode 100644 ostd/specs/task/cpu_core.rs create mode 100644 ostd/specs/task/cpu_local.rs diff --git a/ostd/specs/task/cpu_core.rs b/ostd/specs/task/cpu_core.rs new file mode 100644 index 000000000..dec7241e0 --- /dev/null +++ b/ostd/specs/task/cpu_core.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Proof model for ownership of one CPU's local resources. +//! +//! A [`CpuCoreOwner`] permanently owns the CPU-local resources assigned to one +//! logical CPU. Scheduling changes only the owner's `current_task`; it never +//! transfers those resources to the task. Runtime CPU-local access temporarily +//! opens the owner into a linear [`CpuCoreOwnerHandle`] and its typed local +//! state, then restores that state before returning the owner to the scheduler. +use core::marker::PhantomData; + +use vstd::{prelude::*, resource::Loc}; +use vstd_extra::resource::ghost_resource::excl::ExclusiveGhost; + +use crate::specs::mm::cpu::CpuId; +use crate::specs::task::cpu_local::CpuLocalAuth; + +verus! { + +/// Logical scheduling state carried by a CPU-local resource owner. +pub ghost struct CpuCoreOwnerView { + /// Stable logical CPU represented by this core. + pub cpu: CpuId, + /// Task currently executing on this core, or `None` while the core is idle. + pub current_task: Option, + /// Ordered identities of the CPU-local resources assigned to this core. + pub locals_key: Seq, +} + +/// A typed collection of resources that belongs permanently to one CPU. +/// +/// Implementations may aggregate any number of differently typed CPU-local +/// points-to resources in a tracked struct. The predicate must state that all +/// resources in the aggregate belong to `cpu`. `local_key` must faithfully and +/// stably list their identities: changing, replacing, reordering, adding, or +/// removing a resource must change the key. +pub trait CpuCoreLocalState { + spec fn belongs_to_cpu(self, cpu: CpuId) -> bool; + + /// Ordered identities of the resources comprising this local state. + /// + /// The key must remain unchanged while the payload is detached from its + /// core. Ordering makes two same-typed fields distinguishable. + spec fn local_key(self) -> Seq; +} + +impl CpuCoreLocalState for () { + open spec fn belongs_to_cpu(self, _cpu: CpuId) -> bool { + true + } + + open spec fn local_key(self) -> Seq { + Seq::empty() + } +} + +impl CpuCoreLocalState for (A, B) { + open spec fn belongs_to_cpu(self, cpu: CpuId) -> bool { + self.0.belongs_to_cpu(cpu) && self.1.belongs_to_cpu(cpu) + } + + open spec fn local_key(self) -> Seq { + self.0.local_key() + self.1.local_key() + } +} + +/// Linear identity and scheduling state left while CPU-local resources are +/// temporarily being accessed. +/// +/// A handle cannot be duplicated. Restoring a [`CpuCoreOwner`] requires +/// returning a local-state aggregate of the same type, with the same ordered +/// resource identities, whose resources all belong to this handle's CPU. +pub tracked struct CpuCoreOwnerHandle { + state: ExclusiveGhost, + marker: PhantomData, +} + +/// Scheduler-owned proof state for one CPU's local resources. +/// +/// `L` is deliberately generic instead of type-erased. A subsystem can define +/// a tracked aggregate containing all CPU-local resources it needs and use that +/// aggregate as the owner's payload. +pub tracked struct CpuCoreOwner { + handle: CpuCoreOwnerHandle, + locals: L, +} + +impl View for CpuCoreOwnerHandle { + type V = CpuCoreOwnerView; + + closed spec fn view(&self) -> Self::V { + self.state.view() + } +} + +impl View for CpuCoreOwner { + type V = CpuCoreOwnerView; + + closed spec fn view(&self) -> Self::V { + self.handle@ + } +} + +impl CpuCoreOwnerHandle { + /// Unique identity of this core resource. + pub closed spec fn id(&self) -> Loc { + self.state.id() + } + + /// Stable CPU represented by this handle. + pub closed spec fn cpu(&self) -> CpuId { + self@.cpu + } + + /// Task currently running on this CPU. + pub closed spec fn current_task(&self) -> Option { + self@.current_task + } + + /// Whether no task is currently associated with this core. + pub open spec fn is_idle(&self) -> bool { + self.current_task() is None + } + + /// Internal validity of the exclusive core state. + pub closed spec fn wf(&self) -> bool { + self.state.wf() + } + + /// Ordered resource identities expected when restoring the core. + pub closed spec fn expected_locals_key(&self) -> Seq { + self@.locals_key + } + + /// Restores a complete core after a temporary CPU-local access. + pub proof fn tracked_restore(tracked self, tracked locals: L) -> (tracked res: CpuCoreOwner) + requires + self.wf(), + locals.belongs_to_cpu(self.cpu()), + locals.local_key() == self.expected_locals_key(), + ensures + res.id() == self.id(), + res@ == self@, + res.wf(), + res.locals() == locals, + res.locals().local_key() == self.expected_locals_key(), + { + CpuCoreOwner { handle: self, locals } + } +} + +impl CpuCoreOwner { + /// Creates an idle core with its permanent CPU-local resource aggregate. + pub proof fn new(cpu: CpuId, tracked locals: L) -> (tracked res: Self) + requires + locals.belongs_to_cpu(cpu), + ensures + res.cpu() == cpu, + res.is_idle(), + res.wf(), + res.locals() == locals, + { + let ghost locals_key = locals.local_key(); + let tracked state = ExclusiveGhost::alloc( + CpuCoreOwnerView { cpu, current_task: None, locals_key }, + ); + let tracked handle = CpuCoreOwnerHandle { state, marker: PhantomData }; + CpuCoreOwner { handle, locals } + } + + /// Unique identity of this core resource. + pub closed spec fn id(&self) -> Loc { + self.handle.id() + } + + /// Stable CPU represented by this core. + pub closed spec fn cpu(&self) -> CpuId { + self@.cpu + } + + /// Task currently running on this CPU. + pub closed spec fn current_task(&self) -> Option { + self@.current_task + } + + /// Whether no task is currently associated with this core. + pub open spec fn is_idle(&self) -> bool { + self.current_task() is None + } + + /// CPU-local resource aggregate permanently assigned to this core. + pub closed spec fn locals(&self) -> L { + self.locals + } + + /// Ordered identities of the CPU-local resources assigned to this core. + pub closed spec fn locals_key(&self) -> Seq { + self.handle.expected_locals_key() + } + + /// The core identity is valid and every local resource belongs to its CPU. + pub closed spec fn wf(&self) -> bool { + &&& self.handle.wf() + &&& self.locals().belongs_to_cpu(self.cpu()) + &&& self.locals().local_key() == self.locals_key() + } + + /// Associates a task with an idle CPU core. + pub proof fn tracked_schedule_in(tracked &mut self, task: Loc) + requires + old(self).wf(), + old(self).is_idle(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).current_task() == Some(task), + final(self).locals() == old(self).locals(), + final(self).locals_key() == old(self).locals_key(), + final(self).wf(), + { + let ghost next = CpuCoreOwnerView { + cpu: self.cpu(), + current_task: Some(task), + locals_key: self.locals_key(), + }; + self.handle.state.update(next); + } + + /// Makes this CPU idle and returns the task that was running on it. + pub proof fn tracked_schedule_out(tracked &mut self) -> (task: Loc) + requires + old(self).wf(), + !old(self).is_idle(), + ensures + old(self).current_task() == Some(task), + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).is_idle(), + final(self).locals() == old(self).locals(), + final(self).locals_key() == old(self).locals_key(), + final(self).wf(), + { + let task = self.current_task()->0; + let ghost next = CpuCoreOwnerView { + cpu: self.cpu(), + current_task: None, + locals_key: self.locals_key(), + }; + self.handle.state.update(next); + task + } + + /// Temporarily separates the typed CPU-local state from the core handle. + /// + /// The caller may update the returned resources, but must eventually call + /// [`CpuCoreOwnerHandle::tracked_restore`] with resources that still + /// belong to this CPU. + pub proof fn tracked_open(tracked self) -> (tracked res: (CpuCoreOwnerHandle, L)) + requires + self.wf(), + ensures + res.0.id() == self.id(), + res.0@ == self@, + res.0.wf(), + res.0.expected_locals_key() == self.locals_key(), + res.1 == self.locals(), + res.1.belongs_to_cpu(res.0.cpu()), + res.1.local_key() == res.0.expected_locals_key(), + { + (self.handle, self.locals) + } +} + +/// Regression proof that a CPU-local points-to resource remains owned by the +/// same core across scheduling and a temporary local-state access. +proof fn cpu_core_owns_cpu_local_points_to(initial: Map, cpu: CpuId, new_value: V) + requires + initial.contains_key(cpu), +{ + let tracked (mut auth, mut points_to_set) = CpuLocalAuth::new(initial); + let tracked points_to = points_to_set.tracked_take(cpu); + let tracked mut core = CpuCoreOwner::new(cpu, points_to); + + let ghost task = auth.id(); + core.tracked_schedule_in(task); + let tracked (handle, mut points_to) = core.tracked_open(); + assert(handle.cpu() == cpu); + assert(handle.current_task() == Some(task)); + + points_to.tracked_update(&mut auth, new_value); + let tracked mut core = handle.tracked_restore(points_to); + assert(core.cpu() == cpu); + assert(core.current_task() == Some(task)); + + let finished_task = core.tracked_schedule_out(); + assert(finished_task == task); + assert(core.is_idle()); +} + +} // verus! diff --git a/ostd/specs/task/cpu_local.rs b/ostd/specs/task/cpu_local.rs new file mode 100644 index 000000000..43d1c0172 --- /dev/null +++ b/ostd/specs/task/cpu_local.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Proof model for CPU-local state. +//! +//! A CPU-local object is modeled as one logical value for every CPU in its +//! configured domain. +//! [`CpuLocalAuth`] owns the authoritative map, while +//! [`CpuLocalPointsTo`] is the exclusive points-to resource for one CPU's +//! entry. Distinct CPUs therefore have independently owned resources and may +//! operate on them concurrently. +//! +//! This module only defines the resource algebra used by CPU-local clients. It +//! does not yet connect the resources to executable CPU-local storage, +//! preemption guards, or scheduler transitions. Those layers should keep the +//! authority in an invariant and transfer each points-to resource +//! into the corresponding CPU core's proof state. +use vstd::{ + prelude::*, + resource::{ + Loc, + map::{GhostMapAuth, GhostPointsTo, GhostSubmap}, + }, +}; + +use crate::specs::mm::cpu::CpuId; +use crate::specs::task::cpu_core::CpuCoreLocalState; + +verus! { + +/// Authoritative logical contents of one CPU-local object. +/// +/// The domain is fixed at allocation time. Updating a value requires both this +/// authority and the matching [`CpuLocalPointsTo`], so the executable invariant +/// cannot change a CPU's entry without its exclusive per-CPU permission. +pub tracked struct CpuLocalAuth { + auth: GhostMapAuth, +} + +/// CPU-local points-to resources that have not yet been distributed. +/// +/// A newly allocated model returns all resources in this collection. CPU setup +/// can split them into individual [`CpuLocalPointsTo`] resources and install +/// each resource in the corresponding CPU core's proof state. +pub tracked struct CpuLocalPointsToSet { + points_to: GhostSubmap, +} + +/// Exclusive ownership of one CPU's entry in a CPU-local object. +/// +/// Two live points-to resources associated with the same +/// [`CpuLocalAuth`] necessarily refer to different CPUs. Holding this +/// token does not by itself establish that the holder is currently executing +/// on that CPU; the scheduler glue must additionally bind `cpu()` to its +/// current-CPU token. +pub tracked struct CpuLocalPointsTo { + points_to: GhostPointsTo, +} + +impl CpuCoreLocalState for CpuLocalPointsTo { + open spec fn belongs_to_cpu(self, cpu: CpuId) -> bool { + self.cpu() == cpu + } + + open spec fn local_key(self) -> Seq { + seq![self.id()] + } +} + +impl View for CpuLocalAuth { + type V = Map; + + closed spec fn view(&self) -> Self::V { + self.auth@ + } +} + +impl View for CpuLocalPointsToSet { + type V = Map; + + closed spec fn view(&self) -> Self::V { + self.points_to@ + } +} + +impl CpuLocalAuth { + /// Allocates proof state for CPU-local contents described by `initial`. + /// + /// Allocation returns the authoritative state and exclusive ownership of + /// every points-to resource. No executable storage is allocated by this + /// proof function. + pub proof fn new(initial: Map) -> (tracked res: ( + CpuLocalAuth, + CpuLocalPointsToSet, + )) + ensures + res.0.id() == res.1.id(), + res.0@ == initial, + res.1@ == initial, + res.0.cpus() == initial.dom(), + res.1.cpus() == initial.dom(), + { + let tracked (auth, points_to) = GhostMapAuth::new(initial); + (CpuLocalAuth { auth }, CpuLocalPointsToSet { points_to }) + } + + /// Identity shared by the authority and all of its points-to resources. + pub closed spec fn id(&self) -> Loc { + self.auth.id() + } + + /// CPUs represented by this CPU-local object. + pub open spec fn cpus(&self) -> Set { + self@.dom() + } + + /// The value currently associated with `cpu`. + pub open spec fn value(&self, cpu: CpuId) -> V + recommends + self.cpus().contains(cpu), + { + self@[cpu] + } + + /// Whether this authority contains exactly the configured CPU set. + pub open spec fn covers(&self, cpus: Set) -> bool { + self.cpus() == cpus + } +} + +impl CpuLocalPointsToSet { + /// Identity of the corresponding [`CpuLocalAuth`]. + pub closed spec fn id(&self) -> Loc { + self.points_to.id() + } + + /// CPUs whose exclusive points-to resources are still held here. + pub open spec fn cpus(&self) -> Set { + self@.dom() + } + + /// Whether this collection currently owns `cpu`'s points-to resource. + pub open spec fn contains(&self, cpu: CpuId) -> bool { + self.cpus().contains(cpu) + } + + /// Splits out exclusive ownership of one CPU's entry. + pub proof fn tracked_take(tracked &mut self, cpu: CpuId) -> (tracked res: CpuLocalPointsTo) + requires + old(self).contains(cpu), + ensures + final(self).id() == old(self).id(), + res.id() == final(self).id(), + res.cpu() == cpu, + res.value() == old(self)@[cpu], + final(self)@ == old(self)@.remove(cpu), + final(self).cpus() == old(self).cpus().remove(cpu), + { + let tracked points_to = self.points_to.split_points_to(cpu); + let tracked res = CpuLocalPointsTo { points_to }; + assert(res.value() == old(self)@[cpu]); + res + } + + /// Returns an individual points-to resource to this collection. + pub proof fn tracked_return(tracked &mut self, tracked points_to: CpuLocalPointsTo) + requires + old(self).id() == points_to.id(), + !old(self).contains(points_to.cpu()), + ensures + final(self).id() == old(self).id(), + final(self)@ == old(self)@.insert(points_to.cpu(), points_to.value()), + final(self).cpus() == old(self).cpus().insert(points_to.cpu()), + { + self.points_to.combine_points_to(points_to.points_to); + } +} + +impl CpuLocalPointsTo { + /// Identity of the corresponding [`CpuLocalAuth`]. + pub closed spec fn id(&self) -> Loc { + self.points_to.id() + } + + /// CPU whose entry is owned by this points-to resource. + pub closed spec fn cpu(&self) -> CpuId { + self.points_to.key() + } + + /// Current logical value of this CPU's entry. + pub closed spec fn value(&self) -> V { + self.points_to.value() + } + + /// Establishes agreement with the authoritative CPU-local contents. + pub proof fn lemma_agree(tracked &self, tracked auth: &CpuLocalAuth) + requires + self.id() == auth.id(), + ensures + auth.cpus().contains(self.cpu()), + auth.value(self.cpu()) == self.value(), + { + self.points_to.agree(&auth.auth); + } + + /// Updates this CPU's logical value. + /// + /// Other CPUs' points-to resources remain disjoint and retain their values. + pub proof fn tracked_update(tracked &mut self, tracked auth: &mut CpuLocalAuth, value: V) + requires + old(self).id() == old(auth).id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).value() == value, + final(auth).id() == old(auth).id(), + final(auth).cpus() == old(auth).cpus(), + final(auth)@ == old(auth)@.insert(old(self).cpu(), value), + { + self.points_to.agree(&auth.auth); + let ghost cpu = self.cpu(); + self.points_to.update(&mut auth.auth, value); + assert(auth@ == old(auth)@.insert(cpu, value)); + assert(auth@.dom() == old(auth)@.dom()); + } + + /// Two points-to resources belonging to one authority refer to distinct CPUs. + pub proof fn lemma_distinct(tracked &mut self, tracked other: &CpuLocalPointsTo) + requires + old(self).id() == other.id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).value() == old(self).value(), + final(self).cpu() != other.cpu(), + { + self.points_to.disjoint(&other.points_to); + } + + /// Two live points-to resources for the same CPU cannot belong to the same + /// CPU-local authority. + pub proof fn lemma_same_cpu_has_distinct_auth( + tracked &mut self, + tracked other: &CpuLocalPointsTo, + ) + requires + old(self).cpu() == other.cpu(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).value() == old(self).value(), + final(self).id() != other.id(), + { + if self.id() == other.id() { + self.points_to.disjoint(&other.points_to); + } + } +} + +/// Regression proof for splitting, independently updating, and returning two +/// CPU-local points-to resources. +proof fn cpu_local_points_to_smoke_test( + initial: Map, + cpu1: CpuId, + cpu2: CpuId, + new_value: V, +) + requires + initial.contains_key(cpu1), + initial.contains_key(cpu2), + cpu1 != cpu2, +{ + let tracked (mut auth, mut points_to_set) = CpuLocalAuth::new(initial); + let tracked mut points_to1 = points_to_set.tracked_take(cpu1); + let tracked mut points_to2 = points_to_set.tracked_take(cpu2); + + points_to1.lemma_distinct(&points_to2); + let ghost old_cpu2_value = points_to2.value(); + points_to1.tracked_update(&mut auth, new_value); + points_to2.lemma_agree(&auth); + assert(points_to2.value() == old_cpu2_value); + + points_to_set.tracked_return(points_to1); + points_to_set.tracked_return(points_to2); + assert(points_to_set.cpus() == initial.dom()); +} + +} // verus! diff --git a/ostd/specs/task/mod.rs b/ostd/specs/task/mod.rs index 4d298bfa7..8a4c96235 100644 --- a/ostd/specs/task/mod.rs +++ b/ostd/specs/task/mod.rs @@ -15,3 +15,5 @@ impl InAtomicMode for AnyAtomicGuard { } } // verus! +pub mod cpu_core; +pub mod cpu_local; From 81998f7fc916c6b864c1d8e0eea8299521b0f57f Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 30 Jul 2026 23:21:25 -0400 Subject: [PATCH 31/47] Modeling CPU core's local views --- ostd/specs/task/cpu_core.rs | 321 ++++++++++++++++++++++++++++++++++-- 1 file changed, 310 insertions(+), 11 deletions(-) diff --git a/ostd/specs/task/cpu_core.rs b/ostd/specs/task/cpu_core.rs index dec7241e0..5e04eb25b 100644 --- a/ostd/specs/task/cpu_core.rs +++ b/ostd/specs/task/cpu_core.rs @@ -6,6 +6,21 @@ //! transfers those resources to the task. Runtime CPU-local access temporarily //! opens the owner into a linear [`CpuCoreOwnerHandle`] and its typed local //! state, then restores that state before returning the owner to the scheduler. +//! +//! The proof lifecycle is: +//! +//! 1. [`CpuCoreOwner::tracked_schedule_in`] creates a fresh +//! [`CpuExecutionToken`]. +//! 2. [`CpuExecutionToken::tracked_disable_preempt`] increments the session's +//! preemption depth and returns a [`CpuPreemptGuardToken`]. +//! 3. [`CpuCoreOwner::tracked_open_current`] uses that guard to open only the +//! CPU-local resources belonging to the pinned CPU. +//! 4. The caller restores the resources, consumes every preemption guard, and +//! calls [`CpuCoreOwner::tracked_schedule_out`] at depth zero. +//! +//! This module is still a pure proof model. Connecting these tokens to the +//! executable scheduler and [`crate::task::DisabledPreemptGuard`] is a separate +//! refinement step. use core::marker::PhantomData; use vstd::{prelude::*, resource::Loc}; @@ -22,10 +37,40 @@ pub ghost struct CpuCoreOwnerView { pub cpu: CpuId, /// Task currently executing on this core, or `None` while the core is idle. pub current_task: Option, + /// Identity of the current execution session. + /// + /// Every schedule-in creates a fresh session. Keeping its identity in the + /// core prevents a preemption guard from an older session from authorizing + /// CPU-local access after a context switch. + pub current_execution: Option, /// Ordered identities of the CPU-local resources assigned to this core. pub locals_key: Seq, } +/// State of one task's execution session on a CPU. +pub ghost struct CpuExecutionView { + /// Identity of the [`CpuCoreOwner`] on which the task is running. + pub core_id: Loc, + /// CPU on which this execution session is pinned. + pub cpu: CpuId, + /// Task running in this execution session. + pub task: Loc, + /// Number of live preemption guards in this execution session. + pub preempt_depth: nat, +} + +/// Logical identity carried by a live preemption guard. +pub ghost struct CpuPreemptGuardView { + /// Execution session in which preemption was disabled. + pub execution_id: Loc, + /// Core owner associated with the execution session. + pub core_id: Loc, + /// CPU on which the guard pins execution. + pub cpu: CpuId, + /// Task that disabled preemption. + pub task: Loc, +} + /// A typed collection of resources that belongs permanently to one CPU. /// /// Implementations may aggregate any number of differently typed CPU-local @@ -84,6 +129,24 @@ pub tracked struct CpuCoreOwner { locals: L, } +/// Linear ownership of one task's current execution session. +/// +/// The scheduler creates this token when scheduling a task in, keeps it in the +/// current CPU's proof context, and consumes it when scheduling the task out. +/// A context switch is only permitted when `preempt_depth()` is zero. +pub tracked struct CpuExecutionToken { + state: ExclusiveGhost, +} + +/// Linear proof counterpart of an executable disabled-preemption guard. +/// +/// Each token contributes one unit to its execution session's preemption +/// depth. Returning it through [`CpuExecutionToken::tracked_enable_preempt`] +/// removes that unit. +pub tracked struct CpuPreemptGuardToken { + state: ExclusiveGhost, +} + impl View for CpuCoreOwnerHandle { type V = CpuCoreOwnerView; @@ -100,6 +163,22 @@ impl View for CpuCoreOwner { } } +impl View for CpuExecutionToken { + type V = CpuExecutionView; + + closed spec fn view(&self) -> Self::V { + self.state@ + } +} + +impl View for CpuPreemptGuardToken { + type V = CpuPreemptGuardView; + + closed spec fn view(&self) -> Self::V { + self.state@ + } +} + impl CpuCoreOwnerHandle { /// Unique identity of this core resource. pub closed spec fn id(&self) -> Loc { @@ -116,14 +195,20 @@ impl CpuCoreOwnerHandle { self@.current_task } + /// Current execution session on this CPU. + pub closed spec fn current_execution(&self) -> Option { + self@.current_execution + } + /// Whether no task is currently associated with this core. pub open spec fn is_idle(&self) -> bool { - self.current_task() is None + self.current_task() is None && self.current_execution() is None } /// Internal validity of the exclusive core state. pub closed spec fn wf(&self) -> bool { - self.state.wf() + &&& self.state.wf() + &&& (self.current_task() is None) == (self.current_execution() is None) } /// Ordered resource identities expected when restoring the core. @@ -161,7 +246,7 @@ impl CpuCoreOwner { { let ghost locals_key = locals.local_key(); let tracked state = ExclusiveGhost::alloc( - CpuCoreOwnerView { cpu, current_task: None, locals_key }, + CpuCoreOwnerView { cpu, current_task: None, current_execution: None, locals_key }, ); let tracked handle = CpuCoreOwnerHandle { state, marker: PhantomData }; CpuCoreOwner { handle, locals } @@ -182,9 +267,14 @@ impl CpuCoreOwner { self@.current_task } + /// Current execution session on this CPU. + pub closed spec fn current_execution(&self) -> Option { + self@.current_execution + } + /// Whether no task is currently associated with this core. pub open spec fn is_idle(&self) -> bool { - self.current_task() is None + self.current_task() is None && self.current_execution() is None } /// CPU-local resource aggregate permanently assigned to this core. @@ -204,8 +294,10 @@ impl CpuCoreOwner { &&& self.locals().local_key() == self.locals_key() } - /// Associates a task with an idle CPU core. - pub proof fn tracked_schedule_in(tracked &mut self, task: Loc) + /// Associates a task with an idle CPU core and starts a fresh execution + /// session. + pub proof fn tracked_schedule_in(tracked &mut self, task: Loc) -> (tracked res: + CpuExecutionToken) requires old(self).wf(), old(self).is_idle(), @@ -213,25 +305,48 @@ impl CpuCoreOwner { final(self).id() == old(self).id(), final(self).cpu() == old(self).cpu(), final(self).current_task() == Some(task), + final(self).current_execution() == Some(res.id()), final(self).locals() == old(self).locals(), final(self).locals_key() == old(self).locals_key(), final(self).wf(), + res.wf(), + res.core_id() == final(self).id(), + res.cpu() == final(self).cpu(), + res.task() == task, + res.preempt_depth() == 0, + res.matches_core(final(self)), { + let tracked execution_state = ExclusiveGhost::alloc( + CpuExecutionView { core_id: self.id(), cpu: self.cpu(), task, preempt_depth: 0 }, + ); + let tracked execution = CpuExecutionToken { state: execution_state }; let ghost next = CpuCoreOwnerView { cpu: self.cpu(), current_task: Some(task), + current_execution: Some(execution.id()), locals_key: self.locals_key(), }; self.handle.state.update(next); + execution } - /// Makes this CPU idle and returns the task that was running on it. - pub proof fn tracked_schedule_out(tracked &mut self) -> (task: Loc) + /// Ends the current execution session and makes this CPU idle. + /// + /// Requiring zero preemption depth rules out a context switch while any + /// [`CpuPreemptGuardToken`] from this session remains live. + pub proof fn tracked_schedule_out( + tracked &mut self, + tracked execution: CpuExecutionToken, + ) -> (task: Loc) requires old(self).wf(), !old(self).is_idle(), + execution.wf(), + execution.matches_core(old(self)), + execution.preempt_depth() == 0, ensures old(self).current_task() == Some(task), + task == execution.task(), final(self).id() == old(self).id(), final(self).cpu() == old(self).cpu(), final(self).is_idle(), @@ -243,6 +358,7 @@ impl CpuCoreOwner { let ghost next = CpuCoreOwnerView { cpu: self.cpu(), current_task: None, + current_execution: None, locals_key: self.locals_key(), }; self.handle.state.update(next); @@ -268,6 +384,182 @@ impl CpuCoreOwner { { (self.handle, self.locals) } + + /// Opens CPU-local resources while execution is pinned to this CPU. + /// + /// This is the client-facing form of [`Self::tracked_open`]. The + /// preemption token proves that the current execution session cannot be + /// scheduled out while the returned CPU-local resources are being used. + pub proof fn tracked_open_current( + tracked self, + tracked preempt_guard: &CpuPreemptGuardToken, + ) -> (tracked res: (CpuCoreOwnerHandle, L)) + requires + self.wf(), + preempt_guard.wf(), + preempt_guard.matches_core(&self), + ensures + res.0.id() == self.id(), + res.0@ == self@, + res.0.wf(), + res.0.cpu() == preempt_guard.cpu(), + res.0.current_task() == Some(preempt_guard.task()), + res.0.current_execution() == Some(preempt_guard.execution_id()), + res.0.expected_locals_key() == self.locals_key(), + res.1 == self.locals(), + res.1.belongs_to_cpu(preempt_guard.cpu()), + res.1.local_key() == res.0.expected_locals_key(), + { + (self.handle, self.locals) + } +} + +impl CpuExecutionToken { + /// Unique identity of this execution session. + pub closed spec fn id(&self) -> Loc { + self.state.id() + } + + /// Identity of the core owner running this session. + pub closed spec fn core_id(&self) -> Loc { + self@.core_id + } + + /// CPU on which this session executes. + pub closed spec fn cpu(&self) -> CpuId { + self@.cpu + } + + /// Task running in this session. + pub closed spec fn task(&self) -> Loc { + self@.task + } + + /// Number of live preemption guards in this session. + pub closed spec fn preempt_depth(&self) -> nat { + self@.preempt_depth + } + + /// Internal validity of the execution token. + pub closed spec fn wf(&self) -> bool { + self.state.wf() + } + + /// Whether this is the current execution session of `core`. + pub open spec fn matches_core(&self, core: &CpuCoreOwner) -> bool { + &&& self.core_id() == core.id() + &&& self.cpu() == core.cpu() + &&& core.current_task() == Some(self.task()) + &&& core.current_execution() == Some(self.id()) + } + + /// Disables preemption once and returns the corresponding linear guard. + pub proof fn tracked_disable_preempt(tracked &mut self) -> (tracked res: CpuPreemptGuardToken) + requires + old(self).wf(), + ensures + final(self).id() == old(self).id(), + final(self).core_id() == old(self).core_id(), + final(self).cpu() == old(self).cpu(), + final(self).task() == old(self).task(), + final(self).preempt_depth() == old(self).preempt_depth() + 1, + final(self).wf(), + res.wf(), + res.execution_id() == final(self).id(), + res.core_id() == final(self).core_id(), + res.cpu() == final(self).cpu(), + res.task() == final(self).task(), + res.matches_execution(final(self)), + { + let ghost next = CpuExecutionView { + core_id: self.core_id(), + cpu: self.cpu(), + task: self.task(), + preempt_depth: self.preempt_depth() + 1, + }; + self.state.update(next); + let tracked state = ExclusiveGhost::alloc( + CpuPreemptGuardView { + execution_id: self.id(), + core_id: self.core_id(), + cpu: self.cpu(), + task: self.task(), + }, + ); + CpuPreemptGuardToken { state } + } + + /// Re-enables one level of preemption by consuming its guard. + pub proof fn tracked_enable_preempt(tracked &mut self, tracked guard: CpuPreemptGuardToken) + requires + old(self).wf(), + guard.wf(), + guard.matches_execution(old(self)), + old(self).preempt_depth() > 0, + ensures + final(self).id() == old(self).id(), + final(self).core_id() == old(self).core_id(), + final(self).cpu() == old(self).cpu(), + final(self).task() == old(self).task(), + final(self).preempt_depth() + 1 == old(self).preempt_depth(), + final(self).wf(), + { + let ghost next = CpuExecutionView { + core_id: self.core_id(), + cpu: self.cpu(), + task: self.task(), + preempt_depth: (self.preempt_depth() - 1) as nat, + }; + self.state.update(next); + } +} + +impl CpuPreemptGuardToken { + /// Identity of this guard token. + pub closed spec fn id(&self) -> Loc { + self.state.id() + } + + /// Execution session in which preemption was disabled. + pub closed spec fn execution_id(&self) -> Loc { + self@.execution_id + } + + /// Identity of the core owner associated with this guard. + pub closed spec fn core_id(&self) -> Loc { + self@.core_id + } + + /// CPU to which this guard pins execution. + pub closed spec fn cpu(&self) -> CpuId { + self@.cpu + } + + /// Task that owns this guard. + pub closed spec fn task(&self) -> Loc { + self@.task + } + + /// Internal validity of the guard token. + pub closed spec fn wf(&self) -> bool { + self.state.wf() + } + + /// Whether this guard belongs to `execution`. + pub open spec fn matches_execution(&self, execution: &CpuExecutionToken) -> bool { + &&& self.execution_id() == execution.id() + &&& self.core_id() == execution.core_id() + &&& self.cpu() == execution.cpu() + &&& self.task() == execution.task() + } + + /// Whether this guard pins the current execution session of `core`. + pub open spec fn matches_core(&self, core: &CpuCoreOwner) -> bool { + &&& self.core_id() == core.id() + &&& self.cpu() == core.cpu() + &&& core.current_task() == Some(self.task()) + &&& core.current_execution() == Some(self.execution_id()) + } } /// Regression proof that a CPU-local points-to resource remains owned by the @@ -281,8 +573,11 @@ proof fn cpu_core_owns_cpu_local_points_to(initial: Map, cpu: CpuId let tracked mut core = CpuCoreOwner::new(cpu, points_to); let ghost task = auth.id(); - core.tracked_schedule_in(task); - let tracked (handle, mut points_to) = core.tracked_open(); + let tracked mut execution = core.tracked_schedule_in(task); + let tracked outer_guard = execution.tracked_disable_preempt(); + let tracked inner_guard = execution.tracked_disable_preempt(); + assert(execution.preempt_depth() == 2); + let tracked (handle, mut points_to) = core.tracked_open_current(&inner_guard); assert(handle.cpu() == cpu); assert(handle.current_task() == Some(task)); @@ -291,7 +586,11 @@ proof fn cpu_core_owns_cpu_local_points_to(initial: Map, cpu: CpuId assert(core.cpu() == cpu); assert(core.current_task() == Some(task)); - let finished_task = core.tracked_schedule_out(); + execution.tracked_enable_preempt(inner_guard); + assert(execution.preempt_depth() == 1); + execution.tracked_enable_preempt(outer_guard); + assert(execution.preempt_depth() == 0); + let finished_task = core.tracked_schedule_out(execution); assert(finished_task == task); assert(core.is_idle()); } From 62c86d94365e3d1e03036153e4d956d429461644 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Sun, 2 Aug 2026 22:20:03 -0400 Subject: [PATCH 32/47] WIP on weak-memory: 81998f7fc Modeling CPU core's local views --- ostd/specs/sync/mod.rs | 1 + ostd/specs/sync/rcu.rs | 729 ++++++++++++++---- ostd/specs/sync/weak_memory.rs | 300 ++++++- ostd/src/sync/rcu/mod.rs | 626 ++++++++++----- ostd/src/sync/rcu/monitor.rs | 565 +++++++++++++- ostd/src/task/preempt/guard.rs | 397 +++++++++- ostd/src/task/scheduler/mod.rs | 325 ++++++-- verified_libs/vstd_extra/src/atomic_weak.rs | 24 +- verified_libs/vstd_extra/src/rcu_read_pool.rs | 673 +++++++++++++++- 9 files changed, 3175 insertions(+), 465 deletions(-) diff --git a/ostd/specs/sync/mod.rs b/ostd/specs/sync/mod.rs index 9a433356c..3a616868b 100644 --- a/ostd/specs/sync/mod.rs +++ b/ostd/specs/sync/mod.rs @@ -3,5 +3,6 @@ pub mod abstract_lock; pub mod mutex; //pub mod mutex_verussync; pub mod rcu; +pub mod rcu_cpu; pub mod sc_model; pub mod weak_memory; diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 15c8fc16f..787fdb8d3 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -15,14 +15,17 @@ //! `BlockInfo` and the guard's `address -> AId` protection map. This distinction //! is required to handle stale weak-memory messages after address reuse. //! -//! The module remains proof-only. Pointer replacement turns the detached -//! traversal-retire permission and its root-history removal observation into -//! the callback monitor's erased safety certificate. The remaining executable -//! connection is the paper's `Guard-seen-retired` rule: reader guards must -//! retain persistent removal observations for their start snapshot so a -//! readable atomic timestamp can be proved inconsistent with an already -//! observed removal. Grace-period synchronization must then make those -//! observations available to later readers before physical reclamation. +//! The module remains proof-only. The executable `Rcu

` adapter is a +//! direct-root specialization: replacing its atomic publication detaches the +//! owned `P` handle from that root. This must not be reused as the detach rule +//! for an internal node with arbitrary incoming links; such nodes require the +//! tracked `RcuPointedBy` transition from the traversal layer. +//! +//! Two paper-level connections remain deliberately incomplete. +//! `Guard-seen-retired` still needs a persistent start-snapshot resource, not +//! only the guard's pure `expired` set, and monitor completion still needs the +//! per-CPU closed-generation resources. Until both are connected, callback +//! completion is not an end-to-end reclamation-safety theorem. use core::marker::PhantomData; use crate::specs::mm::cpu::CpuId; @@ -49,7 +52,12 @@ pub ghost struct RcuReaderContext { pub task: Loc, pub session: Loc, pub cpu: CpuId, - /// Quiescent interval in which this reader started. + /// Implementation generation in which this reader started. + /// + /// For a [`super::rcu_cpu::CpuRcuReadGuardToken`], this is required to + /// equal the persistent CPU participant generation. The legacy + /// task-session generation must not be substituted here when the two + /// authorities have not been connected. pub generation: nat, } @@ -63,18 +71,46 @@ pub ghost struct RcuReaderContext { /// after unlink and before the callback is enqueued; completion must eventually /// prove that every CPU report has advanced beyond this view. pub ghost struct RcuCallbackSummary { + /// Scheduler whose CPU participants must complete the grace period. + pub scheduler: Loc, /// The RCU protection domain whose grace period governs this callback. pub domain: Loc, /// Logical identity of the retired object inside `domain`. pub obj: nat, /// Root-atomic removal observation retained from `Retired(a, Q)`. pub removal: RcuRemovalObservation, + /// Authoritative observation map that recorded `removal`. + pub retire_observation_registry: Loc, /// The domain-local epoch in which `obj` was retired. pub retire_epoch: nat, /// Weak-memory observations that must precede safe reclamation. pub retire_view: WmView, } +/// Persistent identity of one completed base-retirement transition. +/// +/// The observation-registry identity is part of the record. A domain-local +/// allocation ID and a numerically equal removal observation are not enough to +/// compare resources unless they also belong to the same authoritative +/// observation map. +pub ghost struct RcuRetiredRecord { + pub domain: Loc, + pub obj: nat, + pub removal: RcuRemovalObservation, + pub retire_observation_registry: Loc, +} + +impl RcuCallbackSummary { + pub open spec fn retired_record(self) -> RcuRetiredRecord { + RcuRetiredRecord { + domain: self.domain, + obj: self.obj, + removal: self.removal, + retire_observation_registry: self.retire_observation_registry, + } + } +} + /// The paper's detachment observation `Q` for a root publication. /// /// A view observes this fact once it has advanced to at least `timestamp` in @@ -176,7 +212,7 @@ impl RcuRetiredOwnedObject { (self.object, self.retired, self.ownership) } - pub closed spec fn wf(self) -> bool { + pub open spec fn wf(self) -> bool { &&& self.object().domain() == self.retired().domain() &&& self.object().obj() == self.retired().obj() &&& self.object().ptr() == self.retired().ptr() @@ -251,9 +287,10 @@ pub open spec fn current_registration_matches( /// not the history index `i`. /// /// This state intentionally does not contain the traversal removed set or a -/// grace-period epoch. In the paper, removal belongs to `SeenRemoved` and link -/// histories, while expiration/reclamation belongs to the base RCU protocol. -/// A root store publishes a pointer, but cannot by itself prove that a node is +/// grace-period epoch. In the paper, removal of an internal node belongs to +/// `SeenRemoved` and incoming-link histories, while expiration/reclamation +/// belongs to the base RCU protocol. A root store only detaches the directly +/// owned root publication; it cannot prove that an arbitrary internal node is /// unreachable from every incoming link. pub tracked struct RcuRootGhost { domain: RcuDomainAuth, @@ -367,6 +404,9 @@ impl RcuRootGhost { final(self).domain_auth().reader_registry() == old( self, ).domain_auth().reader_registry(), + final(self).domain_auth().retire_observation_registry() == old( + self, + ).domain_auth().retire_observation_registry(), final(self).domain_auth().retired() == old(self).domain_auth().retired(), final(self).domain_auth().retire_observations() == old( self, @@ -457,6 +497,9 @@ impl RcuRootGhost { final(self).domain_auth().reader_registry() == old( self, ).domain_auth().reader_registry(), + final(self).domain_auth().retire_observation_registry() == old( + self, + ).domain_auth().retire_observation_registry(), final(self).domain_auth().retired() == old(self).domain_auth().retired(), final(self).domain_auth().retire_observations() == old( self, @@ -550,6 +593,7 @@ pub ghost struct RcuRootKey { pub nullable: bool, pub domain: Loc, pub reader_registry: Loc, + pub retire_observation_registry: Loc, } /// Typed ownership state paired with one executable RCU root atomic. @@ -583,6 +627,10 @@ impl RcuRootOwnedGhost { self.root().domain_auth().reader_registry() } + pub closed spec fn retire_observation_registry(self) -> Loc { + self.root().domain_auth().retire_observation_registry() + } + pub open spec fn published_at(self, ts: nat) -> Option recommends ts < self.publications().len(), @@ -777,11 +825,37 @@ impl RcuRootOwnedGhost { assert(self.root().domain_auth().retire_observations().contains_key(obj)); } - /// Registers and starts one fresh paper reader while preserving root state. + /// Relates an observed persistent retirement-fact collection to this + /// root's entry-time expired set. + pub proof fn lemma_retired_facts_observed( + tracked &self, + history: History<*mut T>, + tracked facts: &RcuRetiredFacts, + root: Loc, + view: WmView, + ) + requires + rcu_owned_root_history_inv(history, *self), + facts.observed_by(view), + ensures + forall|record: RcuRetiredRecord| #[trigger] + facts.records().contains(record) && record.domain == self.domain() + && record.retire_observation_registry == self.retire_observation_registry() + && record.removal.root == root ==> self.root().domain_auth().observed_retired( + root, + view, + ).contains(record.obj), + { + facts.lemma_matching_records_observed_retired(&self.root.domain, root, view); + } + + /// Registers and starts one fresh logical reader instance. /// - /// Reader slots are proof-only and currently allocated per critical - /// section. `tracked_stop_reader` consumes the live slot again; no runtime - /// reader counter is introduced. + /// The paper leaves `TId` abstract. This implementation allocates one + /// proof-only identity per critical section so nested kernel readers remain + /// distinguishable. It implements the paper's base specification, but is + /// not the fixed `LOCALS[tid]` identity used by its concrete epoch + /// algorithm. pub proof fn tracked_start_reader( tracked &mut self, history: History<*mut T>, @@ -795,6 +869,7 @@ impl RcuRootOwnedGhost { rcu_owned_root_history_inv(history, *final(self)), final(self).domain() == old(self).domain(), final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).current_owned() == old(self).current_owned(), final(self).publications() == old(self).publications(), final(self).infos() == old(self).infos(), @@ -807,6 +882,11 @@ impl RcuRootOwnedGhost { res.domain() == final(self).domain(), res.reader_registry() == final(self).reader_registry(), res.reader() == reader, + res.root() == root, + res.start_view() == start_view, + res.retire_observation_registry() + == final(self).root().domain_auth().retire_observation_registry(), + res.retire_observation_registry() == old(self).retire_observation_registry(), res.expired() == final(self).root().domain_auth().observed_retired(root, start_view), { let tracked inactive = self.root.domain.tracked_register_reader(reader); @@ -816,29 +896,6 @@ impl RcuRootOwnedGhost { guard } - /// Ends a reader started by `tracked_start_reader`. - pub proof fn tracked_stop_reader( - tracked &mut self, - history: History<*mut T>, - tracked guard: RcuBaseGuard, - ) - requires - rcu_owned_root_history_inv(history, *old(self)), - guard.wf(), - guard.domain() == old(self).domain(), - guard.reader_registry() == old(self).reader_registry(), - ensures - rcu_owned_root_history_inv(history, *final(self)), - final(self).domain() == old(self).domain(), - final(self).reader_registry() == old(self).reader_registry(), - final(self).current_owned() == old(self).current_owned(), - { - assert(guard.belongs_to(self.root.domain)); - let tracked _inactive = self.root.domain.tracked_guard_stop(guard); - assert(current_registration_matches(self.root(), self.current_registration())); - assert(self.infos_wf()); - } - /// Initializes root history and retains the initial registration as the /// current unique ownership resource. pub proof fn tracked_initial(ptr: *mut T, tracked ownership: Option) -> (tracked res: Self) @@ -895,10 +952,10 @@ impl RcuRootOwnedGhost { /// Publishes a fresh allocation and retires the previously current root. /// - /// A root replacement is also the complete traversal-removal proof for - /// the old root: this cell was its only incoming root edge. The old base - /// retire permission is therefore consumed before ownership leaves the - /// atomic invariant. + /// In this direct-root specialization, replacement is the complete removal + /// event for the old owned publication: this cell is the only managed edge + /// for that `P` handle. This rule is not the paper's general + /// `RcuPointedBy-detach` rule for internal nodes. pub proof fn tracked_push_fresh( tracked &mut self, prev: History<*mut T>, @@ -918,6 +975,7 @@ impl RcuRootOwnedGhost { rcu_owned_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), match detached { Some(detached) => { &&& old(self).current_registration() is Some @@ -1001,7 +1059,7 @@ impl RcuRootOwnedGhost { removed: Set::empty().insert(object.obj()), link_view: RcuLinkView::empty(), }; - let tracked retire = lift_retire_perm(base, seen_removed); + let tracked retire = lift_direct_root_retire_perm(base, seen_removed); let tracked retired = self.root.domain.tracked_retire(retire, removal); Some(RcuRetiredOwnedObject { object, retired, ownership: old_ownership }) }, @@ -1098,6 +1156,7 @@ impl RcuRootOwnedGhost { rcu_owned_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).current_registration() == old(self).current_registration(), { let tracked owned = self.current.tracked_take(); @@ -1222,6 +1281,7 @@ impl WeakAtomicInvariantPredicate< ) -> bool { &&& key.domain == g.domain() &&& key.reader_registry == g.reader_registry() + &&& key.retire_observation_registry == g.retire_observation_registry() &&& rcu_history_inv(key.nullable, history) &&& rcu_owned_root_history_inv(history, g) &&& rcu_current_ownership_inv::(g) @@ -1580,10 +1640,14 @@ impl RcuSeenRemoved { /// Authoritative ghost handle for one RCU protection domain. /// /// The concrete implementation owns this token in its invariant. We keep the -/// fields private so clients cannot manufacture domain authority. +/// fields private so clients cannot manufacture domain authority. `readers` +/// only registers fresh logical reader-instance identities. Active/inactive +/// phase is represented linearly by [`RcuInactive`] and [`RcuBaseGuard`], so +/// ending a critical section does not need this authority. pub tracked struct RcuDomainAuth { objects: GhostMapAuth, retire_perms: GhostMapAuth, + retire_observation_cells: GhostMapAuth>, readers: GhostMapAuth, ghost next_obj: nat, ghost next_reader: nat, @@ -1636,17 +1700,31 @@ impl RcuDomainAuth { self.retire_perms.id() } + /// Resource registry that agrees every retired object with its unique + /// detachment observation. + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retire_observation_cells.id() + } + pub closed spec fn next_reader(self) -> nat { self.next_reader } - /// Internal consistency of the two resource algebras used by the base RCU - /// model. The first map backs persistent `BlockInfo`; the second map backs - /// the unique retire capability. + /// Internal consistency of the resource algebras used by the base RCU + /// model. pub closed spec fn wf(self) -> bool { &&& self.objects@ == self.retire_perms@ + &&& self.retire_observation_cells@.dom() == self.objects@.dom() + &&& forall|obj: nat| #[trigger] + self.objects@.contains_key(obj) ==> { + match self.retire_observation_cells@[obj] { + Some(removal) => self.retire_observations().contains_pair(obj, removal), + None => !self.retire_observations().contains_key(obj), + } + } &&& forall|obj: nat| #[trigger] self.objects@.contains_key(obj) ==> obj < self.next_obj() &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> tid < self.next_reader() + &&& forall|tid: nat| #[trigger] self.readers@.contains_key(tid) ==> !self.readers@[tid] &&& self.retired().subset_of(self.objects().dom()) &&& self.retire_observations().dom() == self.retired() } @@ -1662,10 +1740,14 @@ impl RcuDomainAuth { { let tracked (objects, _objects_entries) = GhostMapAuth::new(Map::empty()); let tracked (retire_perms, _retire_entries) = GhostMapAuth::new(Map::empty()); + let tracked (retire_observation_cells, _retire_observation_entries) = GhostMapAuth::new( + Map::empty(), + ); let tracked (readers, _reader_entries) = GhostMapAuth::new(Map::empty()); RcuDomainAuth { objects, retire_perms, + retire_observation_cells, readers, next_obj: 0, next_reader: 0, @@ -1691,6 +1773,7 @@ impl RcuDomainAuth { final(self).id() == old(self).id(), final(self).retire_registry() == old(self).retire_registry(), final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).next_obj() == old(self).next_obj() + 1, final(self).retired() == old(self).retired(), final(self).retire_observations() == old(self).retire_observations(), @@ -1713,6 +1796,7 @@ impl RcuDomainAuth { let tracked object = self.objects.insert(obj, ptr.addr()); let tracked block_info = object.persist(); let tracked retire_perm = self.retire_perms.insert(obj, ptr.addr()); + let tracked retire_observation = self.retire_observation_cells.insert(obj, None); self.next_obj = self.next_obj + 1; assert forall|registered: nat| #[trigger] @@ -1724,7 +1808,12 @@ impl RcuDomainAuth { ( RcuBlockInfo { info: block_info, ptr }, - RcuBaseRetirePerm { domain: self.id(), perm: retire_perm, ptr }, + RcuBaseRetirePerm { + domain: self.id(), + perm: retire_perm, + observation: retire_observation, + ptr, + }, ) } @@ -1753,6 +1842,7 @@ impl RcuDomainAuth { final(self).id() == old(self).id(), final(self).retire_registry() == old(self).retire_registry(), final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired(), final(self).retire_observations() == old(self).retire_observations(), @@ -1772,6 +1862,12 @@ impl RcuDomainAuth { assert(old(self).readers@.contains_key(registered)); } }; + assert forall|registered: nat| #[trigger] + self.readers@.contains_key(registered) implies !self.readers@[registered] by { + if registered != tid { + assert(old(self).readers@.contains_key(registered)); + } + }; RcuInactive { domain: self.id(), state, reader } } @@ -1784,89 +1880,41 @@ impl RcuDomainAuth { /// observe a newly retired stale pointer while that retirement's grace /// period is still in progress. pub proof fn tracked_guard_start( - tracked &mut self, - tracked mut inactive: RcuInactive, + tracked &self, + tracked inactive: RcuInactive, root: Loc, start_view: WmView, ) -> (tracked res: RcuBaseGuard) requires - old(self).wf(), - inactive.belongs_to(*old(self)), + self.wf(), + inactive.belongs_to(*self), inactive.wf(), ensures - final(self).wf(), - final(self).id() == old(self).id(), - final(self).retire_registry() == old(self).retire_registry(), - final(self).reader_registry() == old(self).reader_registry(), - final(self).objects() == old(self).objects(), - final(self).retired() == old(self).retired(), - final(self).retire_observations() == old(self).retire_observations(), - res.belongs_to(*final(self)), + res.belongs_to(*self), res.tid() == inactive.tid(), res.reader() == inactive.reader(), - res.expired() == old(self).observed_retired(root, start_view), + res.root() == root, + res.start_view() == start_view, + res.retire_observation_registry() == self.retire_observation_registry(), + res.expired() == self.observed_retired(root, start_view), res.protected() == Map::::empty(), res.wf(), { let ghost tid = inactive.tid(); inactive.state.agree(&self.readers); assert(self.readers@.contains_key(tid)); - inactive.state.update(&mut self.readers, true); - assert forall|registered: nat| #[trigger] - self.readers@.contains_key(registered) implies registered < self.next_reader by { - if registered == tid { - assert(old(self).readers@.contains_key(registered)); - } else { - assert(old(self).readers@.contains_key(registered)); - } - }; RcuBaseGuard { domain: self.id(), state: inactive.state, reader: inactive.reader, + root, + start_view, + retire_observation_registry: self.retire_observation_registry(), expired: self.observed_retired(root, start_view), protected: Map::empty(), } } - /// Ends a read-side critical section and returns the unique inactive token - /// for the same reader slot. - pub proof fn tracked_guard_stop( - tracked &mut self, - tracked mut guard: RcuBaseGuard, - ) -> (tracked res: RcuInactive) - requires - old(self).wf(), - guard.belongs_to(*old(self)), - guard.wf(), - ensures - final(self).wf(), - final(self).id() == old(self).id(), - final(self).retire_registry() == old(self).retire_registry(), - final(self).reader_registry() == old(self).reader_registry(), - final(self).objects() == old(self).objects(), - final(self).retired() == old(self).retired(), - final(self).retire_observations() == old(self).retire_observations(), - res.belongs_to(*final(self)), - res.tid() == guard.tid(), - res.reader() == guard.reader(), - res.wf(), - { - let ghost tid = guard.tid(); - guard.state.agree(&self.readers); - assert(self.readers@.contains_key(tid)); - guard.state.update(&mut self.readers, false); - assert forall|registered: nat| #[trigger] - self.readers@.contains_key(registered) implies registered < self.next_reader by { - if registered == tid { - assert(old(self).readers@.contains_key(registered)); - } else { - assert(old(self).readers@.contains_key(registered)); - } - }; - RcuInactive { domain: self.id(), state: guard.state, reader: guard.reader } - } - /// Implements the base `rcu-retire` transition by adding the detached AId /// to `RcuState.R` and consuming its unique traversal retire permission. pub proof fn tracked_retire( @@ -1884,6 +1932,7 @@ impl RcuDomainAuth { final(self).id() == old(self).id(), final(self).retire_registry() == old(self).retire_registry(), final(self).reader_registry() == old(self).reader_registry(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).objects() == old(self).objects(), final(self).retired() == old(self).retired().insert(retire.obj()), final(self).retire_observations() == old(self).retire_observations().insert( @@ -1900,13 +1949,46 @@ impl RcuDomainAuth { let ghost obj = retire.obj(); let ghost ptr = retire.ptr(); retire.base.perm.agree(&self.retire_perms); + retire.base.observation.agree(&self.retire_observation_cells); assert(self.objects().contains_key(obj)); + let tracked mut observation = retire.base.observation; + observation.update(&mut self.retire_observation_cells, Some(removal)); + let tracked observation = observation.persist(); self.retired = self.retired.insert(obj); self.retire_observations = self.retire_observations.insert(obj, removal); + assert(self.objects@ == old(self).objects@); + assert(self.retire_perms@ == old(self).retire_perms@); + assert(self.readers@ == old(self).readers@); + assert(self.next_obj() == old(self).next_obj()); + assert(self.next_reader() == old(self).next_reader()); + assert(self.retire_observation_cells@ == old(self).retire_observation_cells@.insert( + obj, + Some(removal), + )); + assert(self.retire_observation_cells@.dom() == old(self).retire_observation_cells@.dom()); + assert(self.retire_observation_cells@.dom() == self.objects@.dom()); + assert forall|registered: nat| #[trigger] self.objects@.contains_key(registered) implies { + match self.retire_observation_cells@[registered] { + Some(recorded) => self.retire_observations().contains_pair(registered, recorded), + None => !self.retire_observations().contains_key(registered), + } + } by { + if registered == obj { + assert(self.retire_observation_cells@[registered] == Some(removal)); + assert(self.retire_observations().contains_pair(registered, removal)); + } else { + assert(self.retire_observation_cells@[registered] == old( + self, + ).retire_observation_cells@[registered]); + assert(self.retire_observations().contains_key(registered) == old( + self, + ).retire_observations().contains_key(registered)); + } + }; assert(self.retired().subset_of(self.objects().dom())); assert(self.retire_observations().dom() == self.retired()); let tracked fact = retire.base.perm.persist(); - RcuRetired { fact: RcuRetiredFact { domain, fact, removal }, ptr } + RcuRetired { fact: RcuRetiredFact { domain, fact, observation }, ptr } } } @@ -1944,10 +2026,16 @@ impl RcuInactive { /// /// `expired` is the start-time snapshot `X`. `protected[addr] = a` is the /// mutable protection map `G` populated by successful protect operations. +/// `root` and `start_view` are implementation-refinement metadata: they are +/// not additional assumptions in the paper's abstract `Guard(tid, X, G)`, but +/// retain the witness from which `X` was computed. pub tracked struct RcuBaseGuard { ghost domain: Loc, state: GhostPointsTo, ghost reader: RcuReaderContext, + ghost root: Loc, + ghost start_view: WmView, + ghost retire_observation_registry: Loc, ghost expired: Set, ghost protected: Map, } @@ -1969,6 +2057,20 @@ impl RcuBaseGuard { self.reader } + /// Weak atomic whose history determined this guard's expired snapshot. + pub closed spec fn root(self) -> Loc { + self.root + } + + /// Thread view captured immediately before the guarded root load. + pub closed spec fn start_view(self) -> WmView { + self.start_view + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retire_observation_registry + } + pub closed spec fn expired(self) -> Set { self.expired } @@ -1978,18 +2080,37 @@ impl RcuBaseGuard { } pub closed spec fn wf(self) -> bool { - self.state.value() + !self.state.value() } pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { &&& self.domain() == domain.id() &&& self.state.id() == domain.reader_registry() + &&& self.retire_observation_registry() == domain.retire_observation_registry() } pub closed spec fn protects(self, addr: usize, obj: nat) -> bool { self.protected().contains_pair(addr, obj) } + /// Implements the paper's local `Guard -> Inactive` unlock rule. + /// + /// The domain registry only certifies the reader-instance identity; it + /// does not track critical-section phase. Consequently this transition + /// consumes the linear guard without opening `RcuState` or the root atomic + /// invariant. + pub proof fn tracked_stop(tracked self) -> (tracked res: RcuInactive) + requires + self.wf(), + ensures + res.domain() == self.domain(), + res.tid() == self.tid(), + res.reader() == self.reader(), + res.wf(), + { + RcuInactive { domain: self.domain, state: self.state, reader: self.reader } + } + /// Implements the base `Guard-protect` update. An object already in the /// guard's start snapshot `X` cannot be newly protected by this guard. pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) @@ -2004,6 +2125,9 @@ impl RcuBaseGuard { final(self).tid() == old(self).tid(), final(self).reader_registry() == old(self).reader_registry(), final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).expired() == old(self).expired(), final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), final(self).protects(info.addr(), info.obj()), @@ -2042,7 +2166,7 @@ impl RcuBlockInfo { self.info.value() } - pub open spec fn wf(self) -> bool { + pub closed spec fn wf(self) -> bool { &&& self.addr() == self.ptr().addr() &&& self.ptr().addr() != 0 } @@ -2170,6 +2294,7 @@ pub proof fn owned_root_replacement_retires_previous_registration( pub tracked struct RcuBaseRetirePerm { ghost domain: Loc, perm: GhostPointsTo, + observation: GhostPointsTo>, ghost ptr: *mut T, } @@ -2190,13 +2315,16 @@ impl RcuBaseRetirePerm { self.perm.value() } - pub open spec fn wf(self) -> bool { - self.addr() == self.ptr().addr() + pub closed spec fn wf(self) -> bool { + &&& self.addr() == self.ptr().addr() + &&& self.observation.key() == self.obj() + &&& self.observation.value() is None } pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { &&& self.domain() == domain.id() &&& self.perm.id() == domain.retire_registry() + &&& self.observation.id() == domain.retire_observation_registry() } } @@ -2246,9 +2374,16 @@ impl RcuRetirePerm { } } -/// Lift a base retire permission once the caller has observed the object in the -/// removed set. -pub proof fn lift_retire_perm( +/// Internal bridge for the directly owned root-pointer specialization. +/// +/// This is deliberately not a public traversal rule. `RcuSeenRemoved` is only +/// a logical view and can be constructed freely, so exposing this function +/// would let a client claim detachment without owning the paper's +/// `RcuPointedBy`/incoming-link authority. The executable `Rcu

` root uses +/// this bridge only while replacing its directly owned root publication. A +/// general linked structure must instead obtain a retire permission from a +/// future tracked traversal-state transition. +proof fn lift_direct_root_retire_perm( tracked base: RcuBaseRetirePerm, seen_removed: RcuSeenRemoved, ) -> (tracked perm: RcuRetirePerm) @@ -2270,15 +2405,16 @@ pub proof fn lift_retire_perm( /// Persistent, type-erased evidence that one allocation passed the base /// `rcu-retire` transition. /// -/// The points-to fact comes from consuming the allocation's unique -/// `BaseRetirePerm`. Its fields are private, so clients cannot manufacture a -/// retirement fact from a `(domain, obj)` pair. The fact remains duplicable -/// after callback type erasure and can therefore be retained in the final -/// reclaim permit. +/// `fact` comes from consuming the allocation's unique `BaseRetirePerm`. +/// `observation` comes from updating that permission's domain-owned +/// observation cell from `None` to `Some(removal)`. Their keys agree, so +/// clients cannot attach an unrelated detachment observation to a registered +/// object. Both facts are persistent and remain duplicable after callback type +/// erasure. pub tracked struct RcuRetiredFact { ghost domain: Loc, fact: GhostPersistentPointsTo, - ghost removal: RcuRemovalObservation, + observation: GhostPersistentPointsTo>, } impl RcuRetiredFact { @@ -2295,24 +2431,313 @@ impl RcuRetiredFact { } pub closed spec fn removal(self) -> RcuRemovalObservation { - self.removal + self.observation.value()->Some_0 + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.observation.id() + } + + pub closed spec fn wf(self) -> bool { + &&& self.fact.key() == self.observation.key() + &&& self.observation.value() is Some } pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { &&& summary.domain == self.domain() &&& summary.obj == self.obj() &&& summary.removal == self.removal() + &&& summary.retire_observation_registry == self.retire_observation_registry() + } + + pub closed spec fn record(self) -> RcuRetiredRecord { + RcuRetiredRecord { + domain: self.domain(), + obj: self.obj(), + removal: self.removal(), + retire_observation_registry: self.retire_observation_registry(), + } } pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + requires + self.wf(), ensures + res.wf(), res.domain() == self.domain(), res.obj() == self.obj(), res.addr() == self.addr(), res.removal() == self.removal(), + res.retire_observation_registry() == self.retire_observation_registry(), { let tracked fact = self.fact.duplicate(); - RcuRetiredFact { domain: self.domain, fact, removal: self.removal } + let tracked observation = self.observation.duplicate(); + RcuRetiredFact { domain: self.domain, fact, observation } + } + + /// Establishes that this callback fact contains the observation recorded by + /// the corresponding domain authority. + pub proof fn lemma_observation_agrees(tracked &self, tracked domain: &RcuDomainAuth) + requires + self.wf(), + domain.wf(), + self.domain() == domain.id(), + self.retire_observation_registry() == domain.retire_observation_registry(), + ensures + domain.retired().contains(self.obj()), + domain.retire_observations().contains_pair(self.obj(), self.removal()), + { + self.observation.agree(&domain.retire_observation_cells); + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + self.wf() + } +} + +/// A finite collection of persistent retirement facts. +/// +/// The map key is the complete retirement record, rather than just an AId. +/// This lets CPU-generation state accumulate facts from independent RCU +/// domains without assuming that domain IDs determine observation-registry +/// identities by pure equality alone. +pub tracked struct RcuRetiredFacts { + facts: Map, +} + +impl RcuRetiredFacts { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + forall|record: RcuRetiredRecord| #[trigger] + self.facts.contains_key(record) ==> { + &&& self.facts[record].wf() + &&& self.facts[record].record() == record + } + } + + /// Complete set of certified retirement records in this collection. + pub closed spec fn records(self) -> Set { + self.facts.dom() + } + + pub closed spec fn contains(self, record: RcuRetiredRecord) -> bool { + self.facts.contains_key(record) + } + + /// Every retained detachment observation is covered by `view`. + /// + /// The retirement facts themselves are persistent, but this predicate is + /// the separate weak-memory premise needed before a CPU report may publish + /// them to readers in a later quiescent generation. + pub open spec fn observed_by(self, view: WmView) -> bool { + forall|record: RcuRetiredRecord| #[trigger] + self.records().contains(record) ==> record.removal.observed_by(view) + } + + proof fn lemma_matching_subset_observed_retired( + tracked &self, + tracked domain: &RcuDomainAuth, + root: Loc, + view: WmView, + records: Set, + ) + requires + domain.wf(), + self.observed_by(view), + records.subset_of(self.records()), + ensures + forall|record: RcuRetiredRecord| #[trigger] + records.contains(record) && record.domain == domain.id() + && record.retire_observation_registry == domain.retire_observation_registry() + && record.removal.root == root ==> domain.observed_retired(root, view).contains( + record.obj, + ), + decreases records.len(), + { + if !records.is_empty() { + let ghost record = records.choose(); + let ghost rest = records.remove(record); + let tracked fact = self.tracked_borrow(record); + assert(self.records().contains(record)); + assert(record.removal.observed_by(view)); + if record.domain == domain.id() && record.retire_observation_registry + == domain.retire_observation_registry() && record.removal.root == root { + fact.lemma_observation_agrees(domain); + assert(domain.retired().contains(record.obj)); + assert(domain.retire_observations().contains_pair(record.obj, record.removal)); + } + Self::lemma_matching_subset_observed_retired(self, domain, root, view, rest); + assert forall|candidate: RcuRetiredRecord| #[trigger] + records.contains(candidate) && candidate.domain == domain.id() + && candidate.retire_observation_registry == domain.retire_observation_registry() + && candidate.removal.root == root implies domain.observed_retired( + root, + view, + ).contains(candidate.obj) by { + if candidate == record { + } else { + assert(rest.contains(candidate)); + } + }; + } + } + + /// Converts persistent retirement facts whose detachments have been + /// observed into membership in the domain's entry-time expired set. + /// + /// This is the paper's `Retired(a, Q)` plus observation-of-`Q` step. The + /// retirement fact alone is deliberately insufficient. + pub proof fn lemma_matching_records_observed_retired( + tracked &self, + tracked domain: &RcuDomainAuth, + root: Loc, + view: WmView, + ) + requires + domain.wf(), + self.observed_by(view), + ensures + forall|record: RcuRetiredRecord| #[trigger] + self.records().contains(record) && record.domain == domain.id() + && record.retire_observation_registry == domain.retire_observation_registry() + && record.removal.root == root ==> domain.observed_retired(root, view).contains( + record.obj, + ), + { + Self::lemma_matching_subset_observed_retired(self, domain, root, view, self.records()); + } + + /// Creates an empty retirement-fact collection. + pub proof fn empty() -> (tracked res: Self) + ensures + res.records() == Set::::empty(), + { + RcuRetiredFacts { facts: Map::tracked_empty() } + } + + /// Borrows the persistent fact for one record. + pub proof fn tracked_borrow(tracked &self, record: RcuRetiredRecord) -> (tracked res: + &RcuRetiredFact) + requires + self.records().contains(record), + ensures + res.wf(), + res.record() == record, + { + use_type_invariant(self); + let tracked res = self.facts.tracked_borrow(record); + res + } + + /// Inserts a persistent copy of `fact`. + pub proof fn tracked_insert(tracked &mut self, tracked fact: &RcuRetiredFact) + requires + fact.wf(), + ensures + final(self).records() == old(self).records().insert(fact.record()), + final(self).contains(fact.record()), + { + use_type_invariant(&*self); + use_type_invariant(fact); + let ghost record = fact.record(); + let ghost old_records = self.facts.dom(); + if !self.contains(record) { + let tracked duplicate = fact.tracked_duplicate(); + self.facts.tracked_insert(record, duplicate); + assert forall|saved: RcuRetiredRecord| #[trigger] + self.facts.contains_key(saved) implies { + &&& self.facts[saved].wf() + &&& self.facts[saved].record() == saved + } by { + if saved == record { + } else { + assert(old_records.contains(saved)); + } + }; + } + } + + proof fn tracked_duplicate_keys( + tracked source: &Map, + keys: Set, + ) -> (tracked duplicates: Map) + requires + keys.subset_of(source.dom()), + forall|record: RcuRetiredRecord| #[trigger] + source.dom().contains(record) ==> { + &&& source[record].wf() + &&& source[record].record() == record + }, + ensures + duplicates.dom() == keys, + forall|record: RcuRetiredRecord| #[trigger] + keys.contains(record) ==> { + &&& duplicates[record].wf() + &&& duplicates[record].record() == record + }, + decreases keys.len(), + { + if keys.is_empty() { + Map::tracked_empty() + } else { + let ghost record = keys.choose(); + let ghost rest = keys.remove(record); + let tracked mut duplicates = Self::tracked_duplicate_keys(source, rest); + let tracked fact = source.tracked_borrow(record); + let tracked duplicate = fact.tracked_duplicate(); + duplicates.tracked_insert(record, duplicate); + assert(keys == rest.insert(record)); + assert forall|saved: RcuRetiredRecord| #[trigger] keys.contains(saved) implies { + &&& duplicates[saved].wf() + &&& duplicates[saved].record() == saved + } by { + if saved == record { + } else { + assert(rest.contains(saved)); + } + }; + duplicates + } + } + + /// Duplicates this persistent fact collection. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.records() == self.records(), + { + use_type_invariant(self); + let tracked facts = Self::tracked_duplicate_keys(&self.facts, self.facts.dom()); + RcuRetiredFacts { facts } + } + + proof fn tracked_merge_keys( + tracked target: &mut RcuRetiredFacts, + tracked source: &RcuRetiredFacts, + keys: Set, + ) + requires + keys.subset_of(source.records()), + ensures + final(target).records() == old(target).records().union(keys), + decreases keys.len(), + { + if !keys.is_empty() { + let ghost record = keys.choose(); + let ghost rest = keys.remove(record); + let tracked fact = source.tracked_borrow(record); + target.tracked_insert(fact); + Self::tracked_merge_keys(target, source, rest); + assert(keys == rest.insert(record)); + } + } + + /// Adds persistent copies of all facts in `other`. + pub proof fn tracked_merge(tracked &mut self, tracked other: &RcuRetiredFacts) + ensures + final(self).records() == old(self).records().union(other.records()), + { + Self::tracked_merge_keys(self, other, other.records()); } } @@ -2347,8 +2772,9 @@ impl RcuRetired { self.fact.removal() } - pub open spec fn wf(self) -> bool { - self.addr() == self.ptr().addr() + pub closed spec fn wf(self) -> bool { + &&& self.fact.wf() + &&& self.addr() == self.ptr().addr() } proof fn tracked_into_fact(tracked self) -> (tracked res: RcuRetiredFact) @@ -2392,7 +2818,7 @@ pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> removed: Set::empty().insert(info.obj()), link_view: RcuLinkView::empty(), }; - let tracked retire = lift_retire_perm(base, seen_removed); + let tracked retire = lift_direct_root_retire_perm(base, seen_removed); let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 1 }; let tracked _retired = domain.tracked_retire(retire, removal); @@ -2428,7 +2854,7 @@ pub proof fn observed_retired_object_enters_guard_expired(ptr: *mut T) -> (tr removed: Set::empty().insert(info.obj()), link_view: RcuLinkView::empty(), }; - let tracked retire = lift_retire_perm(base, seen_removed); + let tracked retire = lift_direct_root_retire_perm(base, seen_removed); let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 0 }; let tracked _retired = domain.tracked_retire(retire, removal); @@ -2461,6 +2887,10 @@ impl RcuCallbackSafety { self.retired.removal() } + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retired.retire_observation_registry() + } + /// The monitor may assign any future batch generation, but it cannot /// change the retired object's domain or allocation identity. pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { @@ -2474,6 +2904,7 @@ impl RcuCallbackSafety { summary.domain == self.domain(), summary.obj == self.obj(), summary.removal == self.removal(), + summary.retire_observation_registry == self.retire_observation_registry(), ensures self.matches(summary), { @@ -2486,11 +2917,15 @@ impl RcuCallbackSafety { requires self.matches(summary), ensures + res.wf(), res.domain() == self.domain(), res.obj() == self.obj(), res.removal() == self.removal(), + res.retire_observation_registry() == self.retire_observation_registry(), res.matches(summary), + res.record() == summary.retired_record(), { + use_type_invariant(&self.retired); self.retired.tracked_duplicate() } } @@ -2551,6 +2986,18 @@ impl RcuReadGuardToken { self.base.reader() } + pub closed spec fn root(self) -> Loc { + self.base.root() + } + + pub closed spec fn start_view(self) -> WmView { + self.base.start_view() + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.base.retire_observation_registry() + } + pub closed spec fn expired(self) -> Set { self.base.expired() } @@ -2584,6 +3031,16 @@ impl RcuReadGuardToken { self.base.belongs_to(domain) } + /// Exposes the traversal-side consequence of a well-formed guard without + /// opening the guard representation in client modules. + pub proof fn lemma_expired_is_removed(tracked &self) + requires + self.wf(), + ensures + self.expired().subset_of(self.seen_removed().removed), + { + } + /// Preconditions of the paper's base `Guard-protect` rule. /// /// This deliberately says only that the allocation was not already @@ -2618,6 +3075,9 @@ impl RcuReadGuardToken { res.tid() == base.tid(), res.reader_registry() == base.reader_registry(), res.reader() == base.reader(), + res.root() == base.root(), + res.start_view() == base.start_view(), + res.retire_observation_registry() == base.retire_observation_registry(), res.expired() == base.expired(), res.protected() == base.protected(), res.seen_removed() == seen_removed, @@ -2636,6 +3096,9 @@ impl RcuReadGuardToken { res.tid() == base.tid(), res.reader_registry() == base.reader_registry(), res.reader() == base.reader(), + res.root() == base.root(), + res.start_view() == base.start_view(), + res.retire_observation_registry() == base.retire_observation_registry(), res.expired() == base.expired(), res.seen_removed().removed == base.expired(), res.link_view() == RcuLinkView::::empty(), @@ -2658,6 +3121,9 @@ impl RcuReadGuardToken { res.tid() == self.tid(), res.reader_registry() == self.reader_registry(), res.reader() == self.reader(), + res.root() == self.root(), + res.start_view() == self.start_view(), + res.retire_observation_registry() == self.retire_observation_registry(), res.expired() == self.expired(), res.protected() == self.protected(), { @@ -2674,6 +3140,9 @@ impl RcuReadGuardToken { final(self).tid() == old(self).tid(), final(self).reader_registry() == old(self).reader_registry(), final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).expired() == old(self).expired(), final(self).seen_removed() == old(self).seen_removed(), final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 95255d062..2c0bb807a 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -8,7 +8,7 @@ pub use vstd_extra::atomic_weak::*; pub use vstd_extra::weak_atomic_with_ghost; -use super::rcu as rcu_spec; +use super::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec}; use vstd::prelude::*; verus! { @@ -101,6 +101,7 @@ impl RcuWeakAtomicPtr where requires self.well_formed(), ensures + old(tv)@.spec_le(final(tv)@), !self.constant().nullable ==> !res.0.is_null(), match (res.2@, res.3@) { (None, None) => res.0.addr() == 0, @@ -117,6 +118,7 @@ impl RcuWeakAtomicPtr where }, { let result; + let ghost start_view = tv@; proof { use_type_invariant(self); } @@ -130,6 +132,11 @@ impl RcuWeakAtomicPtr where } let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { + start_view.lemma_acquire( + self.id(), + loaded.1@, + hist.msg_at(loaded.1@).view, + ); assert(hist.valid_ts(loaded.1@)); assert(loaded.1@ < hist.history().len()); assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); @@ -170,9 +177,10 @@ impl RcuWeakAtomicPtr where /// The ghost reader transition occurs in the same invariant opening as the /// real acquire load. Executably this is identical to `load_acquire_rcu`. #[inline(always)] - pub fn load_acquire_rcu_guarded( + pub fn load_acquire_rcu_guarded_with_retired( &self, Ghost(reader): Ghost, + Tracked(retired_facts): Tracked<&rcu_spec::RcuRetiredFacts>, Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ( *mut T, @@ -183,12 +191,23 @@ impl RcuWeakAtomicPtr where )) requires self.well_formed(), + retired_facts.observed_by(old(tv)@), ensures + old(tv)@.spec_le(final(tv)@), !self.constant().nullable ==> !res.0.is_null(), res.4@.wf(), res.4@.domain() == self.constant().domain, res.4@.reader_registry() == self.constant().reader_registry, + res.4@.retire_observation_registry() == self.constant().retire_observation_registry, res.4@.reader() == reader, + res.4@.root() == self.id(), + res.4@.start_view() == old(tv)@, + forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + retired_facts.records().contains(record) && record.domain == res.4@.domain() + && record.retire_observation_registry == res.4@.retire_observation_registry() + && record.removal.root == res.4@.root() ==> res.4@.expired().contains( + record.obj, + ), match (res.2@, res.3@) { (None, None) => res.0.addr() == 0, (Some(object), Some(info)) => { @@ -218,13 +237,33 @@ impl RcuWeakAtomicPtr where assert(hist.id() == self.id()); assert(raw_atomic.id() == self.id()); assert(hist.id() == raw_atomic.id()); + assert(rcu_spec::RcuOwnedWeakAtomicInv::::atomic_inv( + self.constant(), + hist.history(), + g, + )); + assert(g.retire_observation_registry() + == self.constant().retire_observation_registry); } proof_decl! { let tracked base_guard = g.tracked_start_reader(hist.history(), self.id(), start_view, reader); } + proof { + g.lemma_retired_facts_observed( + hist.history(), + retired_facts, + self.id(), + start_view, + ); + } let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); proof { + start_view.lemma_acquire( + self.id(), + loaded.1@, + hist.msg_at(loaded.1@).view, + ); assert(hist.valid_ts(loaded.1@)); assert(loaded.1@ < hist.history().len()); assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); @@ -262,6 +301,12 @@ impl RcuWeakAtomicPtr where } assert(base_guard.domain() == self.constant().domain); assert(base_guard.reader_registry() == self.constant().reader_registry); + assert(base_guard.retire_observation_registry() + == g.retire_observation_registry()); + assert(g.retire_observation_registry() + == self.constant().retire_observation_registry); + assert(base_guard.retire_observation_registry() + == self.constant().retire_observation_registry); assert(rcu_spec::rcu_current_ownership_inv::(g)); } proof_decl! { @@ -329,6 +374,188 @@ impl RcuWeakAtomicPtr where result } + /// Acquire-load an RCU root while starting a paper read-side guard. + /// + /// This compatibility entry point has no CPU-generation retirement + /// history, so it starts the guard with only the root invariant's directly + /// observed retirements. + #[inline(always)] + pub fn load_acquire_rcu_guarded( + &self, + Ghost(reader): Ghost, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + Tracked>, + )) + requires + self.well_formed(), + ensures + old(tv)@.spec_le(final(tv)@), + !self.constant().nullable ==> !res.0.is_null(), + res.4@.wf(), + res.4@.domain() == self.constant().domain, + res.4@.reader_registry() == self.constant().reader_registry, + res.4@.retire_observation_registry() == self.constant().retire_observation_registry, + res.4@.reader() == reader, + res.4@.root() == self.id(), + res.4@.start_view() == old(tv)@, + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == res.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + &&& !res.4@.expired().contains(info.obj()) + &&& res.4@.protects(info.addr(), info.obj()) + }, + _ => false, + }, + { + proof_decl! { + let tracked retired_facts = rcu_spec::RcuRetiredFacts::empty(); + } + self.load_acquire_rcu_guarded_with_retired( + Ghost(reader), + Tracked(&retired_facts), + Tracked(tv), + ) + } + + /// Acquire-load an RCU root while retaining the CPU implementation + /// fragment in the returned guard. + /// + /// The caller must split `cpu_reader` after disabling preemption and before + /// calling this method. The fragment is therefore live before the first + /// protected load, while the participant view bound ensures that the paper + /// guard starts no earlier than the CPU state from which it was split. + #[inline(always)] + pub fn load_acquire_rcu_guarded_cpu( + &self, + Ghost(reader): Ghost, + Tracked(cpu_reader): Tracked, + Tracked(binding): Tracked, + Tracked(tv): Tracked<&mut ThreadView>, + ) -> (res: ( + *mut T, + Ghost, + Ghost>, + Tracked>>, + Tracked>, + )) + requires + self.well_formed(), + cpu_reader.wf(), + reader.cpu == cpu_reader.cpu(), + reader.generation == cpu_reader.generation(), + binding.scheduler() == reader.scheduler, + binding.cpu() == cpu_reader.cpu(), + binding.participant_id() == cpu_reader.participant_id(), + cpu_reader.participant_view().spec_le(old(tv)@), + ensures + old(tv)@.spec_le(final(tv)@), + !self.constant().nullable ==> !res.0.is_null(), + res.4@.wf(), + res.4@.participant_id() == cpu_reader.participant_id(), + res.4@.cpu() == cpu_reader.cpu(), + res.4@.generation() == cpu_reader.generation(), + res.4@.participant_view() == cpu_reader.participant_view(), + res.4@.reader_fragment() == cpu_reader, + res.4@.scheduler() == binding.scheduler(), + res.4@.domain() == self.constant().domain, + res.4@.reader_registry() == self.constant().reader_registry, + res.4@.retire_observation_registry() == self.constant().retire_observation_registry, + res.4@.reader_context() == reader, + res.4@.root() == self.id(), + res.4@.start_view() == old(tv)@, + match (res.2@, res.3@) { + (None, None) => res.0.addr() == 0, + (Some(object), Some(info)) => { + &&& res.0.addr() != 0 + &&& object.addr == res.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == res.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), res.0) + &&& !res.4@.expired().contains(info.obj()) + &&& res.4@.protects(info.addr(), info.obj()) + }, + _ => false, + }, + { + let loaded = { + proof_decl! { + let tracked retired_facts = + cpu_reader.tracked_retired_facts_observed_by(tv@); + } + let loaded = self.load_acquire_rcu_guarded_with_retired( + Ghost(reader), + Tracked(retired_facts), + Tracked(tv), + ); + proof { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + cpu_reader.known_retired().contains(record) && record.domain + == loaded.4@.domain() && record.retire_observation_registry + == loaded.4@.retire_observation_registry() && record.removal.root + == loaded.4@.root() implies loaded.4@.expired().contains(record.obj) by { + assert(retired_facts.records().contains(record)); + }; + } + loaded + }; + proof { + assert(loaded.4@.reader() == reader); + assert(match (loaded.2@, loaded.3@) { + (None, None) => loaded.0.addr() == 0, + (Some(object), Some(info)) => { + &&& loaded.0.addr() != 0 + &&& object.addr == loaded.0.addr() + &&& info.wf() + &&& info.domain() == object.domain + &&& info.domain() == loaded.4@.domain() + &&& info.obj() == object.obj + &&& info.addr() == object.addr + &&& equal(info.ptr(), loaded.0) + &&& !loaded.4@.expired().contains(info.obj()) + &&& loaded.4@.protects(info.addr(), info.obj()) + }, + _ => false, + }); + } + let (ptr, timestamp, published, info, Tracked(paper_guard)) = loaded; + proof_decl! { + let tracked guard = + rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new(paper_guard, cpu_reader, binding); + } + proof { + assert(guard.reader_context() == reader); + assert(guard.reader_fragment() == cpu_reader); + match (&published@, &info@) { + (Some(object), Some(info)) => { + assert(info.domain() == guard.domain()); + assert(!guard.expired().contains(info.obj())); + assert(guard.protects(info.addr(), info.obj())); + }, + (None, None) => { + assert(ptr.addr() == 0); + }, + _ => assert(false), + } + } + (ptr, timestamp, published, info, Tracked(guard)) + } + /// End a paper read-side guard without executing another atomic operation. #[inline(always)] pub fn stop_rcu_reader(&self, Tracked(guard): Tracked>) @@ -337,26 +564,51 @@ impl RcuWeakAtomicPtr where guard.wf(), guard.domain() == self.constant().domain, guard.reader_registry() == self.constant().reader_registry, + guard.retire_observation_registry() == self.constant().retire_observation_registry, { proof_decl! { let tracked base_guard = guard.tracked_into_base(); + let tracked _inactive = base_guard.tracked_stop(); } - let credit = vstd::invariant::create_open_invariant_credit(); proof { use_type_invariant(self); - vstd::invariant::open_atomic_invariant_in_proof!( - credit.get() => self.tracked_atomic_inv() => pair => { - let tracked (hist, mut g) = pair; - assert(g.domain() == self.constant().domain); - assert(g.reader_registry() == self.constant().reader_registry); - g.tracked_stop_reader(hist.history(), base_guard); - assert(rcu_spec::rcu_current_ownership_inv::(g)); - pair = (hist, g); - } - ); } } + /// Ends a CPU-refined reader and returns its linear CPU fragment. + /// + /// The fragment is intentionally returned instead of dropped. The standard + /// guard destruction path must join it back into the current CPU's + /// participant before executable preemption is re-enabled. + #[inline(always)] + pub fn stop_cpu_rcu_reader( + &self, + Tracked(guard): Tracked>, + ) -> (res: Tracked) + requires + self.well_formed(), + guard.wf(), + guard.domain() == self.constant().domain, + guard.root() == self.id(), + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + res@.wf(), + res@ == guard.reader_fragment(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + opens_invariants none + no_unwind + { + proof_decl! { + let tracked (_inactive, reader) = guard.tracked_stop(); + } + proof { + use_type_invariant(self); + } + Tracked(reader) + } + /// Release-swap helper for a freshly introduced RCU root pointer. /// /// The new registration remains owned by the atomic invariant. The return @@ -381,6 +633,7 @@ impl RcuWeakAtomicPtr where None => value.is_null(), }, ensures + old(tv)@.spec_le(final(tv)@), (res.1@ is Some) == !res.0.is_null(), res.1@ is Some ==> res.1@->Some_0.object().wf(), res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), @@ -390,6 +643,7 @@ impl RcuWeakAtomicPtr where res.1@ is Some ==> OwnPred::owns(res.0, res.1@->Some_0.ownership()), { let result; + let ghost start_view = tv@; proof_decl! { let tracked retired_ownership; } @@ -410,6 +664,7 @@ impl RcuWeakAtomicPtr where let snap = swap.1; let ghost next = hist.history(); proof { + start_view.lemma_observe(self.id(), prev.len()); assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); assert(rcu_spec::rcu_current_ownership_inv::(g)); rcu_spec::lemma_current_owned_resources::(prev, &g); @@ -487,6 +742,7 @@ impl RcuWeakAtomicPtr where None => new.is_null(), }, ensures + old(tv)@.spec_le(final(tv)@), res.0 is Err ==> res.2@.0 is None, res.0 is Err ==> res.2@.1 == new_ownership, res.0 is Ok ==> res.2@.1 is None, @@ -499,6 +755,7 @@ impl RcuWeakAtomicPtr where res.2@.0 is Some ==> OwnPred::owns(res.0->Ok_0, res.2@.0->Some_0.ownership()), { let result; + let ghost start_view = tv@; proof_decl! { let tracked retired_ownership; } @@ -523,6 +780,17 @@ impl RcuWeakAtomicPtr where result = (cas_result.0, cas_result.1); let ghost next = hist.history(); proof { + let ghost read_view = prev[cas_result.1@ as int].view; + let ghost after_read = + start_view.observe(self.id(), cas_result.1@).join(read_view); + start_view.lemma_acquire(self.id(), cas_result.1@, read_view); + if cas_result.0 is Ok { + after_read.lemma_observe(self.id(), prev.len()); + start_view.lemma_spec_le_transitive( + after_read, + after_read.observe(self.id(), prev.len()), + ); + } assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); assert(rcu_spec::rcu_current_ownership_inv::(g)); rcu_spec::lemma_current_owned_resources::(prev, &g); @@ -636,6 +904,8 @@ impl RcuMonitorWeakAtomicBool { )) requires self.well_formed(), + ensures + old(tv)@.spec_le(final(tv)@), { self.inner.load_relaxed(Tracked(tv)) } @@ -679,7 +949,10 @@ impl RcuMonitorWeakAtomicBool { self.well_formed(), state.wf(), !value ==> state.no_pending_work(), + ensures + old(tv)@.spec_le(final(tv)@), { + let ghost start_view = tv@; proof { use_type_invariant(self); } @@ -695,6 +968,7 @@ impl RcuMonitorWeakAtomicBool { let snap = raw_atomic.store_relaxed(Tracked(&mut hist), Tracked(tv), value); let ghost next = hist.history(); proof { + start_view.lemma_observe(self.id(), prev.len()); assert(snap@.msg().value == value); rcu_spec::preserve_rcu_monitor_flag_inv_on_push( prev, diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index d38d0002c..3a2141c47 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -41,7 +41,8 @@ //! removed set. The domain's base `rcu-retire` transition then records it in //! `RcuState.R` as `RcuRetired`. //! - `RcuCallbackSafety` compresses that recorded retire proof into an erased -//! `RcuCallbackSummary { domain, obj, removal, retire_epoch, retire_view }`, +//! summary containing the domain, AId, removal observation, +//! observation-registry identity, retire epoch, and retire view, //! which is what the monitor stores next to a type-erased executable //! callback. `removal` is the paper's `Retired(a, Q)` detachment observation: //! it records the root atomic and the first timestamp after the object was @@ -94,15 +95,15 @@ //! return the old raw pointer and matching ownership, and route a certified //! callback into the monitor. Scheduler handoff now preserves a per-CPU //! `ThreadView`: schedule-out joins the departing task's observations into the -//! CPU view, and schedule-in imports that view into the incoming task. -//! `RunningTaskContext` retains the CPU identity, so a quiescent report is tied -//! to the CPU whose persistent view it reports. +//! CPU view, and schedule-in imports that view into the incoming task together +//! with the CPU's canonical `CpuRcuParticipant`. //! //! An executable `read()` now performs the paper's `Inactive -> Guard` //! transition while opening the root weak-atomic invariant. The resulting -//! `RcuReadGuardToken` and exact historical `BlockInfo` remain in the -//! executable read guard until destruction or consuming CAS performs -//! `Guard -> Inactive`. +//! `CpuRcuReadGuardToken`, its fractional CPU reader fragment, and the exact +//! historical `BlockInfo` remain in the executable read guard until +//! destruction or consuming CAS performs `Guard -> Inactive` and returns the +//! fragment before re-enabling preemption. //! //! A guarded weak load now installs the loaded root's exact `BlockInfo` in the //! guard's protection map. The proof derives the guard's expired set from the @@ -113,21 +114,23 @@ //! pointer's physical reference permission; `assume_shared_ref` still stands //! in for that final argument. //! -//! Each reader token now records its scheduler, task, CPU, preemption-session -//! identity, and quiescent generation. The generation is part of the -//! fractional preemption resource. A monitor report advances it only while the -//! running context owns the full resource, so a live preemption-disabled -//! reader from the reported generation makes that transition impossible. New -//! readers in the same session carry the next generation and retain the -//! report's weak-memory view; schedule-out similarly requires full ownership -//! and publishes that view to the CPU view imported by the next session. +//! The proof-only `rcu_cpu` module now defines the required persistent +//! `CpuRcuParticipant`: a reader splits a fractional fragment, and a quiescent +//! report requires the full fraction before advancing the CPU generation and +//! view. Monitor completion retains one `CpuRcuClosedGeneration` for every +//! online CPU and duplicates those persistent resources into each callback's +//! `RcuReclaimPermit`. A live guard that coexists with such a permit is +//! necessarily from a later CPU generation and its start view includes the +//! callback's removal observation. The matching retirement record therefore +//! belongs to the guard's expired set. Since traversal well-formedness embeds +//! expired objects in `SeenRemoved`, a callback permit and a guard-protected +//! pointer to the same object are proved mutually exclusive. //! -//! The remaining scheduler boundary is to verify the executable -//! `processor::switch_to_task` call site against this tracked transition and -//! package the per-session argument into a global theorem over every CPU -//! report. Until that and the physical-reference boundary are closed, the -//! reclaim permit remains a monitor-level authorization rather than the final -//! end-to-end memory-safety authority. +//! The remaining end-to-end boundary is physical reference ownership. Guarded +//! loads must split an `RcuReadLease`, guard destruction must +//! return it, and reclamation must recover the whole pool before invoking the +//! callback. Until that is connected, `assume_shared_ref` remains the explicit +//! reference-permission bypass. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::prelude::*; @@ -138,13 +141,13 @@ use vstd_extra::rcu_read_pool::RcuReadLease; use crate::{ specs::{ sync::{ - rcu as rcu_spec, + rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, weak_memory::{RcuWeakAtomicPtr, ThreadView}, }, task::InAtomicMode, }, sync::Once, - task::{disable_preempt_in_context, DisabledPreemptGuard, RunningTaskContext}, + task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, }; use non_null::{NonNullPtr, NonNullPtrRef}; @@ -229,9 +232,13 @@ pub struct RcuInner { struct RcuReadGuardInner<'a, P: NonNullPtr> { obj_ptr: *mut

::Target, rcu: &'a RcuInner

, + proof_active: bool, _inner_guard: DisabledPreemptGuard, tracked_info: Tracked::Target>>>, - tracked_guard: Tracked::Target>>, + tracked_guard: Tracked< + Option::Target>>, + >, + tracked_session: Tracked>, } /// Sized callback payload that retains the physical ownership of one detached @@ -344,6 +351,7 @@ impl RcuInner

{ nullable: true, domain: root_ghost.domain(), reader_registry: root_ghost.reader_registry(), + retire_observation_registry: root_ghost.retire_observation_registry(), }; } let ptr = RcuAtomicPtr::

::new(Ghost(key), core::ptr::null_mut(), Tracked(root_ghost)); @@ -375,6 +383,7 @@ impl RcuInner

{ nullable, domain: root_ghost.domain(), reader_registry: root_ghost.reader_registry(), + retire_observation_registry: root_ghost.retire_observation_registry(), }; } let ptr = RcuAtomicPtr::

::new(Ghost(key), raw_ptr, Tracked(root_ghost)); @@ -393,6 +402,7 @@ impl RcuInner

{ requires self.type_inv(), ensures + old(tv)@.spec_le(final(tv)@), !self.is_nullable() ==> !res.0.is_null(), match res.1@ { None => res.0.is_null(), @@ -420,20 +430,38 @@ impl RcuInner

{ fn load_ptr_acquire_guarded( &self, Ghost(reader): Ghost, + Tracked(cpu_reader): Tracked, + Tracked(binding): Tracked, Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ( *mut

::Target, Tracked::Target>>>, - Tracked::Target>>, + Tracked::Target>>, )) requires self.type_inv(), + cpu_reader.wf(), + reader.cpu == cpu_reader.cpu(), + reader.generation == cpu_reader.generation(), + binding.scheduler() == reader.scheduler, + binding.cpu() == cpu_reader.cpu(), + binding.participant_id() == cpu_reader.participant_id(), + cpu_reader.participant_view().spec_le(old(tv)@), ensures + old(tv)@.spec_le(final(tv)@), !self.is_nullable() ==> !res.0.is_null(), res.2@.wf(), + res.2@.participant_id() == cpu_reader.participant_id(), + res.2@.cpu() == cpu_reader.cpu(), + res.2@.generation() == cpu_reader.generation(), + res.2@.participant_view() == cpu_reader.participant_view(), + res.2@.reader_fragment() == cpu_reader, + res.2@.scheduler() == binding.scheduler(), res.2@.domain() == self.ptr.constant().domain, res.2@.reader_registry() == self.ptr.constant().reader_registry, - res.2@.reader() == reader, + res.2@.retire_observation_registry() == self.ptr.constant().retire_observation_registry, + res.2@.root() == self.ptr.id(), + res.2@.reader_context() == reader, match res.1@ { None => res.0.is_null(), Some(info) => { @@ -449,7 +477,12 @@ impl RcuInner

{ proof { assert(self.ptr.constant().nullable == self.is_nullable()); } - let res = self.ptr.load_acquire_rcu_guarded(Ghost(reader), Tracked(tv)); + let res = self.ptr.load_acquire_rcu_guarded_cpu( + Ghost(reader), + Tracked(cpu_reader), + Tracked(binding), + Tracked(tv), + ); proof { if !self.is_nullable() { assert(!self.ptr.constant().nullable); @@ -488,6 +521,7 @@ impl RcuInner

{ None => new_ptr.is_null(), }, ensures + old(tv)@.spec_le(final(tv)@), (res.1@ is Some) == !res.0.is_null(), res.1@ is Some ==> res.1@->Some_0.object().wf(), res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), @@ -551,10 +585,10 @@ impl RcuInner

{ } } - fn read(&self, Tracked(session): Tracked<&mut RunningTaskContext>) -> (res: RcuReadGuardInner< - '_, - P, - >) + fn read<'a>( + &'a self, + Tracked(session): Tracked<&'a mut RunningTaskContext>, + ) -> (res: RcuReadGuardInner<'a, P>) requires self.type_inv(), old(session).wf(), @@ -562,22 +596,40 @@ impl RcuInner

{ ensures res.type_inv(), res.rcu.is_nullable() == self.is_nullable(), - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() + 1 == old(session).available_fractions(), - final(session).preempt_depth() == old(session).preempt_depth() + 1, - res.matches_context(*final(session)), + res.proof_active, { + let ghost context_before_disable = *session; let inner_guard = disable_preempt_in_context(Tracked(session)); + proof { + assert(session.wf()); + session.lemma_rcu_participant_view_le(); + } + let ghost context_before_reader = *session; + proof_decl! { + let tracked cpu_reader = session.tracked_start_rcu_reader(); + let tracked rcu_binding = session.tracked_rcu_binding(); + } + proof { + inner_guard.lemma_matches_context_preserved(context_before_reader, session); + assert(session.rcu_participant_id() == context_before_disable.rcu_participant_id()); + assert(session.rcu_generation() == context_before_disable.rcu_generation()); + assert(session.rcu_participant_view() == context_before_disable.rcu_participant_view()); + assert(context_before_reader.wf()); + assert(context_before_reader.rcu_participant_view().spec_le( + context_before_reader.view(), + )); + assert(cpu_reader.participant_view() == context_before_reader.rcu_participant_view()); + assert(session.view() == context_before_reader.view()); + assert(cpu_reader.participant_view().spec_le(session.view())); + } let ghost reader = rcu_spec::RcuReaderContext { scheduler: session.scheduler(), task: session.task(), session: session.session_id(), cpu: session.cpu(), - generation: session.quiescent_generation(), + generation: session.rcu_generation(), }; + let ghost context_before_load = *session; proof_decl! { let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( session, @@ -586,17 +638,192 @@ impl RcuInner

{ } let (obj_ptr, tracked_info, tracked_guard) = self.load_ptr_acquire_guarded( Ghost(reader), + Tracked(cpu_reader), + Tracked(rcu_binding), Tracked(tv), ); - RcuReadGuardInner { + proof { + assert(session.rcu_participant_id() == context_before_disable.rcu_participant_id()); + assert(session.rcu_generation() == context_before_disable.rcu_generation()); + assert(session.rcu_participant_view() == context_before_disable.rcu_participant_view()); + assert(tracked_guard@.participant_id() == cpu_reader.participant_id()); + assert(cpu_reader.participant_id() == context_before_reader.rcu_participant_id()); + assert(session.rcu_participant_id() == context_before_reader.rcu_participant_id()); + assert(session.wf()); + inner_guard.lemma_matches_context_preserved(context_before_load, session); + assert(inner_guard.matches_context(*session)); + assert(inner_guard.has_resource()); + assert(tracked_guard@.wf()); + assert(tracked_guard@.domain() == self.ptr.constant().domain); + assert(tracked_guard@.root() == self.ptr.id()); + assert(tracked_guard@.reader_registry() == self.ptr.constant().reader_registry); + assert(tracked_guard@.retire_observation_registry() + == self.ptr.constant().retire_observation_registry); + assert(tracked_guard@.cpu() == session.cpu()); + assert(tracked_guard@.generation() == session.rcu_generation()); + assert(cpu_reader.fraction() == context_before_reader.rcu_fraction() / 2real); + assert(context_before_load.rcu_fraction() + == context_before_reader.rcu_fraction() / 2real); + assert(session.rcu_fraction() == context_before_load.rcu_fraction()); + assert(tracked_guard@.reader_fragment().fraction() == cpu_reader.fraction()); + assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction()); + assert(tracked_guard@.reader_context() == (rcu_spec::RcuReaderContext { + scheduler: session.scheduler(), + task: session.task(), + session: session.session_id(), + cpu: session.cpu(), + generation: session.rcu_generation(), + })); + match tracked_info@ { + None => assert(obj_ptr.is_null()), + Some(info) => { + assert(!obj_ptr.is_null()); + assert(info.wf()); + assert(info.domain() == tracked_guard@.domain()); + assert(equal(info.ptr(), obj_ptr)); + assert(!tracked_guard@.expired().contains(info.obj())); + assert(tracked_guard@.protects(info.addr(), info.obj())); + }, + } + } + let res = RcuReadGuardInner { obj_ptr, rcu: self, + proof_active: true, _inner_guard: inner_guard, tracked_info, - tracked_guard, + tracked_guard: Tracked(Some(tracked_guard.get())), + tracked_session: Tracked(Some(session)), + }; + proof { + let ghost stored_context = *res.tracked_session@->Some_0; + assert(res._inner_guard.matches_context(stored_context)); + assert(res.guard_token().participant_id() == stored_context.rcu_participant_id()); + assert(res.guard_token().cpu() == stored_context.cpu()); + assert(res.guard_token().generation() == stored_context.rcu_generation()); + assert(res.guard_token().reader_fragment().fraction() + == stored_context.rcu_fraction()); + assert(res.guard_token().reader_context() == reader); + assert(res.matches_context(stored_context)); } + res + } + +} + +/// Detaches the proof-only reader state while leaving the executable +/// preemption guard in place. +/// +/// The surrounding guard enters a private transitional state that still owns +/// the preemption resource. Normal completion returns the updated session +/// before the executable guard can be observed again. +fn take_reader_state<'a, T>( + proof_active: &mut bool, + Tracked(guard_slot): Tracked< + &mut Tracked>>, + >, + Tracked(session_slot): Tracked< + &mut Tracked>, + >, +) -> (res: Tracked<( + rcu_cpu_spec::CpuRcuReadGuardToken, + &'a mut RunningTaskContext, +)>) + requires + *old(proof_active), + old(guard_slot)@ is Some, + old(session_slot)@ is Some, + ensures + !*final(proof_active), + final(guard_slot)@ is None, + final(session_slot)@ is None, + res@.0 == old(guard_slot)@->Some_0, + equal(*res@.1, *old(session_slot)@->Some_0), + opens_invariants none + no_unwind +{ + proof_decl! { + let tracked guard = guard_slot.borrow_mut().tracked_take(); + let tracked session = session_slot.borrow_mut().tracked_take(); + } + *proof_active = false; + Tracked((guard, session)) +} + +/// Completes `Guard -> Inactive` and returns both reader fractions. +fn finish_reader_state<'a, P: NonNullPtr>( + rcu: &RcuInner

, + inner_guard: &mut DisabledPreemptGuard, + Tracked(guard): Tracked< + rcu_cpu_spec::CpuRcuReadGuardToken<

::Target>, + >, + Tracked(session): Tracked<&'a mut RunningTaskContext>, +) -> (res: Tracked<&'a mut RunningTaskContext>) + requires + rcu.type_inv(), + old(session).wf(), + old(inner_guard).matches_context(*old(session)), + guard.wf(), + guard.domain() == rcu.ptr.constant().domain, + guard.root() == rcu.ptr.id(), + guard.retire_observation_registry() + == rcu.ptr.constant().retire_observation_registry, + guard.participant_id() == old(session).rcu_participant_id(), + guard.reader_fragment().fraction() == old(session).rcu_fraction(), + ensures + !final(inner_guard).has_resource(), + (*res@).wf(), + (*res@).task() == old(session).task(), + (*res@).scheduler() == old(session).scheduler(), + (*res@).cpu() == old(session).cpu(), + (*res@).view() == old(session).view(), + (*res@).session_id() == old(session).session_id(), + (*res@).quiescent_generation() == old(session).quiescent_generation(), + (*res@).available_fractions() == old(session).available_fractions() + 1, + (*res@).preempt_depth() + 1 == old(session).preempt_depth(), + (*res@).rcu_participant_id() == old(session).rcu_participant_id(), + (*res@).rcu_generation() == old(session).rcu_generation(), + (*res@).rcu_participant_view() == old(session).rcu_participant_view(), + (*res@).rcu_fraction() == old(session).rcu_fraction() * 2real, + opens_invariants none + no_unwind +{ + let ghost context_before_stop = *session; + let Tracked(cpu_reader) = rcu.ptr.stop_cpu_rcu_reader(Tracked(guard)); + proof { + inner_guard.lemma_matches_context_depth(session); + session.tracked_stop_rcu_reader(cpu_reader); + inner_guard.lemma_matches_context_preserved(context_before_stop, session); + inner_guard.lemma_matches_context_depth(session); + } + inner_guard.release_in_place_to_context(Tracked(session)); + Tracked(session) +} + +fn restore_reader_session<'a>( + Tracked(session_slot): Tracked< + &mut Tracked>, + >, + Tracked(session): Tracked<&'a mut RunningTaskContext>, + Ghost(restored): Ghost, +) + requires + old(session_slot)@ is None, + old(session).wf(), + *old(session) == restored, + ensures + final(session_slot)@ is Some, + (*final(session_slot)@->Some_0).wf(), + *final(session_slot)@->Some_0 == restored, + opens_invariants none + no_unwind +{ + proof_decl! { + *session_slot.borrow_mut() = Some(session); } +} +impl RcuInner

{ #[inline] pub fn read_with<'a, A: InAtomicMode>( &'a self, @@ -644,30 +871,29 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { fn compare_exchange( self, new_ptr: Option

, - Tracked(session): Tracked<&mut RunningTaskContext>, ) -> (res: Result<(), Option

>) requires self.rcu.is_nullable() || new_ptr is Some, - old(session).wf(), - self.matches_context(*old(session)), + self.type_inv(), + self.proof_active, ensures new_ptr is Some && res is Err ==> res->Err_0 is Some, - final(session).wf(), - final(session).task() == old(session).task(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).session_id() == old(session).session_id(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() == old(session).available_fractions() + 1, - final(session).preempt_depth() + 1 == old(session).preempt_depth(), { - let expected = self.obj_ptr; - let rcu = self.rcu; - + let mut this = self; proof { - use_type_invariant(&self); + use_type_invariant(&this); } - + let expected = this.obj_ptr; + let rcu = this.rcu; + let tracked_state = take_reader_state::<

::Target>( + &mut this.proof_active, + Tracked(&mut this.tracked_guard), + Tracked(&mut this.tracked_session), + ); + proof_decl! { + let tracked (guard, session) = tracked_state.get(); + } + let ghost context_at_entry = *session; proof_decl! { let ghost new_ptr_is_some = new_ptr is Some; } @@ -683,17 +909,15 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { assert(new_ptr_is_some); } assert(rcu.is_nullable() || !new_raw.is_null()); - } - - proof { assert(rcu.ptr.constant().nullable == rcu.is_nullable()); assert(rcu.ptr.constant().nullable || !new_raw.is_null()); } + let cas_res = { proof_decl! { let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( session, - &self._inner_guard, + &this._inner_guard, ); } rcu.ptr.compare_exchange_acqrel_acquire_rcu( @@ -705,7 +929,8 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { }; let ghost context_before_enqueue = *session; proof { - assert(self._inner_guard.matches_context(context_before_enqueue)); + this._inner_guard.lemma_matches_context_preserved(context_at_entry, session); + assert(this._inner_guard.matches_context(context_before_enqueue)); } proof_decl! { let tracked (detached, rejected_new_perm) = cas_res.2.get(); @@ -737,42 +962,80 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { }, }; proof { - self._inner_guard.lemma_matches_context_preserved(context_before_enqueue, session); - self._inner_guard.lemma_matches_context_depth(session); - } - proof_decl! { - let tracked guard = self.tracked_guard.get(); + this._inner_guard.lemma_matches_context_preserved(context_before_enqueue, session); } - rcu.ptr.stop_rcu_reader(Tracked(guard)); - self._inner_guard.release_to_context(Tracked(session)); + let Tracked(session) = finish_reader_state( + rcu, + &mut this._inner_guard, + Tracked(guard), + Tracked(session), + ); + let ghost restored = *session; + restore_reader_session( + Tracked(&mut this.tracked_session), + Tracked(session), + Ghost(restored), + ); res } +} - #[inline] - fn drop(self, Tracked(session): Tracked<&mut RunningTaskContext>) - requires - old(session).wf(), - self.matches_context(*old(session)), +impl<'a, P: NonNullPtr> Drop for RcuReadGuardInner<'a, P> { + fn drop(&mut self) ensures - final(session).wf(), - final(session).task() == old(session).task(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).view() == old(session).view(), - final(session).session_id() == old(session).session_id(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() == old(session).available_fractions() + 1, - final(session).preempt_depth() + 1 == old(session).preempt_depth(), + !final(self).is_active(), + old(self).is_active() ==> { + &&& final(self).stored_context().wf() + &&& final(self).stored_context().task() == old(self).stored_context().task() + &&& final(self).stored_context().scheduler() + == old(self).stored_context().scheduler() + &&& final(self).stored_context().cpu() == old(self).stored_context().cpu() + &&& final(self).stored_context().view() == old(self).stored_context().view() + &&& final(self).stored_context().session_id() + == old(self).stored_context().session_id() + &&& final(self).stored_context().quiescent_generation() + == old(self).stored_context().quiescent_generation() + &&& final(self).stored_context().available_fractions() + == old(self).stored_context().available_fractions() + 1 + &&& final(self).stored_context().preempt_depth() + 1 + == old(self).stored_context().preempt_depth() + &&& final(self).stored_context().rcu_participant_id() + == old(self).stored_context().rcu_participant_id() + &&& final(self).stored_context().rcu_generation() + == old(self).stored_context().rcu_generation() + &&& final(self).stored_context().rcu_participant_view() + == old(self).stored_context().rcu_participant_view() + &&& final(self).stored_context().rcu_fraction() + == old(self).stored_context().rcu_fraction() * 2real + }, + opens_invariants none + no_unwind { proof { - use_type_invariant(&self); - self._inner_guard.lemma_matches_context_depth(session); + use_type_invariant(&*self); } - proof_decl! { - let tracked guard = self.tracked_guard.get(); + if self.proof_active { + let tracked_state = take_reader_state::<

::Target>( + &mut self.proof_active, + Tracked(&mut self.tracked_guard), + Tracked(&mut self.tracked_session), + ); + proof_decl! { + let tracked (guard, session) = tracked_state.get(); + } + let Tracked(session) = finish_reader_state( + self.rcu, + &mut self._inner_guard, + Tracked(guard), + Tracked(session), + ); + let ghost restored = *session; + restore_reader_session( + Tracked(&mut self.tracked_session), + Tracked(session), + Ghost(restored), + ); } - self.rcu.ptr.stop_rcu_reader(Tracked(guard)); - self._inner_guard.release_to_context(Tracked(session)); } } @@ -838,20 +1101,12 @@ impl Rcu

{ #[inline] #[verus_spec(res => with - Tracked(session): Tracked<&mut RunningTaskContext>, + Tracked(session): Tracked<&'a mut RunningTaskContext>, requires old(session).wf(), old(session).available_fractions() > 1, - ensures - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() + 1 == old(session).available_fractions(), - final(session).preempt_depth() == old(session).preempt_depth() + 1, - res.matches_context(*final(session)), )] - pub fn read(&self) -> RcuReadGuard<'_, P> { + pub fn read<'a>(&'a self) -> RcuReadGuard<'a, P> { proof { use_type_invariant(self); } @@ -906,20 +1161,12 @@ impl RcuOption

{ #[inline] #[verus_spec(res => with - Tracked(session): Tracked<&mut RunningTaskContext>, + Tracked(session): Tracked<&'a mut RunningTaskContext>, requires old(session).wf(), old(session).available_fractions() > 1, - ensures - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() + 1 == old(session).available_fractions(), - final(session).preempt_depth() == old(session).preempt_depth() + 1, - res.matches_context(*final(session)), )] - pub fn read(&self) -> RcuOptionReadGuard<'_, P> { + pub fn read<'a>(&'a self) -> RcuOptionReadGuard<'a, P> { proof { use_type_invariant(self); } @@ -953,22 +1200,7 @@ impl RcuOption

{ #[verus_verify] impl RcuReadGuard<'_, P> { #[inline] - #[verus_spec( - with - Tracked(session): Tracked<&mut RunningTaskContext>, - requires - old(session).wf(), - self.matches_context(*old(session)), - ensures - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() == old(session).available_fractions() + 1, - final(session).preempt_depth() + 1 == old(session).preempt_depth(), - )] pub fn drop(self) { - self.0.drop(Tracked(session)); } #[inline] @@ -982,22 +1214,11 @@ impl RcuReadGuard<'_, P> { /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] - #[verus_spec( - with - Tracked(session): Tracked<&mut RunningTaskContext>, - requires - old(session).wf(), - self.matches_context(*old(session)), - ensures - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() == old(session).available_fractions() + 1, - final(session).preempt_depth() + 1 == old(session).preempt_depth(), - )] pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { - self.0.compare_exchange(Some(new_ptr), Tracked(session)).map_err( + proof { + use_type_invariant(&self); + } + self.0.compare_exchange(Some(new_ptr)).map_err( |err| requires err is Some, @@ -1009,22 +1230,7 @@ impl RcuReadGuard<'_, P> { #[verus_verify] impl RcuOptionReadGuard<'_, P> { #[inline] - #[verus_spec( - with - Tracked(session): Tracked<&mut RunningTaskContext>, - requires - old(session).wf(), - self.matches_context(*old(session)), - ensures - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() == old(session).available_fractions() + 1, - final(session).preempt_depth() + 1 == old(session).preempt_depth(), - )] pub fn drop(self) { - self.0.drop(Tracked(session)); } #[inline] @@ -1039,25 +1245,11 @@ impl RcuOptionReadGuard<'_, P> { /// Tries to replace the pointer using AcqRel/Acquire CAS. #[inline] - #[verus_spec( - with - Tracked(session): Tracked<&mut RunningTaskContext>, - requires - old(session).wf(), - self.matches_context(*old(session)), - ensures - final(session).wf(), - final(session).scheduler() == old(session).scheduler(), - final(session).cpu() == old(session).cpu(), - final(session).quiescent_generation() == old(session).quiescent_generation(), - final(session).available_fractions() == old(session).available_fractions() + 1, - final(session).preempt_depth() + 1 == old(session).preempt_depth(), - )] pub fn compare_exchange(self, new_ptr: Option

) -> Result<(), Option

> { proof { use_type_invariant(&self); } - self.0.compare_exchange(new_ptr, Tracked(session)) + self.0.compare_exchange(new_ptr) } } @@ -1176,6 +1368,7 @@ impl<'a, P: NonNullPtr> RcuReadGuard<'a, P> { closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() &&& !self.0.rcu.is_nullable() + &&& self.0.is_active() } } @@ -1190,38 +1383,103 @@ impl<'a, P: NonNullPtr> RcuOptionReadGuard<'a, P> { closed spec fn type_inv(self) -> bool { &&& self.0.type_inv() &&& self.0.rcu.is_nullable() + &&& self.0.is_active() } } impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + pub closed spec fn is_active(self) -> bool { + self.proof_active + } + + pub closed spec fn has_stored_context(self) -> bool { + self.tracked_session@ is Some + } + + pub closed spec fn stored_context(self) -> RunningTaskContext + recommends + self.has_stored_context(), + { + *self.tracked_session@->Some_0 + } + + closed spec fn guard_token( + self, + ) -> rcu_cpu_spec::CpuRcuReadGuardToken<

::Target> + recommends + self.tracked_guard@ is Some, + { + self.tracked_guard@->Some_0 + } + closed spec fn matches_context(self, session: RunningTaskContext) -> bool { + &&& self.proof_active + &&& self.tracked_guard@ is Some &&& self._inner_guard.matches_context(session) - &&& self.tracked_guard@.reader() == (rcu_spec::RcuReaderContext { + &&& self.guard_token().participant_id() == session.rcu_participant_id() + &&& self.guard_token().cpu() == session.cpu() + &&& self.guard_token().generation() == session.rcu_generation() + &&& self.guard_token().reader_context() == (rcu_spec::RcuReaderContext { scheduler: session.scheduler(), task: session.task(), session: session.session_id(), cpu: session.cpu(), - generation: session.quiescent_generation(), + generation: session.rcu_generation(), }) } + proof fn lemma_matches_context_preserved( + &self, + before: RunningTaskContext, + tracked after: &RunningTaskContext, + ) + requires + self.matches_context(before), + after.wf(), + after.task() == before.task(), + after.scheduler() == before.scheduler(), + after.cpu() == before.cpu(), + after.session_id() == before.session_id(), + after.quiescent_generation() == before.quiescent_generation(), + after.available_fractions() == before.available_fractions(), + after.preempt_depth() == before.preempt_depth(), + after.rcu_participant_id() == before.rcu_participant_id(), + after.rcu_generation() == before.rcu_generation(), + ensures + self.matches_context(*after), + { + self._inner_guard.lemma_matches_context_preserved(before, after); + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.rcu.type_inv() - &&& self.tracked_guard@.wf() - &&& self.tracked_guard@.domain() == self.rcu.ptr.constant().domain - &&& self.tracked_guard@.reader_registry() == self.rcu.ptr.constant().reader_registry &&& !self.rcu.is_nullable() ==> !self.obj_ptr.is_null() - &&& match self.tracked_info@ { - None => self.obj_ptr.is_null(), - Some(info) => { - &&& !self.obj_ptr.is_null() - &&& info.wf() - &&& info.domain() == self.tracked_guard@.domain() - &&& equal(info.ptr(), self.obj_ptr) - &&& !self.tracked_guard@.expired().contains(info.obj()) - &&& self.tracked_guard@.protects(info.addr(), info.obj()) - }, + &&& self.proof_active == (self.tracked_guard@ is Some) + &&& self.proof_active ==> self.tracked_session@ is Some + &&& self.tracked_session@ is Some ==> self.stored_context().wf() + &&& self.proof_active ==> { + &&& self._inner_guard.has_resource() + &&& self.guard_token().wf() + &&& self.guard_token().domain() == self.rcu.ptr.constant().domain + &&& self.guard_token().root() == self.rcu.ptr.id() + &&& self.guard_token().reader_registry() == self.rcu.ptr.constant().reader_registry + &&& self.guard_token().retire_observation_registry() + == self.rcu.ptr.constant().retire_observation_registry + &&& self.matches_context(self.stored_context()) + &&& self.guard_token().reader_fragment().fraction() + == self.stored_context().rcu_fraction() + &&& match self.tracked_info@ { + None => self.obj_ptr.is_null(), + Some(info) => { + &&& !self.obj_ptr.is_null() + &&& info.wf() + &&& info.domain() == self.guard_token().domain() + &&& equal(info.ptr(), self.obj_ptr) + &&& !self.guard_token().expired().contains(info.obj()) + &&& self.guard_token().protects(info.addr(), info.obj()) + }, + } } } } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index f3508a2f1..2011fcd5c 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -10,6 +10,7 @@ use crate::specs::{ sync::{ rcu as rcu_spec, rcu::{GracePeriodView, MonitorStateView}, + rcu_cpu as rcu_cpu_spec, weak_memory::{History, RcuMonitorWeakAtomicBool, ThreadView, WmView}, }, }; @@ -27,50 +28,78 @@ type MonitorAtomicBool = RcuMonitorWeakAtomicBool; /// Evidence captured at a call site where the current task is quiescent. /// /// This token deliberately does not claim that a complete RCU grace period has -/// elapsed. It records the task and weak-memory view at one CPU-local -/// observation; the monitor binds it to the currently active generation below. -/// A later scheduler proof must additionally show that this unsafe entrypoint -/// is reached by the required context-switch path. +/// elapsed. It records one CPU-local quiescent boundary and carries the +/// resource proving that the CPU's previous reader generation is closed. The +/// monitor must still collect one such resource from every online CPU. tracked struct RcuQuiescentContext { ghost cpu: CpuId, ghost task: Loc, ghost scheduler: Loc, ghost session: Loc, + ghost participant: Loc, ghost generation: nat, ghost view: WmView, + closed: rcu_cpu_spec::CpuRcuClosedGeneration, } impl RcuQuiescentContext { + closed spec fn wf(self) -> bool { + &&& self.closed.wf() + &&& self.closed.scheduler() == self.scheduler + &&& self.closed.participant_id() == self.participant + &&& self.closed.cpu() == self.cpu + &&& self.closed.closed_generation() == self.generation + &&& self.closed.view() == self.view + } + proof fn tracked_from_running_context( tracked context: &mut RunningTaskContext, cpu: CpuId, + tracked retired_facts: &rcu_spec::RcuRetiredFacts, ) -> (tracked res: Self) requires old(context).wf(), old(context).is_quiescent(), cpu == old(context).cpu(), + retired_facts.observed_by(old(context).view()), ensures res.cpu == cpu, res.task == old(context).task(), res.scheduler == old(context).scheduler(), res.session == old(context).session_id(), - res.generation == old(context).quiescent_generation(), + res.participant == old(context).rcu_participant_id(), + res.generation == old(context).rcu_generation(), res.view == old(context).view(), + res.closed.wf(), + res.closed.scheduler() == res.scheduler, + res.closed.participant_id() == res.participant, + res.closed.cpu() == cpu, + res.closed.closed_generation() == res.generation, + res.closed.view() == res.view, + retired_facts.records().subset_of(res.closed.known_retired()), + res.wf(), final(context).wf(), final(context).is_quiescent(), final(context).task() == old(context).task(), final(context).scheduler() == old(context).scheduler(), final(context).cpu() == old(context).cpu(), final(context).session_id() == old(context).session_id(), - final(context).quiescent_generation() == res.generation + 1, + final(context).quiescent_generation() == old(context).quiescent_generation() + 1, + final(context).rcu_participant_id() == res.participant, + final(context).rcu_generation() == res.generation + 1, + final(context).rcu_participant_view() == res.view, + final(context).rcu_fraction() == 1real, final(context).view() == old(context).view(), { let ghost task = context.task(); let ghost scheduler = context.scheduler(); let ghost session = context.session_id(); + let ghost participant = context.rcu_participant_id(); let ghost view = context.view(); - let ghost generation = context.tracked_record_quiescent(); - RcuQuiescentContext { cpu, task, scheduler, session, generation, view } + let ghost generation = context.rcu_generation(); + let tracked closed = context.tracked_report_rcu_quiescent_with(retired_facts); + let ghost _session_generation = context.tracked_record_quiescent(); + RcuQuiescentContext { cpu, task, scheduler, session, participant, generation, view, closed } } } @@ -80,6 +109,8 @@ ghost struct RcuCpuQuiescentReport { task: Loc, scheduler: Loc, session: Loc, + /// Stable identity of the scheduler-owned CPU participant. + participant: Loc, /// Last reader generation closed by this quiescent transition. generation: nat, view: WmView, @@ -87,16 +118,65 @@ ghost struct RcuCpuQuiescentReport { } impl RcuCpuQuiescentReport { - /// Whether this report is the quiescent boundary immediately following a - /// reader generation in the same scheduler session. - closed spec fn closes_same_session_generation( - self, - reader: rcu_spec::RcuReaderContext, - ) -> bool { - &&& self.cpu == reader.cpu - &&& self.scheduler == reader.scheduler - &&& self.session == reader.session - &&& reader.generation <= self.generation + closed spec fn matches_closed(self, closed: rcu_cpu_spec::CpuRcuClosedGeneration) -> bool { + &&& closed.wf() + &&& closed.scheduler() == self.scheduler + &&& closed.participant_id() == self.participant + &&& closed.cpu() == self.cpu + &&& closed.closed_generation() == self.generation + &&& closed.view() == self.view + } +} + +/// Copies persistent closed-generation facts for a finite CPU set without +/// removing the originals from the grace-period invariant. +proof fn duplicate_closed_generations( + tracked source: &Map, + keys: Set, +) -> (tracked duplicates: Map) + requires + keys.subset_of(source.dom()), + forall|cpu: CpuId| #[trigger] source.dom().contains(cpu) ==> source[cpu].wf(), + ensures + duplicates.dom() == keys, + forall|cpu: CpuId| #[trigger] + keys.contains(cpu) ==> { + &&& duplicates[cpu].wf() + &&& duplicates[cpu].participant_id() == source[cpu].participant_id() + &&& duplicates[cpu].cpu() == source[cpu].cpu() + &&& duplicates[cpu].closed_generation() == source[cpu].closed_generation() + &&& duplicates[cpu].view() == source[cpu].view() + &&& duplicates[cpu].known_retired() == source[cpu].known_retired() + &&& duplicates[cpu].scheduler() == source[cpu].scheduler() + }, + decreases keys.len(), +{ + if keys.is_empty() { + Map::tracked_empty() + } else { + let ghost cpu = keys.choose(); + let ghost rest = keys.remove(cpu); + let tracked mut duplicates = duplicate_closed_generations(source, rest); + let tracked closed = source.tracked_borrow(cpu); + let tracked duplicate = closed.tracked_duplicate_from_ref(); + duplicates.tracked_insert(cpu, duplicate); + assert(keys == rest.insert(cpu)); + assert(duplicates.dom() == keys); + assert forall|other: CpuId| #[trigger] keys.contains(other) implies { + &&& duplicates[other].wf() + &&& duplicates[other].participant_id() == source[other].participant_id() + &&& duplicates[other].cpu() == source[other].cpu() + &&& duplicates[other].closed_generation() == source[other].closed_generation() + &&& duplicates[other].view() == source[other].view() + &&& duplicates[other].known_retired() == source[other].known_retired() + &&& duplicates[other].scheduler() == source[other].scheduler() + } by { + if other == cpu { + } else { + assert(rest.contains(other)); + } + }; + duplicates } } @@ -129,23 +209,28 @@ impl RcuCallback { Tracked(cert): Tracked, Ghost(retire_epoch): Ghost, Ghost(retire_view): Ghost, + Ghost(scheduler): Ghost, ) -> (res: Self) requires cert.removal().observed_by(retire_view), ensures res.wf(), res@ == (rcu_spec::RcuCallbackSummary { + scheduler, domain: cert.domain(), obj: cert.obj(), removal: cert.removal(), + retire_observation_registry: cert.retire_observation_registry(), retire_epoch, retire_view, }), { let ghost summary = rcu_spec::RcuCallbackSummary { + scheduler, domain: cert.domain(), obj: cert.obj(), removal: cert.removal(), + retire_observation_registry: cert.retire_observation_registry(), retire_epoch, retire_view, }; @@ -157,6 +242,12 @@ impl RcuCallback { /// Runs the underlying callback once the monitor has completed the grace /// period that contained this callback's retire summary. + /// + /// This remains an executable type-erasure boundary. `RcuReclaimPermit` + /// proves batch membership, weak-memory view coverage, and carries a + /// closed-generation resource for every online CPU. Recovering the + /// callback object's physical permission still depends on the separate + /// read-lease protocol. #[inline] #[verifier::external_body] unsafe fn call_once(self, Tracked(permit): Tracked) @@ -174,6 +265,20 @@ impl RcuCallback { &&& self@.removal.observed_by(self@.retire_view) } + /// Duplicates the persistent base-retirement fact retained by this + /// type-erased callback. + proof fn tracked_retired_fact(tracked &self) -> (tracked fact: rcu_spec::RcuRetiredFact) + requires + self.wf(), + ensures + fact.wf(), + fact.record() == self@.retired_record(), + { + let tracked fact = self.safety.borrow().tracked_retired_fact(self@); + assert(fact.matches(self@)); + fact + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { self.wf() @@ -190,6 +295,7 @@ tracked struct CompletedGracePeriod { callbacks: Ghost>, reported_cpus: Ghost>, reports: Ghost>, + closed_generations: Map, } /// Object-level authorization to execute one reclamation callback. @@ -199,25 +305,146 @@ tracked struct CompletedGracePeriod { /// completion of the batch containing that callback. Keeping its constructor /// private prevents executable callback code from treating batch membership /// alone as proof that an arbitrary object was retired safely. +/// +/// `authorizes` includes one closed-generation resource for every reported CPU. +/// Those resources classify every coexisting executable guard as a later +/// reader whose start view observes the callback's removal. Physical +/// permission recovery remains the responsibility of the read-lease layer. tracked struct RcuReclaimPermit { summary: Ghost, retired: rcu_spec::RcuRetiredFact, reports: Ghost>, + closed_generations: Map, } impl RcuReclaimPermit { + closed spec fn closed_generations(self) -> Map { + self.closed_generations + } + closed spec fn authorizes(self, callback: rcu_spec::RcuCallbackSummary) -> bool { &&& self.summary@ == callback &&& self.retired.matches(callback) &&& self.reports@.dom() == online_cpus() + &&& self.closed_generations().dom() == self.reports@.dom() &&& forall|cpu: CpuId| #[trigger] self.reports@.contains_key(cpu) ==> { &&& self.reports@[cpu].cpu == cpu &&& self.reports@[cpu].epoch == callback.retire_epoch + &&& self.reports@[cpu].matches_closed(self.closed_generations()[cpu]) &&& callback.retire_view.spec_le(self.reports@[cpu].view) &&& callback.removal.observed_by(self.reports@[cpu].view) + &&& self.closed_generations()[cpu].known_retired().contains( + callback.retired_record(), + ) } } + + /// Classifies any still-live guard on a reported CPU as a later reader. + /// + /// The old-reader branch is excluded by resource validity: a guard from + /// the closed generation cannot coexist with the report token in this + /// permit. The surviving branch starts after the report and therefore + /// observes the callback's root-removal message. + proof fn tracked_later_guard( + tracked &self, + callback: rcu_spec::RcuCallbackSummary, + cpu: CpuId, + tracked guard: rcu_cpu_spec::CpuRcuReadGuardToken, + ) -> (tracked res: rcu_cpu_spec::CpuRcuReadGuardToken) + requires + self.authorizes(callback), + self.reports@.contains_key(cpu), + guard.wf(), + guard.cpu() == cpu, + guard.participant_id() == self.closed_generations()[cpu].participant_id(), + ensures + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.reader_fragment() == guard.reader_fragment(), + res.scheduler() == guard.scheduler(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.domain() == guard.domain(), + res.root() == guard.root(), + res.retire_observation_registry() == guard.retire_observation_registry(), + res.start_view() == guard.start_view(), + res.expired() == guard.expired(), + res.seen_removed() == guard.seen_removed(), + self.reports@[cpu].generation < res.generation(), + self.reports@[cpu].view.spec_le(res.start_view()), + callback.removal.observed_by(res.start_view()), + res.known_retired().contains(callback.retired_record()), + { + let tracked closed = self.closed_generations.tracked_borrow(cpu); + assert(self.reports@[cpu].matches_closed(*closed)); + assert(closed.known_retired().contains(callback.retired_record())); + let tracked guard = closed.lemma_later_guard(guard); + assert(closed.known_retired().subset_of(guard.known_retired())); + assert(guard.known_retired().contains(callback.retired_record())); + assert(callback.removal.observed_by(self.reports@[cpu].view)); + assert(self.reports@[cpu].view.spec_le(guard.start_view())); + assert(callback.removal.timestamp <= self.reports@[cpu].view.seen_at( + callback.removal.root, + )); + assert(self.reports@[cpu].view.seen_at(callback.removal.root) <= guard.start_view().seen_at( + callback.removal.root, + )); + guard + } + + /// End-to-end safety statement for one completed callback and one live + /// protected pointer on a reported CPU. + /// + /// A pre-existing reader cannot coexist with the closed-generation + /// resource. A coexisting later reader imports the callback's observed + /// retirement fact, so the reclaimed object is in its expired/removed set + /// and cannot simultaneously be protected. + proof fn tracked_excludes_protected_callback_object( + tracked &self, + callback: rcu_spec::RcuCallbackSummary, + cpu: CpuId, + tracked guard: rcu_cpu_spec::CpuRcuReadGuardToken, + tracked protected: rcu_spec::RcuProtectedPtr, + ) + requires + self.authorizes(callback), + self.reports@.contains_key(cpu), + guard.wf(), + guard.cpu() == cpu, + guard.participant_id() == self.closed_generations()[cpu].participant_id(), + callback.domain == guard.domain(), + callback.retire_observation_registry == guard.retire_observation_registry(), + callback.removal.root == guard.root(), + protected.obj() == callback.obj, + protected.protected_by(guard.paper_guard()), + ensures + false, + { + let ghost guard_domain = guard.domain(); + let ghost guard_root = guard.root(); + let ghost guard_retire_observation_registry = guard.retire_observation_registry(); + let ghost protected_guard = guard.paper_guard(); + assert(protected.protected_by(protected_guard)); + let tracked guard = self.tracked_later_guard(callback, cpu, guard); + assert(guard.paper_guard() == protected_guard); + assert(guard.domain() == guard_domain); + assert(guard.root() == guard_root); + assert(guard.retire_observation_registry() == guard_retire_observation_registry); + assert(guard.known_retired().contains(callback.retired_record())); + assert(callback.retired_record().domain == callback.domain); + assert(callback.retired_record().obj == callback.obj); + assert(callback.retired_record().removal == callback.removal); + assert(callback.retired_record().retire_observation_registry + == callback.retire_observation_registry); + guard.lemma_known_retired_expired(callback.retired_record()); + assert(guard.expired().contains(callback.obj)); + assert(protected.protected_by(guard.paper_guard())); + guard.lemma_protected_not_expired(&protected); + assert(!guard.expired().contains(protected.obj())); + assert(false); + } } impl CompletedGracePeriod { @@ -237,19 +464,29 @@ impl CompletedGracePeriod { self.reports@ } + closed spec fn closed_generations(self) -> Map { + self.closed_generations + } + closed spec fn reports_wf(self) -> bool { &&& self.reports().dom() == self.reported_cpus() + &&& self.closed_generations().dom() == self.reported_cpus() &&& forall|cpu: CpuId| #[trigger] self.reports().contains_key(cpu) ==> { &&& self.reports()[cpu].cpu == cpu &&& self.reports()[cpu].epoch == self.epoch() + &&& self.reports()[cpu].matches_closed(self.closed_generations()[cpu]) } } closed spec fn callbacks_covered(self) -> bool { forall|i: int, cpu: CpuId| - 0 <= i < self.callbacks().len() && #[trigger] self.reports().contains_key(cpu) ==> ( - #[trigger] self.callbacks()[i]).retire_view.spec_le(self.reports()[cpu].view) + 0 <= i < self.callbacks().len() && #[trigger] self.reports().contains_key(cpu) ==> { + &&& (#[trigger] self.callbacks()[i]).retire_view.spec_le(self.reports()[cpu].view) + &&& self.closed_generations()[cpu].known_retired().contains( + self.callbacks()[i].retired_record(), + ) + } } closed spec fn covers(self, callback: rcu_spec::RcuCallbackSummary) -> bool { @@ -258,9 +495,12 @@ impl CompletedGracePeriod { &&& self.reported_cpus() == online_cpus() &&& self.reports_wf() &&& forall|cpu: CpuId| #[trigger] - self.reports().contains_key(cpu) ==> callback.retire_view.spec_le( - self.reports()[cpu].view, - ) + self.reports().contains_key(cpu) ==> { + &&& callback.retire_view.spec_le(self.reports()[cpu].view) + &&& self.closed_generations()[cpu].known_retired().contains( + callback.retired_record(), + ) + } } /// Combines traversal retirement with monitor completion for one callback. @@ -277,13 +517,42 @@ impl CompletedGracePeriod { permit.authorizes(callback), { let tracked retired = safety.tracked_retired_fact(callback); + assert forall|cpu: CpuId| #[trigger] + self.closed_generations().dom().contains( + cpu, + ) implies self.closed_generations()[cpu].wf() by { + assert(self.reports().contains_key(cpu)); + assert(self.reports()[cpu].matches_closed(self.closed_generations()[cpu])); + }; + let tracked closed_generations = duplicate_closed_generations( + &self.closed_generations, + self.closed_generations().dom(), + ); assert forall|cpu: CpuId| #[trigger] self.reports().contains_key(cpu) implies callback.removal.observed_by( self.reports()[cpu].view, ) by { assert(callback.retire_view.spec_le(self.reports()[cpu].view)); }; - RcuReclaimPermit { summary: Ghost(callback), retired, reports: Ghost(self.reports()) } + assert forall|cpu: CpuId| #[trigger] + self.reports().contains_key( + cpu, + ) implies self.closed_generations()[cpu].known_retired().contains( + callback.retired_record(), + ) by {}; + assert forall|cpu: CpuId| #[trigger] + self.reports().contains_key(cpu) implies self.reports()[cpu].matches_closed( + closed_generations[cpu], + ) by { + assert(self.closed_generations().dom().contains(cpu)); + assert(self.reports()[cpu].matches_closed(self.closed_generations()[cpu])); + }; + RcuReclaimPermit { + summary: Ghost(callback), + retired, + reports: Ghost(self.reports()), + closed_generations, + } } } @@ -404,6 +673,7 @@ pub(super) struct GracePeriod { cpu_mask: AtomicCpuSet, tracked_cpu_mask: Tracked, ghost_reports: Ghost>, + tracked_closed_generations: Tracked>, is_complete: bool, ghost_epoch: Ghost, } @@ -437,12 +707,14 @@ impl GracePeriod { let mut cpu_mask = AtomicCpuSet::new(empty_cpu_set); proof_decl! { let tracked cpu_mask_token = cpu_mask.tracked_take_token(); + let tracked closed_generations = Map::tracked_empty(); } let res = Self { callbacks, cpu_mask, tracked_cpu_mask: Tracked(cpu_mask_token), ghost_reports: Ghost(Map::empty()), + tracked_closed_generations: Tracked(closed_generations), is_complete: true, ghost_epoch: Ghost(0), }; @@ -451,6 +723,7 @@ impl GracePeriod { assert(res.cpu_mask.initial_cpus() == Set::::empty()); assert(res.ghost_reports@.dom() == Set::::empty()); assert(res.tracked_cpu_mask@.cpus() == Set::::empty()); + assert(res.tracked_closed_generations@.dom() == Set::::empty()); } res } @@ -462,6 +735,7 @@ impl GracePeriod { fn restart(&mut self, callbacks: Callbacks, Ghost(epoch): Ghost) requires old(self).wf(), + callback_summaries(callbacks).len() > 0, forall|i: int| 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( callbacks, @@ -473,6 +747,13 @@ impl GracePeriod { final(self).wf(), no_unwind { + proof { + let tracked mut empty_closed = Map::tracked_empty(); + vstd::modes::tracked_swap( + self.tracked_closed_generations.borrow_mut(), + &mut empty_closed, + ); + } self.is_complete = false; self.callbacks = callbacks; self.ghost_epoch = Ghost(epoch); @@ -495,10 +776,17 @@ impl GracePeriod { old(self).wf(), online_cpus().contains(this_cpu), context.cpu == this_cpu, + context.wf(), + old(self).tracked_closed_generations@.dom() == old(self).ghost_reports@.dom(), forall|i: int| 0 <= i < old(self).callback_summaries().len() ==> (#[trigger] old( self, ).callback_summaries()[i]).retire_view.spec_le(context.view), + forall|i: int| + 0 <= i < old(self).callback_summaries().len() + ==> context.closed.known_retired().contains( + (#[trigger] old(self).callback_summaries()[i]).retired_record(), + ), ensures final(self).wf(), final(self)@ == old(self)@, @@ -506,17 +794,34 @@ impl GracePeriod { no_unwind { let ghost old_reports = self.ghost_reports@; + let ghost old_closed_generations = self.tracked_closed_generations@; let ghost report = RcuCpuQuiescentReport { cpu: this_cpu, task: context.task, scheduler: context.scheduler, session: context.session, + participant: context.participant, generation: context.generation, view: context.view, epoch: self@.epoch, }; proof { + let tracked closed = context.closed; + assert(report.matches_closed(closed)); + self.tracked_closed_generations.borrow_mut().tracked_insert(this_cpu, closed); + assert(old_closed_generations.dom().insert(this_cpu) == old_reports.dom().insert( + this_cpu, + )); + assert(self.tracked_closed_generations@ == old_closed_generations.insert( + this_cpu, + closed, + )); self.ghost_reports = Ghost(self.ghost_reports@.insert(this_cpu, report)); + assert(self.tracked_closed_generations@.dom() == old_closed_generations.dom().insert( + this_cpu, + )); + assert(self.ghost_reports@.dom() == old_reports.dom().insert(this_cpu)); + assert(self.tracked_closed_generations@.dom() == self.ghost_reports@.dom()); } #[verus_spec(with Tracked(self.tracked_cpu_mask.borrow_mut()))] self.cpu_mask.add(this_cpu, Ordering::Relaxed); @@ -525,34 +830,67 @@ impl GracePeriod { let complete = cpu_mask.is_full(); proof { assert(self.ghost_reports@ == old_reports.insert(this_cpu, report)); + assert(self.tracked_closed_generations@ == old_closed_generations.insert( + this_cpu, + self.tracked_closed_generations@[this_cpu], + )); + assert(self.tracked_closed_generations@.dom() == self.ghost_reports@.dom()); assert(self.callback_summaries() == old(self).callback_summaries()); assert forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) implies { &&& self.ghost_reports@[cpu].cpu == cpu &&& self.ghost_reports@[cpu].epoch == self@.epoch + &&& self.ghost_reports@[cpu].matches_closed(self.tracked_closed_generations@[cpu]) &&& forall|i: int| 0 <= i < self.callback_summaries().len() ==> ( #[trigger] self.callback_summaries()[i]).retire_view.spec_le( self.ghost_reports@[cpu].view, ) + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() + ==> self.tracked_closed_generations@[cpu].known_retired().contains( + (#[trigger] self.callback_summaries()[i]).retired_record(), + ) } by { + assert(self.tracked_closed_generations@.contains_key(cpu)); if cpu == this_cpu { assert(self.ghost_reports@[cpu] == report); + assert(self.ghost_reports@[cpu].matches_closed( + self.tracked_closed_generations@[cpu], + )); assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( #[trigger] self.callback_summaries()[i]).retire_view.spec_le( self.ghost_reports@[cpu].view, ) by { assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); }; + assert forall|i: int| + 0 <= i + < self.callback_summaries().len() implies self.tracked_closed_generations@[cpu].known_retired().contains( + self.callback_summaries()[i].retired_record()) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; } else { assert(self.ghost_reports@[cpu] == old_reports[cpu]); assert(old(self).ghost_reports@.contains_key(cpu)); assert(self.ghost_reports@[cpu] == old(self).ghost_reports@[cpu]); + assert(self.tracked_closed_generations@[cpu] == old( + self, + ).tracked_closed_generations@[cpu]); + assert(self.ghost_reports@[cpu].matches_closed( + self.tracked_closed_generations@[cpu], + )); assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( #[trigger] self.callback_summaries()[i]).retire_view.spec_le( self.ghost_reports@[cpu].view, ) by { assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); }; + assert forall|i: int| + 0 <= i + < self.callback_summaries().len() implies self.tracked_closed_generations@[cpu].known_retired().contains( + self.callback_summaries()[i].retired_record()) by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; } }; assert(self.wf()); @@ -577,6 +915,11 @@ impl GracePeriod { &&& self.tracked_cpu_mask@.id() == self.cpu_mask.id() &&& self.tracked_cpu_mask@.wf() &&& self.ghost_reports@.dom() == self.tracked_cpu_mask@.cpus() + &&& self.tracked_closed_generations@.dom() == self.ghost_reports@.dom() + &&& forall|cpu: CpuId| #[trigger] + self.ghost_reports@.contains_key(cpu) ==> self.ghost_reports@[cpu].matches_closed( + self.tracked_closed_generations@[cpu], + ) &&& forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) ==> { &&& self.ghost_reports@[cpu].cpu == cpu @@ -586,6 +929,11 @@ impl GracePeriod { #[trigger] self.callback_summaries()[i]).retire_view.spec_le( self.ghost_reports@[cpu].view, ) + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() + ==> self.tracked_closed_generations@[cpu].known_retired().contains( + (#[trigger] self.callback_summaries()[i]).retired_record(), + ) } } } @@ -593,6 +941,9 @@ impl GracePeriod { pub(super) struct State { current_gp: GracePeriod, next_callbacks: Callbacks, + /// Monotonic registry of persistent `Retired(a, Q)` facts for callbacks + /// observed through this monitor. + tracked_retired_facts: Tracked, /// Release view of the monitor lock. /// /// This proof-only token is updated before unlocking and imported after @@ -686,6 +1037,16 @@ impl State { old(self).next_callbacks, )[i].retire_view.lemma_spec_le_transitive(old_lock_view, final(self).lock_view()); }; + assert(self.tracked_retired_facts@.observed_by(final(self).lock_view())) by { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + self.tracked_retired_facts@.records().contains( + record, + ) implies record.removal.observed_by(final(self).lock_view()) by { + assert(record.removal.observed_by(old_lock_view)); + assert(old_lock_view.seen_at(record.removal.root) + <= final(self).lock_view().seen_at(record.removal.root)); + }; + }; } } @@ -700,8 +1061,14 @@ impl State { let next_callbacks = Callbacks::new(); proof_decl! { let tracked lock_view = ThreadView::new(); + let tracked retired_facts = rcu_spec::RcuRetiredFacts::empty(); } - let res = Self { current_gp, next_callbacks, tracked_lock_view: Tracked(lock_view) }; + let res = Self { + current_gp, + next_callbacks, + tracked_retired_facts: Tracked(retired_facts), + tracked_lock_view: Tracked(lock_view), + }; proof { callback_summaries_empty(res.next_callbacks); } @@ -731,6 +1098,11 @@ impl State { { proof { use_type_invariant(&*self); + let tracked fact = callback.tracked_retired_fact(); + self.tracked_retired_facts.borrow_mut().tracked_insert(&fact); + assert(callback@.retired_record() == fact.record()); + assert(callback@.retire_view.spec_le(self.lock_view())); + assert(callback@.removal.observed_by(self.lock_view())); } let ghost callback_epoch = self.next_callback_epoch(); if self.current_gp.is_complete { @@ -787,12 +1159,19 @@ impl State { ) -> (complete: bool) requires old(self).wf(), + !old(self).current_gp.is_complete, online_cpus().contains(this_cpu), context.cpu == this_cpu, + context.wf(), forall|i: int| 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( self, ).current_gp.callback_summaries()[i]).retire_view.spec_le(context.view), + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() + ==> context.closed.known_retired().contains( + (#[trigger] old(self).current_gp.callback_summaries()[i]).retired_record(), + ), ensures final(self).wf(), final(self)@ == old(self)@, @@ -802,6 +1181,48 @@ impl State { self.current_gp.record_quiescent_state(this_cpu, Tracked(context)) } + /// Bridges the lock-protected persistent retirement registry into the + /// scheduler-owned CPU participant at a quiescent boundary. + fn make_quiescent_context( + &self, + cpu: CpuId, + Tracked(context): Tracked<&mut RunningTaskContext>, + ) -> (res: Tracked) + requires + self.wf(), + old(context).wf(), + old(context).is_quiescent(), + cpu == old(context).cpu(), + self.tracked_retired_facts@.observed_by(old(context).view()), + ensures + res@.cpu == cpu, + res@.view == old(context).view(), + res@.wf(), + self.tracked_retired_facts@.records().subset_of(res@.closed.known_retired()), + final(context).wf(), + final(context).is_quiescent(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation() + 1, + final(context).rcu_generation() == old(context).rcu_generation() + 1, + final(context).rcu_fraction() == 1real, + final(context).view() == old(context).view(), + no_unwind + { + proof_decl! { + let tracked retired_facts = self.tracked_retired_facts.borrow(); + let tracked quiescent_context = + RcuQuiescentContext::tracked_from_running_context( + context, + cpu, + retired_facts, + ); + } + Tracked(quiescent_context) + } + /// Records a quiescent state for the current CPU, returns the callbacks /// that become reclaimable if this completes the grace period, and /// immediately starts the next grace period if callbacks accumulated while @@ -823,10 +1244,16 @@ impl State { requires online_cpus().contains(this_cpu), context.cpu == this_cpu, + context.wf(), forall|i: int| 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( self, ).current_gp.callback_summaries()[i]).retire_view.spec_le(context.view), + forall|i: int| + 0 <= i < old(self).current_gp.callback_summaries().len() + ==> context.closed.known_retired().contains( + (#[trigger] old(self).current_gp.callback_summaries()[i]).retired_record(), + ), ensures final(self).wf(), completed_token@.callbacks() == callback_summaries(completed_callbacks), @@ -870,6 +1297,13 @@ impl State { let mut completed_gp = false; let ghost mut completed_cpu_mask = Set::::empty(); let ghost mut completed_reports = Map::::empty(); + let ghost mut completed_closed_generations_view = Map::< + CpuId, + rcu_cpu_spec::CpuRcuClosedGeneration, + >::empty(); + proof_decl! { + let tracked mut completed_closed_generations = Map::tracked_empty(); + } if !self.current_gp.is_complete { let is_complete = self.record_quiescent_state(this_cpu, Tracked(context)); if is_complete { @@ -877,7 +1311,17 @@ impl State { proof { completed_cpu_mask = self.current_gp.tracked_cpu_mask@.cpus(); completed_reports = self.current_gp.ghost_reports@; + completed_closed_generations_view = self.current_gp.tracked_closed_generations@; assert(completed_cpu_mask == online_cpus()); + assert(completed_closed_generations_view.dom() == completed_reports.dom()); + assert forall|cpu: CpuId| #[trigger] + completed_reports.contains_key( + cpu, + ) implies completed_reports[cpu].matches_closed( + completed_closed_generations_view[cpu], + ) by { + assert(self.current_gp.wf()); + }; assert(self.current_gp.callback_summaries() == initial_current_callbacks); assert forall|i: int, cpu: CpuId| 0 <= i < initial_current_callbacks.len() @@ -892,6 +1336,18 @@ impl State { proof { assert(callback_summaries(completed_callbacks) == initial_current_callbacks); callback_summaries_empty(self.current_gp.callbacks); + assert forall|cpu: CpuId| #[trigger] + self.current_gp.tracked_closed_generations@.dom().contains( + cpu, + ) implies self.current_gp.tracked_closed_generations@[cpu].wf() by { + assert(self.current_gp.ghost_reports@.contains_key(cpu)); + assert(self.current_gp.wf()); + }; + completed_closed_generations = + duplicate_closed_generations( + self.current_gp.tracked_closed_generations.borrow(), + self.current_gp.tracked_closed_generations@.dom(), + ); } if self.next_callbacks.len() > 0 { let mut next_callbacks = Callbacks::new(); @@ -918,6 +1374,7 @@ impl State { callbacks: Ghost(callback_summaries(completed_callbacks)), reported_cpus: Ghost(completed_cpu_mask), reports: Ghost(completed_reports), + closed_generations: completed_closed_generations, }; } proof { @@ -935,12 +1392,18 @@ impl State { assert(completed.callbacks_covered()) by { assert forall|i: int, cpu: CpuId| 0 <= i < completed.callbacks().len() - && #[trigger] completed.reports().contains_key(cpu) implies ( - #[trigger] completed.callbacks()[i]).retire_view.spec_le( - completed.reports()[cpu].view, - ) by { + && #[trigger] completed.reports().contains_key(cpu) implies { + &&& (#[trigger] completed.callbacks()[i]).retire_view.spec_le( + completed.reports()[cpu].view, + ) + &&& completed.closed_generations()[cpu].known_retired().contains( + completed.callbacks()[i].retired_record(), + ) + } by { assert(completed.callbacks()[i] == initial_current_callbacks[i]); assert(completed.reports() == completed_reports); + assert(completed.closed_generations()[cpu].known_retired() + == completed_closed_generations_view[cpu].known_retired()); }; }; assert forall|i: int| @@ -1001,6 +1464,17 @@ impl State { #[trigger] callback_summaries(self.next_callbacks)[i]).retire_view.spec_le( self.lock_view(), ) + &&& self.tracked_retired_facts@.observed_by(self.lock_view()) + &&& forall|i: int| + 0 <= i < self.current_gp.callback_summaries().len() + ==> self.tracked_retired_facts@.records().contains( + (#[trigger] self.current_gp.callback_summaries()[i]).retired_record(), + ) + &&& forall|i: int| + 0 <= i < callback_summaries(self.next_callbacks).len() + ==> self.tracked_retired_facts@.records().contains( + (#[trigger] callback_summaries(self.next_callbacks)[i]).retired_record(), + ) } #[verifier::type_invariant] @@ -1134,6 +1608,8 @@ impl RcuMonitor { self.wf(), state.wf(), state.has_pending_work() ==> value, + ensures + old(tv)@.spec_le(final(tv)@), { proof { use_type_invariant(self); @@ -1163,6 +1639,10 @@ impl RcuMonitor { final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions(), final(session).preempt_depth() == old(session).preempt_depth(), + final(session).rcu_participant_id() == old(session).rcu_participant_id(), + final(session).rcu_generation() == old(session).rcu_generation(), + final(session).rcu_participant_view() == old(session).rcu_participant_view(), + final(session).rcu_fraction() == old(session).rcu_fraction(), )] pub(super) fn after_grace_period( &self, @@ -1198,6 +1678,7 @@ impl RcuMonitor { Tracked(cert), Ghost(retire_epoch), Ghost(retire_view), + Ghost(session.scheduler()), ); let started_gp = state.enqueue_after_grace_period(callback); if started_gp { @@ -1268,10 +1749,22 @@ impl RcuMonitor { return; } let this_cpu = CpuId::current(Tracked(&*session)); - proof_decl! { - let tracked quiescent_context = - RcuQuiescentContext::tracked_from_running_context(session, this_cpu); + proof { + assert(state.value().tracked_retired_facts@.observed_by(session.view())) by { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + state.value().tracked_retired_facts@.records().contains( + record, + ) implies record.removal.observed_by(session.view()) by { + assert(record.removal.observed_by(state.value().lock_view())); + assert(state.value().lock_view().seen_at(record.removal.root) + <= session.view().seen_at(record.removal.root)); + }; + }; } + let Tracked(quiescent_context) = state.make_quiescent_context( + CpuId::current(Tracked(&*session)), + Tracked(session), + ); proof { assert forall|i: int| 0 <= i < state.value().current_gp.callback_summaries().len() implies ( @@ -1284,6 +1777,14 @@ impl RcuMonitor { session.view(), ); }; + assert forall|i: int| + 0 <= i + < state.value().current_gp.callback_summaries().len() implies quiescent_context.closed.known_retired().contains( + (#[trigger] state.value().current_gp.callback_summaries()[i]).retired_record()) by { + assert(state.value().tracked_retired_facts@.records().contains( + state.value().current_gp.callback_summaries()[i].retired_record(), + )); + }; } let (completed_gp, completed_callbacks, Tracked(completed)) = state.finish_grace_period( this_cpu, diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index c533d0ab3..049ac5056 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -3,7 +3,14 @@ use vstd::{prelude::*, resource::Loc}; use vstd_extra::resource::ghost_resource::{count::CountGhost, tokens::CountGhostResource}; use crate::{ - specs::sync::weak_memory::{ThreadView, WmView}, + specs::sync::{ + rcu::RcuRetiredFacts, + rcu_cpu::{ + CpuRcuClosedGeneration, CpuRcuParticipant, CpuRcuParticipantBinding, + CpuRcuReaderFragment, + }, + weak_memory::{ThreadView, WmView}, + }, sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ task::scheduler::{SchedulerView, TaskThreadView}, }; @@ -354,6 +361,8 @@ impl PreemptThreadViewSession { /// the scheduler when no guard remains live. pub tracked struct RunningTaskContext { session: PreemptThreadViewSession, + rcu_participant: CpuRcuParticipant, + rcu_binding: CpuRcuParticipantBinding, preempt_depth: Ghost, cpu: Ghost, } @@ -362,11 +371,23 @@ impl RunningTaskContext { /// Starts a running interval for a checked-out task view. pub proof fn new( tracked task_view: TaskThreadView, + tracked rcu_participant: CpuRcuParticipant, + tracked rcu_binding: CpuRcuParticipantBinding, sched_view: SchedulerView, cpu: crate::specs::mm::cpu::CpuId, ) -> (tracked res: Self) requires task_view.wf(sched_view), + rcu_participant.wf(), + rcu_participant.cpu() == cpu, + rcu_participant.fraction() == 1real, + rcu_participant.view().spec_le(task_view.view()), + rcu_binding.scheduler() == task_view.scheduler(), + rcu_binding.cpu() == cpu, + rcu_binding.participant_id() == rcu_participant.id(), + sched_view.cpu_has_rcu_participant(cpu), + sched_view.cpu_rcu_participant_id(cpu) == rcu_participant.id(), + !sched_view.cpu_rcu_participant_is_stored(cpu), sched_view.current.contains_key(cpu), sched_view.current[cpu] == Some(task_view.task()), ensures @@ -377,12 +398,25 @@ impl RunningTaskContext { res.preempt_depth() == 0, res.quiescent_generation() == 0, res.available_fractions() == PREEMPT_SESSION_FRACTIONS, + res.rcu_participant_id() == rcu_participant.id(), + res.rcu_generation() == rcu_participant.generation(), + res.rcu_participant_view() == rcu_participant.view(), + res.rcu_fraction() == 1real, + res.rcu_binding().scheduler() == res.scheduler(), + res.rcu_binding().cpu() == cpu, + res.rcu_binding().participant_id() == res.rcu_participant_id(), res.wf(), res.is_quiescent(), res.wf_scheduler(sched_view), { let tracked session = PreemptThreadViewSession::new(task_view, sched_view); - let tracked res = RunningTaskContext { session, preempt_depth: Ghost(0), cpu: Ghost(cpu) }; + let tracked res = RunningTaskContext { + session, + rcu_participant, + rcu_binding, + preempt_depth: Ghost(0), + cpu: Ghost(cpu), + }; assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); assert(res.wf()); assert(res.session.wf(sched_view)); @@ -426,9 +460,45 @@ impl RunningTaskContext { self.preempt_depth@ } + pub closed spec fn rcu_participant_id(self) -> Loc { + self.rcu_participant.id() + } + + pub closed spec fn rcu_generation(self) -> nat { + self.rcu_participant.generation() + } + + pub closed spec fn rcu_participant_view(self) -> WmView { + self.rcu_participant.view() + } + + pub closed spec fn rcu_fraction(self) -> real { + self.rcu_participant.fraction() + } + + pub closed spec fn rcu_binding(self) -> CpuRcuParticipantBinding { + self.rcu_binding + } + pub closed spec fn wf(self) -> bool { &&& self.session.wf_session_resource() &&& self.available_fractions() + self.preempt_depth() == PREEMPT_SESSION_FRACTIONS + &&& self.rcu_participant.wf() + &&& self.rcu_binding().scheduler() == self.scheduler() + &&& self.rcu_binding().cpu() == self.cpu() + &&& self.rcu_binding().participant_id() == self.rcu_participant_id() + &&& self.rcu_participant.cpu() == self.cpu() + &&& self.rcu_participant_view().spec_le(self.view()) + } + + /// The checked-out task view includes the persistent view of this CPU's + /// RCU participant. + pub proof fn lemma_rcu_participant_view_le(tracked &self) + requires + self.wf(), + ensures + self.rcu_participant_view().spec_le(self.view()), + { } /// Relates this running context to the scheduler snapshot from which its @@ -438,6 +508,9 @@ impl RunningTaskContext { &&& self.session.wf(sched_view) &&& sched_view.current.contains_key(self.cpu()) &&& sched_view.current[self.cpu()] == Some(self.task()) + &&& sched_view.cpu_has_rcu_participant(self.cpu()) + &&& sched_view.cpu_rcu_participant_id(self.cpu()) == self.rcu_participant_id() + &&& !sched_view.cpu_rcu_participant_is_stored(self.cpu()) } /// Re-establishes the scheduler relation after the checked-out task view @@ -453,6 +526,9 @@ impl RunningTaskContext { sched_view.task_views[self.task()] == self.view(), sched_view.current.contains_key(self.cpu()), sched_view.current[self.cpu()] == Some(self.task()), + sched_view.cpu_has_rcu_participant(self.cpu()), + sched_view.cpu_rcu_participant_id(self.cpu()) == self.rcu_participant_id(), + !sched_view.cpu_rcu_participant_is_stored(self.cpu()), ensures self.wf_scheduler(sched_view), { @@ -464,6 +540,7 @@ impl RunningTaskContext { &&& self.preempt_depth() == 0 &&& self.available_fractions() == PREEMPT_SESSION_FRACTIONS &&& self.has_full_authority() + &&& self.rcu_fraction() == 1real } /// Borrows the running task's persistent weak-memory view. @@ -480,12 +557,165 @@ impl RunningTaskContext { final(self).available_fractions() == old(self).available_fractions(), final(self).has_full_authority() == old(self).has_full_authority(), final(self).preempt_depth() == old(self).preempt_depth(), - final(self).wf(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), final(self).view() == (*final(tv))@, + old(self).view().spec_le((*final(tv))@) ==> final(self).wf(), { self.session.tracked_borrow_thread_view_mut() } + /// Starts one RCU reader from this CPU's persistent participant. + /// + /// Preemption must already be disabled. The returned fragment names the + /// participant's current CPU generation and remains live until the reader + /// guard is destroyed. + pub proof fn tracked_start_rcu_reader(tracked &mut self) -> (tracked reader: + CpuRcuReaderFragment) + requires + old(self).wf(), + old(self).preempt_depth() > 0, + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction() / 2real, + reader.wf(), + reader.participant_id() == old(self).rcu_participant_id(), + reader.cpu() == old(self).cpu(), + reader.generation() == old(self).rcu_generation(), + reader.participant_view() == old(self).rcu_participant_view(), + reader.fraction() == old(self).rcu_fraction() / 2real, + { + self.rcu_participant.tracked_start_reader_in_place(self.view()) + } + + /// Copies the persistent scheduler binding for a guard or quiescent report. + pub proof fn tracked_rcu_binding(tracked &self) -> (tracked binding: + CpuRcuParticipantBinding) + requires + self.wf(), + ensures + binding.scheduler() == self.scheduler(), + binding.cpu() == self.cpu(), + binding.participant_id() == self.rcu_participant_id(), + { + self.rcu_binding.tracked_duplicate() + } + + /// Returns a completed reader to this CPU's persistent participant. + pub proof fn tracked_stop_rcu_reader(tracked &mut self, tracked reader: CpuRcuReaderFragment) + requires + old(self).wf(), + old(self).preempt_depth() > 0, + reader.wf(), + reader.participant_id() == old(self).rcu_participant_id(), + ensures + final(self).wf(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction() + reader.fraction(), + { + self.rcu_participant.tracked_stop_reader_in_place(reader); + } + + /// Closes the current CPU participation generation at a quiescent point. + /// + /// Unlike [`Self::tracked_record_quiescent`], this transition is backed by + /// the persistent CPU participant PCM and returns an unforgeable token + /// that conflicts with every reader fragment from the closed generation. + pub proof fn tracked_report_rcu_quiescent(tracked &mut self) -> (tracked closed: + CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).is_quiescent(), + ensures + closed.wf(), + closed.scheduler() == old(self).scheduler(), + closed.participant_id() == old(self).rcu_participant_id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).rcu_generation(), + closed.view() == old(self).view(), + final(self).wf(), + final(self).is_quiescent(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation() + 1, + final(self).rcu_participant_view() == old(self).view(), + final(self).rcu_fraction() == 1real, + { + let tracked binding = self.rcu_binding.tracked_duplicate(); + self.rcu_participant.tracked_report_quiescent_in_place(binding, self.view()) + } + + /// Closes the current CPU generation while publishing retirement facts + /// whose detachment observations are covered by this task's current view. + pub proof fn tracked_report_rcu_quiescent_with( + tracked &mut self, + tracked learned: &RcuRetiredFacts, + ) -> (tracked closed: CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).is_quiescent(), + learned.observed_by(old(self).view()), + ensures + closed.wf(), + closed.scheduler() == old(self).scheduler(), + closed.participant_id() == old(self).rcu_participant_id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).rcu_generation(), + closed.view() == old(self).view(), + learned.records().subset_of(closed.known_retired()), + final(self).wf(), + final(self).is_quiescent(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == old(self).view(), + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation() + 1, + final(self).rcu_participant_view() == old(self).view(), + final(self).rcu_fraction() == 1real, + { + let tracked binding = self.rcu_binding.tracked_duplicate(); + self.rcu_participant.tracked_report_quiescent_with_in_place( + binding, + self.view(), + learned, + ) + } + /// Records one quiescent boundary for this running session. /// /// The returned generation names the interval that has just ended. The @@ -505,6 +735,10 @@ impl RunningTaskContext { final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), final(self).wf(), final(self).is_quiescent(), { @@ -513,17 +747,28 @@ impl RunningTaskContext { /// Ends a running interval and returns the updated task view to scheduler /// ownership. The full-fraction requirement rules out live preempt guards. - pub proof fn tracked_into_task_view(tracked self) -> (tracked res: TaskThreadView) + pub proof fn tracked_into_task_view(tracked self) -> (tracked res: ( + TaskThreadView, + CpuRcuParticipant, + )) requires self.wf(), self.preempt_depth() == 0, + self.rcu_fraction() == 1real, ensures - res.scheduler() == self.scheduler(), - res.task() == self.task(), - res.view() == self.view(), + res.0.scheduler() == self.scheduler(), + res.0.task() == self.task(), + res.0.view() == self.view(), + res.1.id() == self.rcu_participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.rcu_generation(), + res.1.view() == self.rcu_participant_view(), + res.1.fraction() == 1real, + res.1.view().spec_le(res.0.view()), + res.1.wf(), { assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); - self.session.tracked_into_task_view() + (self.session.tracked_into_task_view(), self.rcu_participant) } /// Scheduler-facing form of `tracked_into_task_view` that preserves the @@ -531,21 +776,28 @@ impl RunningTaskContext { pub proof fn tracked_into_task_view_for_scheduler( tracked self, sched_view: SchedulerView, - ) -> (tracked res: TaskThreadView) + ) -> (tracked res: (TaskThreadView, CpuRcuParticipant)) requires self.wf_scheduler(sched_view), self.is_quiescent(), ensures - res.scheduler() == self.scheduler(), - res.task() == self.task(), - res.view() == self.view(), - res.wf(sched_view), + res.0.scheduler() == self.scheduler(), + res.0.task() == self.task(), + res.0.view() == self.view(), + res.0.wf(sched_view), + res.1.id() == self.rcu_participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.rcu_generation(), + res.1.view() == self.rcu_participant_view(), + res.1.fraction() == 1real, + res.1.view().spec_le(res.0.view()), + res.1.wf(), { assert(self.preempt_depth() == 0); assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); assert(self.session.wf(sched_view)); - let tracked res = self.session.tracked_into_task_view_for_scheduler(sched_view); - res + let tracked task_view = self.session.tracked_into_task_view_for_scheduler(sched_view); + (task_view, self.rcu_participant) } } @@ -667,6 +919,10 @@ impl RunningTaskContext { final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() + 1 == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth() + 1, + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), resource.matches_context(*final(self)), resource.is_outermost() <==> old(self).preempt_depth() == 0, resource.is_nested() <==> old(self).preempt_depth() > 0, @@ -702,6 +958,10 @@ impl RunningTaskContext { final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions() + 1, final(self).preempt_depth() + 1 == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), { let ghost old_depth = self.preempt_depth@; resource.tracked_return_to_session(&mut self.session); @@ -722,7 +982,7 @@ pub struct DisabledPreemptGuard { // // The guard only records whether this scope is outermost or nested. The // checked-out `TaskThreadView` is owned by `PreemptThreadViewSession`. - tracked_resource: Tracked, + tracked_resource: Tracked>, } /* impl !Send for DisabledPreemptGuard {} @@ -738,40 +998,54 @@ impl DisabledPreemptGuard { tracked_resource.wf(arbitrary()), ensures res.wf(arbitrary()), - res.tracked_resource@ == tracked_resource, + res.tracked_resource@ == Some(tracked_resource), { // The CPU-local backend is outside the current Verus dependency // closure, but executable builds must still perform the real // preemption-disable transition. #[cfg(not(verus_keep_ghost))] super::cpu_local::inc_guard_count(); - Self { _private: (), tracked_resource: Tracked(tracked_resource) } + Self { _private: (), tracked_resource: Tracked(Some(tracked_resource)) } } } impl DisabledPreemptGuard { + pub(crate) closed spec fn has_resource(&self) -> bool { + self.tracked_resource@ is Some + } + + closed spec fn resource(&self) -> PreemptGuardResource + recommends + self.tracked_resource@ is Some, + { + self.tracked_resource@->Some_0 + } + pub closed spec fn is_outermost(&self) -> bool { - self.tracked_resource@.is_outermost() + self.resource().is_outermost() } pub closed spec fn is_nested(&self) -> bool { - self.tracked_resource@.is_nested() + self.resource().is_nested() } pub closed spec fn wf(&self, sched_view: SchedulerView) -> bool { - self.tracked_resource@.wf(sched_view) + &&& self.tracked_resource@ is Some + &&& self.resource().wf(sched_view) } pub closed spec fn matches_session(&self, session: PreemptThreadViewSession) -> bool { - self.tracked_resource@.matches_session(session) + &&& self.tracked_resource@ is Some + &&& self.resource().matches_session(session) } pub closed spec fn matches_context(&self, context: RunningTaskContext) -> bool { - self.tracked_resource@.matches_context(context) + &&& self.tracked_resource@ is Some + &&& self.resource().matches_context(context) } pub closed spec fn quiescent_generation(&self) -> nat { - self.tracked_resource@.quiescent_generation() + self.resource().quiescent_generation() } /// Extracts the positive preemption depth witnessed by this guard. @@ -817,13 +1091,13 @@ impl DisabledPreemptGuard { { assert(before.session.session_task() == before.task()); assert(after.session.session_task() == after.task()); - assert(self.tracked_resource@.session_token().task() == before.task()); - assert(self.tracked_resource@.session_token().task() == after.task()); - assert(self.tracked_resource@.session_token().quiescent_generation() + assert(self.resource().session_token().task() == before.task()); + assert(self.resource().session_token().task() == after.task()); + assert(self.resource().session_token().quiescent_generation() == before.quiescent_generation()); - assert(self.tracked_resource@.session_token().quiescent_generation() + assert(self.resource().session_token().quiescent_generation() == after.quiescent_generation()); - assert(after.session.token_matches(self.tracked_resource@.session_token())); + assert(after.session.token_matches(self.resource().session_token())); } /// Borrows the running task's view while this guard witnesses that @@ -846,16 +1120,28 @@ impl DisabledPreemptGuard { final(context).available_fractions() == old(context).available_fractions(), final(context).has_full_authority() == old(context).has_full_authority(), final(context).preempt_depth() == old(context).preempt_depth(), - final(context).wf(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), final(context).view() == (*final(tv))@, - guard.matches_context(*final(context)), + old(context).view().spec_le((*final(tv))@) ==> final(context).wf(), + old(context).view().spec_le((*final(tv))@) ==> guard.matches_context(*final(context)), { context.tracked_borrow_thread_view_mut() } - /// Consumes this guard, returns its fractional witness, and decrements the - /// modeled preemption depth. - pub(crate) fn release_to_context(self, Tracked(context): Tracked<&mut RunningTaskContext>) + /// Returns this guard's fractional witness and decrements the modeled + /// preemption depth. + /// + /// The proof resource is stored in an `Option` so a containing guard's + /// standard `Drop::drop(&mut self)` can consume it exactly once. The + /// executable preemption counter is still decremented by this guard's Rust + /// destructor. + pub(crate) fn release_in_place_to_context( + &mut self, + Tracked(context): Tracked<&mut RunningTaskContext>, + ) requires old(context).wf(), old(context).preempt_depth() > 0, @@ -870,14 +1156,50 @@ impl DisabledPreemptGuard { final(context).quiescent_generation() == old(context).quiescent_generation(), final(context).available_fractions() == old(context).available_fractions() + 1, final(context).preempt_depth() + 1 == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + !final(self).has_resource(), + opens_invariants none + no_unwind { proof_decl! { - let tracked resource = self.tracked_resource.get(); + let tracked resource = self.tracked_resource.borrow_mut().tracked_take(); } proof { context.tracked_enable_preempt(resource); } } + + /// Consuming compatibility wrapper for callers that do not need standard + /// destructor integration. + pub(crate) fn release_to_context( + self, + Tracked(context): Tracked<&mut RunningTaskContext>, + ) + requires + old(context).wf(), + old(context).preempt_depth() > 0, + self.matches_context(*old(context)), + ensures + final(context).wf(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).view() == old(context).view(), + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() == old(context).available_fractions() + 1, + final(context).preempt_depth() + 1 == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + { + let mut this = self; + this.release_in_place_to_context(Tracked(context)); + } } } // verus! @@ -932,6 +1254,11 @@ pub(crate) fn disable_preempt_in_context( final(context).quiescent_generation() == old(context).quiescent_generation(), final(context).available_fractions() + 1 == old(context).available_fractions(), final(context).preempt_depth() == old(context).preempt_depth() + 1, + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + res.has_resource(), res.is_outermost() <==> old(context).preempt_depth() == 0, res.is_nested() <==> old(context).preempt_depth() > 0, res.matches_context(*final(context)), diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index e7529bb6f..5d5c581fb 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -69,7 +69,10 @@ use vstd::{map::Map, prelude::*, resource::Loc}; use super::{Task, preempt::RunningTaskContext}; use crate::{ specs::mm::cpu::CpuId, - specs::sync::weak_memory::{ThreadView, WmView}, + specs::sync::{ + rcu_cpu::{CpuRcuParticipant, CpuRcuParticipantBinding}, + weak_memory::{ThreadView, WmView}, + }, sync::{OnceImpl, RoArc, TrivialPred}, }; @@ -131,6 +134,9 @@ pub ghost enum TaskSchedState { /// weak-memory view. `stored_views` records views still owned by the scheduler /// resource; `checked_out_views` records views temporarily held by guards. /// `cpu_views` persists observations across context switches on each CPU. +/// `cpu_rcu_participant_ids` permanently binds each registered CPU to one RCU +/// participant resource. `stored_cpu_rcu_participants` records which complete +/// participant resources are currently owned by the scheduler. pub ghost struct SchedulerView { pub id: Loc, pub runqueues: Map>, @@ -140,6 +146,8 @@ pub ghost struct SchedulerView { pub stored_views: Map, pub checked_out_views: Map, pub cpu_views: Map, + pub cpu_rcu_participant_ids: Map, + pub stored_cpu_rcu_participants: Set, } impl SchedulerView { @@ -154,6 +162,8 @@ impl SchedulerView { stored_views: Map::empty(), checked_out_views: Map::empty(), cpu_views: Map::empty(), + cpu_rcu_participant_ids: Map::empty(), + stored_cpu_rcu_participants: Set::empty(), } } @@ -197,6 +207,21 @@ impl SchedulerView { self.cpu_views[cpu] } + pub open spec fn cpu_has_rcu_participant(self, cpu: CpuId) -> bool { + self.cpu_rcu_participant_ids.contains_key(cpu) + } + + pub open spec fn cpu_rcu_participant_id(self, cpu: CpuId) -> Loc + recommends + self.cpu_has_rcu_participant(cpu), + { + self.cpu_rcu_participant_ids[cpu] + } + + pub open spec fn cpu_rcu_participant_is_stored(self, cpu: CpuId) -> bool { + self.stored_cpu_rcu_participants.contains(cpu) + } + /// The scheduling policy changed no weak-memory ownership state. /// /// This relation deliberately ignores runqueues, current tasks, and task @@ -209,6 +234,8 @@ impl SchedulerView { &&& self.stored_views == other.stored_views &&& self.checked_out_views == other.checked_out_views &&& self.cpu_views == other.cpu_views + &&& self.cpu_rcu_participant_ids == other.cpu_rcu_participant_ids + &&& self.stored_cpu_rcu_participants == other.stored_cpu_rcu_participants } pub open spec fn task_in_runqueue(self, task: Loc) -> bool { @@ -274,23 +301,31 @@ impl SchedulerView { /// /// Scheduler policy may populate that CPU's runqueue/current slot only /// after this transition. The initial empty view carries no observations. - pub open spec fn register_cpu(self, cpu: CpuId) -> SchedulerView + pub open spec fn register_cpu(self, cpu: CpuId, rcu_participant_id: Loc) -> SchedulerView recommends !self.cpu_views.contains_key(cpu), valid_cpu(cpu), { - SchedulerView { cpu_views: self.cpu_views.insert(cpu, WmView::empty()), ..self } + SchedulerView { + cpu_views: self.cpu_views.insert(cpu, WmView::empty()), + cpu_rcu_participant_ids: self.cpu_rcu_participant_ids.insert(cpu, rcu_participant_id), + stored_cpu_rcu_participants: self.stored_cpu_rcu_participants.insert(cpu), + ..self + } } - pub proof fn lemma_register_cpu_preserves_wf(self, cpu: CpuId) + pub proof fn lemma_register_cpu_preserves_wf(self, cpu: CpuId, rcu_participant_id: Loc) requires self.wf(), !self.cpu_views.contains_key(cpu), valid_cpu(cpu), ensures - self.register_cpu(cpu).wf(), - self.register_cpu(cpu).cpu_has_thread_view(cpu), - self.register_cpu(cpu).cpu_thread_view(cpu) == WmView::empty(), + self.register_cpu(cpu, rcu_participant_id).wf(), + self.register_cpu(cpu, rcu_participant_id).cpu_has_thread_view(cpu), + self.register_cpu(cpu, rcu_participant_id).cpu_thread_view(cpu) == WmView::empty(), + self.register_cpu(cpu, rcu_participant_id).cpu_rcu_participant_id(cpu) + == rcu_participant_id, + self.register_cpu(cpu, rcu_participant_id).cpu_rcu_participant_is_stored(cpu), { } @@ -327,6 +362,7 @@ impl SchedulerView { task_views: self.task_views.insert(task, joined), stored_views: self.stored_views.remove(task), checked_out_views: self.checked_out_views.insert(task, joined), + stored_cpu_rcu_participants: self.stored_cpu_rcu_participants.remove(cpu), ..self } } @@ -367,20 +403,38 @@ impl SchedulerView { /// /// The caller must provide the same view that is recorded as checked out; /// this prevents check-in from overwriting the task with an unrelated view. - pub open spec fn checkin_task_view(self, task: Loc, view: WmView) -> SchedulerView + pub open spec fn checkin_task_view(self, cpu: CpuId, task: Loc, view: WmView) -> SchedulerView recommends self.task_view_is_checked_out(task), !self.task_view_is_stored(task), self.checked_out_views[task] == view, + self.current.contains_key(cpu), + self.current[cpu] == Some(task), + !self.cpu_rcu_participant_is_stored(cpu), { SchedulerView { task_views: self.task_views.insert(task, view), stored_views: self.stored_views.insert(task, view), checked_out_views: self.checked_out_views.remove(task), + stored_cpu_rcu_participants: self.stored_cpu_rcu_participants.insert(cpu), ..self } } + pub proof fn lemma_checkin_task_view_preserves_wf(self, cpu: CpuId, task: Loc, view: WmView) + requires + self.wf(), + self.task_view_is_checked_out(task), + !self.task_view_is_stored(task), + self.checked_out_views[task] == view, + self.current.contains_key(cpu), + self.current[cpu] == Some(task), + !self.cpu_rcu_participant_is_stored(cpu), + ensures + self.checkin_task_view(cpu, task, view).wf(), + { + } + /// Publishes the outgoing task's observations into the persistent CPU /// view. A subsequent task scheduled on this CPU imports the result in /// `checkout_task_view`. @@ -410,6 +464,7 @@ impl SchedulerView { { self.cpu_thread_view(cpu).lemma_join_left(view); self.cpu_thread_view(cpu).lemma_join_right(view); + assert(self.publish_cpu_view(cpu, view).cpu_views.dom() == self.cpu_views.dom()); } pub open spec fn wf(self) -> bool { @@ -423,8 +478,18 @@ impl SchedulerView { self.cpu_views.contains_key(cpu) ==> valid_cpu( cpu, ) - // Runqueues contain exactly runnable tasks; current slots contain - // running tasks. + // The complete RCU participant follows the same checkout boundary as + // the current task view, while its identity remains CPU-stable. + &&& self.cpu_rcu_participant_ids.dom() == self.cpu_views.dom() + &&& self.stored_cpu_rcu_participants.subset_of(self.cpu_views.dom()) + &&& forall|cpu: CpuId| #[trigger] + self.cpu_views.contains_key(cpu) ==> (self.stored_cpu_rcu_participants.contains(cpu) + <==> !(self.current.contains_key(cpu) && self.current[cpu] is Some + && self.checked_out_views.contains_key( + self.current[cpu]->0, + ))) + // Runqueues contain exactly runnable tasks; current slots contain + // running tasks. &&& forall|cpu: CpuId, idx: int| #![trigger self.runqueues[cpu][idx]] self.runqueues.contains_key(cpu) && 0 <= idx && idx < self.runqueues[cpu].len() @@ -484,6 +549,8 @@ tracked struct SchedulerThreadViews { scheduler: Ghost, views: Map, cpu_views: Map, + rcu_participants: Map, + rcu_bindings: Map, } /// A checked-out per-task `ThreadView`. @@ -569,10 +636,20 @@ impl SchedulerThreadViews { res.scheduler() == scheduler, res.view() == Map::::empty(), res.cpu_view_map() == Map::::empty(), + res.rcu_participant_id_map() == Map::::empty(), + res.rcu_binding_id_map() == Map::::empty(), { let tracked views = Map::::tracked_empty(); let tracked cpu_views = Map::::tracked_empty(); - SchedulerThreadViews { scheduler: Ghost(scheduler), views, cpu_views } + let tracked rcu_participants = Map::::tracked_empty(); + let tracked rcu_bindings = Map::::tracked_empty(); + SchedulerThreadViews { + scheduler: Ghost(scheduler), + views, + cpu_views, + rcu_participants, + rcu_bindings, + } } closed spec fn scheduler(self) -> Loc { @@ -587,6 +664,14 @@ impl SchedulerThreadViews { Map::new(self.cpu_views.dom(), |cpu: CpuId| self.cpu_views[cpu]@) } + pub closed spec fn rcu_participant_id_map(self) -> Map { + Map::new(self.rcu_participants.dom(), |cpu: CpuId| self.rcu_participants[cpu].id()) + } + + pub closed spec fn rcu_binding_id_map(self) -> Map { + Map::new(self.rcu_bindings.dom(), |cpu: CpuId| self.rcu_bindings[cpu].participant_id()) + } + pub closed spec fn contains(self, task: Loc) -> bool { self.views.contains_key(task) } @@ -609,6 +694,44 @@ impl SchedulerThreadViews { self.cpu_views[cpu]@ } + closed spec fn rcu_participants_wf(self, sched_view: SchedulerView) -> bool { + &&& self.rcu_participants.dom() == sched_view.stored_cpu_rcu_participants + &&& forall|cpu: CpuId| #[trigger] + self.rcu_participants.contains_key(cpu) ==> { + &&& self.rcu_participants[cpu].wf() + &&& self.rcu_participants[cpu].cpu() == cpu + &&& self.rcu_participants[cpu].id() == sched_view.cpu_rcu_participant_ids[cpu] + &&& self.rcu_participants[cpu].fraction() == 1real + &&& self.rcu_participants[cpu].view().spec_le(self.cpu_views[cpu]@) + } + } + + closed spec fn rcu_bindings_wf(self, sched_view: SchedulerView) -> bool { + &&& self.rcu_bindings.dom() == sched_view.cpu_rcu_participant_ids.dom() + &&& forall|cpu: CpuId| #[trigger] + self.rcu_bindings.contains_key(cpu) ==> { + &&& self.rcu_bindings[cpu].scheduler() == sched_view.id + &&& self.rcu_bindings[cpu].cpu() == cpu + &&& self.rcu_bindings[cpu].participant_id() + == sched_view.cpu_rcu_participant_ids[cpu] + } + } + + proof fn lemma_rcu_participants_frame( + tracked &self, + before: SchedulerView, + after: SchedulerView, + ) + requires + self.rcu_participants_wf(before), + before.stored_cpu_rcu_participants == after.stored_cpu_rcu_participants, + before.cpu_rcu_participant_ids == after.cpu_rcu_participant_ids, + before.cpu_views == after.cpu_views, + ensures + self.rcu_participants_wf(after), + { + } + /// The tracked owner contains exactly the views still stored in scheduler /// state. Checked-out views are represented by `TaskThreadView` tokens /// instead, so they are intentionally absent here. @@ -616,41 +739,81 @@ impl SchedulerThreadViews { &&& self.scheduler() == sched_view.id &&& self.view() == sched_view.stored_views &&& self.cpu_view_map() == sched_view.cpu_views + &&& self.rcu_participants_wf(sched_view) + &&& self.rcu_bindings_wf(sched_view) } - proof fn tracked_register_cpu(tracked &mut self, sched_view: SchedulerView, cpu: CpuId) + proof fn tracked_register_cpu( + tracked &mut self, + tracked identity: &mut GhostMapAuth, + sched_view: SchedulerView, + cpu: CpuId, + ) -> (participant_id: Loc) requires old(self).wf(sched_view), sched_view.wf(), !sched_view.cpu_has_thread_view(cpu), valid_cpu(cpu), + old(identity).id() == sched_view.id, + old(identity)@ == sched_view.cpu_rcu_participant_ids, ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), - final(self).cpu_view_map() == sched_view.register_cpu(cpu).cpu_views, - final(self).wf(sched_view.register_cpu(cpu)), + final(self).cpu_view_map() == sched_view.register_cpu(cpu, participant_id).cpu_views, + final(self).wf(sched_view.register_cpu(cpu, participant_id)), final(self).contains_cpu(cpu), final(self).cpu_thread_view(cpu) == WmView::empty(), + final(self).rcu_participant_id_map().contains_key(cpu), + final(self).rcu_participant_id_map()[cpu] == participant_id, + final(self).rcu_binding_id_map() == sched_view.register_cpu( + cpu, + participant_id, + ).cpu_rcu_participant_ids, + final(identity).id() == old(identity).id(), + final(identity)@ == sched_view.register_cpu( + cpu, + participant_id, + ).cpu_rcu_participant_ids, { - sched_view.lemma_register_cpu_preserves_wf(cpu); let tracked cpu_view = ThreadView::new(); + let tracked participant = CpuRcuParticipant::new(cpu, WmView::empty()); + let ghost participant_id = participant.id(); + let tracked entry = identity.insert(cpu, participant_id); + let tracked binding = CpuRcuParticipantBinding::tracked_new(entry); + sched_view.lemma_register_cpu_preserves_wf(cpu, participant_id); self.cpu_views.tracked_insert(cpu, cpu_view); - assert(final(self).cpu_view_map() == sched_view.register_cpu(cpu).cpu_views); - assert(final(self).wf(sched_view.register_cpu(cpu))); + self.rcu_participants.tracked_insert(cpu, participant); + self.rcu_bindings.tracked_insert(cpu, binding); + assert(final(self).cpu_view_map() == sched_view.register_cpu( + cpu, + participant_id, + ).cpu_views); + assert(final(self).wf(sched_view.register_cpu(cpu, participant_id))); + participant_id } /// Inserts a task view created during task registration. /// /// This is separate from check-in: initial insertion creates a new stored /// entry, while check-in returns an existing checked-out view. - proof fn tracked_insert_initial_thread_view(tracked &mut self, tracked token: TaskThreadView) + proof fn tracked_insert_initial_thread_view( + tracked &mut self, + tracked token: TaskThreadView, + rcu_view: SchedulerView, + ) requires !old(self).contains(token.task()), token.scheduler() == old(self).scheduler(), + old(self).rcu_participants_wf(rcu_view), + old(self).rcu_bindings_wf(rcu_view), ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(token.task(), token.view()), final(self).cpu_view_map() == old(self).cpu_view_map(), + final(self).rcu_participants == old(self).rcu_participants, + final(self).rcu_participants_wf(rcu_view), + final(self).rcu_bindings == old(self).rcu_bindings, + final(self).rcu_bindings_wf(rcu_view), { let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; self.views.tracked_insert(task, thread_view); @@ -677,7 +840,8 @@ impl SchedulerThreadViews { sched_view.lemma_register_task_preserves_wf(task); let tracked thread_view = ThreadView::new(); let tracked token = TaskThreadView::new(self.scheduler(), task, thread_view); - self.tracked_insert_initial_thread_view(token); + self.lemma_rcu_participants_frame(sched_view, sched_view.register_task(task)); + self.tracked_insert_initial_thread_view(token, sched_view.register_task(task)); assert(final(self).view() == sched_view.register_task(task).stored_views); assert(final(self).wf(sched_view.register_task(task))); } @@ -690,7 +854,7 @@ impl SchedulerThreadViews { tracked &mut self, sched_view: SchedulerView, cpu: CpuId, - ) -> (tracked token: TaskThreadView) + ) -> (tracked res: (TaskThreadView, CpuRcuParticipant, CpuRcuParticipantBinding)) requires old(self).wf(sched_view), sched_view.wf(), @@ -701,21 +865,31 @@ impl SchedulerThreadViews { old(self).contains(sched_view.current[cpu]->0), old(self).contains_cpu(cpu), ensures - token.task() == sched_view.current[cpu]->0, - token.scheduler() == sched_view.id, - token.view() == old(self).thread_view(sched_view.current[cpu]->0).join( + res.0.task() == sched_view.current[cpu]->0, + res.0.scheduler() == sched_view.id, + res.0.view() == old(self).thread_view(sched_view.current[cpu]->0).join( old(self).cpu_thread_view(cpu), ), + res.1.id() == sched_view.cpu_rcu_participant_id(cpu), + res.1.cpu() == cpu, + res.1.fraction() == 1real, + res.1.view().spec_le(res.0.view()), + res.1.wf(), + res.2.scheduler() == sched_view.id, + res.2.cpu() == cpu, + res.2.participant_id() == res.1.id(), final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), final(self).cpu_view_map() == old(self).cpu_view_map(), final(self).view() == sched_view.checkout_task_view(cpu).stored_views, final(self).wf(sched_view.checkout_task_view(cpu)), - token.wf(sched_view.checkout_task_view(cpu)), + res.0.wf(sched_view.checkout_task_view(cpu)), { let task = sched_view.current[cpu]->0; let tracked mut thread_view = self.views.tracked_remove(task); let tracked cpu_view = self.cpu_views.tracked_borrow(cpu); + let tracked participant = self.rcu_participants.tracked_remove(cpu); + let tracked binding = self.rcu_bindings.tracked_borrow(cpu).tracked_duplicate(); thread_view.tracked_join(cpu_view); let tracked token = TaskThreadView { scheduler: Ghost(self.scheduler()), @@ -726,7 +900,7 @@ impl SchedulerThreadViews { assert(final(self).view() == next.stored_views); assert(final(self).wf(next)); assert(token.wf(next)); - token + (token, participant, binding) } /// Checks out the current task's weak-memory view and starts its running @@ -759,43 +933,14 @@ impl SchedulerThreadViews { final(self).wf(sched_view.checkout_task_view(cpu)), { let ghost next = sched_view.checkout_task_view(cpu); - let tracked task_view = self.tracked_take_current_thread_view(sched_view, cpu); - let tracked context = RunningTaskContext::new(task_view, next, cpu); + let tracked (task_view, participant, binding) = self.tracked_take_current_thread_view( + sched_view, + cpu, + ); + let tracked context = RunningTaskContext::new(task_view, participant, binding, next, cpu); context } - /// Returns a checked-out task view to the tracked owner. - /// - /// The `token.wf(sched_view)` precondition ties this write-back to the - /// scheduler's checked-out partition, so the task id and view cannot be - /// swapped with another task. - proof fn tracked_put_checked_out_thread_view( - tracked &mut self, - sched_view: SchedulerView, - tracked token: TaskThreadView, - ) - requires - old(self).wf(sched_view), - token.wf(sched_view), - !old(self).contains(token.task()), - ensures - final(self).scheduler() == old(self).scheduler(), - final(self).view() == old(self).view().insert(token.task(), token.view()), - final(self).cpu_view_map() == old(self).cpu_view_map(), - final(self).view() == sched_view.checkin_task_view( - token.task(), - token.view(), - ).stored_views, - final(self).wf(sched_view.checkin_task_view(token.task(), token.view())), - { - let ghost view = token.view(); - let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; - self.views.tracked_insert(task, thread_view); - let next = sched_view.checkin_task_view(task, view); - assert(final(self).view() == next.stored_views); - assert(final(self).wf(next)); - } - /// Ends a quiescent running interval and checks its updated task view back /// into scheduler ownership. proof fn tracked_put_running_context( @@ -811,6 +956,8 @@ impl SchedulerThreadViews { sched_view.current.contains_key(context.cpu()), sched_view.current[context.cpu()] == Some(context.task()), sched_view.cpu_has_thread_view(context.cpu()), + sched_view.cpu_has_rcu_participant(context.cpu()), + sched_view.cpu_rcu_participant_id(context.cpu()) == context.rcu_participant_id(), context.wf(), context.is_quiescent(), !old(self).contains(context.task()), @@ -826,7 +973,7 @@ impl SchedulerThreadViews { sched_view.update_checked_out_task_view( context.task(), context.view(), - ).checkin_task_view(context.task(), context.view()).publish_cpu_view( + ).checkin_task_view(context.cpu(), context.task(), context.view()).publish_cpu_view( context.cpu(), context.view(), ), @@ -838,14 +985,35 @@ impl SchedulerThreadViews { sched_view.lemma_update_checked_out_task_view_preserves_wf(task, view); let ghost updated = sched_view.update_checked_out_task_view(task, view); context.lemma_wf_scheduler(updated); - let tracked task_view = context.tracked_into_task_view_for_scheduler(updated); + let tracked (task_view, participant) = context.tracked_into_task_view_for_scheduler( + updated, + ); let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = task_view; let tracked cpu_view = self.cpu_views.tracked_borrow_mut(cpu); cpu_view.tracked_join(&thread_view); self.views.tracked_insert(task, thread_view); - let ghost checked = updated.checkin_task_view(task, view); + self.rcu_participants.tracked_insert(cpu, participant); + updated.lemma_checkin_task_view_preserves_wf(cpu, task, view); + let ghost checked = updated.checkin_task_view(cpu, task, view); + assert(checked.wf()); checked.lemma_publish_cpu_view_preserves_wf(cpu, view); let ghost next = checked.publish_cpu_view(cpu, view); + assert(next.wf()); + assert(participant.view().spec_le(view)); + old(self).cpu_thread_view(cpu).lemma_join_right(view); + participant.view().lemma_spec_le_transitive(view, self.cpu_views[cpu]@); + assert forall|stored_cpu: CpuId| #[trigger] + self.rcu_participants.contains_key(stored_cpu) implies { + &&& self.rcu_participants[stored_cpu].wf() + &&& self.rcu_participants[stored_cpu].cpu() == stored_cpu + &&& self.rcu_participants[stored_cpu].id() == next.cpu_rcu_participant_ids[stored_cpu] + &&& self.rcu_participants[stored_cpu].fraction() == 1real + &&& self.rcu_participants[stored_cpu].view().spec_le(self.cpu_views[stored_cpu]@) + } by { + if stored_cpu == cpu { + assert(self.rcu_participants[stored_cpu] == participant); + } + }; assert(final(self).view() == next.stored_views); assert(final(self).cpu_view_map() == next.cpu_views); assert(final(self).wf(next)); @@ -859,7 +1027,7 @@ impl SchedulerThreadViews { /// layers together, preventing a proof from changing `SchedulerView` without /// moving the corresponding linear token (or vice versa). pub tracked struct SchedulerGhostState { - identity: GhostMapAuth, + identity: GhostMapAuth, view: Ghost, thread_views: SchedulerThreadViews, } @@ -871,7 +1039,7 @@ impl SchedulerGhostState { res.wf(), res.view() == SchedulerView::empty(res.id()), { - let tracked (identity, _entries) = GhostMapAuth::new(Map::::empty()); + let tracked (identity, _entries) = GhostMapAuth::new(Map::::empty()); let ghost id = identity.id(); SchedulerView::lemma_empty_wf(id); let tracked thread_views = SchedulerThreadViews::empty(id); @@ -895,6 +1063,7 @@ impl SchedulerGhostState { pub closed spec fn wf(self) -> bool { &&& self.view().wf() &&& self.view().id == self.id() + &&& self.identity@ == self.view().cpu_rcu_participant_ids &&& self.thread_views.wf(self.view()) } @@ -929,13 +1098,21 @@ impl SchedulerGhostState { ensures final(self).wf(), final(self).id() == old(self).id(), - final(self).view() == old(self).view().register_cpu(cpu), + final(self).view() == old(self).view().register_cpu( + cpu, + final(self).view().cpu_rcu_participant_id(cpu), + ), final(self).view().cpu_has_thread_view(cpu), final(self).view().cpu_thread_view(cpu) == WmView::empty(), + final(self).view().cpu_rcu_participant_is_stored(cpu), { let ghost old_view = self.view@; - self.thread_views.tracked_register_cpu(old_view, cpu); - self.view = Ghost(old_view.register_cpu(cpu)); + let ghost participant_id = self.thread_views.tracked_register_cpu( + &mut self.identity, + old_view, + cpu, + ); + self.view = Ghost(old_view.register_cpu(cpu, participant_id)); assert(self.wf()); } @@ -1000,6 +1177,8 @@ impl SchedulerGhostState { old(self).view().current.contains_key(context.cpu()), old(self).view().current[context.cpu()] == Some(context.task()), old(self).view().cpu_has_thread_view(context.cpu()), + old(self).view().cpu_has_rcu_participant(context.cpu()), + old(self).view().cpu_rcu_participant_id(context.cpu()) == context.rcu_participant_id(), context.wf(), context.is_quiescent(), ensures @@ -1008,7 +1187,7 @@ impl SchedulerGhostState { final(self).view() == old(self).view().update_checked_out_task_view( context.task(), context.view(), - ).checkin_task_view(context.task(), context.view()).publish_cpu_view( + ).checkin_task_view(context.cpu(), context.task(), context.view()).publish_cpu_view( context.cpu(), context.view(), ), @@ -1024,10 +1203,18 @@ impl SchedulerGhostState { let ghost next = old_view.update_checked_out_task_view( task, context_view, - ).checkin_task_view(task, context_view).publish_cpu_view(cpu, context_view); + ).checkin_task_view(cpu, task, context_view).publish_cpu_view(cpu, context_view); self.thread_views.tracked_put_running_context(old_view, context); + old_view.lemma_update_checked_out_task_view_preserves_wf(task, context_view); + let ghost updated = old_view.update_checked_out_task_view(task, context_view); + updated.lemma_checkin_task_view_preserves_wf(cpu, task, context_view); + let ghost checked = updated.checkin_task_view(cpu, task, context_view); + checked.lemma_publish_cpu_view_preserves_wf(cpu, context_view); + assert(next.wf()); self.view = Ghost(next); old_view.cpu_thread_view(cpu).lemma_join_right(context_view); + assert(self.thread_views.wf(next)); + assert(next.id == self.id()); assert(self.wf()); } } diff --git a/verified_libs/vstd_extra/src/atomic_weak.rs b/verified_libs/vstd_extra/src/atomic_weak.rs index 8c5a5da05..0794f2ebc 100644 --- a/verified_libs/vstd_extra/src/atomic_weak.rs +++ b/verified_libs/vstd_extra/src/atomic_weak.rs @@ -131,6 +131,23 @@ impl WmView { { } + /// Observing one location can only advance a thread view. + pub proof fn lemma_observe(self, id: AtomicId, ts: Timestamp) + ensures + self.spec_le(self.observe(id, ts)), + { + } + + /// An acquire read can only advance a thread view. + pub proof fn lemma_acquire(self, id: AtomicId, ts: Timestamp, published: Self) + ensures + self.spec_le(self.observe(id, ts).join(published)), + { + self.lemma_observe(id, ts); + self.observe(id, ts).lemma_join_left(published); + self.lemma_spec_le_transitive(self.observe(id, ts), self.observe(id, ts).join(published)); + } + pub proof fn lemma_spec_le_transitive(self, middle: Self, upper: Self) requires self.spec_le(middle), @@ -460,8 +477,12 @@ macro_rules! declare_weak_atomic_type { pub fn load_relaxed( &self, Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) { + ) -> (res: ($value_ty, Ghost)) + ensures + old(tv)@.spec_le(final(tv)@), + { let result; + let ghost start_view = tv@; proof { use_type_invariant(self); } @@ -474,6 +495,7 @@ macro_rules! declare_weak_atomic_type { } result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); proof { + start_view.lemma_observe(self.id(), result.1@); pair = (hist, g); } }); diff --git a/verified_libs/vstd_extra/src/rcu_read_pool.rs b/verified_libs/vstd_extra/src/rcu_read_pool.rs index 416991ae8..bd47f52c3 100644 --- a/verified_libs/vstd_extra/src/rcu_read_pool.rs +++ b/verified_libs/vstd_extra/src/rcu_read_pool.rs @@ -7,7 +7,7 @@ //! only after all leases have been returned and the pool fraction is whole. use vstd::{ prelude::*, - resource::{frac_opt::Frac, Loc}, + resource::{Loc, frac_opt::Frac}, }; verus! { @@ -30,6 +30,263 @@ pub tracked struct RcuReadPoolRegistry { pools: Map>, } +/// One allocation-indexed lease issued by a tracked registry. +/// +/// The private `lease_id` names the matching active record in the registry. +/// Returning a lease must consume that exact record, so a lease cannot be +/// returned to another allocation that happens to store an equal resource. +pub tracked struct RcuTrackedReadLease { + ghost lease_id: nat, + ghost key: K, + lease: RcuReadLease, +} + +/// Registry-side accounting for one outstanding read lease. +/// +/// `W` is a client-provided linear witness. RCU uses it to retain enough of the +/// reader's CPU-generation authority for a completed grace period to rule out +/// this record before reclamation. +pub tracked struct RcuReadLeaseRecord { + ghost key: K, + ghost pool_id: Loc, + ghost fraction: real, + witness: W, +} + +/// Allocation-indexed pools with explicit accounting for every issued lease. +/// +/// Unlike [`RcuReadPoolRegistry`], this registry records every split and only +/// removes the record when the matching lease is returned. Its invariant says +/// that each pool's owner fraction plus all of that allocation's active lease +/// fractions is exactly one. Consequently, proving that an allocation has no +/// active records is sufficient to recover its stored resource. +pub tracked struct RcuTrackedReadPoolRegistry { + pools: Map>, + active: Map>, + ghost next_lease: nat, +} + +impl RcuReadLeaseRecord { + pub closed spec fn key(self) -> K { + self.key + } + + pub closed spec fn pool_id(self) -> Loc { + self.pool_id + } + + pub closed spec fn fraction(self) -> real { + self.fraction + } + + pub closed spec fn witness(self) -> W { + self.witness + } +} + +impl RcuTrackedReadLease { + pub closed spec fn lease_id(self) -> nat { + self.lease_id + } + + pub closed spec fn key(self) -> K { + self.key + } + + pub closed spec fn pool_id(self) -> Loc { + self.lease.id() + } + + pub closed spec fn resource(self) -> T { + self.lease.resource() + } + + pub closed spec fn fraction(self) -> real { + self.lease.fraction() + } + + pub proof fn borrow(tracked &self) -> (tracked resource: &T) + ensures + *resource == self.resource(), + { + self.lease.borrow() + } +} + +/// Sum of active lease fractions for `key` among record IDs below `upto`. +pub open spec fn active_lease_fraction( + active: Map>, + key: K, + upto: nat, +) -> real + decreases upto, +{ + if upto == 0 { + 0real + } else { + let id = (upto - 1) as nat; + active_lease_fraction(active, key, id) + if active.contains_key(id) && active[id].key() + == key { + active[id].fraction() + } else { + 0real + } + } +} + +proof fn lemma_active_fraction_insert_above( + active: Map>, + inserted: nat, + record: RcuReadLeaseRecord, + key: K, + upto: nat, +) + requires + upto <= inserted, + ensures + active_lease_fraction(active.insert(inserted, record), key, upto) == active_lease_fraction( + active, + key, + upto, + ), + decreases upto, +{ + if upto > 0 { + let id = (upto - 1) as nat; + lemma_active_fraction_insert_above(active, inserted, record, key, id); + assert(id < inserted); + assert(active.insert(inserted, record).contains_key(id) == active.contains_key(id)); + if active.contains_key(id) { + assert(active.insert(inserted, record)[id] == active[id]); + } + } +} + +proof fn lemma_active_fraction_insert_next( + active: Map>, + next: nat, + record: RcuReadLeaseRecord, + key: K, +) + ensures + active_lease_fraction(active.insert(next, record), key, next + 1) == active_lease_fraction( + active, + key, + next, + ) + if record.key() == key { + record.fraction() + } else { + 0real + }, +{ + lemma_active_fraction_insert_above(active, next, record, key, next); +} + +proof fn lemma_active_fraction_remove( + active: Map>, + removed: nat, + key: K, + upto: nat, +) + requires + removed < upto, + active.contains_key(removed), + ensures + active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active, + key, + upto, + ) - if active[removed].key() == key { + active[removed].fraction() + } else { + 0real + }, + decreases upto, +{ + let id = (upto - 1) as nat; + if removed == id { + lemma_active_fraction_remove_above(active, removed, key, id); + assert(!active.remove(removed).contains_key(id)); + assert(active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active.remove(removed), + key, + id, + )); + assert(active_lease_fraction(active, key, upto) == active_lease_fraction(active, key, id) + + if active[removed].key() == key { + active[removed].fraction() + } else { + 0real + }); + } else { + assert(removed < id); + lemma_active_fraction_remove(active, removed, key, id); + assert(active.remove(removed).contains_key(id) == active.contains_key(id)); + if active.contains_key(id) { + assert(active.remove(removed)[id] == active[id]); + } + assert(active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active.remove(removed), + key, + id, + ) + if active.contains_key(id) && active[id].key() == key { + active[id].fraction() + } else { + 0real + }); + assert(active_lease_fraction(active, key, upto) == active_lease_fraction(active, key, id) + + if active.contains_key(id) && active[id].key() == key { + active[id].fraction() + } else { + 0real + }); + } +} + +proof fn lemma_active_fraction_remove_above( + active: Map>, + removed: nat, + key: K, + upto: nat, +) + requires + upto <= removed, + ensures + active_lease_fraction(active.remove(removed), key, upto) == active_lease_fraction( + active, + key, + upto, + ), + decreases upto, +{ + if upto > 0 { + let id = (upto - 1) as nat; + lemma_active_fraction_remove_above(active, removed, key, id); + assert(id < removed); + assert(active.remove(removed).contains_key(id) == active.contains_key(id)); + if active.contains_key(id) { + assert(active.remove(removed)[id] == active[id]); + } + } +} + +proof fn lemma_active_fraction_zero( + active: Map>, + key: K, + upto: nat, +) + requires + forall|id: nat| id < upto && active.contains_key(id) ==> active[id].key() != key, + ensures + active_lease_fraction(active, key, upto) == 0real, + decreases upto, +{ + if upto > 0 { + let id = (upto - 1) as nat; + lemma_active_fraction_zero(active, key, id); + } +} + impl RcuReadPool { /// Stores `resource` and creates a whole read pool. pub proof fn new(tracked resource: T) -> (tracked res: Self) @@ -239,6 +496,420 @@ impl RcuReadPoolRegistry { } } +impl RcuTrackedReadPoolRegistry { + /// Creates an empty tracked registry. + pub proof fn empty() -> (tracked res: Self) + ensures + res.wf(), + res.keys() == Set::::empty(), + res.active_ids() == Set::::empty(), + res.next_lease() == 0, + { + RcuTrackedReadPoolRegistry { + pools: Map::tracked_empty(), + active: Map::tracked_empty(), + next_lease: 0, + } + } + + pub closed spec fn keys(self) -> Set { + self.pools.dom() + } + + pub closed spec fn contains(self, key: K) -> bool { + self.pools.contains_key(key) + } + + pub closed spec fn pool(self, key: K) -> RcuReadPool + recommends + self.contains(key), + { + self.pools[key] + } + + pub closed spec fn active_ids(self) -> Set { + self.active.dom() + } + + /// Ghost snapshot used to state the per-allocation accounting invariant. + pub closed spec fn active_records(self) -> Map> { + self.active + } + + pub closed spec fn next_lease(self) -> nat { + self.next_lease + } + + pub closed spec fn active_record(self, lease_id: nat) -> RcuReadLeaseRecord + recommends + self.active_ids().contains(lease_id), + { + self.active[lease_id] + } + + pub open spec fn has_active(self, key: K) -> bool { + exists|lease_id: nat| + self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == key + } + + pub open spec fn wf(self) -> bool { + &&& forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> { + let record = self.active_record(lease_id); + &&& lease_id < self.next_lease() + &&& self.contains(record.key()) + &&& record.pool_id() == self.pool(record.key()).id() + &&& record.fraction() > 0real + } + &&& forall|key: K| #[trigger] + self.contains(key) ==> self.pool(key).fraction() + active_lease_fraction( + self.active_records(), + key, + self.next_lease(), + ) == 1real + } + + /// Registers one allocation and stores its complete ownership resource. + pub proof fn insert(tracked &mut self, key: K, tracked resource: T) + requires + old(self).wf(), + !old(self).contains(key), + ensures + final(self).wf(), + final(self).keys() == old(self).keys().insert(key), + final(self).active_ids() == old(self).active_ids(), + final(self).next_lease() == old(self).next_lease(), + final(self).contains(key), + final(self).pool(key).resource() == resource, + final(self).pool(key).fraction() == 1real, + forall|other: K| + old(self).contains(other) ==> final(self).pool(other) == old(self).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|lease_id: nat| #[trigger] old(self).active_ids().contains(lease_id) implies { + let record = old(self).active_record(lease_id); + &&& lease_id < old(self).next_lease() + &&& old(self).contains(record.key()) + &&& record.pool_id() == old(self).pool(record.key()).id() + &&& record.fraction() > 0real + } by {}; + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + let tracked pool = RcuReadPool::new(resource); + self.pools.tracked_insert(key, pool); + assert forall|lease_id: nat| self.active_ids().contains(lease_id) implies { + &&& lease_id < self.next_lease() + &&& self.contains(self.active_record(lease_id).key()) + &&& self.active_record(lease_id).pool_id() == self.pool( + self.active_record(lease_id).key(), + ).id() + &&& self.active_record(lease_id).fraction() > 0real + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).active_record(lease_id).key() != key); + }; + assert forall|lease_id: nat| + lease_id < self.next_lease() && self.active_records().contains_key( + lease_id, + ) implies self.active_records()[lease_id].key() != key by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).contains(old(self).active_record(lease_id).key())); + }; + assert(active_lease_fraction(self.active_records(), key, self.next_lease()) == 0real) by { + lemma_active_fraction_zero(self.active_records(), key, self.next_lease()); + }; + assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + if other == key { + assert(self.pool(key).fraction() == 1real); + } else { + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + } + }; + } + + /// Splits a lease and installs its client witness in the active registry. + pub proof fn split_lease(tracked &mut self, key: K, tracked witness: W) -> (tracked lease: + RcuTrackedReadLease) + requires + old(self).wf(), + old(self).contains(key), + ensures + final(self).wf(), + final(self).keys() == old(self).keys(), + final(self).next_lease() == old(self).next_lease() + 1, + lease.lease_id() == old(self).next_lease(), + lease.key() == key, + final(self).active_ids() == old(self).active_ids().insert(lease.lease_id()), + final(self).active_record(lease.lease_id()).key() == key, + final(self).active_record(lease.lease_id()).pool_id() == lease.pool_id(), + final(self).active_record(lease.lease_id()).fraction() == lease.fraction(), + final(self).active_record(lease.lease_id()).witness() == witness, + forall|lease_id: nat| + old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) + == old(self).active_record(lease_id), + lease.pool_id() == old(self).pool(key).id(), + lease.resource() == old(self).pool(key).resource(), + lease.fraction() == old(self).pool(key).fraction() / 2real, + final(self).pool(key).id() == old(self).pool(key).id(), + final(self).pool(key).resource() == old(self).pool(key).resource(), + final(self).pool(key).fraction() == old(self).pool(key).fraction() / 2real, + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + let ghost lease_id = self.next_lease; + let tracked pool = self.pools.tracked_borrow_mut(key); + let tracked lease = pool.split_lease(); + lease.lemma_fraction_bounded(); + let ghost pool_id = lease.id(); + let ghost fraction = lease.fraction(); + let tracked record = RcuReadLeaseRecord { key, pool_id, fraction, witness }; + self.active.tracked_insert(lease_id, record); + self.next_lease = lease_id + 1; + + assert forall|active_id: nat| self.active_ids().contains(active_id) implies { + &&& active_id < self.next_lease() + &&& self.contains(self.active_record(active_id).key()) + &&& self.active_record(active_id).pool_id() == self.pool( + self.active_record(active_id).key(), + ).id() + &&& self.active_record(active_id).fraction() > 0real + } by { + if active_id == lease_id { + assert(self.active_record(active_id).fraction() == fraction); + } else { + assert(old(self).active_ids().contains(active_id)); + assert(self.active_record(active_id) == old(self).active_record(active_id)); + } + }; + + assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + lemma_active_fraction_insert_next( + old(self).active_records(), + lease_id, + self.active_record(lease_id), + other, + ); + if other == key { + assert(old(self).pool(key).fraction() + active_lease_fraction( + old(self).active_records(), + key, + lease_id, + ) == 1real); + } else { + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + assert(old(self).pool(other).fraction() + active_lease_fraction( + old(self).active_records(), + other, + lease_id, + ) == 1real); + } + }; + RcuTrackedReadLease { lease_id, key, lease } + } + + /// Returns one lease and removes exactly its matching active record. + pub proof fn return_lease( + tracked &mut self, + tracked lease: RcuTrackedReadLease, + ) -> (tracked witness: W) + requires + old(self).wf(), + old(self).active_ids().contains(lease.lease_id()), + old(self).active_record(lease.lease_id()).key() == lease.key(), + old(self).active_record(lease.lease_id()).pool_id() == lease.pool_id(), + old(self).active_record(lease.lease_id()).fraction() == lease.fraction(), + ensures + final(self).wf(), + final(self).keys() == old(self).keys(), + final(self).next_lease() == old(self).next_lease(), + final(self).active_ids() == old(self).active_ids().remove(lease.lease_id()), + witness == old(self).active_record(lease.lease_id()).witness(), + forall|lease_id: nat| + lease_id != lease.lease_id() && old(self).active_ids().contains(lease_id) + ==> final(self).active_record(lease_id) == old(self).active_record(lease_id), + final(self).pool(lease.key()).id() == old(self).pool(lease.key()).id(), + final(self).pool(lease.key()).resource() == old(self).pool(lease.key()).resource(), + final(self).pool(lease.key()).fraction() == old(self).pool(lease.key()).fraction() + + lease.fraction(), + forall|other: K| + other != lease.key() && old(self).contains(other) ==> final(self).pool(other) + == old(self).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + let ghost lease_id = lease.lease_id; + let ghost key = lease.key; + let tracked record = self.active.tracked_remove(lease_id); + let tracked pool = self.pools.tracked_borrow_mut(key); + pool.return_lease(lease.lease); + + assert forall|active_id: nat| self.active_ids().contains(active_id) implies { + &&& active_id < self.next_lease() + &&& self.contains(self.active_record(active_id).key()) + &&& self.active_record(active_id).pool_id() == self.pool( + self.active_record(active_id).key(), + ).id() + &&& self.active_record(active_id).fraction() > 0real + } by { + assert(old(self).active_ids().contains(active_id)); + assert(active_id != lease_id); + assert(self.active_record(active_id) == old(self).active_record(active_id)); + }; + + assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + lemma_active_fraction_remove( + old(self).active_records(), + lease_id, + other, + self.next_lease(), + ); + if other == key { + assert(old(self).pool(key).fraction() + active_lease_fraction( + old(self).active_records(), + key, + self.next_lease(), + ) == 1real); + } else { + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + assert(old(self).pool(other).fraction() + active_lease_fraction( + old(self).active_records(), + other, + self.next_lease(), + ) == 1real); + } + }; + record.witness + } + + /// Recovers one allocation after a client proof rules out all active leases. + pub proof fn reclaim(tracked &mut self, key: K) -> (tracked resource: T) + requires + old(self).wf(), + old(self).contains(key), + !old(self).has_active(key), + ensures + final(self).wf(), + final(self).keys() == old(self).keys().remove(key), + final(self).active_ids() == old(self).active_ids(), + final(self).next_lease() == old(self).next_lease(), + !final(self).contains(key), + resource == old(self).pool(key).resource(), + forall|other: K| + other != key && old(self).contains(other) ==> final(self).pool(other) == old( + self, + ).pool(other), + { + reveal(RcuTrackedReadPoolRegistry::active_ids); + reveal(RcuTrackedReadPoolRegistry::active_records); + reveal(RcuTrackedReadPoolRegistry::active_record); + assert forall|old_key: K| #[trigger] old(self).contains(old_key) implies old(self).pool( + old_key, + ).fraction() + active_lease_fraction( + old(self).active_records(), + old_key, + old(self).next_lease(), + ) == 1real by {}; + assert forall|lease_id: nat| + lease_id < self.next_lease() && self.active_records().contains_key( + lease_id, + ) implies self.active_records()[lease_id].key() != key by { + if self.active_records()[lease_id].key() == key { + assert(self.active_ids().contains(lease_id)); + assert(exists|candidate: nat| + self.active_ids().contains(candidate) && self.active_record(candidate).key() + == key) by { + assert(self.active_record(lease_id).key() == key); + }; + assert(self.has_active(key)); + } + }; + lemma_active_fraction_zero(self.active_records(), key, self.next_lease()); + assert(self.pool(key).fraction() == 1real); + let tracked pool = self.pools.tracked_remove(key); + let tracked resource = pool.reclaim(); + assert forall|lease_id: nat| self.active_ids().contains(lease_id) implies { + &&& lease_id < self.next_lease() + &&& self.contains(self.active_record(lease_id).key()) + &&& self.active_record(lease_id).pool_id() == self.pool( + self.active_record(lease_id).key(), + ).id() + &&& self.active_record(lease_id).fraction() > 0real + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).active_record(lease_id).key() != key); + }; + assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { + assert(other != key); + assert(old(self).contains(other)); + assert(self.pool(other) == old(self).pool(other)); + assert(self.active_records() == old(self).active_records()); + assert(old(self).pool(other).fraction() + active_lease_fraction( + old(self).active_records(), + other, + old(self).next_lease(), + ) == 1real); + }; + resource + } +} + +/// Regression proof for the complete indexed split/return/reclaim lifecycle. +proof fn tracked_registry_reclaims_after_returns( + key: K, + tracked resource: T, + tracked first_witness: W, + tracked second_witness: W, +) -> (tracked res: T) + ensures + res == resource, +{ + let tracked mut registry = RcuTrackedReadPoolRegistry::empty(); + registry.insert(key, resource); + let tracked first = registry.split_lease(key, first_witness); + let tracked second = registry.split_lease(key, second_witness); + let tracked _first_witness = registry.return_lease(first); + let tracked _second_witness = registry.return_lease(second); + assert(!registry.has_active(key)); + assert(registry.pool(key).resource() == resource); + let tracked res = registry.reclaim(key); + assert(res == resource); + res +} + /// Regression proof: recursively splitting leases does not require a capacity /// assumption, and returning them restores the whole resource. pub proof fn split_return_reclaims(tracked resource: T) -> (tracked res: T) From 3df1ede74c7a5749d94093a974987e59fccd1b56 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 3 Aug 2026 03:08:02 -0400 Subject: [PATCH 33/47] update --- ostd/specs/sync/rcu.rs | 57 +- ostd/specs/sync/rcu_cpu.rs | 1544 ++++++++++++++++++++++++++++++++ ostd/specs/sync/weak_memory.rs | 8 +- ostd/specs/task/cpu_core.rs | 123 ++- ostd/src/sync/rcu/mod.rs | 108 +-- ostd/src/task/preempt/guard.rs | 148 ++- ostd/src/task/scheduler/mod.rs | 392 +++++--- 7 files changed, 2130 insertions(+), 250 deletions(-) create mode 100644 ostd/specs/sync/rcu_cpu.rs diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 787fdb8d3..b2775c651 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -2012,8 +2012,12 @@ impl RcuInactive { self.reader } + pub closed spec fn reader_registry(self) -> Loc { + self.state.id() + } + pub closed spec fn wf(self) -> bool { - !self.state.value() + true } pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { @@ -2057,16 +2061,17 @@ impl RcuBaseGuard { self.reader } - /// Weak atomic whose history determined this guard's expired snapshot. + /// Root atomic whose removal history determined this guard's expired set. pub closed spec fn root(self) -> Loc { self.root } - /// Thread view captured immediately before the guarded root load. + /// Weak-memory view captured when this read-side critical section began. pub closed spec fn start_view(self) -> WmView { self.start_view } + /// Registry that issued persistent retirement facts for this domain. pub closed spec fn retire_observation_registry(self) -> Loc { self.retire_observation_registry } @@ -2080,7 +2085,11 @@ impl RcuBaseGuard { } pub closed spec fn wf(self) -> bool { - !self.state.value() + &&& !self.state.value() + &&& forall|addr: usize| #[trigger] + self.protected().dom().contains(addr) ==> !self.expired().contains( + self.protected()[addr], + ) } pub closed spec fn belongs_to(self, domain: RcuDomainAuth) -> bool { @@ -2107,6 +2116,7 @@ impl RcuBaseGuard { res.tid() == self.tid(), res.reader() == self.reader(), res.wf(), + res.reader_registry() == self.reader_registry(), { RcuInactive { domain: self.domain, state: self.state, reader: self.reader } } @@ -2133,6 +2143,17 @@ impl RcuBaseGuard { final(self).protects(info.addr(), info.obj()), { self.protected = self.protected.insert(info.addr(), info.obj()); + assert forall|addr: usize| #[trigger] + self.protected().dom().contains(addr) implies !self.expired().contains( + self.protected()[addr], + ) by { + if addr == info.addr() { + assert(self.protected()[addr] == info.obj()); + } else { + assert(old(self).protected().dom().contains(addr)); + assert(self.protected()[addr] == old(self).protected()[addr]); + } + }; } } @@ -2443,27 +2464,28 @@ impl RcuRetiredFact { &&& self.observation.value() is Some } - pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { - &&& summary.domain == self.domain() - &&& summary.obj == self.obj() - &&& summary.removal == self.removal() - &&& summary.retire_observation_registry == self.retire_observation_registry() - } - - pub closed spec fn record(self) -> RcuRetiredRecord { + pub open spec fn record(self) -> RcuRetiredRecord { RcuRetiredRecord { domain: self.domain(), obj: self.obj(), - removal: self.removal(), retire_observation_registry: self.retire_observation_registry(), + removal: self.removal(), } } + pub closed spec fn matches(self, summary: RcuCallbackSummary) -> bool { + &&& summary.domain == self.domain() + &&& summary.obj == self.obj() + &&& summary.removal == self.removal() + &&& summary.retire_observation_registry == self.retire_observation_registry() + } + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) requires self.wf(), ensures res.wf(), + res.record() == self.record(), res.domain() == self.domain(), res.obj() == self.obj(), res.addr() == self.addr(), @@ -3041,6 +3063,15 @@ impl RcuReadGuardToken { { } + pub proof fn lemma_protected_not_expired(tracked &self, addr: usize, obj: nat) + requires + self.wf(), + self.protects(addr, obj), + ensures + !self.expired().contains(obj), + { + } + /// Preconditions of the paper's base `Guard-protect` rule. /// /// This deliberately says only that the allocation was not already diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs new file mode 100644 index 000000000..3aa692c15 --- /dev/null +++ b/ostd/specs/sync/rcu_cpu.rs @@ -0,0 +1,1544 @@ +// SPDX-License-Identifier: MPL-2.0 +//! CPU-local grace-period participation for preemptible RCU. +//! +//! This is the proof component used to refine the paper's abstract reader guard +//! to Asterinas's per-CPU quiescent-state implementation. A +//! [`CpuRcuParticipant`] is intended to stay in one CPU core's tracked local +//! state across task switches. Starting a reader splits a fractional +//! [`CpuRcuReaderFragment`]. Reporting a quiescent state requires the whole +//! fraction, so it cannot race with a live reader from the generation being +//! closed. +//! +//! A report also creates an idempotent [`CpuRcuClosedGeneration`] resource. +//! Resource validity gives the two facts needed by the relaxed-memory proof: +//! +//! - a closed generation is strictly older than every reader that can coexist +//! with its report; +//! - the report's weak-memory view is included in every such later reader's +//! participant view, which [`CpuRcuParticipant::tracked_start_reader`] +//! requires the task's start view to include. +//! +//! # Refinement boundary +//! +//! A participant generation is not the global epoch from the paper's concrete +//! epoch-based implementation. It only names the interval between two +//! quiescent reports on one CPU. A complete grace period must separately prove +//! that every relevant CPU reported after the callback's retire point and that +//! each report view includes the callback's retire view. +//! +//! The end-to-end refinement must enforce all of the following: +//! +//! - preemption is disabled before splitting a reader fragment and remains +//! disabled until that same fragment is returned; +//! - the fragment is created before the first protected load and is retained by +//! the executable RCU guard until guard destruction; +//! - only the scheduler-owned CPU-local participant can issue a report; +//! - a task entering after a report imports the participant's persistent view +//! before starting a reader. +//! +//! The current refinement treats `online_cpus()` as stable for a grace period +//! and covers only readers represented by a task `RunningTaskContext`. +//! CPU-hotplug transitions and interrupt/NMI readers need separate +//! participants before they can be included in the end-to-end theorem. +//! +//! `RunningTaskContext` owns the scheduler-checked-out canonical participant. +//! Its older task/session generation remains a distinct preemption-session +//! counter and is not an authority for this persistent CPU generation. Reader +//! contexts obtain their CPU generation from [`CpuRcuReaderFragment`]. +use crate::specs::{ + mm::cpu::CpuId, + task::cpu_core::{CpuCoreLocalState, CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreRegistration}, +}; +use vstd::{ + modes::tracked_swap, + prelude::*, + resource::{ + Loc, + agree::AgreementRA, + algebra::{Resource, ResourceAlgebra}, + frac::FractionRA, + map::GhostMapAuth, + product::ProductRA, + relations::frame_preserving_update_opt, + }, +}; + +use super::{ + rcu::{ + RcuBlockInfo, RcuInactive, RcuProtectedPtr, RcuReadGuardToken, RcuReaderContext, + RcuRetiredFacts, RcuRetiredRecord, RcuSeenRemoved, + }, + weak_memory::WmView, +}; + +verus! { + +broadcast use vstd::set::group_set_lemmas; + +/// One CPU quiescent report retained by the participant PCM. +pub ghost struct CpuRcuReportView { + pub cpu: CpuId, + pub generation: nat, + pub view: WmView, + pub known_retired: Set, +} + +pub(super) ghost struct CpuRcuStateView { + pub(super) cpu: CpuId, + pub(super) generation: nat, + pub(super) view: WmView, + pub(super) known_retired: Set, +} + +pub(super) type CpuRcuState = ProductRA>; + +pub(super) ghost struct CpuRcuCarrier { + pub(super) state: Option, + pub(super) closed: Set, +} + +impl CpuRcuCarrier { + pub(super) open spec fn records_observed(records: Set, view: WmView) -> bool { + forall|record: RcuRetiredRecord| #[trigger] + records.contains(record) ==> record.removal.observed_by(view) + } + + pub(super) open spec fn state( + cpu: CpuId, + generation: nat, + view: WmView, + known_retired: Set, + fraction: real, + ) -> Self { + CpuRcuCarrier { + state: Some( + ProductRA { + left: FractionRA::Frac(fraction), + right: AgreementRA::Agree( + CpuRcuStateView { cpu, generation, view, known_retired }, + ), + }, + ), + closed: Set::empty(), + } + } + + pub(super) open spec fn closed(report: CpuRcuReportView) -> Self { + CpuRcuCarrier { state: None, closed: Set::empty().insert(report) } + } + + pub(super) open spec fn reports_fit( + state: CpuRcuStateView, + reports: Set, + ) -> bool { + forall|report: CpuRcuReportView| #[trigger] + reports.contains(report) ==> { + &&& report.cpu == state.cpu + &&& report.generation < state.generation + &&& report.view.spec_le(state.view) + &&& report.known_retired.subset_of(state.known_retired) + &&& CpuRcuCarrier::records_observed(report.known_retired, report.view) + } + } + + pub(super) open spec fn state_view(self) -> CpuRcuStateView { + self.state.unwrap().right->Agree_0 + } + + pub(super) open spec fn fraction(self) -> real { + self.state.unwrap().left->Frac_0 + } + + pub(super) open spec fn has_valid_state(self) -> bool { + &&& self.state is Some + &&& self.state.unwrap().left is Frac + &&& self.state.unwrap().right is Agree + &&& self.state.valid() + } +} + +impl ResourceAlgebra for CpuRcuCarrier { + closed spec fn valid(self) -> bool { + match self.state { + Some(ProductRA { left: FractionRA::Frac(_), right: AgreementRA::Agree(state) }) => { + &&& self.state.valid() + &&& CpuRcuCarrier::reports_fit(state, self.closed) + &&& CpuRcuCarrier::records_observed(state.known_retired, state.view) + }, + None => forall|report: CpuRcuReportView| #[trigger] + self.closed.contains(report) ==> CpuRcuCarrier::records_observed( + report.known_retired, + report.view, + ), + _ => false, + } + } + + closed spec fn op(left: Self, right: Self) -> Self { + CpuRcuCarrier { + state: Option::::op(left.state, right.state), + closed: left.closed.union(right.closed), + } + } + + proof fn valid_op(left: Self, right: Self) { + Option::::valid_op(left.state, right.state); + match left.state { + None => { + let ghost combined = CpuRcuCarrier::op(left, right); + assert(combined.valid()); + assert(combined.closed == left.closed.union(right.closed)); + assert forall|report: CpuRcuReportView| #[trigger] + left.closed.contains(report) implies CpuRcuCarrier::records_observed( + report.known_retired, + report.view, + ) by { + assert(combined.closed.contains(report)); + match right.state { + None => { + assert(combined.state is None); + }, + Some( + ProductRA { + left: FractionRA::Frac(_), + right: AgreementRA::Agree(right_state), + }, + ) => { + assert(combined.state == right.state); + assert(combined.state_view() == right_state); + assert(CpuRcuCarrier::reports_fit(right_state, combined.closed)); + }, + _ => {}, + } + }; + }, + Some( + ProductRA { left: FractionRA::Frac(_), right: AgreementRA::Agree(left_state) }, + ) => { + assert forall|report: CpuRcuReportView| #[trigger] + left.closed.contains(report) implies { + &&& report.cpu == left_state.cpu + &&& report.generation < left_state.generation + &&& report.view.spec_le(left_state.view) + &&& report.known_retired.subset_of(left_state.known_retired) + &&& CpuRcuCarrier::records_observed(report.known_retired, report.view) + } by { + assert(left.closed.union(right.closed).contains(report)); + match right.state { + None => {}, + Some( + ProductRA { left: FractionRA::Frac(_), right: AgreementRA::Agree(_) }, + ) => {}, + _ => {}, + } + }; + }, + _ => {}, + } + } + + proof fn commutative(left: Self, right: Self) { + Option::::commutative(left.state, right.state); + assert(left.closed.union(right.closed) =~= right.closed.union(left.closed)); + } + + proof fn associative(left: Self, middle: Self, right: Self) { + Option::::associative(left.state, middle.state, right.state); + assert(left.closed.union(middle.closed.union(right.closed)) =~= left.closed.union( + middle.closed, + ).union(right.closed)); + } +} + +/// CPU-owned fractional authority for one RCU participation generation. +/// +/// This token belongs permanently to the CPU core named by [`Self::cpu`]. +/// It may cross task execution sessions, but it must never migrate to another +/// CPU's local-state aggregate. +pub tracked struct CpuRcuParticipant { + resource: Resource, + known_retired: RcuRetiredFacts, +} + +/// Generic CPU-core registration evidence specialized to the RCU local state. +pub type CpuRcuCoreBinding = CpuCoreOwnerBinding; + +/// Linear witness that one reader is live in a CPU participation generation. +pub tracked struct CpuRcuReaderFragment { + resource: Resource, + known_retired: RcuRetiredFacts, +} + +/// Idempotent proof that one CPU generation has passed a quiescent boundary. +/// +/// The resource can be split into two identical copies because its PCM element +/// is idempotent. It contains no executable state. +pub tracked struct CpuRcuClosedGeneration { + resource: Resource, + known_retired: RcuRetiredFacts, + binding: CpuRcuCoreBinding, +} + +/// Refinement of the paper guard with one live CPU reader fragment. +/// +/// The wrapped [`RcuReadGuardToken`] remains the reusable abstract +/// `Guard(tid, X, G)`. The fragment is the Asterinas implementation witness +/// that prevents this CPU from reporting a quiescent boundary until the guard +/// is destroyed. Keeping both resources in one linear token prevents the +/// executable guard from ending only the abstract critical section while +/// silently losing its CPU participation. +#[verifier::reject_recursive_types(T)] +pub tracked struct CpuRcuReadGuardToken { + paper_guard: RcuReadGuardToken, + reader: CpuRcuReaderFragment, + binding: CpuRcuCoreBinding, +} + +proof fn lemma_choose_singleton_report(report: CpuRcuReportView) + ensures + (choose|candidate: CpuRcuReportView| Set::empty().insert(report).contains(candidate)) + == report, +{ + let ghost reports = Set::empty().insert(report); + assert(reports.contains(report)); + let ghost chosen = choose|candidate: CpuRcuReportView| reports.contains(candidate); + assert(reports.contains(chosen)); + assert(chosen == report); +} + +impl CpuRcuParticipant { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().has_valid_state() + &&& self.resource.value().closed.is_empty() + &&& self.resource.value().state_view().known_retired == self.known_retired.records() + &&& CpuRcuCarrier::records_observed( + self.resource.value().state_view().known_retired, + self.resource.value().state_view().view, + ) + } + + /// Creates generation zero for one CPU. + pub proof fn new(cpu: CpuId, view: WmView) -> (tracked res: Self) + ensures + res.cpu() == cpu, + res.generation() == 0, + res.view() == view, + res.fraction() == 1real, + res.wf(), + { + let tracked known_retired = RcuRetiredFacts::empty(); + let tracked resource = Resource::alloc( + CpuRcuCarrier::state(cpu, 0, view, known_retired.records(), 1real), + ); + CpuRcuParticipant { resource, known_retired } + } + + /// Stable identity of this CPU's RCU participant. + pub closed spec fn id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn cpu(self) -> CpuId { + self.resource.value().state_view().cpu + } + + pub closed spec fn generation(self) -> nat { + self.resource.value().state_view().generation + } + + pub closed spec fn view(self) -> WmView { + self.resource.value().state_view().view + } + + /// Persistent retirement facts known before this CPU generation started. + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub closed spec fn fraction(self) -> real { + self.resource.value().fraction() + } + + pub open spec fn wf(self) -> bool { + 0real < self.fraction() <= 1real + } + + /// Starts a reader in the current CPU generation. + /// + /// `start_view` is the task view after it has imported the persistent CPU + /// view. The caller chooses a positive rational `reader_fraction`, so this + /// protocol imposes no fixed bound on the number of readers. + pub proof fn tracked_start_reader( + tracked self, + start_view: WmView, + reader_fraction: real, + ) -> (tracked res: (CpuRcuParticipant, CpuRcuReaderFragment)) + requires + self.wf(), + self.view().spec_le(start_view), + 0real < reader_fraction < self.fraction(), + ensures + res.0.id() == self.id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation(), + res.0.view() == self.view(), + res.0.known_retired() == self.known_retired(), + res.0.fraction() == self.fraction() - reader_fraction, + res.0.wf(), + res.1.participant_id() == self.id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + res.1.participant_view() == self.view(), + res.1.known_retired() == self.known_retired(), + res.1.fraction() == reader_fraction, + res.1.wf(), + { + use_type_invariant(&self); + let ghost participant = CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.view(), + self.known_retired(), + self.fraction() - reader_fraction, + ); + let ghost reader = CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.view(), + self.known_retired(), + reader_fraction, + ); + assert(self.resource.value() == CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.view(), + self.known_retired(), + self.fraction(), + )); + assert(0real < self.fraction() - reader_fraction <= 1real); + assert(0real < reader_fraction <= 1real); + assert(FractionRA::op( + FractionRA::Frac(self.fraction() - reader_fraction), + FractionRA::Frac(reader_fraction), + ) == FractionRA::Frac(self.fraction())); + assert(AgreementRA::op( + AgreementRA::Agree(self.resource.value().state_view()), + AgreementRA::Agree(self.resource.value().state_view()), + ) == AgreementRA::Agree(self.resource.value().state_view())); + assert(Option::::op(participant.state, reader.state) + == self.resource.value().state); + assert(participant.closed.union(reader.closed).is_empty()); + assert(self.resource.value() == CpuRcuCarrier::op(participant, reader)); + let tracked (participant_resource, reader_resource) = self.resource.split( + participant, + reader, + ); + let tracked reader_known_retired = self.known_retired.tracked_duplicate(); + ( + CpuRcuParticipant { resource: participant_resource, known_retired: self.known_retired }, + CpuRcuReaderFragment { resource: reader_resource, known_retired: reader_known_retired }, + ) + } + + /// Splits half of the participant's current rational fraction in place. + /// + /// Repeated nested reads therefore remain unbounded: each live reader gets + /// a positive fraction, while the participant retains a positive fraction + /// for subsequent splits. The complete fraction can be recovered only by + /// returning every fragment. + pub proof fn tracked_start_reader_in_place( + tracked &mut self, + start_view: WmView, + ) -> (tracked reader: CpuRcuReaderFragment) + requires + old(self).wf(), + old(self).view().spec_le(start_view), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation(), + final(self).view() == old(self).view(), + final(self).known_retired() == old(self).known_retired(), + final(self).fraction() == old(self).fraction() / 2real, + final(self).wf(), + reader.participant_id() == old(self).id(), + reader.cpu() == old(self).cpu(), + reader.generation() == old(self).generation(), + reader.participant_view() == old(self).view(), + reader.known_retired() == old(self).known_retired(), + reader.fraction() == old(self).fraction() / 2real, + reader.wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let ghost reader_fraction = self.fraction() / 2real; + assert(0real < reader_fraction < self.fraction()); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked (mut participant, reader) = owned.tracked_start_reader( + start_view, + reader_fraction, + ); + tracked_swap(self, &mut participant); + reader + } + + /// Returns a reader fragment to its CPU-local participant. + pub proof fn tracked_stop_reader( + tracked self, + tracked reader: CpuRcuReaderFragment, + ) -> (tracked res: CpuRcuParticipant) + requires + self.wf(), + reader.wf(), + self.id() == reader.participant_id(), + ensures + res.id() == self.id(), + res.cpu() == self.cpu(), + res.generation() == self.generation(), + res.view() == self.view(), + res.known_retired() == self.known_retired(), + res.fraction() == self.fraction() + reader.fraction(), + res.wf(), + { + use_type_invariant(&self); + use_type_invariant(&reader); + let tracked mut participant_resource = self.resource; + participant_resource.validate_2(&reader.resource); + let tracked resource = participant_resource.join(reader.resource); + CpuRcuParticipant { resource, known_retired: self.known_retired } + } + + /// Returns a reader fragment to this participant in place. + pub proof fn tracked_stop_reader_in_place( + tracked &mut self, + tracked reader: CpuRcuReaderFragment, + ) + requires + old(self).wf(), + reader.wf(), + old(self).id() == reader.participant_id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation(), + final(self).view() == old(self).view(), + final(self).known_retired() == old(self).known_retired(), + final(self).fraction() == old(self).fraction() + reader.fraction(), + final(self).wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked mut participant = owned.tracked_stop_reader(reader); + tracked_swap(self, &mut participant); + } + + /// Closes the current generation and imports retirement facts observed by + /// the report view. + /// + /// A whole fraction is incompatible with every live reader fragment. + /// The returned participant starts the next generation and retains the + /// report view and persistent retirement facts for task sessions that run + /// later on this CPU. + pub proof fn tracked_report_quiescent_with( + tracked self, + tracked binding: CpuRcuCoreBinding, + report_view: WmView, + tracked learned: &RcuRetiredFacts, + ) -> (tracked res: (CpuRcuParticipant, CpuRcuClosedGeneration)) + requires + self.wf(), + self.fraction() == 1real, + self.view().spec_le(report_view), + learned.observed_by(report_view), + binding.cpu() == self.cpu(), + binding.single_local_id() == self.id(), + ensures + res.0.id() == self.id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation() + 1, + res.0.view() == report_view, + res.0.known_retired() == self.known_retired().union(learned.records()), + res.0.fraction() == 1real, + res.0.wf(), + res.1.participant_id() == self.id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.generation(), + res.1.view() == report_view, + res.1.known_retired() == self.known_retired().union(learned.records()), + res.1.scheduler() == binding.registry(), + res.1.wf(), + { + use_type_invariant(&self); + let ghost old_cpu = self.cpu(); + let ghost old_generation = self.generation(); + let ghost old_view = self.view(); + let ghost old_known_retired = self.known_retired(); + let ghost merged_records = old_known_retired.union(learned.records()); + assert(self.resource.value() == CpuRcuCarrier::state( + old_cpu, + old_generation, + old_view, + old_known_retired, + 1real, + )); + assert(CpuRcuCarrier::records_observed(old_known_retired, old_view)); + assert(CpuRcuCarrier::records_observed(merged_records, report_view)) by { + assert forall|record: RcuRetiredRecord| #[trigger] + merged_records.contains(record) implies record.removal.observed_by(report_view) by { + if old_known_retired.contains(record) { + assert(record.removal.observed_by(old_view)); + assert(old_view.seen_at(record.removal.root) <= report_view.seen_at( + record.removal.root, + )); + } else { + assert(learned.records().contains(record)); + } + }; + }; + let ghost report = CpuRcuReportView { + cpu: old_cpu, + generation: old_generation, + view: report_view, + known_retired: merged_records, + }; + let ghost next_state = CpuRcuCarrier::state( + old_cpu, + old_generation + 1, + report_view, + merged_records, + 1real, + ); + let ghost next = CpuRcuCarrier { + state: next_state.state, + closed: Set::empty().insert(report), + }; + assert forall|frame: Option| + #![trigger Option::::op(Some(self.resource.value()), frame).valid()] + Option::::op( + Some(self.resource.value()), + frame, + ).valid() implies Option::::op(Some(next), frame).valid() by { + match frame { + Some(CpuRcuCarrier { state: None, closed }) => { + let ghost frame_carrier = CpuRcuCarrier { state: None, closed }; + let ghost combined = CpuRcuCarrier::op(self.resource.value(), frame_carrier); + assert(combined.valid()); + assert(Option::::op(self.resource.value().state, None) + == self.resource.value().state); + assert(self.resource.value().closed.is_empty()); + assert(self.resource.value().closed.union(closed) =~= closed); + assert(combined == CpuRcuCarrier { + state: self.resource.value().state, + closed, + }); + assert(combined.state_view() == CpuRcuStateView { + cpu: old_cpu, + generation: old_generation, + view: old_view, + known_retired: old_known_retired, + }); + assert(CpuRcuCarrier::reports_fit( + CpuRcuStateView { + cpu: old_cpu, + generation: old_generation, + view: old_view, + known_retired: old_known_retired, + }, + closed, + )); + assert forall|old_report: CpuRcuReportView| #[trigger] + closed.contains(old_report) implies { + &&& old_report.cpu == old_cpu + &&& old_report.generation < old_generation + 1 + &&& old_report.view.spec_le(report_view) + &&& old_report.known_retired.subset_of(merged_records) + } by { + assert(old_report.generation < old_generation); + old_report.view.lemma_spec_le_transitive(old_view, report_view); + assert(old_report.known_retired.subset_of(old_known_retired)); + assert(old_known_retired.subset_of(merged_records)); + }; + }, + None => { + assert(next.valid()); + }, + _ => {}, + } + }; + let tracked combined = self.resource.update(next); + let ghost participant = CpuRcuCarrier::state( + old_cpu, + old_generation + 1, + report_view, + merged_records, + 1real, + ); + let ghost closed = CpuRcuCarrier::closed(report); + assert(Option::::op(participant.state, closed.state) == participant.state); + assert(participant.closed.union(closed.closed) =~= Set::empty().insert(report)); + assert(next == CpuRcuCarrier::op(participant, closed)); + let tracked (participant_resource, closed_resource) = combined.split(participant, closed); + let tracked mut known_retired = self.known_retired; + known_retired.tracked_merge(learned); + let tracked closed_known_retired = known_retired.tracked_duplicate(); + assert(known_retired.records() == merged_records); + lemma_choose_singleton_report(report); + ( + CpuRcuParticipant { resource: participant_resource, known_retired }, + CpuRcuClosedGeneration { + resource: closed_resource, + known_retired: closed_known_retired, + binding, + }, + ) + } + + /// Closes the current generation without importing additional retirement + /// facts. + pub proof fn tracked_report_quiescent( + tracked self, + tracked binding: CpuRcuCoreBinding, + report_view: WmView, + ) -> (tracked res: (CpuRcuParticipant, CpuRcuClosedGeneration)) + requires + self.wf(), + self.fraction() == 1real, + self.view().spec_le(report_view), + binding.cpu() == self.cpu(), + binding.single_local_id() == self.id(), + ensures + res.0.id() == self.id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation() + 1, + res.0.view() == report_view, + res.0.known_retired() == self.known_retired(), + res.0.fraction() == 1real, + res.0.wf(), + res.1.participant_id() == self.id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.generation(), + res.1.view() == report_view, + res.1.known_retired() == self.known_retired(), + res.1.scheduler() == binding.registry(), + res.1.wf(), + { + let tracked empty = RcuRetiredFacts::empty(); + let tracked res = self.tracked_report_quiescent_with(binding, report_view, &empty); + assert(empty.records() == Set::::empty()); + assert(self.known_retired().union(empty.records()) =~= self.known_retired()); + res + } + + /// Closes the current generation while retaining this CPU's canonical + /// participant in place. + /// + /// The full-fraction requirement is the resource-level statement that no + /// reader fragment from the generation being closed remains live. + pub proof fn tracked_report_quiescent_in_place( + tracked &mut self, + tracked binding: CpuRcuCoreBinding, + report_view: WmView, + ) -> (tracked closed: CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).fraction() == 1real, + old(self).view().spec_le(report_view), + binding.cpu() == old(self).cpu(), + binding.single_local_id() == old(self).id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation() + 1, + final(self).view() == report_view, + final(self).known_retired() == old(self).known_retired(), + final(self).fraction() == 1real, + final(self).wf(), + closed.participant_id() == old(self).id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).generation(), + closed.view() == report_view, + closed.known_retired() == old(self).known_retired(), + closed.scheduler() == binding.registry(), + closed.wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked (mut participant, closed) = owned.tracked_report_quiescent( + binding, + report_view, + ); + tracked_swap(self, &mut participant); + closed + } + + /// In-place form of [`Self::tracked_report_quiescent_with`]. + pub proof fn tracked_report_quiescent_with_in_place( + tracked &mut self, + tracked binding: CpuRcuCoreBinding, + report_view: WmView, + tracked learned: &RcuRetiredFacts, + ) -> (tracked closed: CpuRcuClosedGeneration) + requires + old(self).wf(), + old(self).fraction() == 1real, + old(self).view().spec_le(report_view), + learned.observed_by(report_view), + binding.cpu() == old(self).cpu(), + binding.single_local_id() == old(self).id(), + ensures + final(self).id() == old(self).id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation() + 1, + final(self).view() == report_view, + final(self).known_retired() == old(self).known_retired().union(learned.records()), + final(self).fraction() == 1real, + final(self).wf(), + closed.participant_id() == old(self).id(), + closed.cpu() == old(self).cpu(), + closed.closed_generation() == old(self).generation(), + closed.view() == report_view, + closed.known_retired() == old(self).known_retired().union(learned.records()), + closed.scheduler() == binding.registry(), + closed.wf(), + { + let ghost old_cpu = self.cpu(); + let ghost old_view = self.view(); + let tracked mut owned = CpuRcuParticipant::new(old_cpu, old_view); + tracked_swap(self, &mut owned); + let tracked (mut participant, closed) = owned.tracked_report_quiescent_with( + binding, + report_view, + learned, + ); + tracked_swap(self, &mut participant); + closed + } +} + +impl CpuCoreLocalState for CpuRcuParticipant { + open spec fn belongs_to_cpu(self, cpu: CpuId) -> bool { + self.cpu() == cpu + } + + open spec fn local_key(self) -> Seq { + seq![self.id()] + } +} + +impl CpuRcuParticipant { + /// Exposes the participant's generic CPU-local identity to scheduler + /// clients without requiring them to unfold this module's trait impl. + pub proof fn lemma_cpu_core_local_state(tracked &self) + ensures + self.belongs_to_cpu(self.cpu()), + self.local_key() == seq![self.id()], + { + } +} + +impl CpuRcuReaderFragment { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().has_valid_state() + &&& self.resource.value().closed.is_empty() + &&& self.resource.value().state_view().known_retired == self.known_retired.records() + &&& CpuRcuCarrier::records_observed( + self.resource.value().state_view().known_retired, + self.resource.value().state_view().view, + ) + } + + pub closed spec fn participant_id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn cpu(self) -> CpuId { + self.resource.value().state_view().cpu + } + + pub closed spec fn generation(self) -> nat { + self.resource.value().state_view().generation + } + + pub closed spec fn participant_view(self) -> WmView { + self.resource.value().state_view().view + } + + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub proof fn tracked_known_retired(tracked &self, record: RcuRetiredRecord) -> (tracked res: + &super::rcu::RcuRetiredFact) + requires + self.known_retired().contains(record), + ensures + res.wf(), + res.record() == record, + { + use_type_invariant(self); + reveal(CpuRcuReaderFragment::known_retired); + assert(self.known_retired() == self.known_retired.records()); + assert(self.known_retired.records().contains(record)); + self.known_retired.tracked_borrow(record) + } + + /// Borrows the persistent retirement facts known at this reader's + /// generation, after lifting their observations to `view`. + pub proof fn tracked_retired_facts_observed_by(tracked &self, view: WmView) -> (tracked res: + &RcuRetiredFacts) + requires + self.participant_view().spec_le(view), + ensures + res.records() == self.known_retired(), + res.observed_by(view), + { + use_type_invariant(self); + assert forall|record: RcuRetiredRecord| #[trigger] + self.known_retired.records().contains(record) implies record.removal.observed_by( + view, + ) by { + assert(self.resource.value().state_view().known_retired.contains(record)); + assert(record.removal.observed_by(self.participant_view())); + assert(self.participant_view().seen_at(record.removal.root) <= view.seen_at( + record.removal.root, + )); + }; + &self.known_retired + } + + pub closed spec fn fraction(self) -> real { + self.resource.value().fraction() + } + + pub open spec fn wf(self) -> bool { + 0real < self.fraction() <= 1real + } +} + +impl CpuRcuReadGuardToken { + pub closed spec fn paper_guard(self) -> RcuReadGuardToken { + self.paper_guard + } + + pub closed spec fn reader_fragment(self) -> CpuRcuReaderFragment { + self.reader + } + + pub closed spec fn binding(self) -> CpuRcuCoreBinding { + self.binding + } + + pub closed spec fn scheduler(self) -> Loc { + self.binding().registry() + } + + pub closed spec fn participant_id(self) -> Loc { + self.reader_fragment().participant_id() + } + + pub closed spec fn cpu(self) -> CpuId { + self.reader_fragment().cpu() + } + + pub closed spec fn generation(self) -> nat { + self.reader_fragment().generation() + } + + pub closed spec fn participant_view(self) -> WmView { + self.reader_fragment().participant_view() + } + + pub closed spec fn known_retired(self) -> Set { + self.reader_fragment().known_retired() + } + + pub closed spec fn domain(self) -> Loc { + self.paper_guard().domain() + } + + pub closed spec fn reader_context(self) -> RcuReaderContext { + self.paper_guard().reader() + } + + pub closed spec fn root(self) -> Loc { + self.paper_guard().root() + } + + pub closed spec fn start_view(self) -> WmView { + self.paper_guard().start_view() + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.paper_guard().retire_observation_registry() + } + + pub closed spec fn reader_registry(self) -> Loc { + self.paper_guard().reader_registry() + } + + pub closed spec fn expired(self) -> Set { + self.paper_guard().expired() + } + + pub closed spec fn protected(self) -> Map { + self.paper_guard().protected() + } + + pub closed spec fn seen_removed(self) -> RcuSeenRemoved { + self.paper_guard().seen_removed() + } + + pub open spec fn protects(self, addr: usize, obj: nat) -> bool { + self.paper_guard().protects(addr, obj) + } + + pub open spec fn protects_pointer(self, ptr: RcuProtectedPtr) -> bool { + ptr.protected_by(self.paper_guard()) + } + + /// Instantiates the entry-time expiration guarantee for one known + /// retirement record belonging to this guard's RCU root. + pub proof fn lemma_known_retired_expired(tracked &self, record: RcuRetiredRecord) + requires + self.wf(), + self.known_retired().contains(record), + record.domain == self.domain(), + record.retire_observation_registry == self.retire_observation_registry(), + record.removal.root == self.root(), + ensures + self.expired().contains(record.obj), + { + } + + /// Forwards the paper guard's traversal-side expiration consequence. + pub proof fn lemma_expired_is_removed(tracked &self) + requires + self.wf(), + ensures + self.expired().subset_of(self.seen_removed().removed), + { + self.paper_guard.lemma_expired_is_removed(); + } + + /// A pointer protected by this guard cannot belong to its entry-time + /// expired set. + pub proof fn lemma_protected_not_expired(tracked &self, tracked protected: &RcuProtectedPtr) + requires + self.wf(), + protected.protected_by(self.paper_guard()), + ensures + !self.expired().contains(protected.obj()), + { + self.paper_guard.lemma_protected_not_expired(protected.ptr().addr(), protected.obj()); + } + + /// Agreement between the abstract guard and its concrete CPU participant. + pub open spec fn wf(self) -> bool { + &&& self.paper_guard().wf() + &&& self.reader_fragment().wf() + &&& self.binding().single_local_id() == self.participant_id() + &&& self.binding().cpu() == self.cpu() + &&& self.scheduler() == self.reader_context().scheduler + &&& self.reader_context().cpu == self.cpu() + &&& self.reader_context().generation == self.generation() + &&& self.participant_view().spec_le(self.start_view()) + &&& forall|record: RcuRetiredRecord| #[trigger] + self.known_retired().contains(record) && record.domain == self.domain() + && record.retire_observation_registry == self.retire_observation_registry() + && record.removal.root == self.root() ==> self.expired().contains(record.obj) + } + + /// Attaches the CPU implementation fragment to a freshly started paper + /// guard. + pub proof fn tracked_new( + tracked paper_guard: RcuReadGuardToken, + tracked reader: CpuRcuReaderFragment, + tracked binding: CpuRcuCoreBinding, + ) -> (tracked res: Self) + requires + paper_guard.wf(), + reader.wf(), + paper_guard.reader().cpu == reader.cpu(), + paper_guard.reader().generation == reader.generation(), + binding.registry() == paper_guard.reader().scheduler, + binding.cpu() == reader.cpu(), + binding.single_local_id() == reader.participant_id(), + reader.participant_view().spec_le(paper_guard.start_view()), + forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry + == paper_guard.retire_observation_registry() && record.removal.root + == paper_guard.root() ==> paper_guard.expired().contains(record.obj), + ensures + res.wf(), + res.paper_guard() == paper_guard, + res.reader_fragment() == reader, + res.reader_context() == paper_guard.reader(), + res.scheduler() == binding.registry(), + res.binding().cpu() == binding.cpu(), + res.binding().single_local_id() == binding.single_local_id(), + res.participant_id() == reader.participant_id(), + res.cpu() == reader.cpu(), + res.generation() == reader.generation(), + res.participant_view() == reader.participant_view(), + res.known_retired() == reader.known_retired(), + res.domain() == paper_guard.domain(), + res.root() == paper_guard.root(), + res.start_view() == paper_guard.start_view(), + res.reader_registry() == paper_guard.reader_registry(), + res.retire_observation_registry() == paper_guard.retire_observation_registry(), + res.expired() == paper_guard.expired(), + res.protected() == paper_guard.protected(), + { + CpuRcuReadGuardToken { paper_guard, reader, binding } + } + + /// Separates the implementation fragment from the abstract guard. + /// + /// This is intentionally consuming. The normal destruction path should use + /// [`Self::tracked_stop`] so the abstract `Guard -> Inactive` transition + /// cannot be forgotten. + pub proof fn tracked_into_parts(tracked self) -> (tracked res: ( + RcuReadGuardToken, + CpuRcuReaderFragment, + CpuRcuCoreBinding, + )) + requires + self.wf(), + ensures + res.0 == self.paper_guard(), + res.1 == self.reader_fragment(), + res.2.registry() == self.scheduler(), + res.2.cpu() == self.cpu(), + res.2.single_local_id() == self.participant_id(), + res.0.wf(), + res.1.wf(), + res.0.reader().cpu == res.1.cpu(), + res.1.participant_view().spec_le(res.0.start_view()), + forall|record: RcuRetiredRecord| #[trigger] + res.1.known_retired().contains(record) && record.domain == res.0.domain() + && record.retire_observation_registry == res.0.retire_observation_registry() + && record.removal.root == res.0.root() ==> res.0.expired().contains(record.obj), + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + let tracked CpuRcuReadGuardToken { paper_guard, reader, binding } = self; + assert(reader.known_retired() == known_retired); + assert(paper_guard.domain() == domain); + assert(paper_guard.retire_observation_registry() == retire_observation_registry); + assert(paper_guard.root() == root); + assert(paper_guard.expired() == expired); + assert forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry == paper_guard.retire_observation_registry() + && record.removal.root == paper_guard.root() implies paper_guard.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + (paper_guard, reader, binding) + } + + /// Ends the paper guard locally and returns the CPU fragment that Drop must + /// join back into the CPU-local participant. + pub proof fn tracked_stop(tracked self) -> (tracked res: (RcuInactive, CpuRcuReaderFragment)) + requires + self.wf(), + ensures + res.0.wf(), + res.0.domain() == self.domain(), + res.0.reader() == self.reader_context(), + res.1 == self.reader_fragment(), + res.1.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + { + let tracked (paper_guard, reader, _binding) = self.tracked_into_parts(); + let tracked base = paper_guard.tracked_into_base(); + let tracked inactive = base.tracked_stop(); + (inactive, reader) + } + + /// Applies the paper's `Guard-protect` rule without changing CPU + /// participation or the captured start view. + pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) + requires + old(self).wf(), + old(self).paper_guard().can_protect(*info), + ensures + final(self).wf(), + final(self).reader_fragment() == old(self).reader_fragment(), + final(self).participant_id() == old(self).participant_id(), + final(self).cpu() == old(self).cpu(), + final(self).generation() == old(self).generation(), + final(self).participant_view() == old(self).participant_view(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected().insert(info.addr(), info.obj()), + final(self).protects(info.addr(), info.obj()), + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + self.paper_guard.tracked_protect(info); + assert(self.known_retired() == known_retired); + assert(self.domain() == domain); + assert(self.retire_observation_registry() == retire_observation_registry); + assert(self.root() == root); + assert(self.expired() == expired); + assert forall|record: RcuRetiredRecord| #[trigger] + self.known_retired().contains(record) && record.domain == self.domain() + && record.retire_observation_registry == self.retire_observation_registry() + && record.removal.root == self.root() implies self.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + } +} + +impl CpuRcuClosedGeneration { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().state is None + &&& self.resource.value().closed =~= Set::empty().insert(self.report()) + &&& self.report().known_retired == self.known_retired.records() + &&& self.binding.single_local_id() == self.resource.loc() + &&& self.binding.cpu() == self.report().cpu + &&& CpuRcuCarrier::records_observed(self.report().known_retired, self.report().view) + } + + closed spec fn report(self) -> CpuRcuReportView { + choose|report: CpuRcuReportView| self.resource.value().closed.contains(report) + } + + pub closed spec fn participant_id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn binding(self) -> CpuRcuCoreBinding { + self.binding + } + + pub closed spec fn scheduler(self) -> Loc { + self.binding().registry() + } + + pub closed spec fn cpu(self) -> CpuId { + self.report().cpu + } + + pub closed spec fn closed_generation(self) -> nat { + self.report().generation + } + + pub closed spec fn view(self) -> WmView { + self.report().view + } + + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub closed spec fn wf(self) -> bool { + &&& self.resource.value() == CpuRcuCarrier::closed(self.report()) + &&& self.binding().single_local_id() == self.participant_id() + &&& self.binding().cpu() == self.cpu() + } + + /// Splits the idempotent closed-generation fact. + pub proof fn tracked_duplicate(tracked self) -> (tracked res: ( + CpuRcuClosedGeneration, + CpuRcuClosedGeneration, + )) + requires + self.wf(), + ensures + res.0.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.0.closed_generation() == self.closed_generation(), + res.0.view() == self.view(), + res.0.known_retired() == self.known_retired(), + res.0.scheduler() == self.scheduler(), + res.0.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.closed_generation(), + res.1.view() == self.view(), + res.1.known_retired() == self.known_retired(), + res.1.scheduler() == self.scheduler(), + res.1.wf(), + { + use_type_invariant(&self); + let ghost report = self.report(); + let ghost records = self.known_retired.records(); + let ghost carrier = CpuRcuCarrier::closed(self.report()); + assert(carrier.closed.union(carrier.closed) =~= carrier.closed); + assert(carrier == CpuRcuCarrier::op(carrier, carrier)); + let tracked (left, right) = self.resource.split(carrier, carrier); + let tracked right_known_retired = self.known_retired.tracked_duplicate(); + let tracked right_binding = self.binding.tracked_duplicate(); + assert(left.value() == carrier); + assert(right.value() == carrier); + lemma_choose_singleton_report(report); + assert((choose|candidate: CpuRcuReportView| left.value().closed.contains(candidate)) + == report); + assert((choose|candidate: CpuRcuReportView| right.value().closed.contains(candidate)) + == report); + assert(report.known_retired == records); + assert(right_known_retired.records() == records); + ( + CpuRcuClosedGeneration { + resource: left, + known_retired: self.known_retired, + binding: self.binding, + }, + CpuRcuClosedGeneration { + resource: right, + known_retired: right_known_retired, + binding: right_binding, + }, + ) + } + + /// Creates another copy of this persistent closed-generation fact. + /// + /// This is admissible because composing the idempotent carrier with itself + /// leaves the carrier unchanged. The monitor uses this operation when one + /// completed grace period authorizes multiple callbacks. + pub proof fn tracked_duplicate_from_ref(tracked &self) -> (tracked duplicate: + CpuRcuClosedGeneration) + requires + self.wf(), + ensures + duplicate.participant_id() == self.participant_id(), + duplicate.cpu() == self.cpu(), + duplicate.closed_generation() == self.closed_generation(), + duplicate.view() == self.view(), + duplicate.known_retired() == self.known_retired(), + duplicate.scheduler() == self.scheduler(), + duplicate.wf(), + { + use_type_invariant(self); + let ghost report = self.report(); + let ghost records = self.known_retired.records(); + let ghost carrier = CpuRcuCarrier::closed(self.report()); + assert(carrier.closed.union(carrier.closed) =~= carrier.closed); + assert(CpuRcuCarrier::op(carrier, carrier) == carrier); + assert(frame_preserving_update_opt::( + carrier, + CpuRcuCarrier::op(carrier, carrier), + )) by { + assert forall|frame: Option| + #![trigger Option::::op(Some(carrier), frame), + Option::::op( + Some(CpuRcuCarrier::op(carrier, carrier)), + frame, + )] + Option::::op(Some(carrier), frame).valid() implies Option::< + CpuRcuCarrier, + >::op(Some(CpuRcuCarrier::op(carrier, carrier)), frame).valid() by { + assert(CpuRcuCarrier::op(carrier, carrier) == carrier); + }; + }; + let tracked resource = self.resource.duplicate_previous(carrier); + let tracked known_retired = self.known_retired.tracked_duplicate(); + let tracked binding = self.binding.tracked_duplicate(); + assert(resource.value() == carrier); + lemma_choose_singleton_report(report); + assert((choose|candidate: CpuRcuReportView| resource.value().closed.contains(candidate)) + == report); + assert(report.known_retired == records); + assert(known_retired.records() == records); + CpuRcuClosedGeneration { resource, known_retired, binding } + } + + /// Any reader coexisting with this report started in a later generation + /// and carries a participant view that includes the report view. + pub proof fn lemma_later_reader( + tracked &self, + tracked mut reader: CpuRcuReaderFragment, + ) -> (tracked res: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + self.participant_id() == reader.participant_id(), + ensures + res == reader, + res.wf(), + self.closed_generation() < res.generation(), + self.view().spec_le(res.participant_view()), + self.known_retired().subset_of(res.known_retired()), + { + use_type_invariant(&reader); + use_type_invariant(self); + assert(reader.resource.value().state_view().known_retired + == reader.known_retired.records()); + assert(self.report().known_retired == self.known_retired.records()); + reader.resource.validate_2(&self.resource); + let ghost report = self.report(); + let ghost reader_state = reader.resource.value().state_view(); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()).valid()); + assert(Option::::op(reader.resource.value().state, None) + == reader.resource.value().state); + assert(reader.resource.value().closed.is_empty()); + assert(reader.resource.value().closed.union(Set::empty().insert(report)) + =~= Set::empty().insert(report)); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()) == CpuRcuCarrier { + state: reader.resource.value().state, + closed: Set::empty().insert(report), + }); + assert(CpuRcuCarrier::reports_fit(reader_state, Set::empty().insert(report))); + assert(Set::empty().insert(report).contains(report)); + assert(report.generation < reader_state.generation); + assert(self.view().spec_le(reader.participant_view())); + assert(report.known_retired.subset_of(reader_state.known_retired)); + assert(self.known_retired().subset_of(reader.known_retired())); + reader + } + + /// Lifts [`Self::lemma_later_reader`] to the task view used to start a + /// reader. + /// + /// This is the paper's later-reader branch: once the task imports the + /// persistent CPU view, it also observes every detachment observation + /// carried by an earlier report. + pub proof fn lemma_later_reader_start_view( + tracked &self, + tracked reader: CpuRcuReaderFragment, + start_view: WmView, + ) -> (tracked res: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + self.participant_id() == reader.participant_id(), + reader.participant_view().spec_le(start_view), + ensures + res == reader, + res.wf(), + self.closed_generation() < res.generation(), + self.view().spec_le(start_view), + self.known_retired().subset_of(res.known_retired()), + { + let tracked reader = self.lemma_later_reader(reader); + self.view().lemma_spec_le_transitive(reader.participant_view(), start_view); + reader + } + + /// A reader from a generation covered by this report cannot remain live. + pub proof fn lemma_excludes_old_reader(tracked &self, tracked reader: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + self.participant_id() == reader.participant_id(), + reader.generation() <= self.closed_generation(), + ensures + false, + { + let tracked _reader = self.lemma_later_reader(reader); + } + + /// A complete refined guard coexisting with this report necessarily + /// started after the generation closed by the report. + pub proof fn lemma_later_guard( + tracked &self, + tracked guard: CpuRcuReadGuardToken, + ) -> (tracked res: CpuRcuReadGuardToken) + requires + self.wf(), + guard.wf(), + self.participant_id() == guard.participant_id(), + self.cpu() == guard.cpu(), + ensures + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.reader_fragment() == guard.reader_fragment(), + res.scheduler() == guard.scheduler(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.domain() == guard.domain(), + res.root() == guard.root(), + res.retire_observation_registry() == guard.retire_observation_registry(), + res.start_view() == guard.start_view(), + res.expired() == guard.expired(), + res.seen_removed() == guard.seen_removed(), + self.closed_generation() < res.generation(), + self.view().spec_le(res.start_view()), + self.known_retired().subset_of(res.known_retired()), + { + let tracked (paper_guard, reader, binding) = guard.tracked_into_parts(); + let tracked reader = self.lemma_later_reader_start_view(reader, paper_guard.start_view()); + CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding) + } + + /// The old-reader branch of the paper proof: a guard from a generation + /// covered by this report cannot still own its CPU reader fragment. + pub proof fn lemma_excludes_old_guard(tracked &self, tracked guard: CpuRcuReadGuardToken) + requires + self.wf(), + guard.wf(), + self.scheduler() == guard.scheduler(), + self.cpu() == guard.cpu(), + guard.generation() <= self.closed_generation(), + ensures + false, + { + self.binding.lemma_same_cpu_agree(&guard.binding); + assert(self.participant_id() == guard.participant_id()); + let tracked (_paper_guard, reader, _binding) = guard.tracked_into_parts(); + self.lemma_excludes_old_reader(reader); + } +} + +/// Regression proof for both sides of the grace-period reader dichotomy. +proof fn cpu_rcu_generation_smoke_test(cpu: CpuId, initial: WmView, later: WmView) + requires + initial.spec_le(later), +{ + let tracked participant = CpuRcuParticipant::new(cpu, initial); + let tracked core = CpuCoreOwner::new(cpu, participant); + let ghost registration = core.registration(); + let tracked (mut registry, _entries) = GhostMapAuth::new( + Map::::empty(), + ); + let tracked entry = registry.insert(cpu, registration); + let tracked binding = CpuCoreOwnerBinding::tracked_new(entry, &core); + let tracked (handle, participant) = core.tracked_open(); + let tracked (participant, reader) = participant.tracked_start_reader(initial, 0.5real); + let tracked participant = participant.tracked_stop_reader(reader); + assert(participant.fraction() == 1real); + let tracked (participant, closed) = participant.tracked_report_quiescent(binding, later); + let tracked (participant, new_reader) = participant.tracked_start_reader(later, 0.5real); + let tracked new_reader = closed.lemma_later_reader_start_view(new_reader, later); + assert(closed.closed_generation() < new_reader.generation()); + assert(closed.view().spec_le(later)); + let tracked participant = participant.tracked_stop_reader(new_reader); + let tracked _core = handle.tracked_restore(participant); +} + +} // verus! diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 2c0bb807a..f579ad99f 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -442,7 +442,7 @@ impl RcuWeakAtomicPtr where &self, Ghost(reader): Ghost, Tracked(cpu_reader): Tracked, - Tracked(binding): Tracked, + Tracked(binding): Tracked, Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ( *mut T, @@ -456,9 +456,9 @@ impl RcuWeakAtomicPtr where cpu_reader.wf(), reader.cpu == cpu_reader.cpu(), reader.generation == cpu_reader.generation(), - binding.scheduler() == reader.scheduler, + binding.registry() == reader.scheduler, binding.cpu() == cpu_reader.cpu(), - binding.participant_id() == cpu_reader.participant_id(), + binding.single_local_id() == cpu_reader.participant_id(), cpu_reader.participant_view().spec_le(old(tv)@), ensures old(tv)@.spec_le(final(tv)@), @@ -469,7 +469,7 @@ impl RcuWeakAtomicPtr where res.4@.generation() == cpu_reader.generation(), res.4@.participant_view() == cpu_reader.participant_view(), res.4@.reader_fragment() == cpu_reader, - res.4@.scheduler() == binding.scheduler(), + res.4@.scheduler() == binding.registry(), res.4@.domain() == self.constant().domain, res.4@.reader_registry() == self.constant().reader_registry, res.4@.retire_observation_registry() == self.constant().retire_observation_registry, diff --git a/ostd/specs/task/cpu_core.rs b/ostd/specs/task/cpu_core.rs index dec7241e0..ff85d4935 100644 --- a/ostd/specs/task/cpu_core.rs +++ b/ostd/specs/task/cpu_core.rs @@ -8,7 +8,13 @@ //! state, then restores that state before returning the owner to the scheduler. use core::marker::PhantomData; -use vstd::{prelude::*, resource::Loc}; +use vstd::{ + prelude::*, + resource::{ + Loc, + map::{GhostPersistentPointsTo, GhostPointsTo}, + }, +}; use vstd_extra::resource::ghost_resource::excl::ExclusiveGhost; use crate::specs::mm::cpu::CpuId; @@ -26,6 +32,17 @@ pub ghost struct CpuCoreOwnerView { pub locals_key: Seq, } +/// Stable identity of one scheduler-owned CPU core and its local aggregate. +/// +/// A scheduler registry stores this value under the corresponding [`CpuId`]. +/// The core ID distinguishes independently allocated owners, while +/// `locals_key` identifies the exact ordered CPU-local resources installed in +/// the owner. +pub ghost struct CpuCoreRegistration { + pub owner_id: Loc, + pub locals_key: Seq, +} + /// A typed collection of resources that belongs permanently to one CPU. /// /// Implementations may aggregate any number of differently typed CPU-local @@ -84,6 +101,17 @@ pub tracked struct CpuCoreOwner { locals: L, } +/// Persistent evidence for the canonical core registered for one CPU. +/// +/// This is deliberately defined in the generic CPU-core model rather than in +/// an individual CPU-local client. A reader, scheduler transition, or +/// quiescent report may duplicate the evidence and later establish that they +/// refer to the same core and the same local-resource aggregate. +pub tracked struct CpuCoreOwnerBinding { + entry: GhostPersistentPointsTo, + marker: PhantomData, +} + impl View for CpuCoreOwnerHandle { type V = CpuCoreOwnerView; @@ -100,6 +128,80 @@ impl View for CpuCoreOwner { } } +impl CpuCoreOwnerBinding { + /// Persists one entry from the scheduler's authoritative core registry. + pub proof fn tracked_new( + tracked entry: GhostPointsTo, + tracked core: &CpuCoreOwner, + ) -> (tracked res: Self) + requires + core.wf(), + entry.key() == core.cpu(), + entry.value() == core.registration(), + ensures + res.registry() == entry.id(), + res.cpu() == core.cpu(), + res.owner_id() == core.id(), + res.locals_key() == core.locals_key(), + { + let tracked entry = entry.persist(); + CpuCoreOwnerBinding { entry, marker: PhantomData } + } + + /// Identity of the authoritative scheduler core registry. + pub closed spec fn registry(self) -> Loc { + self.entry.id() + } + + /// CPU whose canonical core is recorded by this entry. + pub closed spec fn cpu(self) -> CpuId { + self.entry.key() + } + + /// Stable identity of the registered [`CpuCoreOwner`]. + pub closed spec fn owner_id(self) -> Loc { + self.entry.value().owner_id + } + + /// Ordered identities of the registered core's CPU-local resources. + pub closed spec fn locals_key(self) -> Seq { + self.entry.value().locals_key + } + + /// Identity of the only local resource in a singleton aggregate. + pub open spec fn single_local_id(self) -> Loc + recommends + self.locals_key().len() == 1, + { + self.locals_key()[0] + } + + /// Creates another persistent copy for a CPU-local client. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.registry() == self.registry(), + res.cpu() == self.cpu(), + res.owner_id() == self.owner_id(), + res.locals_key() == self.locals_key(), + { + CpuCoreOwnerBinding { entry: self.entry.duplicate(), marker: PhantomData } + } + + /// Entries for the same CPU in one registry agree on the complete core + /// registration. + pub proof fn lemma_same_cpu_agree(tracked &self, tracked other: &Self) + requires + self.registry() == other.registry(), + self.cpu() == other.cpu(), + ensures + self.owner_id() == other.owner_id(), + self.locals_key() == other.locals_key(), + { + let tracked mut duplicate = self.entry.duplicate(); + duplicate.intersection_agrees(&other.entry); + } +} + impl CpuCoreOwnerHandle { /// Unique identity of this core resource. pub closed spec fn id(&self) -> Loc { @@ -140,6 +242,13 @@ impl CpuCoreOwnerHandle { ensures res.id() == self.id(), res@ == self@, + res.cpu() == self.cpu(), + res.current_task() == self.current_task(), + res.locals_key() == self.expected_locals_key(), + res.registration() == (CpuCoreRegistration { + owner_id: self.id(), + locals_key: self.expected_locals_key(), + }), res.wf(), res.locals() == locals, res.locals().local_key() == self.expected_locals_key(), @@ -158,6 +267,9 @@ impl CpuCoreOwner { res.is_idle(), res.wf(), res.locals() == locals, + res.locals_key() == locals.local_key(), + res.registration().owner_id == res.id(), + res.registration().locals_key == locals.local_key(), { let ghost locals_key = locals.local_key(); let tracked state = ExclusiveGhost::alloc( @@ -197,6 +309,11 @@ impl CpuCoreOwner { self.handle.expected_locals_key() } + /// Stable value stored in the scheduler's canonical CPU-core registry. + pub closed spec fn registration(&self) -> CpuCoreRegistration { + CpuCoreRegistration { owner_id: self.id(), locals_key: self.locals_key() } + } + /// The core identity is valid and every local resource belongs to its CPU. pub closed spec fn wf(&self) -> bool { &&& self.handle.wf() @@ -215,6 +332,7 @@ impl CpuCoreOwner { final(self).current_task() == Some(task), final(self).locals() == old(self).locals(), final(self).locals_key() == old(self).locals_key(), + final(self).registration() == old(self).registration(), final(self).wf(), { let ghost next = CpuCoreOwnerView { @@ -237,6 +355,7 @@ impl CpuCoreOwner { final(self).is_idle(), final(self).locals() == old(self).locals(), final(self).locals_key() == old(self).locals_key(), + final(self).registration() == old(self).registration(), final(self).wf(), { let task = self.current_task()->0; @@ -261,6 +380,8 @@ impl CpuCoreOwner { res.0.id() == self.id(), res.0@ == self@, res.0.wf(), + res.0.cpu() == self.cpu(), + res.0.current_task() == self.current_task(), res.0.expected_locals_key() == self.locals_key(), res.1 == self.locals(), res.1.belongs_to_cpu(res.0.cpu()), diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 3a2141c47..ee53bef32 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -235,9 +235,7 @@ struct RcuReadGuardInner<'a, P: NonNullPtr> { proof_active: bool, _inner_guard: DisabledPreemptGuard, tracked_info: Tracked::Target>>>, - tracked_guard: Tracked< - Option::Target>>, - >, + tracked_guard: Tracked::Target>>>, tracked_session: Tracked>, } @@ -431,7 +429,7 @@ impl RcuInner

{ &self, Ghost(reader): Ghost, Tracked(cpu_reader): Tracked, - Tracked(binding): Tracked, + Tracked(binding): Tracked, Tracked(tv): Tracked<&mut ThreadView>, ) -> (res: ( *mut

::Target, @@ -443,9 +441,9 @@ impl RcuInner

{ cpu_reader.wf(), reader.cpu == cpu_reader.cpu(), reader.generation == cpu_reader.generation(), - binding.scheduler() == reader.scheduler, + binding.registry() == reader.scheduler, binding.cpu() == cpu_reader.cpu(), - binding.participant_id() == cpu_reader.participant_id(), + binding.single_local_id() == cpu_reader.participant_id(), cpu_reader.participant_view().spec_le(old(tv)@), ensures old(tv)@.spec_le(final(tv)@), @@ -456,7 +454,7 @@ impl RcuInner

{ res.2@.generation() == cpu_reader.generation(), res.2@.participant_view() == cpu_reader.participant_view(), res.2@.reader_fragment() == cpu_reader, - res.2@.scheduler() == binding.scheduler(), + res.2@.scheduler() == binding.registry(), res.2@.domain() == self.ptr.constant().domain, res.2@.reader_registry() == self.ptr.constant().reader_registry, res.2@.retire_observation_registry() == self.ptr.constant().retire_observation_registry, @@ -585,10 +583,8 @@ impl RcuInner

{ } } - fn read<'a>( - &'a self, - Tracked(session): Tracked<&'a mut RunningTaskContext>, - ) -> (res: RcuReadGuardInner<'a, P>) + fn read<'a>(&'a self, Tracked(session): Tracked<&'a mut RunningTaskContext>) -> (res: + RcuReadGuardInner<'a, P>) requires self.type_inv(), old(session).wf(), @@ -662,8 +658,8 @@ impl RcuInner

{ assert(tracked_guard@.cpu() == session.cpu()); assert(tracked_guard@.generation() == session.rcu_generation()); assert(cpu_reader.fraction() == context_before_reader.rcu_fraction() / 2real); - assert(context_before_load.rcu_fraction() - == context_before_reader.rcu_fraction() / 2real); + assert(context_before_load.rcu_fraction() == context_before_reader.rcu_fraction() + / 2real); assert(session.rcu_fraction() == context_before_load.rcu_fraction()); assert(tracked_guard@.reader_fragment().fraction() == cpu_reader.fraction()); assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction()); @@ -701,14 +697,12 @@ impl RcuInner

{ assert(res.guard_token().participant_id() == stored_context.rcu_participant_id()); assert(res.guard_token().cpu() == stored_context.cpu()); assert(res.guard_token().generation() == stored_context.rcu_generation()); - assert(res.guard_token().reader_fragment().fraction() - == stored_context.rcu_fraction()); + assert(res.guard_token().reader_fragment().fraction() == stored_context.rcu_fraction()); assert(res.guard_token().reader_context() == reader); assert(res.matches_context(stored_context)); } res } - } /// Detaches the proof-only reader state while leaving the executable @@ -719,16 +713,9 @@ impl RcuInner

{ /// before the executable guard can be observed again. fn take_reader_state<'a, T>( proof_active: &mut bool, - Tracked(guard_slot): Tracked< - &mut Tracked>>, - >, - Tracked(session_slot): Tracked< - &mut Tracked>, - >, -) -> (res: Tracked<( - rcu_cpu_spec::CpuRcuReadGuardToken, - &'a mut RunningTaskContext, -)>) + Tracked(guard_slot): Tracked<&mut Tracked>>>, + Tracked(session_slot): Tracked<&mut Tracked>>, +) -> (res: Tracked<(rcu_cpu_spec::CpuRcuReadGuardToken, &'a mut RunningTaskContext)>) requires *old(proof_active), old(guard_slot)@ is Some, @@ -746,7 +733,7 @@ fn take_reader_state<'a, T>( let tracked guard = guard_slot.borrow_mut().tracked_take(); let tracked session = session_slot.borrow_mut().tracked_take(); } - *proof_active = false; + * proof_active = false; Tracked((guard, session)) } @@ -754,9 +741,7 @@ fn take_reader_state<'a, T>( fn finish_reader_state<'a, P: NonNullPtr>( rcu: &RcuInner

, inner_guard: &mut DisabledPreemptGuard, - Tracked(guard): Tracked< - rcu_cpu_spec::CpuRcuReadGuardToken<

::Target>, - >, + Tracked(guard): Tracked::Target>>, Tracked(session): Tracked<&'a mut RunningTaskContext>, ) -> (res: Tracked<&'a mut RunningTaskContext>) requires @@ -766,8 +751,7 @@ fn finish_reader_state<'a, P: NonNullPtr>( guard.wf(), guard.domain() == rcu.ptr.constant().domain, guard.root() == rcu.ptr.id(), - guard.retire_observation_registry() - == rcu.ptr.constant().retire_observation_registry, + guard.retire_observation_registry() == rcu.ptr.constant().retire_observation_registry, guard.participant_id() == old(session).rcu_participant_id(), guard.reader_fragment().fraction() == old(session).rcu_fraction(), ensures @@ -801,9 +785,7 @@ fn finish_reader_state<'a, P: NonNullPtr>( } fn restore_reader_session<'a>( - Tracked(session_slot): Tracked< - &mut Tracked>, - >, + Tracked(session_slot): Tracked<&mut Tracked>>, Tracked(session): Tracked<&'a mut RunningTaskContext>, Ghost(restored): Ghost, ) @@ -868,10 +850,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { res } - fn compare_exchange( - self, - new_ptr: Option

, - ) -> (res: Result<(), Option

>) + fn compare_exchange(self, new_ptr: Option

) -> (res: Result<(), Option

>) requires self.rcu.is_nullable() || new_ptr is Some, self.type_inv(), @@ -987,26 +966,35 @@ impl<'a, P: NonNullPtr> Drop for RcuReadGuardInner<'a, P> { old(self).is_active() ==> { &&& final(self).stored_context().wf() &&& final(self).stored_context().task() == old(self).stored_context().task() - &&& final(self).stored_context().scheduler() - == old(self).stored_context().scheduler() + &&& final(self).stored_context().scheduler() == old( + self, + ).stored_context().scheduler() &&& final(self).stored_context().cpu() == old(self).stored_context().cpu() &&& final(self).stored_context().view() == old(self).stored_context().view() - &&& final(self).stored_context().session_id() - == old(self).stored_context().session_id() - &&& final(self).stored_context().quiescent_generation() - == old(self).stored_context().quiescent_generation() - &&& final(self).stored_context().available_fractions() - == old(self).stored_context().available_fractions() + 1 - &&& final(self).stored_context().preempt_depth() + 1 - == old(self).stored_context().preempt_depth() - &&& final(self).stored_context().rcu_participant_id() - == old(self).stored_context().rcu_participant_id() - &&& final(self).stored_context().rcu_generation() - == old(self).stored_context().rcu_generation() - &&& final(self).stored_context().rcu_participant_view() - == old(self).stored_context().rcu_participant_view() - &&& final(self).stored_context().rcu_fraction() - == old(self).stored_context().rcu_fraction() * 2real + &&& final(self).stored_context().session_id() == old( + self, + ).stored_context().session_id() + &&& final(self).stored_context().quiescent_generation() == old( + self, + ).stored_context().quiescent_generation() + &&& final(self).stored_context().available_fractions() == old( + self, + ).stored_context().available_fractions() + 1 + &&& final(self).stored_context().preempt_depth() + 1 == old( + self, + ).stored_context().preempt_depth() + &&& final(self).stored_context().rcu_participant_id() == old( + self, + ).stored_context().rcu_participant_id() + &&& final(self).stored_context().rcu_generation() == old( + self, + ).stored_context().rcu_generation() + &&& final(self).stored_context().rcu_participant_view() == old( + self, + ).stored_context().rcu_participant_view() + &&& final(self).stored_context().rcu_fraction() == old( + self, + ).stored_context().rcu_fraction() * 2real }, opens_invariants none no_unwind @@ -1403,9 +1391,9 @@ impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { *self.tracked_session@->Some_0 } - closed spec fn guard_token( - self, - ) -> rcu_cpu_spec::CpuRcuReadGuardToken<

::Target> + closed spec fn guard_token(self) -> rcu_cpu_spec::CpuRcuReadGuardToken< +

::Target, + > recommends self.tracked_guard@ is Some, { diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 049ac5056..9774b86c7 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -6,11 +6,11 @@ use crate::{ specs::sync::{ rcu::RcuRetiredFacts, rcu_cpu::{ - CpuRcuClosedGeneration, CpuRcuParticipant, CpuRcuParticipantBinding, - CpuRcuReaderFragment, + CpuRcuClosedGeneration, CpuRcuCoreBinding, CpuRcuParticipant, CpuRcuReaderFragment, }, weak_memory::{ThreadView, WmView}, }, + specs::task::cpu_core::{CpuCoreOwner, CpuCoreOwnerHandle, CpuCoreRegistration}, sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ task::scheduler::{SchedulerView, TaskThreadView}, }; @@ -361,8 +361,9 @@ impl PreemptThreadViewSession { /// the scheduler when no guard remains live. pub tracked struct RunningTaskContext { session: PreemptThreadViewSession, + core_handle: CpuCoreOwnerHandle, rcu_participant: CpuRcuParticipant, - rcu_binding: CpuRcuParticipantBinding, + rcu_binding: CpuRcuCoreBinding, preempt_depth: Ghost, cpu: Ghost, } @@ -371,22 +372,31 @@ impl RunningTaskContext { /// Starts a running interval for a checked-out task view. pub proof fn new( tracked task_view: TaskThreadView, + tracked core_handle: CpuCoreOwnerHandle, tracked rcu_participant: CpuRcuParticipant, - tracked rcu_binding: CpuRcuParticipantBinding, + tracked rcu_binding: CpuRcuCoreBinding, sched_view: SchedulerView, cpu: crate::specs::mm::cpu::CpuId, ) -> (tracked res: Self) requires task_view.wf(sched_view), + core_handle.wf(), + core_handle.cpu() == cpu, + core_handle.current_task() == Some(task_view.task()), + core_handle.expected_locals_key() == seq![rcu_participant.id()], rcu_participant.wf(), rcu_participant.cpu() == cpu, rcu_participant.fraction() == 1real, rcu_participant.view().spec_le(task_view.view()), - rcu_binding.scheduler() == task_view.scheduler(), + rcu_binding.registry() == task_view.scheduler(), rcu_binding.cpu() == cpu, - rcu_binding.participant_id() == rcu_participant.id(), + rcu_binding.owner_id() == core_handle.id(), + rcu_binding.locals_key() == core_handle.expected_locals_key(), + rcu_binding.single_local_id() == rcu_participant.id(), sched_view.cpu_has_rcu_participant(cpu), sched_view.cpu_rcu_participant_id(cpu) == rcu_participant.id(), + sched_view.cpu_core_registration(cpu).owner_id == core_handle.id(), + sched_view.cpu_core_registration(cpu).locals_key == core_handle.expected_locals_key(), !sched_view.cpu_rcu_participant_is_stored(cpu), sched_view.current.contains_key(cpu), sched_view.current[cpu] == Some(task_view.task()), @@ -395,6 +405,7 @@ impl RunningTaskContext { res.task() == task_view.task(), res.view() == task_view.view(), res.cpu() == cpu, + res.core_owner_id() == core_handle.id(), res.preempt_depth() == 0, res.quiescent_generation() == 0, res.available_fractions() == PREEMPT_SESSION_FRACTIONS, @@ -402,9 +413,9 @@ impl RunningTaskContext { res.rcu_generation() == rcu_participant.generation(), res.rcu_participant_view() == rcu_participant.view(), res.rcu_fraction() == 1real, - res.rcu_binding().scheduler() == res.scheduler(), + res.rcu_binding().registry() == res.scheduler(), res.rcu_binding().cpu() == cpu, - res.rcu_binding().participant_id() == res.rcu_participant_id(), + res.rcu_binding().single_local_id() == res.rcu_participant_id(), res.wf(), res.is_quiescent(), res.wf_scheduler(sched_view), @@ -412,6 +423,7 @@ impl RunningTaskContext { let tracked session = PreemptThreadViewSession::new(task_view, sched_view); let tracked res = RunningTaskContext { session, + core_handle, rcu_participant, rcu_binding, preempt_depth: Ghost(0), @@ -440,6 +452,10 @@ impl RunningTaskContext { self.cpu@ } + pub closed spec fn core_owner_id(self) -> Loc { + self.core_handle.id() + } + pub closed spec fn session_id(self) -> Loc { self.session.session_id() } @@ -476,17 +492,23 @@ impl RunningTaskContext { self.rcu_participant.fraction() } - pub closed spec fn rcu_binding(self) -> CpuRcuParticipantBinding { + pub closed spec fn rcu_binding(self) -> CpuRcuCoreBinding { self.rcu_binding } pub closed spec fn wf(self) -> bool { &&& self.session.wf_session_resource() &&& self.available_fractions() + self.preempt_depth() == PREEMPT_SESSION_FRACTIONS + &&& self.core_handle.wf() + &&& self.core_handle.cpu() == self.cpu() + &&& self.core_handle.current_task() == Some(self.task()) + &&& self.core_handle.expected_locals_key() == seq![self.rcu_participant_id()] &&& self.rcu_participant.wf() - &&& self.rcu_binding().scheduler() == self.scheduler() + &&& self.rcu_binding().registry() == self.scheduler() &&& self.rcu_binding().cpu() == self.cpu() - &&& self.rcu_binding().participant_id() == self.rcu_participant_id() + &&& self.rcu_binding().owner_id() == self.core_owner_id() + &&& self.rcu_binding().locals_key() == self.core_handle.expected_locals_key() + &&& self.rcu_binding().single_local_id() == self.rcu_participant_id() &&& self.rcu_participant.cpu() == self.cpu() &&& self.rcu_participant_view().spec_le(self.view()) } @@ -510,6 +532,10 @@ impl RunningTaskContext { &&& sched_view.current[self.cpu()] == Some(self.task()) &&& sched_view.cpu_has_rcu_participant(self.cpu()) &&& sched_view.cpu_rcu_participant_id(self.cpu()) == self.rcu_participant_id() + &&& sched_view.cpu_core_registration(self.cpu()).owner_id == self.core_owner_id() + &&& sched_view.cpu_core_registration(self.cpu()).locals_key == seq![ + self.rcu_participant_id(), + ] &&& !sched_view.cpu_rcu_participant_is_stored(self.cpu()) } @@ -528,6 +554,10 @@ impl RunningTaskContext { sched_view.current[self.cpu()] == Some(self.task()), sched_view.cpu_has_rcu_participant(self.cpu()), sched_view.cpu_rcu_participant_id(self.cpu()) == self.rcu_participant_id(), + sched_view.cpu_core_registration(self.cpu()).owner_id == self.core_owner_id(), + sched_view.cpu_core_registration(self.cpu()).locals_key == seq![ + self.rcu_participant_id(), + ], !sched_view.cpu_rcu_participant_is_stored(self.cpu()), ensures self.wf_scheduler(sched_view), @@ -602,14 +632,15 @@ impl RunningTaskContext { } /// Copies the persistent scheduler binding for a guard or quiescent report. - pub proof fn tracked_rcu_binding(tracked &self) -> (tracked binding: - CpuRcuParticipantBinding) + pub proof fn tracked_rcu_binding(tracked &self) -> (tracked binding: CpuRcuCoreBinding) requires self.wf(), ensures - binding.scheduler() == self.scheduler(), + binding.registry() == self.scheduler(), binding.cpu() == self.cpu(), - binding.participant_id() == self.rcu_participant_id(), + binding.owner_id() == self.core_owner_id(), + binding.locals_key() == seq![self.rcu_participant_id()], + binding.single_local_id() == self.rcu_participant_id(), { self.rcu_binding.tracked_duplicate() } @@ -709,11 +740,7 @@ impl RunningTaskContext { final(self).rcu_fraction() == 1real, { let tracked binding = self.rcu_binding.tracked_duplicate(); - self.rcu_participant.tracked_report_quiescent_with_in_place( - binding, - self.view(), - learned, - ) + self.rcu_participant.tracked_report_quiescent_with_in_place(binding, self.view(), learned) } /// Records one quiescent boundary for this running session. @@ -749,7 +776,7 @@ impl RunningTaskContext { /// ownership. The full-fraction requirement rules out live preempt guards. pub proof fn tracked_into_task_view(tracked self) -> (tracked res: ( TaskThreadView, - CpuRcuParticipant, + CpuCoreOwner, )) requires self.wf(), @@ -759,16 +786,40 @@ impl RunningTaskContext { res.0.scheduler() == self.scheduler(), res.0.task() == self.task(), res.0.view() == self.view(), - res.1.id() == self.rcu_participant_id(), + res.1.id() == self.core_owner_id(), res.1.cpu() == self.cpu(), - res.1.generation() == self.rcu_generation(), - res.1.view() == self.rcu_participant_view(), - res.1.fraction() == 1real, - res.1.view().spec_le(res.0.view()), + res.1.current_task() == Some(self.task()), + res.1.locals_key() == seq![self.rcu_participant_id()], + res.1.registration() == (CpuCoreRegistration { + owner_id: self.core_owner_id(), + locals_key: seq![self.rcu_participant_id()], + }), + res.1.locals().id() == self.rcu_participant_id(), + res.1.locals().generation() == self.rcu_generation(), + res.1.locals().view() == self.rcu_participant_view(), + res.1.locals().fraction() == 1real, + res.1.locals().view().spec_le(res.0.view()), res.1.wf(), { assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); - (self.session.tracked_into_task_view(), self.rcu_participant) + let ghost cpu = self.cpu(); + let ghost task = self.task(); + let ghost core_owner_id = self.core_owner_id(); + let ghost rcu_participant_id = self.rcu_participant_id(); + let ghost core_view = self.core_handle@; + assert(self.core_handle.cpu() == cpu); + assert(self.core_handle.current_task() == Some(task)); + let tracked core = self.core_handle.tracked_restore(self.rcu_participant); + assert(core@ == core_view); + assert(core.id() == core_owner_id); + assert(core.cpu() == cpu); + assert(core.current_task() == Some(task)); + assert(core.locals_key() == seq![rcu_participant_id]); + assert(core.registration() == (CpuCoreRegistration { + owner_id: core_owner_id, + locals_key: seq![rcu_participant_id], + })); + (self.session.tracked_into_task_view(), core) } /// Scheduler-facing form of `tracked_into_task_view` that preserves the @@ -776,7 +827,7 @@ impl RunningTaskContext { pub proof fn tracked_into_task_view_for_scheduler( tracked self, sched_view: SchedulerView, - ) -> (tracked res: (TaskThreadView, CpuRcuParticipant)) + ) -> (tracked res: (TaskThreadView, CpuCoreOwner)) requires self.wf_scheduler(sched_view), self.is_quiescent(), @@ -785,19 +836,43 @@ impl RunningTaskContext { res.0.task() == self.task(), res.0.view() == self.view(), res.0.wf(sched_view), - res.1.id() == self.rcu_participant_id(), + res.1.id() == self.core_owner_id(), res.1.cpu() == self.cpu(), - res.1.generation() == self.rcu_generation(), - res.1.view() == self.rcu_participant_view(), - res.1.fraction() == 1real, - res.1.view().spec_le(res.0.view()), + res.1.current_task() == Some(self.task()), + res.1.locals_key() == seq![self.rcu_participant_id()], + res.1.registration() == (CpuCoreRegistration { + owner_id: self.core_owner_id(), + locals_key: seq![self.rcu_participant_id()], + }), + res.1.locals().id() == self.rcu_participant_id(), + res.1.locals().generation() == self.rcu_generation(), + res.1.locals().view() == self.rcu_participant_view(), + res.1.locals().fraction() == 1real, + res.1.locals().view().spec_le(res.0.view()), res.1.wf(), { assert(self.preempt_depth() == 0); assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); assert(self.session.wf(sched_view)); + let ghost cpu = self.cpu(); + let ghost task = self.task(); + let ghost core_owner_id = self.core_owner_id(); + let ghost rcu_participant_id = self.rcu_participant_id(); + let ghost core_view = self.core_handle@; + assert(self.core_handle.cpu() == cpu); + assert(self.core_handle.current_task() == Some(task)); let tracked task_view = self.session.tracked_into_task_view_for_scheduler(sched_view); - (task_view, self.rcu_participant) + let tracked core = self.core_handle.tracked_restore(self.rcu_participant); + assert(core@ == core_view); + assert(core.id() == core_owner_id); + assert(core.cpu() == cpu); + assert(core.current_task() == Some(task)); + assert(core.locals_key() == seq![rcu_participant_id]); + assert(core.registration() == (CpuCoreRegistration { + owner_id: core_owner_id, + locals_key: seq![rcu_participant_id], + })); + (task_view, core) } } @@ -1174,10 +1249,7 @@ impl DisabledPreemptGuard { /// Consuming compatibility wrapper for callers that do not need standard /// destructor integration. - pub(crate) fn release_to_context( - self, - Tracked(context): Tracked<&mut RunningTaskContext>, - ) + pub(crate) fn release_to_context(self, Tracked(context): Tracked<&mut RunningTaskContext>) requires old(context).wf(), old(context).preempt_depth() > 0, diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 5d5c581fb..e1f0e39ac 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -68,10 +68,15 @@ use vstd::{map::Map, prelude::*, resource::Loc}; use super::{Task, preempt::RunningTaskContext}; use crate::{ - specs::mm::cpu::CpuId, - specs::sync::{ - rcu_cpu::{CpuRcuParticipant, CpuRcuParticipantBinding}, - weak_memory::{ThreadView, WmView}, + specs::{ + mm::cpu::CpuId, + sync::{ + rcu_cpu::CpuRcuParticipant, + weak_memory::{ThreadView, WmView}, + }, + task::cpu_core::{ + CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreOwnerHandle, CpuCoreRegistration, + }, }, sync::{OnceImpl, RoArc, TrivialPred}, }; @@ -134,9 +139,9 @@ pub ghost enum TaskSchedState { /// weak-memory view. `stored_views` records views still owned by the scheduler /// resource; `checked_out_views` records views temporarily held by guards. /// `cpu_views` persists observations across context switches on each CPU. -/// `cpu_rcu_participant_ids` permanently binds each registered CPU to one RCU -/// participant resource. `stored_cpu_rcu_participants` records which complete -/// participant resources are currently owned by the scheduler. +/// `cpu_core_registrations` permanently binds each registered CPU to one core +/// owner and its stable CPU-local aggregate. `stored_cpu_cores` records which +/// complete owners are currently held by the scheduler. pub ghost struct SchedulerView { pub id: Loc, pub runqueues: Map>, @@ -146,8 +151,8 @@ pub ghost struct SchedulerView { pub stored_views: Map, pub checked_out_views: Map, pub cpu_views: Map, - pub cpu_rcu_participant_ids: Map, - pub stored_cpu_rcu_participants: Set, + pub cpu_core_registrations: Map, + pub stored_cpu_cores: Set, } impl SchedulerView { @@ -162,8 +167,8 @@ impl SchedulerView { stored_views: Map::empty(), checked_out_views: Map::empty(), cpu_views: Map::empty(), - cpu_rcu_participant_ids: Map::empty(), - stored_cpu_rcu_participants: Set::empty(), + cpu_core_registrations: Map::empty(), + stored_cpu_cores: Set::empty(), } } @@ -208,18 +213,26 @@ impl SchedulerView { } pub open spec fn cpu_has_rcu_participant(self, cpu: CpuId) -> bool { - self.cpu_rcu_participant_ids.contains_key(cpu) + self.cpu_core_registrations.contains_key(cpu) + && self.cpu_core_registrations[cpu].locals_key.len() == 1 } pub open spec fn cpu_rcu_participant_id(self, cpu: CpuId) -> Loc recommends self.cpu_has_rcu_participant(cpu), { - self.cpu_rcu_participant_ids[cpu] + self.cpu_core_registrations[cpu].locals_key[0] } pub open spec fn cpu_rcu_participant_is_stored(self, cpu: CpuId) -> bool { - self.stored_cpu_rcu_participants.contains(cpu) + self.stored_cpu_cores.contains(cpu) + } + + pub open spec fn cpu_core_registration(self, cpu: CpuId) -> CpuCoreRegistration + recommends + self.cpu_core_registrations.contains_key(cpu), + { + self.cpu_core_registrations[cpu] } /// The scheduling policy changed no weak-memory ownership state. @@ -234,8 +247,8 @@ impl SchedulerView { &&& self.stored_views == other.stored_views &&& self.checked_out_views == other.checked_out_views &&& self.cpu_views == other.cpu_views - &&& self.cpu_rcu_participant_ids == other.cpu_rcu_participant_ids - &&& self.stored_cpu_rcu_participants == other.stored_cpu_rcu_participants + &&& self.cpu_core_registrations == other.cpu_core_registrations + &&& self.stored_cpu_cores == other.stored_cpu_cores } pub open spec fn task_in_runqueue(self, task: Loc) -> bool { @@ -301,31 +314,42 @@ impl SchedulerView { /// /// Scheduler policy may populate that CPU's runqueue/current slot only /// after this transition. The initial empty view carries no observations. - pub open spec fn register_cpu(self, cpu: CpuId, rcu_participant_id: Loc) -> SchedulerView + pub open spec fn register_cpu( + self, + cpu: CpuId, + registration: CpuCoreRegistration, + ) -> SchedulerView recommends !self.cpu_views.contains_key(cpu), valid_cpu(cpu), + registration.locals_key.len() == 1, { SchedulerView { cpu_views: self.cpu_views.insert(cpu, WmView::empty()), - cpu_rcu_participant_ids: self.cpu_rcu_participant_ids.insert(cpu, rcu_participant_id), - stored_cpu_rcu_participants: self.stored_cpu_rcu_participants.insert(cpu), + cpu_core_registrations: self.cpu_core_registrations.insert(cpu, registration), + stored_cpu_cores: self.stored_cpu_cores.insert(cpu), ..self } } - pub proof fn lemma_register_cpu_preserves_wf(self, cpu: CpuId, rcu_participant_id: Loc) + pub proof fn lemma_register_cpu_preserves_wf( + self, + cpu: CpuId, + registration: CpuCoreRegistration, + ) requires self.wf(), !self.cpu_views.contains_key(cpu), valid_cpu(cpu), + registration.locals_key.len() == 1, ensures - self.register_cpu(cpu, rcu_participant_id).wf(), - self.register_cpu(cpu, rcu_participant_id).cpu_has_thread_view(cpu), - self.register_cpu(cpu, rcu_participant_id).cpu_thread_view(cpu) == WmView::empty(), - self.register_cpu(cpu, rcu_participant_id).cpu_rcu_participant_id(cpu) - == rcu_participant_id, - self.register_cpu(cpu, rcu_participant_id).cpu_rcu_participant_is_stored(cpu), + self.register_cpu(cpu, registration).wf(), + self.register_cpu(cpu, registration).cpu_has_thread_view(cpu), + self.register_cpu(cpu, registration).cpu_thread_view(cpu) == WmView::empty(), + self.register_cpu(cpu, registration).cpu_core_registration(cpu) == registration, + self.register_cpu(cpu, registration).cpu_rcu_participant_id(cpu) + == registration.locals_key[0], + self.register_cpu(cpu, registration).cpu_rcu_participant_is_stored(cpu), { } @@ -362,7 +386,7 @@ impl SchedulerView { task_views: self.task_views.insert(task, joined), stored_views: self.stored_views.remove(task), checked_out_views: self.checked_out_views.insert(task, joined), - stored_cpu_rcu_participants: self.stored_cpu_rcu_participants.remove(cpu), + stored_cpu_cores: self.stored_cpu_cores.remove(cpu), ..self } } @@ -416,7 +440,7 @@ impl SchedulerView { task_views: self.task_views.insert(task, view), stored_views: self.stored_views.insert(task, view), checked_out_views: self.checked_out_views.remove(task), - stored_cpu_rcu_participants: self.stored_cpu_rcu_participants.insert(cpu), + stored_cpu_cores: self.stored_cpu_cores.insert(cpu), ..self } } @@ -480,11 +504,14 @@ impl SchedulerView { ) // The complete RCU participant follows the same checkout boundary as // the current task view, while its identity remains CPU-stable. - &&& self.cpu_rcu_participant_ids.dom() == self.cpu_views.dom() - &&& self.stored_cpu_rcu_participants.subset_of(self.cpu_views.dom()) + &&& self.cpu_core_registrations.dom() == self.cpu_views.dom() + &&& forall|cpu: CpuId| #[trigger] + self.cpu_core_registrations.contains_key(cpu) + ==> self.cpu_core_registrations[cpu].locals_key.len() == 1 + &&& self.stored_cpu_cores.subset_of(self.cpu_views.dom()) &&& forall|cpu: CpuId| #[trigger] - self.cpu_views.contains_key(cpu) ==> (self.stored_cpu_rcu_participants.contains(cpu) - <==> !(self.current.contains_key(cpu) && self.current[cpu] is Some + self.cpu_views.contains_key(cpu) ==> (self.stored_cpu_cores.contains(cpu) <==> !( + self.current.contains_key(cpu) && self.current[cpu] is Some && self.checked_out_views.contains_key( self.current[cpu]->0, ))) @@ -549,8 +576,8 @@ tracked struct SchedulerThreadViews { scheduler: Ghost, views: Map, cpu_views: Map, - rcu_participants: Map, - rcu_bindings: Map, + cpu_cores: Map>, + core_bindings: Map>, } /// A checked-out per-task `ThreadView`. @@ -636,19 +663,22 @@ impl SchedulerThreadViews { res.scheduler() == scheduler, res.view() == Map::::empty(), res.cpu_view_map() == Map::::empty(), - res.rcu_participant_id_map() == Map::::empty(), - res.rcu_binding_id_map() == Map::::empty(), + res.core_registration_map() == Map::::empty(), + res.binding_registration_map() == Map::::empty(), { let tracked views = Map::::tracked_empty(); let tracked cpu_views = Map::::tracked_empty(); - let tracked rcu_participants = Map::::tracked_empty(); - let tracked rcu_bindings = Map::::tracked_empty(); + let tracked cpu_cores = Map::>::tracked_empty(); + let tracked core_bindings = Map::< + CpuId, + CpuCoreOwnerBinding, + >::tracked_empty(); SchedulerThreadViews { scheduler: Ghost(scheduler), views, cpu_views, - rcu_participants, - rcu_bindings, + cpu_cores, + core_bindings, } } @@ -664,12 +694,19 @@ impl SchedulerThreadViews { Map::new(self.cpu_views.dom(), |cpu: CpuId| self.cpu_views[cpu]@) } - pub closed spec fn rcu_participant_id_map(self) -> Map { - Map::new(self.rcu_participants.dom(), |cpu: CpuId| self.rcu_participants[cpu].id()) + pub closed spec fn core_registration_map(self) -> Map { + Map::new(self.cpu_cores.dom(), |cpu: CpuId| self.cpu_cores[cpu].registration()) } - pub closed spec fn rcu_binding_id_map(self) -> Map { - Map::new(self.rcu_bindings.dom(), |cpu: CpuId| self.rcu_bindings[cpu].participant_id()) + pub closed spec fn binding_registration_map(self) -> Map { + Map::new( + self.core_bindings.dom(), + |cpu: CpuId| + CpuCoreRegistration { + owner_id: self.core_bindings[cpu].owner_id(), + locals_key: self.core_bindings[cpu].locals_key(), + }, + ) } pub closed spec fn contains(self, task: Loc) -> bool { @@ -694,41 +731,43 @@ impl SchedulerThreadViews { self.cpu_views[cpu]@ } - closed spec fn rcu_participants_wf(self, sched_view: SchedulerView) -> bool { - &&& self.rcu_participants.dom() == sched_view.stored_cpu_rcu_participants + closed spec fn cpu_cores_wf(self, sched_view: SchedulerView) -> bool { + &&& self.cpu_cores.dom() == sched_view.stored_cpu_cores &&& forall|cpu: CpuId| #[trigger] - self.rcu_participants.contains_key(cpu) ==> { - &&& self.rcu_participants[cpu].wf() - &&& self.rcu_participants[cpu].cpu() == cpu - &&& self.rcu_participants[cpu].id() == sched_view.cpu_rcu_participant_ids[cpu] - &&& self.rcu_participants[cpu].fraction() == 1real - &&& self.rcu_participants[cpu].view().spec_le(self.cpu_views[cpu]@) + self.cpu_cores.contains_key(cpu) ==> { + &&& self.cpu_cores[cpu].wf() + &&& self.cpu_cores[cpu].cpu() == cpu + &&& self.cpu_cores[cpu].is_idle() + &&& self.cpu_cores[cpu].id() == sched_view.cpu_core_registrations[cpu].owner_id + &&& self.cpu_cores[cpu].locals_key() + == sched_view.cpu_core_registrations[cpu].locals_key + &&& self.cpu_cores[cpu].registration() == sched_view.cpu_core_registrations[cpu] + &&& self.cpu_cores[cpu].locals().fraction() == 1real + &&& self.cpu_cores[cpu].locals().view().spec_le(self.cpu_views[cpu]@) } } - closed spec fn rcu_bindings_wf(self, sched_view: SchedulerView) -> bool { - &&& self.rcu_bindings.dom() == sched_view.cpu_rcu_participant_ids.dom() + closed spec fn core_bindings_wf(self, sched_view: SchedulerView) -> bool { + &&& self.core_bindings.dom() == sched_view.cpu_core_registrations.dom() &&& forall|cpu: CpuId| #[trigger] - self.rcu_bindings.contains_key(cpu) ==> { - &&& self.rcu_bindings[cpu].scheduler() == sched_view.id - &&& self.rcu_bindings[cpu].cpu() == cpu - &&& self.rcu_bindings[cpu].participant_id() - == sched_view.cpu_rcu_participant_ids[cpu] + self.core_bindings.contains_key(cpu) ==> { + &&& self.core_bindings[cpu].registry() == sched_view.id + &&& self.core_bindings[cpu].cpu() == cpu + &&& self.core_bindings[cpu].owner_id() + == sched_view.cpu_core_registrations[cpu].owner_id + &&& self.core_bindings[cpu].locals_key() + == sched_view.cpu_core_registrations[cpu].locals_key } } - proof fn lemma_rcu_participants_frame( - tracked &self, - before: SchedulerView, - after: SchedulerView, - ) + proof fn lemma_cpu_cores_frame(tracked &self, before: SchedulerView, after: SchedulerView) requires - self.rcu_participants_wf(before), - before.stored_cpu_rcu_participants == after.stored_cpu_rcu_participants, - before.cpu_rcu_participant_ids == after.cpu_rcu_participant_ids, + self.cpu_cores_wf(before), + before.stored_cpu_cores == after.stored_cpu_cores, + before.cpu_core_registrations == after.cpu_core_registrations, before.cpu_views == after.cpu_views, ensures - self.rcu_participants_wf(after), + self.cpu_cores_wf(after), { } @@ -739,57 +778,57 @@ impl SchedulerThreadViews { &&& self.scheduler() == sched_view.id &&& self.view() == sched_view.stored_views &&& self.cpu_view_map() == sched_view.cpu_views - &&& self.rcu_participants_wf(sched_view) - &&& self.rcu_bindings_wf(sched_view) + &&& self.cpu_cores_wf(sched_view) + &&& self.core_bindings_wf(sched_view) } proof fn tracked_register_cpu( tracked &mut self, - tracked identity: &mut GhostMapAuth, + tracked identity: &mut GhostMapAuth, sched_view: SchedulerView, cpu: CpuId, - ) -> (participant_id: Loc) + ) -> (registration: CpuCoreRegistration) requires old(self).wf(sched_view), sched_view.wf(), !sched_view.cpu_has_thread_view(cpu), valid_cpu(cpu), old(identity).id() == sched_view.id, - old(identity)@ == sched_view.cpu_rcu_participant_ids, + old(identity)@ == sched_view.cpu_core_registrations, ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), - final(self).cpu_view_map() == sched_view.register_cpu(cpu, participant_id).cpu_views, - final(self).wf(sched_view.register_cpu(cpu, participant_id)), + registration.locals_key.len() == 1, + final(self).cpu_view_map() == sched_view.register_cpu(cpu, registration).cpu_views, + final(self).wf(sched_view.register_cpu(cpu, registration)), final(self).contains_cpu(cpu), final(self).cpu_thread_view(cpu) == WmView::empty(), - final(self).rcu_participant_id_map().contains_key(cpu), - final(self).rcu_participant_id_map()[cpu] == participant_id, - final(self).rcu_binding_id_map() == sched_view.register_cpu( + final(self).core_registration_map().contains_key(cpu), + final(self).core_registration_map()[cpu] == registration, + final(self).binding_registration_map() == sched_view.register_cpu( cpu, - participant_id, - ).cpu_rcu_participant_ids, + registration, + ).cpu_core_registrations, final(identity).id() == old(identity).id(), - final(identity)@ == sched_view.register_cpu( - cpu, - participant_id, - ).cpu_rcu_participant_ids, + final(identity)@ == sched_view.register_cpu(cpu, registration).cpu_core_registrations, { let tracked cpu_view = ThreadView::new(); let tracked participant = CpuRcuParticipant::new(cpu, WmView::empty()); + participant.lemma_cpu_core_local_state(); let ghost participant_id = participant.id(); - let tracked entry = identity.insert(cpu, participant_id); - let tracked binding = CpuRcuParticipantBinding::tracked_new(entry); - sched_view.lemma_register_cpu_preserves_wf(cpu, participant_id); + let tracked core = CpuCoreOwner::new(cpu, participant); + let ghost registration = core.registration(); + assert(core.locals_key() == seq![participant_id]); + assert(registration.locals_key.len() == 1); + let tracked entry = identity.insert(cpu, registration); + let tracked binding = CpuCoreOwnerBinding::tracked_new(entry, &core); + sched_view.lemma_register_cpu_preserves_wf(cpu, registration); self.cpu_views.tracked_insert(cpu, cpu_view); - self.rcu_participants.tracked_insert(cpu, participant); - self.rcu_bindings.tracked_insert(cpu, binding); - assert(final(self).cpu_view_map() == sched_view.register_cpu( - cpu, - participant_id, - ).cpu_views); - assert(final(self).wf(sched_view.register_cpu(cpu, participant_id))); - participant_id + self.cpu_cores.tracked_insert(cpu, core); + self.core_bindings.tracked_insert(cpu, binding); + assert(final(self).cpu_view_map() == sched_view.register_cpu(cpu, registration).cpu_views); + assert(final(self).wf(sched_view.register_cpu(cpu, registration))); + registration } /// Inserts a task view created during task registration. @@ -804,16 +843,16 @@ impl SchedulerThreadViews { requires !old(self).contains(token.task()), token.scheduler() == old(self).scheduler(), - old(self).rcu_participants_wf(rcu_view), - old(self).rcu_bindings_wf(rcu_view), + old(self).cpu_cores_wf(rcu_view), + old(self).core_bindings_wf(rcu_view), ensures final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().insert(token.task(), token.view()), final(self).cpu_view_map() == old(self).cpu_view_map(), - final(self).rcu_participants == old(self).rcu_participants, - final(self).rcu_participants_wf(rcu_view), - final(self).rcu_bindings == old(self).rcu_bindings, - final(self).rcu_bindings_wf(rcu_view), + final(self).cpu_cores == old(self).cpu_cores, + final(self).cpu_cores_wf(rcu_view), + final(self).core_bindings == old(self).core_bindings, + final(self).core_bindings_wf(rcu_view), { let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = token; self.views.tracked_insert(task, thread_view); @@ -840,7 +879,7 @@ impl SchedulerThreadViews { sched_view.lemma_register_task_preserves_wf(task); let tracked thread_view = ThreadView::new(); let tracked token = TaskThreadView::new(self.scheduler(), task, thread_view); - self.lemma_rcu_participants_frame(sched_view, sched_view.register_task(task)); + self.lemma_cpu_cores_frame(sched_view, sched_view.register_task(task)); self.tracked_insert_initial_thread_view(token, sched_view.register_task(task)); assert(final(self).view() == sched_view.register_task(task).stored_views); assert(final(self).wf(sched_view.register_task(task))); @@ -854,7 +893,12 @@ impl SchedulerThreadViews { tracked &mut self, sched_view: SchedulerView, cpu: CpuId, - ) -> (tracked res: (TaskThreadView, CpuRcuParticipant, CpuRcuParticipantBinding)) + ) -> (tracked res: ( + TaskThreadView, + CpuCoreOwnerHandle, + CpuRcuParticipant, + CpuCoreOwnerBinding, + )) requires old(self).wf(sched_view), sched_view.wf(), @@ -870,14 +914,21 @@ impl SchedulerThreadViews { res.0.view() == old(self).thread_view(sched_view.current[cpu]->0).join( old(self).cpu_thread_view(cpu), ), - res.1.id() == sched_view.cpu_rcu_participant_id(cpu), - res.1.cpu() == cpu, - res.1.fraction() == 1real, - res.1.view().spec_le(res.0.view()), res.1.wf(), - res.2.scheduler() == sched_view.id, + res.1.cpu() == cpu, + res.1.current_task() == Some(res.0.task()), + res.1.id() == sched_view.cpu_core_registration(cpu).owner_id, + res.1.expected_locals_key() == seq![res.2.id()], + res.1.expected_locals_key() == sched_view.cpu_core_registration(cpu).locals_key, + res.2.id() == sched_view.cpu_rcu_participant_id(cpu), res.2.cpu() == cpu, - res.2.participant_id() == res.1.id(), + res.2.fraction() == 1real, + res.2.view().spec_le(res.0.view()), + res.2.wf(), + res.3.registry() == sched_view.id, + res.3.cpu() == cpu, + res.3.owner_id() == res.1.id(), + res.3.locals_key() == seq![res.2.id()], final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view().remove(sched_view.current[cpu]->0), final(self).cpu_view_map() == old(self).cpu_view_map(), @@ -888,8 +939,15 @@ impl SchedulerThreadViews { let task = sched_view.current[cpu]->0; let tracked mut thread_view = self.views.tracked_remove(task); let tracked cpu_view = self.cpu_views.tracked_borrow(cpu); - let tracked participant = self.rcu_participants.tracked_remove(cpu); - let tracked binding = self.rcu_bindings.tracked_borrow(cpu).tracked_duplicate(); + let tracked mut core = self.cpu_cores.tracked_remove(cpu); + assert(core.is_idle()); + assert(core.id() == sched_view.cpu_core_registration(cpu).owner_id); + assert(core.locals_key() == sched_view.cpu_core_registration(cpu).locals_key); + assert(core.registration() == sched_view.cpu_core_registration(cpu)); + core.tracked_schedule_in(task); + assert(core.registration() == sched_view.cpu_core_registration(cpu)); + let tracked (core_handle, participant) = core.tracked_open(); + let tracked binding = self.core_bindings.tracked_borrow(cpu).tracked_duplicate(); thread_view.tracked_join(cpu_view); let tracked token = TaskThreadView { scheduler: Ghost(self.scheduler()), @@ -897,10 +955,17 @@ impl SchedulerThreadViews { thread_view, }; let next = sched_view.checkout_task_view(cpu); + assert(core_handle.cpu() == cpu); + assert(core_handle.current_task() == Some(task)); + assert(core_handle.id() == sched_view.cpu_core_registration(cpu).owner_id); + assert(participant.cpu() == cpu); + assert(core_handle.expected_locals_key() == seq![participant.id()]); + assert(sched_view.cpu_core_registration(cpu).locals_key == seq![participant.id()]); + assert(participant.id() == sched_view.cpu_rcu_participant_id(cpu)); assert(final(self).view() == next.stored_views); assert(final(self).wf(next)); assert(token.wf(next)); - (token, participant, binding) + (token, core_handle, participant, binding) } /// Checks out the current task's weak-memory view and starts its running @@ -933,11 +998,16 @@ impl SchedulerThreadViews { final(self).wf(sched_view.checkout_task_view(cpu)), { let ghost next = sched_view.checkout_task_view(cpu); - let tracked (task_view, participant, binding) = self.tracked_take_current_thread_view( - sched_view, + let tracked (task_view, core_handle, participant, binding) = + self.tracked_take_current_thread_view(sched_view, cpu); + let tracked context = RunningTaskContext::new( + task_view, + core_handle, + participant, + binding, + next, cpu, ); - let tracked context = RunningTaskContext::new(task_view, participant, binding, next, cpu); context } @@ -982,36 +1052,88 @@ impl SchedulerThreadViews { let ghost task = context.task(); let ghost view = context.view(); let ghost cpu = context.cpu(); + let ghost expected_registration = CpuCoreRegistration { + owner_id: context.core_owner_id(), + locals_key: seq![context.rcu_participant_id()], + }; + let tracked context_binding = context.tracked_rcu_binding(); + let tracked canonical_binding = self.core_bindings.tracked_borrow(cpu); + canonical_binding.lemma_same_cpu_agree(&context_binding); + assert(canonical_binding.owner_id() == sched_view.cpu_core_registration(cpu).owner_id); + assert(canonical_binding.locals_key() == sched_view.cpu_core_registration(cpu).locals_key); + assert(context_binding.owner_id() == context.core_owner_id()); + assert(context_binding.locals_key() == seq![context.rcu_participant_id()]); + assert(expected_registration.owner_id == sched_view.cpu_core_registration(cpu).owner_id); + assert(expected_registration.locals_key == sched_view.cpu_core_registration( + cpu, + ).locals_key); + assert(expected_registration == sched_view.cpu_core_registration(cpu)); sched_view.lemma_update_checked_out_task_view_preserves_wf(task, view); let ghost updated = sched_view.update_checked_out_task_view(task, view); context.lemma_wf_scheduler(updated); - let tracked (task_view, participant) = context.tracked_into_task_view_for_scheduler( - updated, - ); + let tracked (task_view, mut core) = context.tracked_into_task_view_for_scheduler(updated); + let ghost participant_view = core.locals().view(); + let ghost participant_id = core.locals().id(); + let ghost core_registration = core.registration(); + assert(core_registration == expected_registration); + assert(core.id() == core_registration.owner_id); + assert(core.locals_key() == core_registration.locals_key); + let ghost scheduled_task = core.tracked_schedule_out(); + assert(core.registration() == core_registration); + assert(core.id() == core_registration.owner_id); + assert(core.locals_key() == core_registration.locals_key); + assert(scheduled_task == task); let tracked TaskThreadView { scheduler: _, task: Ghost(task), thread_view } = task_view; let tracked cpu_view = self.cpu_views.tracked_borrow_mut(cpu); cpu_view.tracked_join(&thread_view); self.views.tracked_insert(task, thread_view); - self.rcu_participants.tracked_insert(cpu, participant); + self.cpu_cores.tracked_insert(cpu, core); updated.lemma_checkin_task_view_preserves_wf(cpu, task, view); let ghost checked = updated.checkin_task_view(cpu, task, view); assert(checked.wf()); checked.lemma_publish_cpu_view_preserves_wf(cpu, view); let ghost next = checked.publish_cpu_view(cpu, view); assert(next.wf()); - assert(participant.view().spec_le(view)); + assert(participant_view.spec_le(view)); old(self).cpu_thread_view(cpu).lemma_join_right(view); - participant.view().lemma_spec_le_transitive(view, self.cpu_views[cpu]@); + participant_view.lemma_spec_le_transitive(view, self.cpu_views[cpu]@); + assert(self.cpu_cores.dom() == next.stored_cpu_cores); assert forall|stored_cpu: CpuId| #[trigger] - self.rcu_participants.contains_key(stored_cpu) implies { - &&& self.rcu_participants[stored_cpu].wf() - &&& self.rcu_participants[stored_cpu].cpu() == stored_cpu - &&& self.rcu_participants[stored_cpu].id() == next.cpu_rcu_participant_ids[stored_cpu] - &&& self.rcu_participants[stored_cpu].fraction() == 1real - &&& self.rcu_participants[stored_cpu].view().spec_le(self.cpu_views[stored_cpu]@) + self.cpu_cores.contains_key(stored_cpu) implies { + &&& self.cpu_cores[stored_cpu].wf() + &&& self.cpu_cores[stored_cpu].cpu() == stored_cpu + &&& self.cpu_cores[stored_cpu].is_idle() + &&& self.cpu_cores[stored_cpu].id() == next.cpu_core_registrations[stored_cpu].owner_id + &&& self.cpu_cores[stored_cpu].locals_key() + == next.cpu_core_registrations[stored_cpu].locals_key + &&& self.cpu_cores[stored_cpu].registration() == next.cpu_core_registrations[stored_cpu] + &&& self.cpu_cores[stored_cpu].locals().fraction() == 1real + &&& self.cpu_cores[stored_cpu].locals().view().spec_le(self.cpu_views[stored_cpu]@) } by { + assert(next.cpu_core_registrations.contains_key(stored_cpu)); if stored_cpu == cpu { - assert(self.rcu_participants[stored_cpu] == participant); + assert(self.cpu_cores[stored_cpu].registration() == core_registration); + assert(core_registration == next.cpu_core_registrations[stored_cpu]); + assert(self.cpu_cores[stored_cpu].wf()); + assert(self.cpu_cores[stored_cpu].cpu() == stored_cpu); + assert(self.cpu_cores[stored_cpu].is_idle()); + assert(self.cpu_cores[stored_cpu].id() == core_registration.owner_id); + assert(self.cpu_cores[stored_cpu].locals_key() == core_registration.locals_key); + assert(self.cpu_cores[stored_cpu].locals().id() == participant_id); + } else { + assert(old(self).cpu_cores.contains_key(stored_cpu)); + assert(self.cpu_cores[stored_cpu] == old(self).cpu_cores[stored_cpu]); + assert(next.cpu_core_registrations[stored_cpu] + == sched_view.cpu_core_registrations[stored_cpu]); + assert(old(self).cpu_cores[stored_cpu].wf()); + assert(old(self).cpu_cores[stored_cpu].cpu() == stored_cpu); + assert(old(self).cpu_cores[stored_cpu].is_idle()); + assert(old(self).cpu_cores[stored_cpu].id() + == sched_view.cpu_core_registrations[stored_cpu].owner_id); + assert(old(self).cpu_cores[stored_cpu].locals_key() + == sched_view.cpu_core_registrations[stored_cpu].locals_key); + assert(old(self).cpu_cores[stored_cpu].registration() + == sched_view.cpu_core_registrations[stored_cpu]); } }; assert(final(self).view() == next.stored_views); @@ -1027,7 +1149,7 @@ impl SchedulerThreadViews { /// layers together, preventing a proof from changing `SchedulerView` without /// moving the corresponding linear token (or vice versa). pub tracked struct SchedulerGhostState { - identity: GhostMapAuth, + identity: GhostMapAuth, view: Ghost, thread_views: SchedulerThreadViews, } @@ -1039,7 +1161,9 @@ impl SchedulerGhostState { res.wf(), res.view() == SchedulerView::empty(res.id()), { - let tracked (identity, _entries) = GhostMapAuth::new(Map::::empty()); + let tracked (identity, _entries) = GhostMapAuth::new( + Map::::empty(), + ); let ghost id = identity.id(); SchedulerView::lemma_empty_wf(id); let tracked thread_views = SchedulerThreadViews::empty(id); @@ -1063,7 +1187,7 @@ impl SchedulerGhostState { pub closed spec fn wf(self) -> bool { &&& self.view().wf() &&& self.view().id == self.id() - &&& self.identity@ == self.view().cpu_rcu_participant_ids + &&& self.identity@ == self.view().cpu_core_registrations &&& self.thread_views.wf(self.view()) } @@ -1100,19 +1224,19 @@ impl SchedulerGhostState { final(self).id() == old(self).id(), final(self).view() == old(self).view().register_cpu( cpu, - final(self).view().cpu_rcu_participant_id(cpu), + final(self).view().cpu_core_registration(cpu), ), final(self).view().cpu_has_thread_view(cpu), final(self).view().cpu_thread_view(cpu) == WmView::empty(), final(self).view().cpu_rcu_participant_is_stored(cpu), { let ghost old_view = self.view@; - let ghost participant_id = self.thread_views.tracked_register_cpu( + let ghost registration = self.thread_views.tracked_register_cpu( &mut self.identity, old_view, cpu, ); - self.view = Ghost(old_view.register_cpu(cpu, participant_id)); + self.view = Ghost(old_view.register_cpu(cpu, registration)); assert(self.wf()); } From dbe42a63bca9f4d765488943e89b3db3e17836fe Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 4 Aug 2026 04:58:25 -0400 Subject: [PATCH 34/47] Migrate weak memory proofs to native IRC11 --- .github/workflows/ci-macos.yml | 27 +- .github/workflows/ci-upstream-verus.yml | 24 +- .github/workflows/ci.yml | 32 +- .github/workflows/doc.yml | 31 +- Cargo.toml | 2 +- dv | 2 +- ostd/specs/sync/rcu.rs | 769 +++++++++++------- ostd/specs/sync/rcu_cpu.rs | 65 +- ostd/specs/sync/weak_memory.rs | 586 +++++++------ ostd/src/sync/once.rs | 5 + ostd/src/sync/rcu/mod.rs | 120 +-- ostd/src/sync/rcu/monitor.rs | 160 ++-- ostd/src/sync/rcu/non_null/mod.rs | 3 +- ostd/src/sync/rwlock.rs | 5 + ostd/src/sync/rwmutex.rs | 5 + ostd/src/task/preempt/guard.rs | 188 ++++- ostd/src/task/scheduler/mod.rs | 162 ++-- tools/patches/verus-irc11-vstd.patch | 200 +++++ verified_libs/vstd_extra/src/atomic_irc11.rs | 693 ++++++++++++++++ verified_libs/vstd_extra/src/atomic_weak.rs | 29 +- .../vstd_extra/src/external/smart_ptr.rs | 11 + verified_libs/vstd_extra/src/lib.rs | 1 + .../src/resource/ghost_resource/count.rs | 9 + verified_libs/vstd_extra/src/sum.rs | 5 + 24 files changed, 2287 insertions(+), 847 deletions(-) create mode 100644 tools/patches/verus-irc11-vstd.patch create mode 100644 verified_libs/vstd_extra/src/atomic_irc11.rs diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 18ae4782b..cfd2612be 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -14,6 +14,9 @@ jobs: runs-on: macos-14 env: CARGO_TERM_COLOR: always + VERUS_REPOSITORY: https://github.com/verus-lang/verus.git + VERUS_BRANCH: irc11 + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository @@ -56,9 +59,13 @@ jobs: id: verus shell: bash run: | - VERUS_COMMIT=$(git ls-remote https://github.com/asterinas/verus HEAD | cut -f1) + VERUS_COMMIT=$(git ls-remote "$VERUS_REPOSITORY" "refs/heads/$VERUS_BRANCH" | cut -f1) + if [ -z "$VERUS_COMMIT" ]; then + echo "Failed to resolve $VERUS_REPOSITORY branch $VERUS_BRANCH" + exit 1 + fi echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus commit: $VERUS_COMMIT" + echo "Using Verus $VERUS_BRANCH commit: $VERUS_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -74,7 +81,7 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verus-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11-vstd.patch') }} - name: Bootstrap Verus (if needed) shell: bash @@ -82,9 +89,19 @@ jobs: if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus..." + echo "Cache miss, bootstrapping Verus $VERUS_BRANCH..." rm -rf tools/verus - cargo dv bootstrap + cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + fi + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_COMMIT" + + - name: Enable IRC11 alongside existing SC atomics + shell: bash + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" fi - name: Run verification diff --git a/.github/workflows/ci-upstream-verus.yml b/.github/workflows/ci-upstream-verus.yml index 817f1207e..681ea7c8e 100644 --- a/.github/workflows/ci-upstream-verus.yml +++ b/.github/workflows/ci-upstream-verus.yml @@ -1,4 +1,4 @@ -name: Verify VOSTD (Main) with verus-lang/verus +name: Verify VOSTD with verus-lang/verus IRC11 on: push: @@ -23,6 +23,8 @@ jobs: runs-on: ubuntu-24.04 env: CARGO_TERM_COLOR: always + VERUS_BRANCH: irc11 + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Get PR head commit @@ -53,7 +55,7 @@ jobs: sha: '${{ steps.pr.outputs.head_sha }}', state: 'pending', context: 'ci/upstream-verus', - description: 'Upstream Verus verification is running', + description: 'Upstream Verus IRC11 verification is running', target_url: runUrl, }); @@ -68,10 +70,18 @@ jobs: sudo apt update -qq sudo apt install -y build-essential unzip pkg-config libssl-dev llvm - - name: Run dv bootstrap with upstream verus - run: cargo dv bootstrap --upstream-verus + - name: Bootstrap upstream Verus IRC11 + run: cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" - - name: Verify ostd with upstream verus + - name: Enable IRC11 alongside existing SC atomics + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" + fi + + - name: Verify ostd with upstream Verus IRC11 run: make - name: Report upstream Verus verification status @@ -85,8 +95,8 @@ jobs: const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; const state = '${{ job.status }}' === 'success' ? 'success' : 'failure'; const description = state === 'success' - ? 'Upstream Verus verification passed' - : 'Upstream Verus verification failed'; + ? 'Upstream Verus IRC11 verification passed' + : 'Upstream Verus IRC11 verification failed'; await github.rest.repos.createCommitStatus({ owner, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fa79da4c..7099d2856 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,9 @@ jobs: runs-on: ubuntu-24.04 env: CARGO_TERM_COLOR: always + VERUS_REPOSITORY: https://github.com/verus-lang/verus.git + VERUS_BRANCH: irc11 + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository @@ -64,9 +67,13 @@ jobs: - name: Get Verus commit id: verus run: | - VERUS_COMMIT=$(git ls-remote https://github.com/asterinas/verus HEAD | cut -f1) + VERUS_COMMIT=$(git ls-remote "$VERUS_REPOSITORY" "refs/heads/$VERUS_BRANCH" | cut -f1) + if [ -z "$VERUS_COMMIT" ]; then + echo "Failed to resolve $VERUS_REPOSITORY branch $VERUS_BRANCH" + exit 1 + fi echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus commit: $VERUS_COMMIT" + echo "Using Verus $VERUS_BRANCH commit: $VERUS_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -82,31 +89,40 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verus-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt uses: actions/cache@v6 with: path: ~/.cargo/bin/verusfmt - key: ${{ runner.os }}-verusfmt-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }} - name: Bootstrap Verus (if needed) run: | if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus..." + echo "Cache miss, bootstrapping Verus $VERUS_BRANCH..." rm -rf tools/verus - cargo dv bootstrap + cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" fi if ! command -v verusfmt >/dev/null 2>&1; then echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap + cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" fi + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_COMMIT" verusfmt --version + - name: Enable IRC11 alongside existing SC atomics + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" + fi + - name: Run verification run: | set -o pipefail @@ -179,4 +195,4 @@ jobs: else echo "- Verification warnings: ✅ none" fi - } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index b698a592b..e3fd65177 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -21,6 +21,10 @@ concurrency: jobs: build: runs-on: ubuntu-latest + env: + VERUS_REPOSITORY: https://github.com/verus-lang/verus.git + VERUS_BRANCH: irc11 + VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository uses: actions/checkout@v7 @@ -65,9 +69,13 @@ jobs: - name: Get Verus commit id: verus run: | - VERUS_COMMIT=$(git ls-remote https://github.com/asterinas/verus HEAD | cut -f1) + VERUS_COMMIT=$(git ls-remote "$VERUS_REPOSITORY" "refs/heads/$VERUS_BRANCH" | cut -f1) + if [ -z "$VERUS_COMMIT" ]; then + echo "Failed to resolve $VERUS_REPOSITORY branch $VERUS_BRANCH" + exit 1 + fi echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus commit: $VERUS_COMMIT" + echo "Using Verus $VERUS_BRANCH commit: $VERUS_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -83,31 +91,40 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verus-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt uses: actions/cache@v6 with: path: ~/.cargo/bin/verusfmt - key: ${{ runner.os }}-verusfmt-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }} - name: Bootstrap Verus (if needed) run: | if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus..." + echo "Cache miss, bootstrapping Verus $VERUS_BRANCH..." rm -rf tools/verus - cargo dv bootstrap + cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" fi if ! command -v verusfmt >/dev/null 2>&1; then echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap + cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" fi + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_COMMIT" verusfmt --version + - name: Enable IRC11 alongside existing SC atomics + run: | + if git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_PATCH"; then + echo "IRC11 compatibility patch is already applied" + else + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_PATCH" + fi + - name: Build docs run: make doc diff --git a/Cargo.toml b/Cargo.toml index d53e3d290..5be335f4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,4 +49,4 @@ codegen-units = 1 [workspace.dependencies] # Verus -vstd = { path = "tools/verus/source/vstd" } +vstd = { path = "tools/verus/source/vstd", features = ["weak-memory"] } diff --git a/dv b/dv index 00d9bc7b5..504e21936 160000 --- a/dv +++ b/dv @@ -1 +1 @@ -Subproject commit 00d9bc7b52ec3bdb14fe9ca95bbbad901c99cc3b +Subproject commit 504e21936625ef9e65a5b9a0e7e39e7e29a73c39 diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index b2775c651..1c2503e4f 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -30,13 +30,20 @@ use core::marker::PhantomData; use crate::specs::mm::cpu::CpuId; -use super::weak_memory::{History, Msg, Timestamp, WeakAtomicInvariantPredicate, WmView}; +use vstd::invariant::InvariantPredicate; use vstd::prelude::*; use vstd::resource::Loc; use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; +use vstd::thread_view::Objective; +use vstd_extra::atomic_irc11::{ + AtomicHistory as Irc11History, AtomicId as Irc11AtomicId, AtomicPointsTo, + ThreadView as Irc11ThreadView, +}; verus! { +broadcast use {vstd::atomic_weak::group_view_history, vstd::thread_view::group_thread_view_axioms}; + pub type LinkIndex = nat; pub type LinkEdge = (nat, LinkIndex); @@ -84,7 +91,7 @@ pub ghost struct RcuCallbackSummary { /// The domain-local epoch in which `obj` was retired. pub retire_epoch: nat, /// Weak-memory observations that must precede safe reclamation. - pub retire_view: WmView, + pub retire_view: Irc11ThreadView, } /// Persistent identity of one completed base-retirement transition. @@ -119,11 +126,12 @@ impl RcuCallbackSummary { pub ghost struct RcuRemovalObservation { pub root: Loc, pub timestamp: nat, + pub message_view: Irc11ThreadView, } impl RcuRemovalObservation { - pub open spec fn observed_by(self, view: WmView) -> bool { - self.timestamp <= view.seen_at(self.root) + pub open spec fn observed_by(self, view: Irc11ThreadView) -> bool { + view.contains(self.message_view) } } @@ -294,7 +302,8 @@ pub open spec fn current_registration_matches( /// unreachable from every incoming link. pub tracked struct RcuRootGhost { domain: RcuDomainAuth, - ghost publications: Seq>, + ghost publications: Map>, + ghost current_timestamp: nat, } impl RcuRootGhost { @@ -314,15 +323,19 @@ impl RcuRootGhost { self.domain.wf() } - pub closed spec fn publications(self) -> Seq> { + pub closed spec fn publications(self) -> Map> { self.publications } + pub closed spec fn current_timestamp(self) -> nat { + self.current_timestamp + } + pub open spec fn published_at(self, ts: nat) -> Option recommends - ts < self.publications().len(), + self.publications().contains_key(ts), { - match self.publications()[ts as int] { + match self.publications()[ts] { Some(obj) => Some( RcuPublishedObject { domain: self.domain(), obj, addr: self.objects()[obj] }, ), @@ -333,9 +346,9 @@ impl RcuRootGhost { /// Allocation identity carried by the latest atomic message. pub open spec fn current(self) -> Option recommends - self.publications().len() > 0, + self.publications().contains_key(self.current_timestamp()), { - self.published_at((self.publications().len() - 1) as nat) + self.published_at(self.current_timestamp()) } /// Allocate a fresh publication registry containing the initial message. @@ -343,17 +356,21 @@ impl RcuRootGhost { /// A non-null initial value is registered exactly once and the registration /// resources are returned to the caller. The root history retains only the /// allocation ID; it does not consume the unique retire permission. - pub proof fn tracked_initial(ptr: *mut T) -> (tracked res: ( - Self, - Option>, - )) + pub proof fn tracked_initial( + ptr: *mut T, + history: Irc11History<*mut T>, + timestamp: nat, + message_view: Irc11ThreadView, + ) -> (tracked res: (Self, Option>)) + requires + history.is_singleton(timestamp, (ptr, message_view)), ensures - rcu_root_history_inv(seq![Msg { value: ptr, view: WmView::empty() }], res.0), + rcu_root_history_inv(history, res.0), (res.1 is Some) == (ptr.addr() != 0), res.1 is Some ==> res.1->Some_0.0.ptr() == ptr, res.1 is Some ==> res.1->Some_0.0.obj() == res.1->Some_0.1.obj(), res.1 is Some ==> res.1->Some_0.0.domain() == res.0.domain(), - res.1 is Some ==> res.0.publications()[0] == Some(res.1->Some_0.0.obj()), + res.1 is Some ==> res.0.publications()[timestamp] == Some(res.1->Some_0.0.obj()), res.1 is Some ==> res.1->Some_0.0.wf(), match res.1 { Some(registration) => res.0.objects() == Map::empty().insert( @@ -367,14 +384,35 @@ impl RcuRootGhost { res.0.domain_auth().retire_observations() == Map::::empty(), { let tracked mut domain = RcuDomainAuth::tracked_new(); + assert(history.is_max_timestamp(timestamp)); + assert(history.dom() == Set::empty().insert(timestamp)) by { + assert forall|ts: nat| + history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { + if history.dom().contains(ts) { + assert(history.contains_timestamp(ts)); + assert(ts == timestamp); + } + }; + }; if ptr.addr() == 0 { - (RcuRootGhost { domain, publications: seq![None] }, None) + ( + RcuRootGhost { + domain, + publications: Map::empty().insert(timestamp, None), + current_timestamp: timestamp, + }, + None, + ) } else { let tracked (block_info, retire_perm) = domain.tracked_register(ptr); let ghost obj = block_info.obj(); assert(domain.objects().contains_pair(obj, ptr.addr())); ( - RcuRootGhost { domain, publications: seq![Some(obj)] }, + RcuRootGhost { + domain, + publications: Map::empty().insert(timestamp, Some(obj)), + current_timestamp: timestamp, + }, Some((block_info, retire_perm)), ) } @@ -388,13 +426,18 @@ impl RcuRootGhost { /// of the append-only atomic history. pub proof fn tracked_push_fresh( tracked &mut self, - prev: History<*mut T>, - next: History<*mut T>, - msg: Msg<*mut T>, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, ) -> (tracked res: Option>) requires rcu_root_history_inv(prev, *old(self)), - next == prev.push(msg), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), ensures rcu_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), @@ -411,11 +454,12 @@ impl RcuRootGhost { final(self).domain_auth().retire_observations() == old( self, ).domain_auth().retire_observations(), - (res is Some) == (msg.value.addr() != 0), - res is Some ==> res->Some_0.0.ptr() == msg.value, + (res is Some) == (value.addr() != 0), + res is Some ==> res->Some_0.0.ptr() == value, res is Some ==> res->Some_0.0.obj() == res->Some_0.1.obj(), res is Some ==> !old(self).objects().contains_key(res->Some_0.0.obj()), - final(self).publications() == old(self).publications().push( + final(self).publications() == old(self).publications().insert( + new_timestamp, match res { Some(registration) => Some(registration.0.obj()), None => None, @@ -424,44 +468,41 @@ impl RcuRootGhost { match res { Some(registration) => final(self).objects() == old(self).objects().insert( registration.0.obj(), - msg.value.addr(), + value.addr(), ), None => final(self).objects() == old(self).objects(), }, current_registration_matches(*final(self), res), { - let ghost ts = prev.len(); - assert(self.publications().len() == ts); - - let tracked res = if msg.value.addr() == 0 { - self.publications = self.publications.push(None); + let tracked res = if value.addr() == 0 { + self.publications = self.publications.insert(new_timestamp, None); None } else { - let tracked (block_info, retire_perm) = self.domain.tracked_register(msg.value); + let tracked (block_info, retire_perm) = self.domain.tracked_register(value); let ghost obj = block_info.obj(); - self.publications = self.publications.push(Some(obj)); + self.publications = self.publications.insert(new_timestamp, Some(obj)); Some((block_info, retire_perm)) }; + self.current_timestamp = new_timestamp; - assert forall|i: int| 0 <= i < next.len() implies { - match #[trigger] self.publications()[i] { - None => next[i].value.addr() == 0, + assert forall|ts: nat| next.contains_timestamp(ts) implies { + match #[trigger] self.publications()[ts] { + None => next.value(ts).addr() == 0, Some(obj) => { - &&& next[i].value.addr() != 0 - &&& self.objects().contains_pair(obj, next[i].value.addr()) + &&& next.value(ts).addr() != 0 + &&& self.objects().contains_pair(obj, next.value(ts).addr()) }, } } by { - if i == prev.len() { - assert(next[i] == msg); + if ts == new_timestamp { } else { - assert(i < prev.len()); - assert(next[i] == prev[i]); - assert(self.publications()[i] == old(self).publications()[i]); - match self.publications()[i] { + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + assert(self.publications()[ts] == old(self).publications()[ts]); + match self.publications()[ts] { Some(obj) => { - assert(old(self).objects().contains_pair(obj, prev[i].value.addr())); - assert(self.objects().contains_pair(obj, next[i].value.addr())); + assert(old(self).objects().contains_pair(obj, prev.value(ts).addr())); + assert(self.objects().contains_pair(obj, next.value(ts).addr())); }, None => {}, } @@ -477,16 +518,21 @@ impl RcuRootGhost { /// `BlockInfo` therefore carries the same allocation identity. pub proof fn tracked_push_registered( tracked &mut self, - prev: History<*mut T>, - next: History<*mut T>, - msg: Msg<*mut T>, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, tracked info: &RcuBlockInfo, ) requires rcu_root_history_inv(prev, *old(self)), - next == prev.push(msg), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), info.domain() == old(self).domain(), - info.ptr() == msg.value, + info.ptr() == value, info.wf(), ensures rcu_root_history_inv(next, *final(self)), @@ -505,37 +551,38 @@ impl RcuRootGhost { self, ).domain_auth().retire_observations(), final(self).objects() == old(self).objects(), - final(self).publications() == old(self).publications().push(Some(info.obj())), + final(self).publications() == old(self).publications().insert( + new_timestamp, + Some(info.obj()), + ), { - let ghost ts = prev.len(); - assert(self.publications().len() == ts); self.domain.lemma_block_info_agree(info); - self.publications = self.publications.push(Some(info.obj())); + self.publications = self.publications.insert(new_timestamp, Some(info.obj())); + self.current_timestamp = new_timestamp; assert(self.objects() == old(self).objects()); - assert forall|i: int| 0 <= i < next.len() implies { - match #[trigger] self.publications()[i] { - None => next[i].value.addr() == 0, + assert forall|ts: nat| next.contains_timestamp(ts) implies { + match #[trigger] self.publications()[ts] { + None => next.value(ts).addr() == 0, Some(obj) => { - &&& next[i].value.addr() != 0 - &&& self.objects().contains_pair(obj, next[i].value.addr()) + &&& next.value(ts).addr() != 0 + &&& self.objects().contains_pair(obj, next.value(ts).addr()) }, } } by { - if i == prev.len() { - assert(next[i] == msg); - assert(info.addr() == msg.value.addr()); - assert(msg.value.addr() != 0); - assert(self.publications()[i] == Some(info.obj())); - assert(self.objects().contains_pair(info.obj(), next[i].value.addr())); + if ts == new_timestamp { + assert(info.addr() == value.addr()); + assert(value.addr() != 0); + assert(self.publications()[ts] == Some(info.obj())); + assert(self.objects().contains_pair(info.obj(), next.value(ts).addr())); } else { - assert(i < prev.len()); - assert(next[i] == prev[i]); - assert(self.publications()[i] == old(self).publications()[i]); - match self.publications()[i] { + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + assert(self.publications()[ts] == old(self).publications()[ts]); + match self.publications()[ts] { Some(obj) => { - assert(old(self).objects().contains_pair(obj, prev[i].value.addr())); - assert(self.objects().contains_pair(obj, next[i].value.addr())); + assert(old(self).objects().contains_pair(obj, prev.value(ts).addr())); + assert(self.objects().contains_pair(obj, next.value(ts).addr())); }, None => {}, } @@ -545,43 +592,29 @@ impl RcuRootGhost { } /// Agreement between the weak-memory message history and RCU allocation IDs. -pub open spec fn rcu_root_history_inv(history: History<*mut T>, ghost: RcuRootGhost) -> bool { - &&& history.len() >= 1 +pub open spec fn rcu_root_history_inv( + history: Irc11History<*mut T>, + ghost: RcuRootGhost, +) -> bool { &&& ghost.domain_wf() - &&& ghost.publications().len() == history.len() - &&& forall|i: int| - 0 <= i < history.len() ==> { - match #[trigger] ghost.publications()[i] { - None => history[i].value.addr() == 0, + &&& ghost.publications().dom() == history.dom() + &&& history.is_max_timestamp(ghost.current_timestamp()) + &&& forall|ts: nat| + history.contains_timestamp(ts) ==> { + match #[trigger] ghost.publications()[ts] { + None => history.value(ts).addr() == 0, Some(obj) => { - &&& history[i].value.addr() != 0 - &&& ghost.objects().contains_pair(obj, history[i].value.addr()) + &&& history.value(ts).addr() != 0 + &&& ghost.objects().contains_pair(obj, history.value(ts).addr()) }, } } } -/// The weak-memory invariant for the root pointer stored in an executable RCU -/// cell. -/// -/// The key is the cell's nullability: `true` for `RcuOption`, `false` for -/// `Rcu`. The predicate connects each atomic message both to the public -/// nullability contract and to its domain-local allocation identity. Physical -/// ownership, read tokens, and reclamation are deliberately modeled by the -/// traversal/reclaim tokens below and will be wired in later steps. -pub struct RcuWeakAtomicInv; - -pub open spec fn rcu_history_inv(nullable: bool, history: History<*mut T>) -> bool { - &&& history.len() >= 1 - &&& !nullable ==> forall|i: int| - 0 <= i < history.len() ==> #[trigger] history[i].value.addr() != 0 -} - -impl WeakAtomicInvariantPredicate for RcuWeakAtomicInv { - open spec fn atomic_inv(nullable: bool, history: History<*mut T>, g: RcuRootGhost) -> bool { - &&& rcu_history_inv(nullable, history) - &&& rcu_root_history_inv(history, g) - } +pub open spec fn rcu_history_inv(nullable: bool, history: Irc11History<*mut T>) -> bool { + &&& !history.dom().is_empty() + &&& !nullable ==> forall|ts: nat| + history.contains_timestamp(ts) ==> #[trigger] history.value(ts).addr() != 0 } /// Immutable identity carried by an executable RCU root atomic. @@ -610,6 +643,12 @@ pub tracked struct RcuRootOwnedGhost { ghost removals: Map, } +// The root ghost owns only global resource-algebra state. Its payload remains +// objective exactly when the client ownership stored in it is objective. +unsafe impl Objective for RcuRootOwnedGhost { + +} + impl RcuRootOwnedGhost { pub closed spec fn root(self) -> RcuRootGhost { self.root @@ -619,7 +658,7 @@ impl RcuRootOwnedGhost { self.root().domain() } - pub closed spec fn publications(self) -> Seq> { + pub closed spec fn publications(self) -> Map> { self.root().publications() } @@ -633,7 +672,7 @@ impl RcuRootOwnedGhost { pub open spec fn published_at(self, ts: nat) -> Option recommends - ts < self.publications().len(), + self.publications().contains_key(ts), { self.root().published_at(ts) } @@ -690,7 +729,7 @@ impl RcuRootOwnedGhost { /// Once `removals[obj] = ts`, no message at or after `ts` may publish that /// allocation ID again. The currently owned registration is therefore /// never in the removed domain. - pub open spec fn removals_wf(self, history: History<*mut T>) -> bool { + pub open spec fn removals_wf(self, history: Irc11History<*mut T>) -> bool { &&& self.removals().dom().subset_of(self.infos().dom()) &&& match self.current_registration() { Some(registration) => !self.removals().contains_key(registration.0.obj()), @@ -699,9 +738,10 @@ impl RcuRootOwnedGhost { &&& forall|obj: nat| self.removals().contains_key(obj) ==> { let ts = (#[trigger] self.removals()[obj]).timestamp; - &&& 0 < ts < history.len() - &&& forall|i: int| - ts <= i < history.len() ==> #[trigger] self.publications()[i] != Some(obj) + &&& history.contains_timestamp(ts) + &&& forall|later: nat| + history.contains_timestamp(later) && ts <= later + ==> #[trigger] self.publications()[later] != Some(obj) } } @@ -728,29 +768,32 @@ impl RcuRootOwnedGhost { /// This is the proof interface used by weak atomic loads. It keeps the /// root's internal publication and identity maps opaque to the atomic /// wrapper while exporting exact pointer provenance, not just an address. - pub proof fn tracked_info_at(tracked &self, history: History<*mut T>, ts: nat) -> (tracked res: - Option>) + pub proof fn tracked_info_at( + tracked &self, + history: Irc11History<*mut T>, + ts: nat, + ) -> (tracked res: Option>) requires rcu_owned_root_history_inv(history, *self), - ts < history.len(), + history.contains_timestamp(ts), ensures - ts < self.publications().len(), + self.publications().contains_key(ts), match (self.published_at(ts), res) { - (None, None) => history[ts as int].value.addr() == 0, + (None, None) => history.value(ts).addr() == 0, (Some(object), Some(info)) => { &&& object.domain == self.domain() - &&& object.addr == history[ts as int].value.addr() + &&& object.addr == history.value(ts).addr() &&& info.wf() &&& info.domain() == object.domain &&& info.obj() == object.obj &&& info.addr() == object.addr - &&& equal(info.ptr(), history[ts as int].value) + &&& equal(info.ptr(), history.value(ts)) }, _ => false, }, { - assert(ts < self.publications().len()); - match self.publications()[ts as int] { + assert(self.publications().contains_key(ts)); + match self.publications()[ts] { Some(obj) => { let ghost object = RcuPublishedObject { domain: self.domain(), @@ -759,7 +802,7 @@ impl RcuRootOwnedGhost { }; assert(self.published_at(ts) == Some(object)); let tracked info = self.tracked_info_for(object); - assert(equal(info.ptr(), history[ts as int].value)); + assert(equal(info.ptr(), history.value(ts))); Some(info) }, None => { @@ -772,20 +815,20 @@ impl RcuRootOwnedGhost { /// Extracts the allocation ID stored in a non-null root publication. pub proof fn lemma_published_object_id( tracked &self, - history: History<*mut T>, + history: Irc11History<*mut T>, ts: nat, object: RcuPublishedObject, ) requires rcu_owned_root_history_inv(history, *self), - ts < history.len(), + history.contains_timestamp(ts), self.published_at(ts) == Some(object), ensures - self.publications()[ts as int] == Some(object.obj), + self.publications()[ts] == Some(object.obj), { - match self.publications()[ts as int] { + match self.publications()[ts] { Some(obj) => { - assert(self.root().objects().contains_pair(obj, history[ts as int].value.addr())); + assert(self.root().objects().contains_pair(obj, history.value(ts).addr())); assert(self.published_at(ts) == Some( RcuPublishedObject { domain: self.domain(), @@ -804,9 +847,9 @@ impl RcuRootOwnedGhost { /// root-removal observation for that allocation. pub proof fn lemma_observed_retired( tracked &self, - history: History<*mut T>, + history: Irc11History<*mut T>, root: Loc, - view: WmView, + view: Irc11ThreadView, obj: nat, ) requires @@ -829,10 +872,10 @@ impl RcuRootOwnedGhost { /// root's entry-time expired set. pub proof fn lemma_retired_facts_observed( tracked &self, - history: History<*mut T>, + history: Irc11History<*mut T>, tracked facts: &RcuRetiredFacts, root: Loc, - view: WmView, + view: Irc11ThreadView, ) requires rcu_owned_root_history_inv(history, *self), @@ -858,9 +901,9 @@ impl RcuRootOwnedGhost { /// algorithm. pub proof fn tracked_start_reader( tracked &mut self, - history: History<*mut T>, + history: Irc11History<*mut T>, root: Loc, - start_view: WmView, + start_view: Irc11ThreadView, reader: RcuReaderContext, ) -> (tracked res: RcuBaseGuard) requires @@ -898,15 +941,23 @@ impl RcuRootOwnedGhost { /// Initializes root history and retains the initial registration as the /// current unique ownership resource. - pub proof fn tracked_initial(ptr: *mut T, tracked ownership: Option) -> (tracked res: Self) + pub proof fn tracked_initial( + ptr: *mut T, + tracked ownership: Option, + history: Irc11History<*mut T>, + timestamp: nat, + message_view: Irc11ThreadView, + ) -> (tracked res: Self) requires (ownership is Some) == (ptr.addr() != 0), + history.is_singleton(timestamp, (ptr, message_view)), ensures - rcu_owned_root_history_inv(seq![Msg { value: ptr, view: WmView::empty() }], res), + rcu_owned_root_history_inv(history, res), (res.current_registration() is Some) == (ptr.addr() != 0), res.current_registration() is Some ==> res.current_registration()->Some_0.0.ptr() == ptr, res.current_ownership() == ownership, + res.removals() == Map::::empty(), match res.current_owned() { Some(owned) => { &&& ptr.addr() != 0 @@ -919,7 +970,12 @@ impl RcuRootOwnedGhost { }, }, { - let tracked (root, registration) = RcuRootGhost::tracked_initial(ptr); + let tracked (root, registration) = RcuRootGhost::tracked_initial( + ptr, + history, + timestamp, + message_view, + ); let tracked mut infos = Map::>::tracked_empty(); let tracked current = match registration { Some(registration) => { @@ -945,7 +1001,7 @@ impl RcuRootOwnedGhost { }; let tracked res = RcuRootOwnedGhost { root, current, infos, removals: Map::empty() }; assert(res.infos_wf()); - assert(res.removals_wf(seq![Msg { value: ptr, view: WmView::empty() }])); + assert(res.removals_wf(history)); assert(res.removals() == res.root().domain_auth().retire_observations()); res } @@ -958,9 +1014,12 @@ impl RcuRootOwnedGhost { /// `RcuPointedBy-detach` rule for internal nodes. pub proof fn tracked_push_fresh( tracked &mut self, - prev: History<*mut T>, - next: History<*mut T>, - msg: Msg<*mut T>, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, root: Loc, tracked ownership: Option, ) -> (tracked detached: Option>) where @@ -969,8 +1028,10 @@ impl RcuRootOwnedGhost { requires rcu_owned_root_history_inv(prev, *old(self)), rcu_current_ownership_inv::(*old(self)), - next == prev.push(msg), - (ownership is Some) == (msg.value.addr() != 0), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), + (ownership is Some) == (value.addr() != 0), ensures rcu_owned_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), @@ -985,26 +1046,34 @@ impl RcuRootOwnedGhost { &&& detached.retired().ptr() == detached.ptr() &&& detached.retired().removal() == (RcuRemovalObservation { root, - timestamp: prev.len(), + timestamp: new_timestamp, + message_view, }) - &&& equal(detached.ptr(), prev[(prev.len() - 1) as int].value) + &&& equal(detached.ptr(), prev.value(old_timestamp)) &&& old(self).current_ownership() == Some(detached.ownership()) &&& OwnPred::owns(detached.ptr(), detached.ownership()) }, None => old(self).current_registration() is None, }, - (final(self).current_registration() is Some) == (msg.value.addr() != 0), + (final(self).current_registration() is Some) == (value.addr() != 0), final(self).current_registration() is Some - ==> final(self).current_registration()->Some_0.0.ptr() == msg.value, + ==> final(self).current_registration()->Some_0.0.ptr() == value, final(self).current_ownership() == ownership, + final(self).removals() == match detached { + Some(detached) => old(self).removals().insert( + detached.obj(), + detached.retired().removal(), + ), + None => old(self).removals(), + }, match final(self).current_owned() { Some(owned) => { - &&& msg.value.addr() != 0 - &&& equal(owned.block_info().ptr(), msg.value) + &&& value.addr() != 0 + &&& equal(owned.block_info().ptr(), value) &&& ownership == Some(owned.ownership()) }, None => { - &&& msg.value.addr() == 0 + &&& value.addr() == 0 &&& ownership is None }, }, @@ -1019,7 +1088,14 @@ impl RcuRootOwnedGhost { } else { None }; - let tracked new_registration = self.root.tracked_push_fresh(prev, next, msg); + let tracked new_registration = self.root.tracked_push_fresh( + prev, + next, + old_timestamp, + new_timestamp, + value, + message_view, + ); let tracked new_current = match new_registration { Some(registration) => { let ghost obj = registration.0.obj(); @@ -1049,7 +1125,7 @@ impl RcuRootOwnedGhost { None }, }; - let ghost removal = RcuRemovalObservation { root, timestamp: prev.len() }; + let ghost removal = RcuRemovalObservation { root, timestamp: new_timestamp, message_view }; let tracked detached = match old_current { Some(owned) => { let tracked (registration, old_ownership) = owned.tracked_into_parts(); @@ -1092,20 +1168,27 @@ impl RcuRootOwnedGhost { }, } }; + assert(self.removals() == match detached { + Some(detached) => old(self).removals().insert( + detached.obj(), + detached.retired().removal(), + ), + None => old(self).removals(), + }); assert(current_registration_matches(self.root(), self.current_registration())); assert(self.infos_wf()); assert(self.removals_wf(next)) by { assert forall|obj: nat| self.removals().contains_key(obj) implies { let ts = (#[trigger] self.removals()[obj]).timestamp; - &&& 0 < ts < next.len() - &&& forall|i: int| - ts <= i < next.len() ==> #[trigger] self.publications()[i] != Some(obj) + &&& next.contains_timestamp(ts) + &&& forall|later: nat| + next.contains_timestamp(later) && ts <= later + ==> #[trigger] self.publications()[later] != Some(obj) } by { if removed_obj == Some(obj) { assert(self.removals()[obj] == removal); - assert(self.removals()[obj].timestamp == prev.len()); - assert(next.len() == prev.len() + 1); - assert(self.publications()[prev.len() as int] == match new_registration { + assert(self.removals()[obj].timestamp == new_timestamp); + assert(self.publications()[new_timestamp] == match new_registration { Some(registration) => Some(registration.0.obj()), None => None, }); @@ -1118,13 +1201,13 @@ impl RcuRootOwnedGhost { } else { assert(old(self).removals().contains_key(obj)); assert(self.removals()[obj] == old(self).removals()[obj]); - assert forall|i: int| - self.removals()[obj].timestamp <= i - < next.len() implies #[trigger] self.publications()[i] != Some(obj) by { - if i < prev.len() { - assert(self.publications()[i] == old(self).publications()[i]); + assert forall|later: nat| + next.contains_timestamp(later) && self.removals()[obj].timestamp + <= later implies #[trigger] self.publications()[later] != Some(obj) by { + if later != new_timestamp { + assert(prev.contains_timestamp(later)); + assert(self.publications()[later] == old(self).publications()[later]); } else { - assert(i == prev.len()); if new_registration is Some { assert(!old(self).root().objects().contains_key( new_registration->Some_0.0.obj(), @@ -1143,15 +1226,20 @@ impl RcuRootOwnedGhost { /// or releasing its unique retire permission. pub proof fn tracked_republish_current( tracked &mut self, - prev: History<*mut T>, - next: History<*mut T>, - msg: Msg<*mut T>, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + old_timestamp: nat, + new_timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, ) requires rcu_owned_root_history_inv(prev, *old(self)), - next == prev.push(msg), + prev.is_max_timestamp(old_timestamp), + new_timestamp == old_timestamp + 1, + next == prev.insert(new_timestamp, value, message_view), old(self).current_registration() is Some, - old(self).current_registration()->Some_0.0.ptr() == msg.value, + old(self).current_registration()->Some_0.0.ptr() == value, ensures rcu_owned_root_history_inv(next, *final(self)), final(self).domain() == old(self).domain(), @@ -1160,27 +1248,37 @@ impl RcuRootOwnedGhost { final(self).current_registration() == old(self).current_registration(), { let tracked owned = self.current.tracked_take(); - self.root.tracked_push_registered(prev, next, msg, &owned.registration.0); + self.root.tracked_push_registered( + prev, + next, + old_timestamp, + new_timestamp, + value, + message_view, + &owned.registration.0, + ); self.current = Some(owned); assert(current_registration_matches(self.root(), self.current_registration())); assert(self.removals() == self.root().domain_auth().retire_observations()); assert(self.removals_wf(next)) by { assert forall|obj: nat| self.removals().contains_key(obj) implies { let ts = (#[trigger] self.removals()[obj]).timestamp; - &&& 0 < ts < next.len() - &&& forall|i: int| - ts <= i < next.len() ==> #[trigger] self.publications()[i] != Some(obj) + &&& next.contains_timestamp(ts) + &&& forall|later: nat| + next.contains_timestamp(later) && ts <= later + ==> #[trigger] self.publications()[later] != Some(obj) } by { assert(old(self).removals().contains_key(obj)); assert(!old(self).removals().contains_key(owned.registration.0.obj())); assert(obj != owned.registration.0.obj()); - assert forall|i: int| - self.removals()[obj].timestamp <= i - < next.len() implies #[trigger] self.publications()[i] != Some(obj) by { - if i < prev.len() { - assert(self.publications()[i] == old(self).publications()[i]); + assert forall|later: nat| + next.contains_timestamp(later) && self.removals()[obj].timestamp + <= later implies #[trigger] self.publications()[later] != Some(obj) by { + if later != new_timestamp { + assert(prev.contains_timestamp(later)); + assert(self.publications()[later] == old(self).publications()[later]); } else { - assert(i == prev.len()); + assert(self.publications()[later] == Some(owned.registration.0.obj())); } }; }; @@ -1191,7 +1289,7 @@ impl RcuRootOwnedGhost { /// The current ownership resource agrees with the latest publication, while /// older history entries need only agree with persistent registration metadata. pub open spec fn rcu_owned_root_history_inv( - history: History<*mut T>, + history: Irc11History<*mut T>, ghost: RcuRootOwnedGhost, ) -> bool { &&& rcu_root_history_inv(history, ghost.root()) @@ -1199,19 +1297,19 @@ pub open spec fn rcu_owned_root_history_inv( &&& ghost.infos_wf() &&& ghost.removals_wf(history) &&& ghost.removals() == ghost.root().domain_auth().retire_observations() - &&& forall|i: int| - 0 <= i < history.len() ==> { - match #[trigger] ghost.publications()[i] { - Some(obj) => equal(ghost.infos()[obj].ptr(), history[i].value), + &&& forall|ts: nat| + history.contains_timestamp(ts) ==> { + match #[trigger] ghost.publications()[ts] { + Some(obj) => equal(ghost.infos()[obj].ptr(), history.value(ts)), None => true, } } &&& match ghost.current_registration() { Some(registration) => equal( registration.0.ptr(), - history[(history.len() - 1) as int].value, + history.value(ghost.root().current_timestamp()), ), - None => history[(history.len() - 1) as int].value.addr() == 0, + None => history.value(ghost.root().current_timestamp()).addr() == 0, } } @@ -1240,7 +1338,7 @@ pub open spec fn rcu_current_ownership_inv( /// Opens the structural current-ownership relation for atomic clients. pub proof fn lemma_current_owned_resources( - history: History<*mut T>, + history: Irc11History<*mut T>, tracked ghost: &RcuRootOwnedGhost, ) where OwnPred: RcuRootOwnershipPredicate requires @@ -1250,10 +1348,10 @@ pub proof fn lemma_current_owned_resources( match ghost.current_owned() { Some(owned) => { &&& owned.block_info().wf() - &&& equal(owned.block_info().ptr(), history[(history.len() - 1) as int].value) + &&& equal(owned.block_info().ptr(), history.value(ghost.root().current_timestamp())) &&& OwnPred::owns(owned.block_info().ptr(), owned.ownership()) }, - None => history[(history.len() - 1) as int].value.addr() == 0, + None => history.value(ghost.root().current_timestamp()).addr() == 0, }, { match ghost.current_owned() { @@ -1269,22 +1367,28 @@ pub struct RcuOwnedWeakAtomicInv { _marker: PhantomData, } -impl WeakAtomicInvariantPredicate< - RcuRootKey, - *mut T, - RcuRootOwnedGhost, +impl InvariantPredicate< + (RcuRootKey, Irc11AtomicId), + (AtomicPointsTo<*mut T>, RcuRootOwnedGhost), > for RcuOwnedWeakAtomicInv where OwnPred: RcuRootOwnershipPredicate { - open spec fn atomic_inv( - key: RcuRootKey, - history: History<*mut T>, - g: RcuRootOwnedGhost, + open spec fn inv( + key_loc: (RcuRootKey, Irc11AtomicId), + pair: (AtomicPointsTo<*mut T>, RcuRootOwnedGhost), ) -> bool { + let (key, loc) = key_loc; + let (points_to, g) = pair; + &&& points_to.loc() == loc &&& key.domain == g.domain() &&& key.reader_registry == g.reader_registry() &&& key.retire_observation_registry == g.retire_observation_registry() - &&& rcu_history_inv(key.nullable, history) - &&& rcu_owned_root_history_inv(history, g) + &&& rcu_history_inv(key.nullable, points_to.hist()) + &&& rcu_owned_root_history_inv(points_to.hist(), g) &&& rcu_current_ownership_inv::(g) + &&& forall|obj: nat| + g.removals().contains_key(obj) ==> { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } } } @@ -1384,43 +1488,58 @@ pub proof fn monitor_state_no_pending_no_summaries(state: MonitorStateView) /// Ghost summary paired with the RCU monitor's `is_monitoring` flag. /// -/// `states[i]` summarizes the lock-protected monitor state at the moment flag -/// message `i` was appended. This is intentionally a summary: the concrete +/// `states[ts]` summarizes the lock-protected monitor state stored with flag +/// message timestamp `ts`. This is intentionally a summary: the concrete /// callback vectors live in the monitor state protected by its lock, and the -/// agreement between `states[i]` and that state is established by the writer, +/// agreement between `states[ts]` and that state is established by the writer, /// which performs every flag store while holding the monitor lock. pub tracked struct RcuMonitorFlagGhost { - pub ghost states: Seq, + pub ghost states: Map, +} + +unsafe impl Objective for RcuMonitorFlagGhost { + } impl RcuMonitorFlagGhost { - pub open spec fn initial() -> Self { - RcuMonitorFlagGhost { states: seq![MonitorStateView::initial()] } + pub open spec fn initial(timestamp: nat) -> Self { + RcuMonitorFlagGhost { + states: Map::empty().insert(timestamp, MonitorStateView::initial()), + } } /// Proof-mode constructor for the tracked ghost state stored inside the /// monitor flag's weak atomic invariant. - pub proof fn tracked_initial() -> (tracked res: Self) + pub proof fn tracked_initial(timestamp: nat) -> (tracked res: Self) ensures - res == Self::initial(), + res == Self::initial(timestamp), { - RcuMonitorFlagGhost { states: seq![MonitorStateView::initial()] } + RcuMonitorFlagGhost { + states: Map::empty().insert(timestamp, MonitorStateView::initial()), + } } - pub open spec fn push(self, state: MonitorStateView) -> Self { - RcuMonitorFlagGhost { states: self.states.push(state) } + pub open spec fn insert(self, timestamp: nat, state: MonitorStateView) -> Self { + RcuMonitorFlagGhost { states: self.states.insert(timestamp, state) } } - pub proof fn tracked_push(tracked self, state: MonitorStateView) -> (tracked res: Self) + pub proof fn tracked_insert( + tracked self, + timestamp: nat, + state: MonitorStateView, + ) -> (tracked res: Self) ensures - res == self.push(state), + res == self.insert(timestamp, state), { - RcuMonitorFlagGhost { states: self.states.push(state) } + RcuMonitorFlagGhost { states: self.states.insert(timestamp, state) } } - /// Whether the state recorded at flag message `i` still had work pending. - pub open spec fn pending_at(self, i: int) -> bool { - self.states[i].has_pending_work() + /// Whether the state recorded at flag message `timestamp` had work pending. + pub open spec fn pending_at(self, timestamp: nat) -> bool + recommends + self.states.contains_key(timestamp), + { + self.states[timestamp].has_pending_work() } } @@ -1438,80 +1557,105 @@ impl RcuMonitorFlagGhost { /// flag message, so skipping the slow path can only delay their grace period, /// never lose them. pub open spec fn rcu_monitor_flag_history_inv( - history: History, + history: Irc11History, ghost: RcuMonitorFlagGhost, ) -> bool { - &&& history.len() >= 1 - &&& ghost.states.len() == history.len() - &&& forall|i: int| 0 <= i < history.len() ==> (#[trigger] ghost.states[i]).wf() - &&& forall|i: int| - 0 <= i < history.len() ==> { - !(#[trigger] history[i].value) ==> ghost.states[i].no_pending_work() + &&& !history.dom().is_empty() + &&& ghost.states.dom() == history.dom() + &&& forall|timestamp: nat| + history.contains_timestamp(timestamp) ==> (#[trigger] ghost.states[timestamp]).wf() + &&& forall|timestamp: nat| + history.contains_timestamp(timestamp) ==> { + !(#[trigger] history.value(timestamp)) ==> ghost.states[timestamp].no_pending_work() } } pub struct RcuMonitorFlagInv; -impl WeakAtomicInvariantPredicate<(), bool, RcuMonitorFlagGhost> for RcuMonitorFlagInv { - open spec fn atomic_inv(_k: (), history: History, ghost: RcuMonitorFlagGhost) -> bool { - rcu_monitor_flag_history_inv(history, ghost) +impl InvariantPredicate< + Irc11AtomicId, + (AtomicPointsTo, RcuMonitorFlagGhost), +> for RcuMonitorFlagInv { + open spec fn inv( + loc: Irc11AtomicId, + pair: (AtomicPointsTo, RcuMonitorFlagGhost), + ) -> bool { + &&& pair.0.loc() == loc + &&& rcu_monitor_flag_history_inv(pair.0.hist(), pair.1) } } -pub proof fn rcu_monitor_flag_initial_inv() +pub proof fn rcu_monitor_flag_initial_inv( + history: Irc11History, + timestamp: nat, + message_view: Irc11ThreadView, +) + requires + history.is_singleton(timestamp, (false, message_view)), ensures - RcuMonitorFlagInv::atomic_inv( - (), - seq![Msg { value: false, view: WmView::empty() }], - RcuMonitorFlagGhost::initial(), + rcu_monitor_flag_history_inv( + history, + RcuMonitorFlagGhost::initial(timestamp), ), { + assert(history.dom() == Set::empty().insert(timestamp)) by { + assert forall|ts: nat| + history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { + if history.dom().contains(ts) { + assert(history.contains_timestamp(ts)); + assert(ts == timestamp); + } + }; + }; } -/// Pushing one flag message preserves the history invariant, provided the +/// Inserting one flag message preserves the history invariant, provided the /// writer records a well-formed state snapshot and only writes `false` when /// that snapshot has no pending work. /// -/// This is the proof obligation of the future `set_monitoring` helper: it +/// This is the proof obligation discharged by `set_monitoring`: it /// stores the flag while holding the monitor lock, so it can supply the /// lock-protected state view as the snapshot. -pub proof fn preserve_rcu_monitor_flag_inv_on_push( - prev: History, - next: History, - msg: Msg, +pub proof fn preserve_rcu_monitor_flag_inv_on_insert( + prev: Irc11History, + next: Irc11History, + timestamp: nat, + value: bool, + message_view: Irc11ThreadView, prev_ghost: RcuMonitorFlagGhost, next_ghost: RcuMonitorFlagGhost, state: MonitorStateView, ) requires rcu_monitor_flag_history_inv(prev, prev_ghost), - next == prev.push(msg), - next_ghost == prev_ghost.push(state), + !prev.contains_timestamp(timestamp), + next == prev.insert(timestamp, value, message_view), + next_ghost == prev_ghost.insert(timestamp, state), state.wf(), - !msg.value ==> state.no_pending_work(), + !value ==> state.no_pending_work(), ensures rcu_monitor_flag_history_inv(next, next_ghost), { - assert(next.len() >= 1); - assert(next_ghost.states.len() == next.len()); - assert forall|i: int| 0 <= i < next.len() implies (#[trigger] next_ghost.states[i]).wf() by { - if i == prev.len() { - assert(next_ghost.states[i] == state); + assert(next_ghost.states.dom() == next.dom()); + assert forall|ts: nat| + next.contains_timestamp(ts) implies (#[trigger] next_ghost.states[ts]).wf() by { + if ts == timestamp { + assert(next_ghost.states[ts] == state); } else { - assert(i < prev.len()); - assert(next_ghost.states[i] == prev_ghost.states[i]); + assert(prev.contains_timestamp(ts)); + assert(next_ghost.states[ts] == prev_ghost.states[ts]); } }; - assert forall|i: int| 0 <= i < next.len() implies { - !(#[trigger] next[i].value) ==> next_ghost.states[i].no_pending_work() + assert forall|ts: nat| next.contains_timestamp(ts) implies { + !(#[trigger] next.value(ts)) ==> next_ghost.states[ts].no_pending_work() } by { - if i == prev.len() { - assert(next[i] == msg); - assert(next_ghost.states[i] == state); + if ts == timestamp { + assert(next.value(ts) == value); + assert(next_ghost.states[ts] == state); } else { - assert(i < prev.len()); - assert(next[i] == prev[i]); - assert(next_ghost.states[i] == prev_ghost.states[i]); + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); + assert(next_ghost.states[ts] == prev_ghost.states[ts]); } }; } @@ -1520,57 +1664,62 @@ pub proof fn preserve_rcu_monitor_flag_inv_on_push( /// message certifies that the monitor state recorded at that message had no /// queued callbacks and no incomplete grace period. pub proof fn rcu_monitor_flag_false_has_no_pending( - history: History, + history: Irc11History, ghost: RcuMonitorFlagGhost, ts: nat, ) requires rcu_monitor_flag_history_inv(history, ghost), - ts < history.len(), - !history[ts as int].value, + history.contains_timestamp(ts), + !history.value(ts), ensures - ghost.states[ts as int].no_pending_work(), - ghost.states[ts as int].pending_summaries() == Seq::::empty(), - ghost.states[ts as int].current_gp.is_complete, + ghost.states[ts].no_pending_work(), + ghost.states[ts].pending_summaries() == Seq::::empty(), + ghost.states[ts].current_gp.is_complete, { - monitor_state_pending_iff_incomplete(ghost.states[ts as int]); - monitor_state_no_pending_no_summaries(ghost.states[ts as int]); + monitor_state_pending_iff_incomplete(ghost.states[ts]); + monitor_state_no_pending_no_summaries(ghost.states[ts]); } pub proof fn preserve_rcu_history_inv_on_push( nullable: bool, - prev: History<*mut T>, - next: History<*mut T>, - msg: Msg<*mut T>, + prev: Irc11History<*mut T>, + next: Irc11History<*mut T>, + timestamp: nat, + value: *mut T, + message_view: Irc11ThreadView, ) requires rcu_history_inv(nullable, prev), - next == prev.push(msg), - nullable || msg.value.addr() != 0, + !prev.contains_timestamp(timestamp), + next == prev.insert(timestamp, value, message_view), + nullable || value.addr() != 0, ensures rcu_history_inv(nullable, next), { - assert(next.len() >= 1); + assert(!next.dom().is_empty()); if !nullable { - assert forall|i: int| 0 <= i < next.len() implies #[trigger] next[i].value.addr() != 0 by { - if i == prev.len() { - assert(next[i] == msg); + assert forall|ts: nat| next.contains_timestamp(ts) implies #[trigger] next.value(ts).addr() + != 0 by { + if ts == timestamp { + assert(next.value(ts) == value); } else { - assert(i < prev.len()); + assert(prev.contains_timestamp(ts)); + assert(next.value(ts) == prev.value(ts)); } }; } } -pub proof fn rcu_history_inv_read_nonnull(history: History<*mut T>, ts: nat) +pub proof fn rcu_history_inv_read_nonnull(history: Irc11History<*mut T>, ts: nat) requires rcu_history_inv(false, history), - ts < history.len(), + history.contains_timestamp(ts), ensures - history[ts as int].value.addr() != 0, - !history[ts as int].value.is_null(), + history.value(ts).addr() != 0, + !history.value(ts).is_null(), { - assert(history[ts as int].value.addr() != 0); + assert(history.value(ts).addr() != 0); } /// Link view carried by an RCU read-side guard. @@ -1684,7 +1833,7 @@ impl RcuDomainAuth { self.retire_observations } - pub open spec fn observed_retired(self, root: Loc, view: WmView) -> Set { + pub open spec fn observed_retired(self, root: Loc, view: Irc11ThreadView) -> Set { self.retired().filter( |obj: nat| self.retire_observations()[obj].root == root @@ -1883,7 +2032,7 @@ impl RcuDomainAuth { tracked &self, tracked inactive: RcuInactive, root: Loc, - start_view: WmView, + start_view: Irc11ThreadView, ) -> (tracked res: RcuBaseGuard) requires self.wf(), @@ -2038,7 +2187,7 @@ pub tracked struct RcuBaseGuard { state: GhostPointsTo, ghost reader: RcuReaderContext, ghost root: Loc, - ghost start_view: WmView, + ghost start_view: Irc11ThreadView, ghost retire_observation_registry: Loc, ghost expired: Set, ghost protected: Map, @@ -2067,7 +2216,7 @@ impl RcuBaseGuard { } /// Weak-memory view captured when this read-side critical section began. - pub closed spec fn start_view(self) -> WmView { + pub closed spec fn start_view(self) -> Irc11ThreadView { self.start_view } @@ -2254,18 +2403,18 @@ pub proof fn registered_republication_preserves_allocation_id(ptr: *mut T) -> requires ptr.addr() != 0, ensures - res.0.publications().len() == 2, + res.0.publications().dom() == Set::empty().insert(0nat).insert(1nat), res.0.publications()[0] == Some(res.1.0.obj()), res.0.publications()[1] == Some(res.1.0.obj()), res.1.0.domain() == res.0.domain(), res.1.0.obj() == res.1.1.obj(), { - let ghost initial = seq![Msg { value: ptr, view: WmView::empty() }]; - let tracked (mut root, registration_opt) = RcuRootGhost::tracked_initial(ptr); + let ghost view = Irc11ThreadView::empty(); + let ghost initial = Irc11History(Map::empty().insert(0nat, (ptr, view))); + let tracked (mut root, registration_opt) = RcuRootGhost::tracked_initial(ptr, initial, 0, view); let tracked registration = registration_opt.tracked_unwrap(); - let ghost msg = Msg { value: ptr, view: WmView::empty() }; - let ghost next = initial.push(msg); - root.tracked_push_registered(initial, next, msg, ®istration.0); + let ghost next = initial.insert(1, ptr, view); + root.tracked_push_registered(initial, next, 0, 1, ptr, view, ®istration.0); (root, registration) } @@ -2290,14 +2439,23 @@ pub proof fn owned_root_replacement_retires_previous_registration( == res.0.current_registration()->Some_0.1.obj(), res.1.domain() == res.0.domain(), { - let ghost initial = seq![Msg { value: first_ptr, view: WmView::empty() }]; - let tracked mut root = RcuRootOwnedGhost::tracked_initial(first_ptr, Some(())); - let ghost next_msg = Msg { value: next_ptr, view: WmView::empty() }; - let ghost next_history = initial.push(next_msg); + let ghost view = Irc11ThreadView::empty(); + let ghost initial = Irc11History(Map::empty().insert(0nat, (first_ptr, view))); + let tracked mut root = RcuRootOwnedGhost::tracked_initial( + first_ptr, + Some(()), + initial, + 0, + view, + ); + let ghost next_history = initial.insert(1, next_ptr, view); let tracked detached = root.tracked_push_fresh::( initial, next_history, - next_msg, + 0, + 1, + next_ptr, + view, root.domain(), Some(()), ); @@ -2552,7 +2710,7 @@ impl RcuRetiredFacts { /// The retirement facts themselves are persistent, but this predicate is /// the separate weak-memory premise needed before a CPU report may publish /// them to readers in a later quiescent generation. - pub open spec fn observed_by(self, view: WmView) -> bool { + pub open spec fn observed_by(self, view: Irc11ThreadView) -> bool { forall|record: RcuRetiredRecord| #[trigger] self.records().contains(record) ==> record.removal.observed_by(view) } @@ -2561,7 +2719,7 @@ impl RcuRetiredFacts { tracked &self, tracked domain: &RcuDomainAuth, root: Loc, - view: WmView, + view: Irc11ThreadView, records: Set, ) requires @@ -2614,7 +2772,7 @@ impl RcuRetiredFacts { tracked &self, tracked domain: &RcuDomainAuth, root: Loc, - view: WmView, + view: Irc11ThreadView, ) requires domain.wf(), @@ -2836,17 +2994,24 @@ pub proof fn retired_but_unexpired_object_remains_protectable(ptr: *mut T) -> { let tracked mut domain = RcuDomainAuth::tracked_new(); let tracked (info, base) = domain.tracked_register(ptr); + let ghost reader = arbitrary(); + let tracked inactive = domain.tracked_register_reader(reader); + let tracked mut guard = domain.tracked_guard_start( + inactive, + domain.id(), + Irc11ThreadView::empty(), + ); let ghost seen_removed = RcuSeenRemoved { removed: Set::empty().insert(info.obj()), link_view: RcuLinkView::empty(), }; let tracked retire = lift_direct_root_retire_perm(base, seen_removed); - let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 1 }; + let ghost removal = RcuRemovalObservation { + root: domain.id(), + timestamp: 1, + message_view: Irc11ThreadView::empty(), + }; let tracked _retired = domain.tracked_retire(retire, removal); - - let ghost reader = arbitrary(); - let tracked inactive = domain.tracked_register_reader(reader); - let tracked mut guard = domain.tracked_guard_start(inactive, domain.id(), WmView::empty()); assert(guard.expired() == Set::::empty()); assert(!guard.expired().contains(info.obj())); guard.tracked_protect(&info); @@ -2877,13 +3042,17 @@ pub proof fn observed_retired_object_enters_guard_expired(ptr: *mut T) -> (tr link_view: RcuLinkView::empty(), }; let tracked retire = lift_direct_root_retire_perm(base, seen_removed); - let ghost removal = RcuRemovalObservation { root: domain.id(), timestamp: 0 }; + let ghost removal = RcuRemovalObservation { + root: domain.id(), + timestamp: 0, + message_view: Irc11ThreadView::empty(), + }; let tracked _retired = domain.tracked_retire(retire, removal); let ghost reader = arbitrary(); let tracked inactive = domain.tracked_register_reader(reader); - let tracked guard = domain.tracked_guard_start(inactive, domain.id(), WmView::empty()); - assert(removal.observed_by(WmView::empty())); + let tracked guard = domain.tracked_guard_start(inactive, domain.id(), Irc11ThreadView::empty()); + assert(removal.observed_by(Irc11ThreadView::empty())); assert(guard.expired().contains(info.obj())); (guard, info) } @@ -3012,7 +3181,7 @@ impl RcuReadGuardToken { self.base.root() } - pub closed spec fn start_view(self) -> WmView { + pub closed spec fn start_view(self) -> Irc11ThreadView { self.base.start_view() } diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs index 3aa692c15..abed5904e 100644 --- a/ostd/specs/sync/rcu_cpu.rs +++ b/ostd/specs/sync/rcu_cpu.rs @@ -63,30 +63,28 @@ use vstd::{ }, }; -use super::{ - rcu::{ - RcuBlockInfo, RcuInactive, RcuProtectedPtr, RcuReadGuardToken, RcuReaderContext, - RcuRetiredFacts, RcuRetiredRecord, RcuSeenRemoved, - }, - weak_memory::WmView, +use super::rcu::{ + RcuBlockInfo, RcuInactive, RcuProtectedPtr, RcuReadGuardToken, RcuReaderContext, + RcuRetiredFacts, RcuRetiredRecord, RcuSeenRemoved, }; +use vstd_extra::atomic_irc11::{ThreadView as Irc11ThreadView, ThreadViewOrder}; verus! { -broadcast use vstd::set::group_set_lemmas; +broadcast use {vstd::set::group_set_lemmas, vstd::thread_view::group_thread_view_axioms}; /// One CPU quiescent report retained by the participant PCM. pub ghost struct CpuRcuReportView { pub cpu: CpuId, pub generation: nat, - pub view: WmView, + pub view: Irc11ThreadView, pub known_retired: Set, } pub(super) ghost struct CpuRcuStateView { pub(super) cpu: CpuId, pub(super) generation: nat, - pub(super) view: WmView, + pub(super) view: Irc11ThreadView, pub(super) known_retired: Set, } @@ -98,7 +96,10 @@ pub(super) ghost struct CpuRcuCarrier { } impl CpuRcuCarrier { - pub(super) open spec fn records_observed(records: Set, view: WmView) -> bool { + pub(super) open spec fn records_observed( + records: Set, + view: Irc11ThreadView, + ) -> bool { forall|record: RcuRetiredRecord| #[trigger] records.contains(record) ==> record.removal.observed_by(view) } @@ -106,7 +107,7 @@ impl CpuRcuCarrier { pub(super) open spec fn state( cpu: CpuId, generation: nat, - view: WmView, + view: Irc11ThreadView, known_retired: Set, fraction: real, ) -> Self { @@ -319,7 +320,7 @@ impl CpuRcuParticipant { } /// Creates generation zero for one CPU. - pub proof fn new(cpu: CpuId, view: WmView) -> (tracked res: Self) + pub proof fn new(cpu: CpuId, view: Irc11ThreadView) -> (tracked res: Self) ensures res.cpu() == cpu, res.generation() == 0, @@ -347,7 +348,7 @@ impl CpuRcuParticipant { self.resource.value().state_view().generation } - pub closed spec fn view(self) -> WmView { + pub closed spec fn view(self) -> Irc11ThreadView { self.resource.value().state_view().view } @@ -371,7 +372,7 @@ impl CpuRcuParticipant { /// protocol imposes no fixed bound on the number of readers. pub proof fn tracked_start_reader( tracked self, - start_view: WmView, + start_view: Irc11ThreadView, reader_fraction: real, ) -> (tracked res: (CpuRcuParticipant, CpuRcuReaderFragment)) requires @@ -449,7 +450,7 @@ impl CpuRcuParticipant { /// returning every fragment. pub proof fn tracked_start_reader_in_place( tracked &mut self, - start_view: WmView, + start_view: Irc11ThreadView, ) -> (tracked reader: CpuRcuReaderFragment) requires old(self).wf(), @@ -546,7 +547,7 @@ impl CpuRcuParticipant { pub proof fn tracked_report_quiescent_with( tracked self, tracked binding: CpuRcuCoreBinding, - report_view: WmView, + report_view: Irc11ThreadView, tracked learned: &RcuRetiredFacts, ) -> (tracked res: (CpuRcuParticipant, CpuRcuClosedGeneration)) requires @@ -591,9 +592,7 @@ impl CpuRcuParticipant { merged_records.contains(record) implies record.removal.observed_by(report_view) by { if old_known_retired.contains(record) { assert(record.removal.observed_by(old_view)); - assert(old_view.seen_at(record.removal.root) <= report_view.seen_at( - record.removal.root, - )); + old_view.lemma_spec_le_transitive(report_view, report_view); } else { assert(learned.records().contains(record)); } @@ -702,7 +701,7 @@ impl CpuRcuParticipant { pub proof fn tracked_report_quiescent( tracked self, tracked binding: CpuRcuCoreBinding, - report_view: WmView, + report_view: Irc11ThreadView, ) -> (tracked res: (CpuRcuParticipant, CpuRcuClosedGeneration)) requires self.wf(), @@ -741,7 +740,7 @@ impl CpuRcuParticipant { pub proof fn tracked_report_quiescent_in_place( tracked &mut self, tracked binding: CpuRcuCoreBinding, - report_view: WmView, + report_view: Irc11ThreadView, ) -> (tracked closed: CpuRcuClosedGeneration) requires old(self).wf(), @@ -781,7 +780,7 @@ impl CpuRcuParticipant { pub proof fn tracked_report_quiescent_with_in_place( tracked &mut self, tracked binding: CpuRcuCoreBinding, - report_view: WmView, + report_view: Irc11ThreadView, tracked learned: &RcuRetiredFacts, ) -> (tracked closed: CpuRcuClosedGeneration) requires @@ -866,7 +865,7 @@ impl CpuRcuReaderFragment { self.resource.value().state_view().generation } - pub closed spec fn participant_view(self) -> WmView { + pub closed spec fn participant_view(self) -> Irc11ThreadView { self.resource.value().state_view().view } @@ -891,8 +890,10 @@ impl CpuRcuReaderFragment { /// Borrows the persistent retirement facts known at this reader's /// generation, after lifting their observations to `view`. - pub proof fn tracked_retired_facts_observed_by(tracked &self, view: WmView) -> (tracked res: - &RcuRetiredFacts) + pub proof fn tracked_retired_facts_observed_by( + tracked &self, + view: Irc11ThreadView, + ) -> (tracked res: &RcuRetiredFacts) requires self.participant_view().spec_le(view), ensures @@ -906,9 +907,7 @@ impl CpuRcuReaderFragment { ) by { assert(self.resource.value().state_view().known_retired.contains(record)); assert(record.removal.observed_by(self.participant_view())); - assert(self.participant_view().seen_at(record.removal.root) <= view.seen_at( - record.removal.root, - )); + self.participant_view().lemma_spec_le_transitive(view, view); }; &self.known_retired } @@ -951,7 +950,7 @@ impl CpuRcuReadGuardToken { self.reader_fragment().generation() } - pub closed spec fn participant_view(self) -> WmView { + pub closed spec fn participant_view(self) -> Irc11ThreadView { self.reader_fragment().participant_view() } @@ -971,7 +970,7 @@ impl CpuRcuReadGuardToken { self.paper_guard().root() } - pub closed spec fn start_view(self) -> WmView { + pub closed spec fn start_view(self) -> Irc11ThreadView { self.paper_guard().start_view() } @@ -1256,7 +1255,7 @@ impl CpuRcuClosedGeneration { self.report().generation } - pub closed spec fn view(self) -> WmView { + pub closed spec fn view(self) -> Irc11ThreadView { self.report().view } @@ -1430,7 +1429,7 @@ impl CpuRcuClosedGeneration { pub proof fn lemma_later_reader_start_view( tracked &self, tracked reader: CpuRcuReaderFragment, - start_view: WmView, + start_view: Irc11ThreadView, ) -> (tracked res: CpuRcuReaderFragment) requires self.wf(), @@ -1516,7 +1515,7 @@ impl CpuRcuClosedGeneration { } /// Regression proof for both sides of the grace-period reader dichotomy. -proof fn cpu_rcu_generation_smoke_test(cpu: CpuId, initial: WmView, later: WmView) +proof fn cpu_rcu_generation_smoke_test(cpu: CpuId, initial: Irc11ThreadView, later: Irc11ThreadView) requires initial.spec_le(later), { diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index f579ad99f..cce6ab7cd 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -1,43 +1,59 @@ // SPDX-License-Identifier: MPL-2.0 -//! OSTD-specific adapters for the generic weak-memory atomic library. +//! OSTD-specific adapters for Verus' native IRC11 weak-memory atomics. //! -//! The reusable view, history, resource algebra, atomic wrappers, and -//! invariant-opening macro live in [`vstd_extra::atomic_weak`]. This module -//! re-exports that API for existing OSTD callers and keeps only transitions -//! coupled to the RCU root and monitor ghost state. -pub use vstd_extra::atomic_weak::*; -pub use vstd_extra::weak_atomic_with_ghost; +//! This module contains only transitions coupled to the RCU root and monitor +//! ghost state. Generic native primitives are re-exported by +//! [`vstd_extra::atomic_irc11`]. + +use core::sync::atomic::Ordering; use super::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec}; +use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::prelude::*; +use vstd::resource::Loc; +use vstd::thread_view::Objective; +use vstd_extra::atomic_irc11::{ + AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, + PAtomicWeakPtr, ReleaseViewSeen, ThreadView as Irc11ThreadView, + ThreadViewOrder as Irc11ThreadViewOrder, Timestamp, ViewSeen, +}; verus! { +broadcast use {vstd::atomic_weak::group_view_history, vstd::thread_view::group_thread_view_axioms}; + /// OSTD's RCU-specific specialization of the generic weak pointer atomic. /// -/// The inner atomic and its history protocol are reusable; this wrapper adds -/// transitions that manipulate `RcuRootOwnedGhost`. +/// This is an RCU client of Verus' native IRC11 protocol. The only local TCB +/// component is `PAtomicWeakPtr`, needed because upstream does not yet expose +/// a native weak-memory `AtomicPtr`. #[verifier::reject_recursive_types(T)] -pub struct RcuWeakAtomicPtr { - inner: WeakAtomicPtr< - T, - rcu_spec::RcuRootKey, - rcu_spec::RcuRootOwnedGhost, - rcu_spec::RcuOwnedWeakAtomicInv, +pub struct RcuWeakAtomicPtr { + atomic: PAtomicWeakPtr, + tracked_atomic_inv: Tracked< + AtomicInvariant< + (rcu_spec::RcuRootKey, Irc11AtomicId), + (AtomicPointsTo<*mut T>, rcu_spec::RcuRootOwnedGhost), + rcu_spec::RcuOwnedWeakAtomicInv, + >, >, } -impl RcuWeakAtomicPtr { +impl RcuWeakAtomicPtr { pub closed spec fn constant(&self) -> rcu_spec::RcuRootKey { - self.inner.constant() + self.tracked_atomic_inv@.constant().0 + } + + pub closed spec fn id(&self) -> Loc { + self.constant().domain } - pub closed spec fn id(&self) -> AtomicId { - self.inner.id() + pub closed spec fn native_loc(&self) -> Irc11AtomicId { + self.atomic.loc() } pub closed spec fn well_formed(&self) -> bool { - self.inner.well_formed() + self.tracked_atomic_inv@.constant().1 == self.native_loc() } #[verifier::type_invariant] @@ -46,53 +62,97 @@ impl RcuWeakAtomicPtr { } } -impl RcuWeakAtomicPtr where +impl RcuWeakAtomicPtr where OwnPred: rcu_spec::RcuRootOwnershipPredicate, { pub const fn new( - Ghost(k): Ghost, + Ghost(nullable): Ghost, init: *mut T, - Tracked(g): Tracked>, + Tracked(ownership): Tracked>, ) -> (res: Self) requires - rcu_spec::RcuOwnedWeakAtomicInv::::atomic_inv( - k, - seq![Msg { value: init, view: WmView::empty() }], - g, - ), + nullable || !init.is_null(), + (ownership is Some) == !init.is_null(), + ownership is Some ==> OwnPred::owns(init, ownership->Some_0), ensures res.well_formed(), - res.constant() == k, + res.constant().nullable == nullable, { - let inner = WeakAtomicPtr::new(Ghost(k), init, Tracked(g)); - Self { inner } + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + PAtomicWeakPtr::new(init); + let tracked g = rcu_spec::RcuRootOwnedGhost::tracked_initial( + init, + ownership, + points_to.hist(), + timestamp, + initial_view@, + ); + let ghost key = rcu_spec::RcuRootKey { + nullable, + domain: g.domain(), + reader_registry: g.reader_registry(), + retire_observation_registry: g.retire_observation_registry(), + }; + let tracked pair = (points_to, g); + proof { + assert(rcu_spec::rcu_history_inv(nullable, pair.0.hist())) by { + assert(!pair.0.hist().dom().is_empty()); + if !nullable { + assert forall|ts: nat| + pair.0.hist().contains_timestamp(ts) implies #[trigger] pair.0.hist().value( + ts, + ).addr() != 0 by { + assert(ts == timestamp); + assert(equal(pair.0.hist().value(ts), init)); + }; + } + }; + assert(rcu_spec::rcu_current_ownership_inv::(pair.1)) by { + match pair.1.current_owned() { + Some(owned) => { + assert(ownership == Some(owned.ownership())); + assert(equal(owned.block_info().ptr(), init)); + }, + None => {}, + } + }; + assert forall|obj: nat| pair.1.removals().contains_key(obj) implies { + let removal = #[trigger] pair.1.removals()[obj]; + pair.0.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + assert(pair.1.removals() == Map::empty()); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::::inv((key, atomic.loc()), pair)); + } + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), pair, 0); + Self { atomic, tracked_atomic_inv: Tracked(atomic_inv) } } - fn raw_atomic(&self) -> (res: &AtomicPtrW) + fn raw_atomic(&self) -> (res: &PAtomicWeakPtr) requires self.well_formed(), ensures - res.id() == self.id(), + res.loc() == self.native_loc(), { - self.inner.raw_atomic() + &self.atomic } proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &vstd::invariant::AtomicInvariant< - (rcu_spec::RcuRootKey, AtomicId), - (HistAuth<*mut T>, rcu_spec::RcuRootOwnedGhost), - WeakAtomicPredPtr>, + (rcu_spec::RcuRootKey, Irc11AtomicId), + (AtomicPointsTo<*mut T>, rcu_spec::RcuRootOwnedGhost), + rcu_spec::RcuOwnedWeakAtomicInv, >) requires self.well_formed(), ensures - res.constant() == (self.constant(), self.id()), + res.constant() == (self.constant(), self.native_loc()), { - self.inner.tracked_atomic_inv() + self.tracked_atomic_inv.borrow() } /// Acquire-load helper for RCU root pointers. #[inline(always)] - pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + pub fn load_acquire_rcu(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( *mut T, Ghost, Ghost>, @@ -124,34 +184,28 @@ impl RcuWeakAtomicPtr where } let raw_atomic = self.raw_atomic(); vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (hist, g) = pair; + let tracked (points_to, g) = pair; proof { - assert(hist.id() == self.id()); - assert(raw_atomic.id() == self.id()); - assert(hist.id() == raw_atomic.id()); + assert(points_to.loc() == self.native_loc()); } - let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&points_to), + ); proof { - start_view.lemma_acquire( - self.id(), - loaded.1@, - hist.msg_at(loaded.1@).view, - ); - assert(hist.valid_ts(loaded.1@)); - assert(loaded.1@ < hist.history().len()); - assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); + assert(rcu_spec::rcu_owned_root_history_inv(points_to.hist(), g)); } proof_decl! { - let ghost published = g.published_at(loaded.1@); + let ghost timestamp = loaded.2@.timestamp; + let ghost published = g.published_at(timestamp); let tracked loaded_info; } proof { - assert(rcu_spec::rcu_history_inv(self.constant().nullable, hist.history())); - assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); - loaded_info = g.tracked_info_at(hist.history(), loaded.1@); + loaded_info = g.tracked_info_at(points_to.hist(), timestamp); match (published, &loaded_info) { (Some(object), Some(info)) => { - assert(equal(hist.history()[loaded.1@ as int].value, loaded.0)); + assert(equal(points_to.hist().value(timestamp), loaded.0)); assert(equal(info.ptr(), loaded.0)); }, (None, None) => { @@ -160,13 +214,13 @@ impl RcuWeakAtomicPtr where _ => assert(false), }; if !self.constant().nullable { - rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), loaded.1@); + rcu_spec::rcu_history_inv_read_nonnull::(points_to.hist(), timestamp); assert(!loaded.0.is_null()); } } - result = (loaded.0, loaded.1, Ghost(published), Tracked(loaded_info)); + result = (loaded.0, Ghost(timestamp), Ghost(published), Tracked(loaded_info)); proof { - pair = (hist, g); + pair = (points_to, g); } }); result @@ -181,7 +235,7 @@ impl RcuWeakAtomicPtr where &self, Ghost(reader): Ghost, Tracked(retired_facts): Tracked<&rcu_spec::RcuRetiredFacts>, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( *mut T, Ghost, @@ -232,60 +286,46 @@ impl RcuWeakAtomicPtr where let ghost start_view = tv@; let raw_atomic = self.raw_atomic(); vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (hist, mut g) = pair; + let tracked (points_to, mut g) = pair; proof { - assert(hist.id() == self.id()); - assert(raw_atomic.id() == self.id()); - assert(hist.id() == raw_atomic.id()); - assert(rcu_spec::RcuOwnedWeakAtomicInv::::atomic_inv( - self.constant(), - hist.history(), - g, - )); + assert(points_to.loc() == self.native_loc()); assert(g.retire_observation_registry() == self.constant().retire_observation_registry); } proof_decl! { let tracked base_guard = - g.tracked_start_reader(hist.history(), self.id(), start_view, reader); + g.tracked_start_reader(points_to.hist(), self.id(), start_view, reader); } proof { g.lemma_retired_facts_observed( - hist.history(), + points_to.hist(), retired_facts, self.id(), start_view, ); } - let loaded = raw_atomic.load_acquire(Tracked(&hist), Tracked(tv)); + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&points_to), + ); + let ghost timestamp = loaded.2@.timestamp; proof { - start_view.lemma_acquire( - self.id(), - loaded.1@, - hist.msg_at(loaded.1@).view, - ); - assert(hist.valid_ts(loaded.1@)); - assert(loaded.1@ < hist.history().len()); - assert(rcu_spec::rcu_owned_root_history_inv(hist.history(), g)); + assert(rcu_spec::rcu_owned_root_history_inv(points_to.hist(), g)); } proof_decl! { let tracked loaded_info; } proof { - assert(rcu_spec::rcu_history_inv( - self.constant().nullable, - hist.history(), - )); - loaded_info = g.tracked_info_at(hist.history(), loaded.1@); - assert(loaded.1@ < g.publications().len()); + loaded_info = g.tracked_info_at(points_to.hist(), timestamp); } proof_decl! { - let ghost published = g.published_at(loaded.1@); + let ghost published = g.published_at(timestamp); } proof { match (published, &loaded_info) { (Some(object), Some(info)) => { - assert(equal(hist.history()[loaded.1@ as int].value, loaded.0)); + assert(equal(points_to.hist().value(timestamp), loaded.0)); assert(equal(info.ptr(), loaded.0)); assert(info.domain() == g.domain()); assert(info.domain() == base_guard.domain()); @@ -296,7 +336,7 @@ impl RcuWeakAtomicPtr where _ => assert(false), }; if !self.constant().nullable { - rcu_spec::rcu_history_inv_read_nonnull::(hist.history(), loaded.1@); + rcu_spec::rcu_history_inv_read_nonnull::(points_to.hist(), timestamp); assert(!loaded.0.is_null()); } assert(base_guard.domain() == self.constant().domain); @@ -324,7 +364,7 @@ impl RcuWeakAtomicPtr where start_view, ).contains(info.obj())); g.lemma_observed_retired( - hist.history(), + points_to.hist(), self.id(), start_view, info.obj(), @@ -332,26 +372,31 @@ impl RcuWeakAtomicPtr where let ghost removal = g.removals()[info.obj()]; assert(removal.root == self.id()); assert(removal.observed_by(start_view)); - assert(start_view.seen_at(self.id()) <= loaded.1@); - assert(removal.timestamp <= loaded.1@); - assert(g.removals_wf(hist.history())); - assert(removal.timestamp < hist.history().len()); - assert(g.publications()[loaded.1@ as int] != Some(info.obj())); + assert(points_to.get_timestamp(removal.message_view) + == Some(removal.timestamp)); + points_to.get_timestamp_monotonic(start_view, removal.message_view); + assert(points_to.get_timestamp(start_view) is Some); + assert(removal.timestamp + <= points_to.get_timestamp(start_view)->Some_0); + assert(points_to.get_timestamp(start_view)->Some_0 <= timestamp); + assert(removal.timestamp <= timestamp); + assert(g.removals_wf(points_to.hist())); + assert(g.publications()[timestamp] != Some(info.obj())); assert(published == Some(rcu_spec::RcuPublishedObject { domain: info.domain(), obj: info.obj(), addr: info.addr(), })); g.lemma_published_object_id( - hist.history(), - loaded.1@, + points_to.hist(), + timestamp, rcu_spec::RcuPublishedObject { domain: info.domain(), obj: info.obj(), addr: info.addr(), }, ); - assert(g.publications()[loaded.1@ as int] == Some(info.obj())); + assert(g.publications()[timestamp] == Some(info.obj())); assert(false); } assert(guard.can_protect(*info)); @@ -362,13 +407,13 @@ impl RcuWeakAtomicPtr where } result = ( loaded.0, - loaded.1, + Ghost(timestamp), Ghost(published), Tracked(loaded_info), Tracked(guard), ); proof { - pair = (hist, g); + pair = (points_to, g); } }); result @@ -383,7 +428,7 @@ impl RcuWeakAtomicPtr where pub fn load_acquire_rcu_guarded( &self, Ghost(reader): Ghost, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( *mut T, Ghost, @@ -443,7 +488,7 @@ impl RcuWeakAtomicPtr where Ghost(reader): Ghost, Tracked(cpu_reader): Tracked, Tracked(binding): Tracked, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( *mut T, Ghost, @@ -620,7 +665,7 @@ impl RcuWeakAtomicPtr where &self, value: *mut T, Tracked(ownership): Tracked>, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: (*mut T, Tracked>>)) requires self.well_formed(), @@ -652,36 +697,38 @@ impl RcuWeakAtomicPtr where } let raw_atomic = self.raw_atomic(); vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (mut hist, mut g) = pair; + let tracked (mut points_to, mut g) = pair; proof { - assert(hist.id() == self.id()); - assert(raw_atomic.id() == self.id()); - assert(hist.id() == raw_atomic.id()); + assert(points_to.loc() == self.native_loc()); } - let ghost prev = hist.history(); - let swap = raw_atomic.swap_release(Tracked(&mut hist), Tracked(tv), value); + let ghost prev = points_to.hist(); + let ghost previous_removals = g.removals(); + let swap = raw_atomic.swap_release(value, Tracked(tv), Tracked(&mut points_to)); result = swap.0; - let snap = swap.1; - let ghost next = hist.history(); + let ghost update = swap.2@; + let ghost next = points_to.hist(); proof { - start_view.lemma_observe(self.id(), prev.len()); assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); assert(rcu_spec::rcu_current_ownership_inv::(g)); rcu_spec::lemma_current_owned_resources::(prev, &g); if !self.constant().nullable { assert(!value.is_null()); - assert(snap@.msg().value.addr() != 0); } rcu_spec::preserve_rcu_history_inv_on_push( self.constant().nullable, prev, next, - snap@.msg(), + update.load_timestamp + 1, + value, + update.store_message_view, ); let tracked detached = g.tracked_push_fresh::( prev, next, - snap@.msg(), + update.load_timestamp, + update.load_timestamp + 1, + value, + update.store_message_view, self.id(), ownership, ); @@ -694,7 +741,7 @@ impl RcuWeakAtomicPtr where assert(detached is Some ==> detached->Some_0.retired().removal().root == self.id()); assert(detached is Some ==> detached->Some_0.retired().removal().timestamp - == prev.len()); + == update.load_timestamp + 1); assert(detached is Some ==> detached->Some_0.retired().removal().observed_by( tv@, )); @@ -707,8 +754,27 @@ impl RcuWeakAtomicPtr where None => {}, } }; + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + match detached { + Some(detached) => { + if obj == detached.obj() { + assert(g.removals()[obj] == detached.retired().removal()); + } else { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + } + }, + None => { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + }, + } + }; retired_ownership = detached; - pair = (hist, g); + pair = (points_to, g); } }); (result, Tracked(retired_ownership)) @@ -725,7 +791,7 @@ impl RcuWeakAtomicPtr where current: *mut T, new: *mut T, Tracked(new_ownership): Tracked>, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( Result<*mut T, *mut T>, Ghost, @@ -764,115 +830,126 @@ impl RcuWeakAtomicPtr where } let raw_atomic = self.raw_atomic(); vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (mut hist, mut g) = pair; + let tracked (mut points_to, mut g) = pair; proof { - assert(hist.id() == self.id()); - assert(raw_atomic.id() == self.id()); - assert(hist.id() == raw_atomic.id()); + assert(points_to.loc() == self.native_loc()); } - let ghost prev = hist.history(); - let cas_result = raw_atomic.compare_exchange_acqrel_acquire( - Tracked(&mut hist), - Tracked(tv), + let ghost prev = points_to.hist(); + let ghost previous_removals = g.removals(); + proof_decl! { + let tracked release_view = vstd::thread_view::ReleaseViewSeen::new(); + } + let cas_result = raw_atomic.compare_exchange( current, new, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut points_to), ); - result = (cas_result.0, cas_result.1); - let ghost next = hist.history(); + result = (cas_result.0, Ghost(cas_result.2@.load_timestamp)); + let ghost update = cas_result.2@; + let ghost next = points_to.hist(); proof { - let ghost read_view = prev[cas_result.1@ as int].view; - let ghost after_read = - start_view.observe(self.id(), cas_result.1@).join(read_view); - start_view.lemma_acquire(self.id(), cas_result.1@, read_view); - if cas_result.0 is Ok { - after_read.lemma_observe(self.id(), prev.len()); - start_view.lemma_spec_le_transitive( - after_read, - after_read.observe(self.id(), prev.len()), - ); - } assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); assert(rcu_spec::rcu_current_ownership_inv::(g)); rcu_spec::lemma_current_owned_resources::(prev, &g); match cas_result.0 { Result::Ok(_) => { - let tracked snap_opt = cas_result.2.get(); - match snap_opt { - Option::Some(snap) => { - if !self.constant().nullable { - assert(!new.is_null()); - assert(snap.msg().value.addr() != 0); - } - rcu_spec::preserve_rcu_history_inv_on_push( - self.constant().nullable, - prev, - next, - snap.msg(), - ); - let tracked detached = g.tracked_push_fresh::( - prev, - next, - snap.msg(), - self.id(), - new_ownership, - ); - assert(detached is Some ==> detached->Some_0.object().wf()); - assert(detached is Some ==> equal( - detached->Some_0.ptr(), - cas_result.0->Ok_0, - )); - assert(detached is Some ==> OwnPred::owns( - cas_result.0->Ok_0, - detached->Some_0.ownership(), - )); - assert(detached is Some ==> detached->Some_0.retired().removal().root - == self.id()); - assert(detached is Some ==> - detached->Some_0.retired().removal().timestamp == prev.len()); - assert(detached is Some ==> - detached->Some_0.retired().removal().observed_by(tv@)); - assert(rcu_spec::rcu_current_ownership_inv::(g)) by { - match g.current_owned() { - Some(owned) => { - assert(new_ownership == Some(owned.ownership())); - assert(equal(owned.block_info().ptr(), new)); - }, - None => {}, + rcu_spec::preserve_rcu_history_inv_on_push( + self.constant().nullable, + prev, + next, + update.load_timestamp + 1, + new, + update.store_message_view, + ); + let tracked detached = g.tracked_push_fresh::( + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + new, + update.store_message_view, + self.id(), + new_ownership, + ); + assert(detached is Some ==> detached->Some_0.object().wf()); + assert(detached is Some ==> equal( + detached->Some_0.ptr(), + cas_result.0->Ok_0, + )); + assert(detached is Some ==> OwnPred::owns( + cas_result.0->Ok_0, + detached->Some_0.ownership(), + )); + assert(detached is Some ==> detached->Some_0.retired().removal().root + == self.id()); + assert(detached is Some ==> + detached->Some_0.retired().removal().observed_by(tv@)); + assert(rcu_spec::rcu_current_ownership_inv::(g)) by { + match g.current_owned() { + Some(owned) => { + assert(new_ownership == Some(owned.ownership())); + assert(equal(owned.block_info().ptr(), new)); + }, + None => {}, + } + }; + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) + == Some(removal.timestamp) + } by { + match detached { + Some(detached) => { + if obj == detached.obj() { + assert(g.removals()[obj] == detached.retired().removal()); + } else { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); } - }; - retired_ownership = (detached, None); - }, - Option::None => { - assert(false); - retired_ownership = (None, None); - }, - } + }, + None => { + assert(previous_removals.contains_key(obj)); + assert(g.removals()[obj] == previous_removals[obj]); + }, + } + }; + retired_ownership = (detached, None); }, Result::Err(_) => { retired_ownership = (None, new_ownership); assert(next == prev); - assert(rcu_spec::rcu_history_inv(self.constant().nullable, next)); }, } - pair = (hist, g); + pair = (points_to, g); } }); (result.0, result.1, Tracked(retired_ownership)) } } -/// RCU monitor specialization of the generic weak boolean atomic. +/// Native IRC11 weak boolean atomic specialized for the RCU monitor flag. pub struct RcuMonitorWeakAtomicBool { - inner: WeakAtomicBool<(), rcu_spec::RcuMonitorFlagGhost, rcu_spec::RcuMonitorFlagInv>, + atomic: Irc11AtomicBool, + tracked_atomic_inv: Tracked< + AtomicInvariant< + Irc11AtomicId, + (AtomicPointsTo, rcu_spec::RcuMonitorFlagGhost), + rcu_spec::RcuMonitorFlagInv, + >, + >, } impl RcuMonitorWeakAtomicBool { - pub closed spec fn id(&self) -> AtomicId { - self.inner.id() + pub closed spec fn id(&self) -> Irc11AtomicId { + self.atomic.loc() } pub closed spec fn well_formed(&self) -> bool { - self.inner.well_formed() + self.tracked_atomic_inv@.constant() == self.id() } #[verifier::type_invariant] @@ -880,70 +957,92 @@ impl RcuMonitorWeakAtomicBool { self.well_formed() } - pub const fn new( - Ghost(k): Ghost<()>, - init: bool, - Tracked(g): Tracked, - ) -> (res: Self) - requires - rcu_spec::RcuMonitorFlagInv::atomic_inv( - k, - seq![Msg { value: init, view: WmView::empty() }], - g, - ), + pub const fn new() -> (res: Self) ensures res.well_formed(), { - let inner = WeakAtomicBool::new(Ghost(k), init, Tracked(g)); - Self { inner } + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + Irc11AtomicBool::new(false); + let tracked flag_ghost = rcu_spec::RcuMonitorFlagGhost::tracked_initial(timestamp); + proof { + rcu_spec::rcu_monitor_flag_initial_inv( + points_to.hist(), + timestamp, + initial_view@, + ); + assert(rcu_spec::RcuMonitorFlagInv::inv( + atomic.loc(), + (points_to, flag_ghost), + )); + } + let tracked pair = (points_to, flag_ghost); + let tracked atomic_inv = AtomicInvariant::new(atomic.loc(), pair, 0); + Self { atomic, tracked_atomic_inv: Tracked(atomic_inv) } } - pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( bool, - Ghost, + Ghost, )) requires self.well_formed(), ensures old(tv)@.spec_le(final(tv)@), { - self.inner.load_relaxed(Tracked(tv)) + let result; + proof { + use_type_invariant(self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { + let tracked (points_to, flag_ghost) = pair; + let loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&points_to), + ); + result = (loaded.0, Ghost(loaded.2@.timestamp)); + proof { + pair = (points_to, flag_ghost); + } + }); + result } - fn raw_atomic(&self) -> (res: &AtomicBoolW) + fn raw_atomic(&self) -> (res: &Irc11AtomicBool) requires self.well_formed(), ensures - res.id() == self.id(), + res.loc() == self.id(), { - self.inner.raw_atomic() + &self.atomic } proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &vstd::invariant::AtomicInvariant< - ((), AtomicId), - (HistAuth, rcu_spec::RcuMonitorFlagGhost), - WeakAtomicPredBool, + Irc11AtomicId, + (AtomicPointsTo, rcu_spec::RcuMonitorFlagGhost), + rcu_spec::RcuMonitorFlagInv, >) requires self.well_formed(), ensures - res.constant() == ((), self.id()), + res.constant() == self.id(), { - self.inner.tracked_atomic_inv() + self.tracked_atomic_inv.borrow() } /// Relaxed-store helper for the RCU monitor flag. /// /// The executable flag remains a relaxed atomic flag, matching the old /// monitor protocol. The proof-side effect is stronger: each stored flag - /// message appends the lock-protected monitor-state snapshot supplied by + /// message inserts the lock-protected monitor-state snapshot supplied by /// the writer. #[inline(always)] pub fn store_relaxed_rcu_monitor( &self, value: bool, Ghost(state): Ghost, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) requires self.well_formed(), @@ -952,34 +1051,41 @@ impl RcuMonitorWeakAtomicBool { ensures old(tv)@.spec_le(final(tv)@), { - let ghost start_view = tv@; proof { use_type_invariant(self); } let raw_atomic = self.raw_atomic(); vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (mut hist, mut g) = pair; + let tracked (mut points_to, mut flag_ghost) = pair; proof { - assert(hist.id() == self.id()); - assert(raw_atomic.id() == self.id()); - assert(hist.id() == raw_atomic.id()); + assert(points_to.loc() == self.id()); + assert(raw_atomic.loc() == self.id()); } - let ghost prev = hist.history(); - let snap = raw_atomic.store_relaxed(Tracked(&mut hist), Tracked(tv), value); - let ghost next = hist.history(); + let ghost prev = points_to.hist(); + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let store = raw_atomic.store( + value, + Ordering::Relaxed, + Tracked(tv), + Tracked(release_view), + Tracked(&mut points_to), + ); + let ghost next = points_to.hist(); proof { - start_view.lemma_observe(self.id(), prev.len()); - assert(snap@.msg().value == value); - rcu_spec::preserve_rcu_monitor_flag_inv_on_push( + rcu_spec::preserve_rcu_monitor_flag_inv_on_insert( prev, next, - snap@.msg(), - g, - g.push(state), + store@.timestamp, + value, + store@.message_view, + flag_ghost, + flag_ghost.insert(store@.timestamp, state), state, ); - g = g.tracked_push(state); - pair = (hist, g); + flag_ghost = flag_ghost.tracked_insert(store@.timestamp, state); + pair = (points_to, flag_ghost); } }); } diff --git a/ostd/src/sync/once.rs b/ostd/src/sync/once.rs index 1e7f54245..aaf173a1a 100644 --- a/ostd/src/sync/once.rs +++ b/ostd/src/sync/once.rs @@ -3,6 +3,7 @@ use vstd::{ cell::pcell::{PCell, PointsTo}, modes::tracked_static_ref, prelude::*, + thread_view::Objective, }; use super::AtomicDataWithOwner; @@ -27,6 +28,10 @@ pub tracked enum OnceState { Init(&'static PointsTo>), } +unsafe impl Objective for OnceState { + +} + /// A [`Predicate`] is something you're gonna preserve during the lifetime /// of any synchronization primitives like [`Once`]. pub trait Predicate { diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 1b4e440e5..09e161216 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -6,15 +6,12 @@ //! //! # Verification model //! -//! The executable RCU API is being rebuilt around an explicit weak-memory -//! history model. The atomic root pointer is a trusted executable wrapper around -//! Rust atomics, while proofs only rely on the specification in -//! [`specs::sync::weak_memory`]. Each RCU root pointer is represented by a -//! `WeakAtomicPtr` whose history records the messages that may be observed by -//! relaxed/acquire loads and CAS operations. Weak atomic operations borrow the -//! unique `ThreadView` from the current task's `RunningTaskContext`; RCU -//! never mints a fresh view and therefore preserves observations across RCU -//! operations and release publication. +//! The executable RCU API is built on Verus' native IRC11 weak-memory model. +//! The atomic root pointer is a trusted executable `AtomicPtr` adapter, while +//! proofs use native `AtomicPointsTo`, `AtomicHistory`, and operation relations. +//! Weak atomic operations borrow the unique native `ViewSeen` from the current +//! task's `RunningTaskContext`; RCU never mints a fresh per-operation view and +//! therefore preserves observations across RCU operations and publication. //! //! The root-pointer invariant keeps publication metadata for the complete //! atomic history. Each non-null message has a domain-local allocation ID, so @@ -140,15 +137,13 @@ use vstd_extra::rcu_read_pool::RcuReadLease; use crate::{ specs::{ - sync::{ - rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, - weak_memory::{RcuWeakAtomicPtr, ThreadView}, - }, + sync::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, weak_memory::RcuWeakAtomicPtr}, task::InAtomicMode, }, sync::Once, task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, }; +use vstd_extra::atomic_irc11::{ThreadViewOrder, ViewSeen}; use non_null::{NonNullPtr, NonNullPtrRef}; @@ -196,11 +191,6 @@ impl rcu_spec::RcuRootOwnershipPredicate< /// non-null history messages. Its publication registry also assigns every /// non-null message a domain-local allocation identity, matching the paper's /// distinction between physical addresses and allocation IDs. -type RcuAtomicGhost

= rcu_spec::RcuRootOwnedGhost< -

::Target, -

::Permission, ->; - type RcuAtomicPtr

= RcuWeakAtomicPtr<

::Target,

::Permission, @@ -339,20 +329,7 @@ impl RcuInner

{ res.type_inv(), res.is_nullable(), { - proof_decl! { - let tracked root_ghost: RcuAtomicGhost

= - rcu_spec::RcuRootOwnedGhost::tracked_initial( - core::ptr::null_mut::<

::Target>(), - None, - ); - let ghost key = rcu_spec::RcuRootKey { - nullable: true, - domain: root_ghost.domain(), - reader_registry: root_ghost.reader_registry(), - retire_observation_registry: root_ghost.retire_observation_registry(), - }; - } - let ptr = RcuAtomicPtr::

::new(Ghost(key), core::ptr::null_mut(), Tracked(root_ghost)); + let ptr = RcuAtomicPtr::

::new(Ghost(true), core::ptr::null_mut(), Tracked(None)); Self { ptr, ghost_nullable: Ghost(true), @@ -374,17 +351,7 @@ impl RcuInner

{ proof { assert(!raw_ptr.is_null()); } - proof_decl! { - let tracked root_ghost = - rcu_spec::RcuRootOwnedGhost::tracked_initial(raw_ptr, Some(perm)); - let ghost key = rcu_spec::RcuRootKey { - nullable, - domain: root_ghost.domain(), - reader_registry: root_ghost.reader_registry(), - retire_observation_registry: root_ghost.retire_observation_registry(), - }; - } - let ptr = RcuAtomicPtr::

::new(Ghost(key), raw_ptr, Tracked(root_ghost)); + let ptr = RcuAtomicPtr::

::new(Ghost(nullable), raw_ptr, Tracked(Some(perm))); Self { ptr, ghost_nullable: Ghost(nullable), @@ -393,7 +360,7 @@ impl RcuInner

{ } #[inline(always)] - fn load_ptr_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( + fn load_ptr_acquire(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( *mut

::Target, Tracked::Target>>>, )) @@ -430,7 +397,7 @@ impl RcuInner

{ Ghost(reader): Ghost, Tracked(cpu_reader): Tracked, Tracked(binding): Tracked, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( *mut

::Target, Tracked::Target>>>, @@ -495,7 +462,7 @@ impl RcuInner

{ &self, new_ptr: *mut

::Target, Tracked(ownership): Tracked::Permission>>, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( *mut

::Target, Tracked< @@ -567,7 +534,7 @@ impl RcuInner

{ } let (old_raw, Tracked(detached)) = { proof_decl! { - let tracked tv = session.tracked_borrow_thread_view_mut(); + let tracked tv = session.tracked_borrow_irc11_view_mut(); } self.swap_ptr_release(raw, Tracked(perm), Tracked(tv)) }; @@ -612,11 +579,11 @@ impl RcuInner

{ assert(session.rcu_participant_view() == context_before_disable.rcu_participant_view()); assert(context_before_reader.wf()); assert(context_before_reader.rcu_participant_view().spec_le( - context_before_reader.view(), + context_before_reader.irc11_view(), )); assert(cpu_reader.participant_view() == context_before_reader.rcu_participant_view()); assert(session.view() == context_before_reader.view()); - assert(cpu_reader.participant_view().spec_le(session.view())); + assert(cpu_reader.participant_view().spec_le(session.irc11_view())); } let ghost reader = rcu_spec::RcuReaderContext { scheduler: session.scheduler(), @@ -627,7 +594,7 @@ impl RcuInner

{ }; let ghost context_before_load = *session; proof_decl! { - let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( + let tracked tv = DisabledPreemptGuard::tracked_borrow_irc11_view_mut_from_context( session, &inner_guard, ); @@ -824,40 +791,19 @@ impl RcuInner

{ final(session).preempt_depth() == old(session).preempt_depth(), { proof_decl! { - let tracked tv = session.tracked_borrow_thread_view_mut(); + let tracked tv = session.tracked_borrow_irc11_view_mut(); } - let obj_ptr = #[verus_spec(with => Tracked(tracked_ref_perm))] - self.load_read_token(); + let (obj_ptr, _tracked_info) = self.load_ptr_acquire(Tracked(tv)); if obj_ptr.is_null() { return None; } - proof_decl! { - // `read_with` returns only the reference and has no guard object to - // store the read token. For this temporary skeleton, leak the - // verification-only token so the returned ref can borrow it for - // `'a`. The final RCU proof should attach this token to the - // atomic-mode/CPU epoch state instead. - let tracked tracked_ref_perm = tracked_ref_perm.tracked_unwrap(); - let tracked tracked_ref_perm = tracked_static_ref(tracked_ref_perm); - let tracked tracked_ref_perm:

>::RefPermission = - P::borrow_perm_as_ref_perm(tracked_ref_perm.tracked_borrow()); - } // SAFETY: // 1. This pointer is not NULL. // 2. The `_guard` guarantees atomic mode for the duration of lifetime // `'a`, the pointer is valid because other writers won't release the // allocation until this task passes the quiescent state. - NonNull::new(obj_ptr).map( - |ptr| - requires - P::ptr_perm_match( - ptr.view_ptr_mut(), - P::ref_perm_view_permission(tracked_ref_perm), - ), - { - unsafe { P::raw_as_ref(ptr, Tracked(tracked_ref_perm)) } - }, - ) + + NonNull::new(obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }) } } @@ -878,27 +824,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { } } - // SAFETY: The guard ensures that `P` will not be dropped. Thus, `P` - // outlives the lifetime of `&self`. Additionally, during this period, - // it is impossible to create a mutable reference to `P`. - NonNull::new(self.obj_ptr).map( - |ptr| - requires - self.tracked_ref_perm@ is Some, - P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_ref_perm->0.resource()), - { - unsafe { - P::raw_as_ref( - ptr, - Tracked( - P::borrow_perm_as_ref_perm( - self.tracked_ref_perm.tracked_borrow().tracked_borrow(), - ), - ), - ) - } - }, - ) + res } fn compare_exchange(self, new_ptr: Option

) -> (res: Result<(), Option

>) @@ -945,7 +871,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { let cas_res = { proof_decl! { - let tracked tv = DisabledPreemptGuard::tracked_borrow_thread_view_mut_from_context( + let tracked tv = DisabledPreemptGuard::tracked_borrow_irc11_view_mut_from_context( session, &this._inner_guard, ); diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 2011fcd5c..604d2f2a2 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -11,16 +11,21 @@ use crate::specs::{ rcu as rcu_spec, rcu::{GracePeriodView, MonitorStateView}, rcu_cpu as rcu_cpu_spec, - weak_memory::{History, RcuMonitorWeakAtomicBool, ThreadView, WmView}, + weak_memory::RcuMonitorWeakAtomicBool, }, }; use crate::sync::{ AtomicDataWithOwner, LocalIrqDisabled, SpinLock, once::Predicate as OncePredicate, }; use crate::task::RunningTaskContext; +use vstd_extra::atomic_irc11::{ + AtomicHistory as Irc11History, ThreadView, ThreadViewOrder, ThreadViewToken, ViewSeen, +}; verus! { +broadcast use vstd::thread_view::group_thread_view_axioms; + pub type Callbacks = VecDeque; type MonitorAtomicBool = RcuMonitorWeakAtomicBool; @@ -38,7 +43,7 @@ tracked struct RcuQuiescentContext { ghost session: Loc, ghost participant: Loc, ghost generation: nat, - ghost view: WmView, + ghost view: ThreadView, closed: rcu_cpu_spec::CpuRcuClosedGeneration, } @@ -61,7 +66,7 @@ impl RcuQuiescentContext { old(context).wf(), old(context).is_quiescent(), cpu == old(context).cpu(), - retired_facts.observed_by(old(context).view()), + retired_facts.observed_by(old(context).irc11_view()), ensures res.cpu == cpu, res.task == old(context).task(), @@ -69,7 +74,7 @@ impl RcuQuiescentContext { res.session == old(context).session_id(), res.participant == old(context).rcu_participant_id(), res.generation == old(context).rcu_generation(), - res.view == old(context).view(), + res.view == old(context).irc11_view(), res.closed.wf(), res.closed.scheduler() == res.scheduler, res.closed.participant_id() == res.participant, @@ -90,12 +95,13 @@ impl RcuQuiescentContext { final(context).rcu_participant_view() == res.view, final(context).rcu_fraction() == 1real, final(context).view() == old(context).view(), + final(context).irc11_view() == old(context).irc11_view(), { let ghost task = context.task(); let ghost scheduler = context.scheduler(); let ghost session = context.session_id(); let ghost participant = context.rcu_participant_id(); - let ghost view = context.view(); + let ghost view = context.irc11_view(); let ghost generation = context.rcu_generation(); let tracked closed = context.tracked_report_rcu_quiescent_with(retired_facts); let ghost _session_generation = context.tracked_record_quiescent(); @@ -113,7 +119,7 @@ ghost struct RcuCpuQuiescentReport { participant: Loc, /// Last reader generation closed by this quiescent transition. generation: nat, - view: WmView, + view: ThreadView, epoch: nat, } @@ -208,7 +214,7 @@ impl RcuCallback { raw: RawCallback, Tracked(cert): Tracked, Ghost(retire_epoch): Ghost, - Ghost(retire_view): Ghost, + Ghost(retire_view): Ghost, Ghost(scheduler): Ghost, ) -> (res: Self) requires @@ -385,12 +391,7 @@ impl RcuReclaimPermit { assert(guard.known_retired().contains(callback.retired_record())); assert(callback.removal.observed_by(self.reports@[cpu].view)); assert(self.reports@[cpu].view.spec_le(guard.start_view())); - assert(callback.removal.timestamp <= self.reports@[cpu].view.seen_at( - callback.removal.root, - )); - assert(self.reports@[cpu].view.seen_at(callback.removal.root) <= guard.start_view().seen_at( - callback.removal.root, - )); + self.reports@[cpu].view.lemma_spec_le_transitive(guard.start_view(), guard.start_view()); guard } @@ -949,7 +950,7 @@ pub(super) struct State { /// This proof-only token is updated before unlocking and imported after /// locking. It gives the existing executable spin lock the release/acquire /// semantics needed by the RCU proof without changing its runtime layout. - tracked_lock_view: Tracked, + tracked_lock_view: Tracked, } impl View for State { @@ -964,7 +965,7 @@ impl View for State { } impl State { - closed spec fn lock_view(self) -> WmView { + closed spec fn lock_view(self) -> ThreadView { self.tracked_lock_view@@ } @@ -973,7 +974,7 @@ impl State { } /// Imports the view published by the previous monitor-lock holder. - fn tracked_acquire_lock_view(&self, Tracked(thread_view): Tracked<&mut ThreadView>) + fn tracked_acquire_lock_view(&self, Tracked(thread_view): Tracked<&mut ViewSeen>) ensures self.wf(), final(thread_view)@ == old(thread_view)@.join(self.lock_view()), @@ -989,14 +990,14 @@ impl State { proof { let ghost before = thread_view@; let ghost lock_view = self.lock_view(); - thread_view.tracked_join(published_view); + published_view.tracked_join_into_view_seen(thread_view); before.lemma_join_left(lock_view); before.lemma_join_right(lock_view); } } /// Publishes the current holder's observations to the next lock acquirer. - fn tracked_publish_lock_view(&mut self, Tracked(thread_view): Tracked<&ThreadView>) + fn tracked_publish_lock_view(&mut self, Tracked(thread_view): Tracked<&ViewSeen>) requires old(self).wf(), ensures @@ -1009,7 +1010,35 @@ impl State { proof { let ghost old_lock_view = old(self).lock_view(); let ghost holder_view = thread_view@; - self.tracked_lock_view.borrow_mut().tracked_join(thread_view); + let tracked joined = self.tracked_lock_view.borrow().tracked_joined_view_seen( + thread_view, + ); + assert(old_lock_view.spec_le(joined@)); + assert(holder_view.spec_le(joined@)); + assert forall|i: int| 0 <= i < self.current_gp.callback_summaries().len() implies ( + #[trigger] self.current_gp.callback_summaries()[i]).retire_view.spec_le(joined@) by { + self.current_gp.callback_summaries()[i].retire_view.lemma_spec_le_transitive( + old_lock_view, + joined@, + ); + }; + assert forall|i: int| 0 <= i < callback_summaries(self.next_callbacks).len() implies ( + #[trigger] callback_summaries(self.next_callbacks)[i]).retire_view.spec_le(joined@) by { + callback_summaries(self.next_callbacks)[i].retire_view.lemma_spec_le_transitive( + old_lock_view, + joined@, + ); + }; + assert(self.tracked_retired_facts@.observed_by(joined@)) by { + assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] + self.tracked_retired_facts@.records().contains( + record, + ) implies record.removal.observed_by(joined@) by { + assert(record.removal.observed_by(old_lock_view)); + old_lock_view.lemma_spec_le_transitive(joined@, joined@); + }; + }; + self.tracked_lock_view = Tracked(joined); old_lock_view.lemma_join_left(holder_view); old_lock_view.lemma_join_right(holder_view); assert forall|i: int| @@ -1043,8 +1072,10 @@ impl State { record, ) implies record.removal.observed_by(final(self).lock_view()) by { assert(record.removal.observed_by(old_lock_view)); - assert(old_lock_view.seen_at(record.removal.root) - <= final(self).lock_view().seen_at(record.removal.root)); + old_lock_view.lemma_spec_le_transitive( + final(self).lock_view(), + final(self).lock_view(), + ); }; }; } @@ -1060,7 +1091,7 @@ impl State { let current_gp = GracePeriod::new(); let next_callbacks = Callbacks::new(); proof_decl! { - let tracked lock_view = ThreadView::new(); + let tracked lock_view = ThreadViewToken::new(); let tracked retired_facts = rcu_spec::RcuRetiredFacts::empty(); } let res = Self { @@ -1193,10 +1224,10 @@ impl State { old(context).wf(), old(context).is_quiescent(), cpu == old(context).cpu(), - self.tracked_retired_facts@.observed_by(old(context).view()), + self.tracked_retired_facts@.observed_by(old(context).irc11_view()), ensures res@.cpu == cpu, - res@.view == old(context).view(), + res@.view == old(context).irc11_view(), res@.wf(), self.tracked_retired_facts@.records().subset_of(res@.closed.known_retired()), final(context).wf(), @@ -1209,6 +1240,7 @@ impl State { final(context).rcu_generation() == old(context).rcu_generation() + 1, final(context).rcu_fraction() == 1real, final(context).view() == old(context).view(), + final(context).irc11_view() == old(context).irc11_view(), no_unwind { proof_decl! { @@ -1494,7 +1526,7 @@ pub open spec fn monitor_flag_matches_state(flag: bool, state: MonitorStateView) /// View-level form of the monitor flag write obligation. Executable monitor /// code can call this while holding a guard by passing the protected state's /// view, without moving the `State` value out of the lock. -proof fn monitor_flag_view_push_obligation(flag: bool, state: MonitorStateView) +proof fn monitor_flag_view_store_obligation(flag: bool, state: MonitorStateView) requires state.wf(), state.has_pending_work() ==> flag, @@ -1509,17 +1541,17 @@ proof fn monitor_flag_view_push_obligation(flag: bool, state: MonitorStateView) /// it: the weak-memory history invariant implies the per-message relation /// above, for stale messages as well as the latest one. proof fn monitor_flag_message_matches_state( - history: History, + history: Irc11History, flag_ghost: rcu_spec::RcuMonitorFlagGhost, ts: nat, ) requires rcu_spec::rcu_monitor_flag_history_inv(history, flag_ghost), - ts < history.len(), + history.contains_timestamp(ts), ensures - monitor_flag_matches_state(history[ts as int].value, flag_ghost.states[ts as int]), + monitor_flag_matches_state(history.value(ts), flag_ghost.states[ts]), { - if !history[ts as int].value { + if !history.value(ts) { rcu_spec::rcu_monitor_flag_false_has_no_pending(history, flag_ghost, ts); } } @@ -1531,16 +1563,16 @@ proof fn monitor_flag_message_matches_state( /// every flag store happens under the monitor lock and records the /// lock-protected state as its snapshot. proof fn monitor_flag_false_certifies_no_pending( - history: History, + history: Irc11History, flag_ghost: rcu_spec::RcuMonitorFlagGhost, ts: nat, state: State, ) requires rcu_spec::rcu_monitor_flag_history_inv(history, flag_ghost), - ts < history.len(), - !history[ts as int].value, - flag_ghost.states[ts as int] == state@, + history.contains_timestamp(ts), + !history.value(ts), + flag_ghost.states[ts] == state@, ensures state.no_pending_work(), state.pending_summaries() == Seq::::empty(), @@ -1550,9 +1582,9 @@ proof fn monitor_flag_false_certifies_no_pending( /// Bridge for the future `set_monitoring` helper: while holding the monitor /// lock with a well-formed state, writing any flag value that over-approximates -/// the state's pending work discharges the push obligation of -/// [`rcu_spec::preserve_rcu_monitor_flag_inv_on_push`]. -proof fn monitor_flag_push_obligation(flag: bool, state: State) +/// the state's pending work discharges the insertion obligation of +/// [`rcu_spec::preserve_rcu_monitor_flag_inv_on_insert`]. +proof fn monitor_flag_store_obligation(flag: bool, state: State) requires state.wf(), state.has_pending_work() ==> flag, @@ -1561,7 +1593,7 @@ proof fn monitor_flag_push_obligation(flag: bool, state: State) !flag ==> state@.no_pending_work(), monitor_flag_matches_state(flag, state@), { - monitor_flag_view_push_obligation(flag, state@); + monitor_flag_view_store_obligation(flag, state@); } /// A RCU monitor ensures the completion of _grace periods_ by keeping track @@ -1578,13 +1610,7 @@ impl RcuMonitor { /// pending monitor work. pub(super) fn new() -> (res: Self) { let state = State::new(); - proof { - rcu_spec::rcu_monitor_flag_initial_inv(); - } - proof_decl! { - let tracked flag_ghost = rcu_spec::RcuMonitorFlagGhost::tracked_initial(); - } - let is_monitoring = MonitorAtomicBool::new(Ghost(()), false, Tracked(flag_ghost)); + let is_monitoring = MonitorAtomicBool::new(); let state = SpinLock::new(state); proof { use_type_invariant(&is_monitoring); @@ -1602,7 +1628,7 @@ impl RcuMonitor { &self, value: bool, Ghost(state): Ghost, - Tracked(tv): Tracked<&mut ThreadView>, + Tracked(tv): Tracked<&mut ViewSeen>, ) requires self.wf(), @@ -1613,7 +1639,7 @@ impl RcuMonitor { { proof { use_type_invariant(self); - monitor_flag_view_push_obligation(value, state); + monitor_flag_view_store_obligation(value, state); } self.is_monitoring.store_relaxed_rcu_monitor(value, Ghost(state), Tracked(tv)); } @@ -1629,7 +1655,7 @@ impl RcuMonitor { Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), - cert@.removal().observed_by(old(session).view()), + cert@.removal().observed_by(old(session).irc11_view()), ensures final(session).wf(), final(session).task() == old(session).task(), @@ -1652,22 +1678,22 @@ impl RcuMonitor { proof { use_type_invariant(self); } - let ghost retire_view = session.view(); + let ghost retire_view = session.irc11_view(); let mut state = self.state.lock(); - let ghost before_acquire = session.view(); + let ghost before_acquire = session.irc11_view(); proof_decl! { - let tracked acquire_view = session.tracked_borrow_thread_view_mut(); + let tracked acquire_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_acquire_lock_view(Tracked(acquire_view)); proof { - retire_view.lemma_spec_le_transitive(before_acquire, session.view()); + retire_view.lemma_spec_le_transitive(before_acquire, session.irc11_view()); } proof_decl! { - let tracked publish_view = session.tracked_borrow_thread_view_mut(); + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_publish_lock_view(Tracked(&*publish_view)); proof { - retire_view.lemma_spec_le_transitive(session.view(), state.value().lock_view()); + retire_view.lemma_spec_le_transitive(session.irc11_view(), state.value().lock_view()); } let ghost retire_epoch = state.view()@.current_gp.epoch + 1; proof_decl! { @@ -1688,12 +1714,12 @@ impl RcuMonitor { use_type_invariant(self); } proof_decl! { - let tracked tv = session.tracked_borrow_thread_view_mut(); + let tracked tv = session.tracked_borrow_irc11_view_mut(); } self.set_monitoring(true, Ghost(state.view()@), Tracked(tv)); } proof_decl! { - let tracked publish_view = session.tracked_borrow_thread_view_mut(); + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); @@ -1729,7 +1755,7 @@ impl RcuMonitor { use_type_invariant(self); } proof_decl! { - let tracked fast_tv = session.tracked_borrow_thread_view_mut(); + let tracked fast_tv = session.tracked_borrow_irc11_view_mut(); } let is_monitoring = self.is_monitoring.load_relaxed(Tracked(fast_tv)).0; if !is_monitoring { @@ -1737,12 +1763,12 @@ impl RcuMonitor { } let mut state = self.state.lock(); proof_decl! { - let tracked acquire_view = session.tracked_borrow_thread_view_mut(); + let tracked acquire_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_acquire_lock_view(Tracked(acquire_view)); if state.current_gp.is_complete { proof_decl! { - let tracked publish_view = session.tracked_borrow_thread_view_mut(); + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); @@ -1750,14 +1776,16 @@ impl RcuMonitor { } let this_cpu = CpuId::current(Tracked(&*session)); proof { - assert(state.value().tracked_retired_facts@.observed_by(session.view())) by { + assert(state.value().tracked_retired_facts@.observed_by(session.irc11_view())) by { assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] state.value().tracked_retired_facts@.records().contains( record, - ) implies record.removal.observed_by(session.view()) by { + ) implies record.removal.observed_by(session.irc11_view()) by { assert(record.removal.observed_by(state.value().lock_view())); - assert(state.value().lock_view().seen_at(record.removal.root) - <= session.view().seen_at(record.removal.root)); + state.value().lock_view().lemma_spec_le_transitive( + session.irc11_view(), + session.irc11_view(), + ); }; }; } @@ -1774,7 +1802,7 @@ impl RcuMonitor { let ghost callback = state.value().current_gp.callback_summaries()[i]; callback.retire_view.lemma_spec_le_transitive( state.value().lock_view(), - session.view(), + session.irc11_view(), ); }; assert forall|i: int| @@ -1792,7 +1820,7 @@ impl RcuMonitor { ); if !completed_gp { proof_decl! { - let tracked publish_view = session.tracked_borrow_thread_view_mut(); + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); @@ -1806,12 +1834,12 @@ impl RcuMonitor { use_type_invariant(self); } proof_decl! { - let tracked tv = session.tracked_borrow_thread_view_mut(); + let tracked tv = session.tracked_borrow_irc11_view_mut(); } self.set_monitoring(false, Ghost(state.view()@), Tracked(tv)); } proof_decl! { - let tracked publish_view = session.tracked_borrow_thread_view_mut(); + let tracked publish_view = session.tracked_borrow_irc11_view_mut(); } state.tracked_publish_lock_view(Tracked(&*publish_view)); state.drop(); diff --git a/ostd/src/sync/rcu/non_null/mod.rs b/ostd/src/sync/rcu/non_null/mod.rs index 25ca3c959..096324a90 100644 --- a/ostd/src/sync/rcu/non_null/mod.rs +++ b/ostd/src/sync/rcu/non_null/mod.rs @@ -4,6 +4,7 @@ use alloc::{boxed::Box, sync::Arc}; use vstd::prelude::*; use vstd::raw_ptr::*; +use vstd::thread_view::Objective; use vstd_extra::prelude::*; mod either; @@ -41,7 +42,7 @@ pub unsafe trait NonNullPtr: Sized + 'static { where Self: 'a;*/ /// A verification-only permission type that represents the ownership of the memory managed by the pointer. - type Permission: Inv; + type Permission: Inv + Objective; /// The power of two of the pointer alignment. const ALIGN_BITS: u32; diff --git a/ostd/src/sync/rwlock.rs b/ostd/src/sync/rwlock.rs index 93b85d485..3ee86cebf 100644 --- a/ostd/src/sync/rwlock.rs +++ b/ostd/src/sync/rwlock.rs @@ -3,6 +3,7 @@ use vstd::atomic_ghost::*; use vstd::cell::{self, CellId, pcell::*}; use vstd::prelude::*; use vstd::resource::Loc; +use vstd::thread_view::Objective; use vstd_extra::resource::ghost_resource::{count::*, csum::*, excl::*, tokens::*}; use vstd_extra::sum::*; use vstd_extra::{prelude::*, resource}; @@ -71,6 +72,10 @@ tracked struct RwPerms { read_guard_token: CountResource, MAX_READER_U64>, } +unsafe impl Objective for RwPerms { + +} + ghost struct RwId { core_token_id: Loc, frac_id: Loc, diff --git a/ostd/src/sync/rwmutex.rs b/ostd/src/sync/rwmutex.rs index 5018ee08d..dca2f41d4 100644 --- a/ostd/src/sync/rwmutex.rs +++ b/ostd/src/sync/rwmutex.rs @@ -3,6 +3,7 @@ use vstd::atomic_ghost::*; use vstd::cell::{self, CellId, pcell::*}; use vstd::prelude::*; use vstd::resource::Loc; +use vstd::thread_view::Objective; use vstd_extra::resource::ghost_resource::{count::*, csum::*, excl::*, tokens::*}; use vstd_extra::sum::*; @@ -48,6 +49,10 @@ tracked struct RwPerms { read_guard_token: CountResource, MAX_READER_U64>, } +unsafe impl Objective for RwPerms { + +} + ghost struct RwId { core_token_id: Loc, frac_id: Loc, diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 9774b86c7..be67db469 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 +use vstd::thread_view::{ThreadView as Irc11ThreadView, ViewSeen}; use vstd::{prelude::*, resource::Loc}; +use vstd_extra::atomic_irc11::ThreadViewOrder; use vstd_extra::resource::ghost_resource::{count::CountGhost, tokens::CountGhostResource}; use crate::{ @@ -8,7 +10,6 @@ use crate::{ rcu_cpu::{ CpuRcuClosedGeneration, CpuRcuCoreBinding, CpuRcuParticipant, CpuRcuReaderFragment, }, - weak_memory::{ThreadView, WmView}, }, specs::task::cpu_core::{CpuCoreOwner, CpuCoreOwnerHandle, CpuCoreRegistration}, sync::GuardTransfer, /*, task::atomic_mode::InAtomicMode*/ @@ -17,6 +18,8 @@ use crate::{ verus! { +broadcast use vstd::thread_view::group_thread_view_axioms; + pub const PREEMPT_SESSION_FRACTIONS: u64 = 1 << 31; /// Proof token carried by a nested preemption-disable guard. @@ -134,6 +137,7 @@ impl PreemptThreadViewSession { res.scheduler() == task_view.scheduler(), res.task() == task_view.task(), res.view() == task_view.view(), + res.irc11_view() == task_view.irc11_view(), res.session_task() == task_view.task(), res.quiescent_generation() == 0, res.available_fractions() == PREEMPT_SESSION_FRACTIONS, @@ -166,10 +170,14 @@ impl PreemptThreadViewSession { self.task_view.scheduler() } - pub closed spec fn view(self) -> WmView { + pub closed spec fn view(self) -> Irc11ThreadView { self.task_view.view() } + pub open spec fn irc11_view(self) -> Irc11ThreadView { + self.view() + } + pub closed spec fn session_id(self) -> Loc { self.tokens.id() } @@ -221,6 +229,7 @@ impl PreemptThreadViewSession { final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), final(self).quiescent_generation() == old(self).quiescent_generation(), @@ -242,6 +251,7 @@ impl PreemptThreadViewSession { final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).session_task() == old(self).session_task(), final(self).quiescent_generation() == old(self).quiescent_generation(), @@ -268,7 +278,7 @@ impl PreemptThreadViewSession { /// After the borrow mutates the view, the caller must update the scheduler /// snapshot with `SchedulerView::update_checked_out_task_view` before /// relying on `wf` again. - pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ViewSeen) ensures (*tv)@ == old(self).view(), final(self).task() == old(self).task(), @@ -280,8 +290,60 @@ impl PreemptThreadViewSession { final(self).has_full_authority() == old(self).has_full_authority(), final(self).wf_session_resource() == old(self).wf_session_resource(), final(self).view() == (*final(tv))@, + final(self).irc11_view() == (*final(tv))@, { - self.task_view.tracked_borrow_thread_view_mut() + let tracked token = self.task_view.tracked_borrow_thread_view_mut(); + token.tracked_borrow_mut() + } + + /// Borrows the task view while preserving an existing lower bound whenever + /// the atomic operation grows the native view. + proof fn tracked_borrow_thread_view_mut_above( + tracked &mut self, + lower: Irc11ThreadView, + ) -> (tracked tv: &mut ViewSeen) + requires + lower.spec_le(old(self).view()), + ensures + (*tv)@ == old(self).view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).wf_session_resource() == old(self).wf_session_resource(), + final(self).view() == (*final(tv))@, + final(self).irc11_view() == (*final(tv))@, + old(self).view().spec_le((*final(tv))@) ==> lower.spec_le((*final(tv))@), + { + let ghost old_view = self.view(); + let tracked token = self.task_view.tracked_borrow_thread_view_mut(); + let tracked tv = token.tracked_borrow_mut(); + if old_view.spec_le((*final(tv))@) { + lower.lemma_spec_le_transitive(old_view, (*final(tv))@); + } + tv + } + + /// Borrows the native subjective view for an IRC11 atomic operation. + pub proof fn tracked_borrow_irc11_view_mut(tracked &mut self) -> (tracked view: &mut ViewSeen) + ensures + (*view)@ == old(self).irc11_view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == (*final(view))@, + final(self).session_id() == old(self).session_id(), + final(self).session_task() == old(self).session_task(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).wf_session_resource() == old(self).wf_session_resource(), + final(self).irc11_view() == (*final(view))@, + { + let tracked token = self.task_view.tracked_borrow_irc11_view_mut(); + token.tracked_borrow_mut() } /// Advances the session's quiescent boundary. @@ -300,6 +362,7 @@ impl PreemptThreadViewSession { final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions(), final(self).wf_session_resource(), @@ -330,6 +393,7 @@ impl PreemptThreadViewSession { res.scheduler() == self.scheduler(), res.task() == self.task(), res.view() == self.view(), + res.irc11_view() == self.irc11_view(), { self.task_view } @@ -346,6 +410,7 @@ impl PreemptThreadViewSession { res.scheduler() == self.scheduler(), res.task() == self.task(), res.view() == self.view(), + res.irc11_view() == self.irc11_view(), res.wf(sched_view), { self.task_view @@ -387,7 +452,7 @@ impl RunningTaskContext { rcu_participant.wf(), rcu_participant.cpu() == cpu, rcu_participant.fraction() == 1real, - rcu_participant.view().spec_le(task_view.view()), + rcu_participant.view().spec_le(task_view.irc11_view()), rcu_binding.registry() == task_view.scheduler(), rcu_binding.cpu() == cpu, rcu_binding.owner_id() == core_handle.id(), @@ -404,6 +469,7 @@ impl RunningTaskContext { res.scheduler() == task_view.scheduler(), res.task() == task_view.task(), res.view() == task_view.view(), + res.irc11_view() == task_view.irc11_view(), res.cpu() == cpu, res.core_owner_id() == core_handle.id(), res.preempt_depth() == 0, @@ -444,10 +510,14 @@ impl RunningTaskContext { self.session.scheduler() } - pub closed spec fn view(self) -> WmView { + pub closed spec fn view(self) -> Irc11ThreadView { self.session.view() } + pub open spec fn irc11_view(self) -> Irc11ThreadView { + self.view() + } + pub closed spec fn cpu(self) -> crate::specs::mm::cpu::CpuId { self.cpu@ } @@ -484,7 +554,7 @@ impl RunningTaskContext { self.rcu_participant.generation() } - pub closed spec fn rcu_participant_view(self) -> WmView { + pub closed spec fn rcu_participant_view(self) -> Irc11ThreadView { self.rcu_participant.view() } @@ -498,6 +568,7 @@ impl RunningTaskContext { pub closed spec fn wf(self) -> bool { &&& self.session.wf_session_resource() + &&& self.view() == self.irc11_view() &&& self.available_fractions() + self.preempt_depth() == PREEMPT_SESSION_FRACTIONS &&& self.core_handle.wf() &&& self.core_handle.cpu() == self.cpu() @@ -510,7 +581,7 @@ impl RunningTaskContext { &&& self.rcu_binding().locals_key() == self.core_handle.expected_locals_key() &&& self.rcu_binding().single_local_id() == self.rcu_participant_id() &&& self.rcu_participant.cpu() == self.cpu() - &&& self.rcu_participant_view().spec_le(self.view()) + &&& self.rcu_participant_view().spec_le(self.irc11_view()) } /// The checked-out task view includes the persistent view of this CPU's @@ -519,7 +590,7 @@ impl RunningTaskContext { requires self.wf(), ensures - self.rcu_participant_view().spec_le(self.view()), + self.rcu_participant_view().spec_le(self.irc11_view()), { } @@ -574,7 +645,7 @@ impl RunningTaskContext { } /// Borrows the running task's persistent weak-memory view. - pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ViewSeen) requires old(self).wf(), ensures @@ -594,7 +665,35 @@ impl RunningTaskContext { final(self).view() == (*final(tv))@, old(self).view().spec_le((*final(tv))@) ==> final(self).wf(), { - self.session.tracked_borrow_thread_view_mut() + let ghost participant_view = self.rcu_participant_view(); + assert(self.irc11_view() == self.view()); + assert(participant_view.spec_le(self.view())); + self.session.tracked_borrow_thread_view_mut_above(participant_view) + } + + /// Borrows the running task's native IRC11 view. + pub proof fn tracked_borrow_irc11_view_mut(tracked &mut self) -> (tracked view: &mut ViewSeen) + requires + old(self).wf(), + ensures + (*view)@ == old(self).irc11_view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).cpu() == old(self).cpu(), + final(self).view() == (*final(view))@, + final(self).session_id() == old(self).session_id(), + final(self).quiescent_generation() == old(self).quiescent_generation(), + final(self).available_fractions() == old(self).available_fractions(), + final(self).has_full_authority() == old(self).has_full_authority(), + final(self).preempt_depth() == old(self).preempt_depth(), + final(self).rcu_participant_id() == old(self).rcu_participant_id(), + final(self).rcu_generation() == old(self).rcu_generation(), + final(self).rcu_participant_view() == old(self).rcu_participant_view(), + final(self).rcu_fraction() == old(self).rcu_fraction(), + final(self).irc11_view() == (*final(view))@, + old(self).irc11_view().spec_le((*final(view))@) ==> final(self).wf(), + { + self.session.tracked_borrow_irc11_view_mut() } /// Starts one RCU reader from this CPU's persistent participant. @@ -613,6 +712,7 @@ impl RunningTaskContext { final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions(), @@ -628,7 +728,7 @@ impl RunningTaskContext { reader.participant_view() == old(self).rcu_participant_view(), reader.fraction() == old(self).rcu_fraction() / 2real, { - self.rcu_participant.tracked_start_reader_in_place(self.view()) + self.rcu_participant.tracked_start_reader_in_place(self.irc11_view()) } /// Copies the persistent scheduler binding for a guard or quiescent report. @@ -686,24 +786,25 @@ impl RunningTaskContext { closed.participant_id() == old(self).rcu_participant_id(), closed.cpu() == old(self).cpu(), closed.closed_generation() == old(self).rcu_generation(), - closed.view() == old(self).view(), + closed.view() == old(self).irc11_view(), final(self).wf(), final(self).is_quiescent(), final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth(), final(self).rcu_participant_id() == old(self).rcu_participant_id(), final(self).rcu_generation() == old(self).rcu_generation() + 1, - final(self).rcu_participant_view() == old(self).view(), + final(self).rcu_participant_view() == old(self).irc11_view(), final(self).rcu_fraction() == 1real, { let tracked binding = self.rcu_binding.tracked_duplicate(); - self.rcu_participant.tracked_report_quiescent_in_place(binding, self.view()) + self.rcu_participant.tracked_report_quiescent_in_place(binding, self.irc11_view()) } /// Closes the current CPU generation while publishing retirement facts @@ -715,14 +816,14 @@ impl RunningTaskContext { requires old(self).wf(), old(self).is_quiescent(), - learned.observed_by(old(self).view()), + learned.observed_by(old(self).irc11_view()), ensures closed.wf(), closed.scheduler() == old(self).scheduler(), closed.participant_id() == old(self).rcu_participant_id(), closed.cpu() == old(self).cpu(), closed.closed_generation() == old(self).rcu_generation(), - closed.view() == old(self).view(), + closed.view() == old(self).irc11_view(), learned.records().subset_of(closed.known_retired()), final(self).wf(), final(self).is_quiescent(), @@ -730,17 +831,22 @@ impl RunningTaskContext { final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth(), final(self).rcu_participant_id() == old(self).rcu_participant_id(), final(self).rcu_generation() == old(self).rcu_generation() + 1, - final(self).rcu_participant_view() == old(self).view(), + final(self).rcu_participant_view() == old(self).irc11_view(), final(self).rcu_fraction() == 1real, { let tracked binding = self.rcu_binding.tracked_duplicate(); - self.rcu_participant.tracked_report_quiescent_with_in_place(binding, self.view(), learned) + self.rcu_participant.tracked_report_quiescent_with_in_place( + binding, + self.irc11_view(), + learned, + ) } /// Records one quiescent boundary for this running session. @@ -759,6 +865,7 @@ impl RunningTaskContext { final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).available_fractions() == old(self).available_fractions(), final(self).preempt_depth() == old(self).preempt_depth(), @@ -786,6 +893,7 @@ impl RunningTaskContext { res.0.scheduler() == self.scheduler(), res.0.task() == self.task(), res.0.view() == self.view(), + res.0.irc11_view() == self.irc11_view(), res.1.id() == self.core_owner_id(), res.1.cpu() == self.cpu(), res.1.current_task() == Some(self.task()), @@ -798,7 +906,7 @@ impl RunningTaskContext { res.1.locals().generation() == self.rcu_generation(), res.1.locals().view() == self.rcu_participant_view(), res.1.locals().fraction() == 1real, - res.1.locals().view().spec_le(res.0.view()), + res.1.locals().view().spec_le(res.0.irc11_view()), res.1.wf(), { assert(self.available_fractions() == PREEMPT_SESSION_FRACTIONS); @@ -835,6 +943,7 @@ impl RunningTaskContext { res.0.scheduler() == self.scheduler(), res.0.task() == self.task(), res.0.view() == self.view(), + res.0.irc11_view() == self.irc11_view(), res.0.wf(sched_view), res.1.id() == self.core_owner_id(), res.1.cpu() == self.cpu(), @@ -848,7 +957,7 @@ impl RunningTaskContext { res.1.locals().generation() == self.rcu_generation(), res.1.locals().view() == self.rcu_participant_view(), res.1.locals().fraction() == 1real, - res.1.locals().view().spec_le(res.0.view()), + res.1.locals().view().spec_le(res.0.irc11_view()), res.1.wf(), { assert(self.preempt_depth() == 0); @@ -961,6 +1070,7 @@ impl PreemptGuardResource { final(session).task() == old(session).task(), final(session).scheduler() == old(session).scheduler(), final(session).view() == old(session).view(), + final(session).irc11_view() == old(session).irc11_view(), final(session).session_id() == old(session).session_id(), final(session).quiescent_generation() == old(session).quiescent_generation(), final(session).available_fractions() == old(session).available_fractions() + 1, @@ -990,6 +1100,7 @@ impl RunningTaskContext { final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() + 1 == old(self).available_fractions(), @@ -1029,6 +1140,7 @@ impl RunningTaskContext { final(self).scheduler() == old(self).scheduler(), final(self).cpu() == old(self).cpu(), final(self).view() == old(self).view(), + final(self).irc11_view() == old(self).irc11_view(), final(self).session_id() == old(self).session_id(), final(self).quiescent_generation() == old(self).quiescent_generation(), final(self).available_fractions() == old(self).available_fractions() + 1, @@ -1181,7 +1293,7 @@ impl DisabledPreemptGuard { pub proof fn tracked_borrow_thread_view_mut_from_context<'context>( tracked context: &'context mut RunningTaskContext, guard: &DisabledPreemptGuard, - ) -> (tracked tv: &'context mut ThreadView) + ) -> (tracked tv: &'context mut ViewSeen) requires old(context).wf(), guard.matches_context(*old(context)), @@ -1206,6 +1318,38 @@ impl DisabledPreemptGuard { context.tracked_borrow_thread_view_mut() } + /// Borrows the current task's native IRC11 view while preemption is disabled. + pub proof fn tracked_borrow_irc11_view_mut_from_context<'context>( + tracked context: &'context mut RunningTaskContext, + guard: &DisabledPreemptGuard, + ) -> (tracked view: &'context mut ViewSeen) + requires + old(context).wf(), + guard.matches_context(*old(context)), + ensures + (*view)@ == old(context).irc11_view(), + final(context).task() == old(context).task(), + final(context).scheduler() == old(context).scheduler(), + final(context).cpu() == old(context).cpu(), + final(context).view() == (*final(view))@, + final(context).session_id() == old(context).session_id(), + final(context).quiescent_generation() == old(context).quiescent_generation(), + final(context).available_fractions() == old(context).available_fractions(), + final(context).has_full_authority() == old(context).has_full_authority(), + final(context).preempt_depth() == old(context).preempt_depth(), + final(context).rcu_participant_id() == old(context).rcu_participant_id(), + final(context).rcu_generation() == old(context).rcu_generation(), + final(context).rcu_participant_view() == old(context).rcu_participant_view(), + final(context).rcu_fraction() == old(context).rcu_fraction(), + final(context).irc11_view() == (*final(view))@, + old(context).irc11_view().spec_le((*final(view))@) ==> final(context).wf(), + old(context).irc11_view().spec_le((*final(view))@) ==> guard.matches_context( + *final(context), + ), + { + context.tracked_borrow_irc11_view_mut() + } + /// Returns this guard's fractional witness and decrements the modeled /// preemption depth. /// diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index e1f0e39ac..8f6a42964 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -65,15 +65,13 @@ //! as the task's stack and internal state may be corrupted by concurrent modifications. use vstd::resource::map::GhostMapAuth; use vstd::{map::Map, prelude::*, resource::Loc}; +use vstd_extra::atomic_irc11::{ThreadView as Irc11ThreadView, ThreadViewOrder, ThreadViewToken}; use super::{Task, preempt::RunningTaskContext}; use crate::{ specs::{ mm::cpu::CpuId, - sync::{ - rcu_cpu::CpuRcuParticipant, - weak_memory::{ThreadView, WmView}, - }, + sync::rcu_cpu::CpuRcuParticipant, task::cpu_core::{ CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreOwnerHandle, CpuCoreRegistration, }, @@ -83,6 +81,8 @@ use crate::{ verus! { +broadcast use vstd::thread_view::group_thread_view_axioms; + /// A task-like object that can be identified in scheduler ghost state. pub trait Schedulable { spec fn sched_id(&self) -> Loc; @@ -147,10 +147,10 @@ pub ghost struct SchedulerView { pub runqueues: Map>, pub current: Map>, pub state: Map, - pub task_views: Map, - pub stored_views: Map, - pub checked_out_views: Map, - pub cpu_views: Map, + pub task_views: Map, + pub stored_views: Map, + pub checked_out_views: Map, + pub cpu_views: Map, pub cpu_core_registrations: Map, pub stored_cpu_cores: Set, } @@ -186,7 +186,7 @@ impl SchedulerView { self.task_views.contains_key(task) } - pub open spec fn task_thread_view(self, task: Loc) -> WmView + pub open spec fn task_thread_view(self, task: Loc) -> Irc11ThreadView recommends self.task_has_thread_view(task), { @@ -205,7 +205,7 @@ impl SchedulerView { self.cpu_views.contains_key(cpu) } - pub open spec fn cpu_thread_view(self, cpu: CpuId) -> WmView + pub open spec fn cpu_thread_view(self, cpu: CpuId) -> Irc11ThreadView recommends self.cpu_has_thread_view(cpu), { @@ -304,8 +304,8 @@ impl SchedulerView { { SchedulerView { state: self.state.insert(task, TaskSchedState::New), - task_views: self.task_views.insert(task, WmView::empty()), - stored_views: self.stored_views.insert(task, WmView::empty()), + task_views: self.task_views.insert(task, Irc11ThreadView::empty()), + stored_views: self.stored_views.insert(task, Irc11ThreadView::empty()), ..self } } @@ -325,7 +325,7 @@ impl SchedulerView { registration.locals_key.len() == 1, { SchedulerView { - cpu_views: self.cpu_views.insert(cpu, WmView::empty()), + cpu_views: self.cpu_views.insert(cpu, Irc11ThreadView::empty()), cpu_core_registrations: self.cpu_core_registrations.insert(cpu, registration), stored_cpu_cores: self.stored_cpu_cores.insert(cpu), ..self @@ -345,7 +345,7 @@ impl SchedulerView { ensures self.register_cpu(cpu, registration).wf(), self.register_cpu(cpu, registration).cpu_has_thread_view(cpu), - self.register_cpu(cpu, registration).cpu_thread_view(cpu) == WmView::empty(), + self.register_cpu(cpu, registration).cpu_thread_view(cpu) == Irc11ThreadView::empty(), self.register_cpu(cpu, registration).cpu_core_registration(cpu) == registration, self.register_cpu(cpu, registration).cpu_rcu_participant_id(cpu) == registration.locals_key[0], @@ -361,7 +361,7 @@ impl SchedulerView { ensures self.register_task(task).wf(), self.register_task(task).state[task] is New, - self.register_task(task).task_thread_view(task) == WmView::empty(), + self.register_task(task).task_thread_view(task) == Irc11ThreadView::empty(), self.register_task(task).task_view_is_stored(task), !self.register_task(task).task_view_is_checked_out(task), { @@ -396,7 +396,11 @@ impl SchedulerView { /// Weak-memory operations mutate the linear `ThreadView` carried by the /// guard. This transition keeps the logical snapshot and checked-out /// partition synchronized with that updated view. - pub open spec fn update_checked_out_task_view(self, task: Loc, view: WmView) -> SchedulerView + pub open spec fn update_checked_out_task_view( + self, + task: Loc, + view: Irc11ThreadView, + ) -> SchedulerView recommends self.task_view_is_checked_out(task), { @@ -409,7 +413,11 @@ impl SchedulerView { /// Updating the view of the currently checked-out task preserves the /// scheduler ownership partition and all scheduling invariants. - pub proof fn lemma_update_checked_out_task_view_preserves_wf(self, task: Loc, view: WmView) + pub proof fn lemma_update_checked_out_task_view_preserves_wf( + self, + task: Loc, + view: Irc11ThreadView, + ) requires self.wf(), self.task_view_is_checked_out(task), @@ -427,7 +435,12 @@ impl SchedulerView { /// /// The caller must provide the same view that is recorded as checked out; /// this prevents check-in from overwriting the task with an unrelated view. - pub open spec fn checkin_task_view(self, cpu: CpuId, task: Loc, view: WmView) -> SchedulerView + pub open spec fn checkin_task_view( + self, + cpu: CpuId, + task: Loc, + view: Irc11ThreadView, + ) -> SchedulerView recommends self.task_view_is_checked_out(task), !self.task_view_is_stored(task), @@ -445,7 +458,12 @@ impl SchedulerView { } } - pub proof fn lemma_checkin_task_view_preserves_wf(self, cpu: CpuId, task: Loc, view: WmView) + pub proof fn lemma_checkin_task_view_preserves_wf( + self, + cpu: CpuId, + task: Loc, + view: Irc11ThreadView, + ) requires self.wf(), self.task_view_is_checked_out(task), @@ -462,7 +480,7 @@ impl SchedulerView { /// Publishes the outgoing task's observations into the persistent CPU /// view. A subsequent task scheduled on this CPU imports the result in /// `checkout_task_view`. - pub open spec fn publish_cpu_view(self, cpu: CpuId, view: WmView) -> SchedulerView + pub open spec fn publish_cpu_view(self, cpu: CpuId, view: Irc11ThreadView) -> SchedulerView recommends self.cpu_has_thread_view(cpu), { @@ -472,7 +490,7 @@ impl SchedulerView { } } - pub proof fn lemma_publish_cpu_view_preserves_wf(self, cpu: CpuId, view: WmView) + pub proof fn lemma_publish_cpu_view_preserves_wf(self, cpu: CpuId, view: Irc11ThreadView) requires self.wf(), self.cpu_has_thread_view(cpu), @@ -574,8 +592,8 @@ impl SchedulerView { /// entry after all preemption guards have been released. tracked struct SchedulerThreadViews { scheduler: Ghost, - views: Map, - cpu_views: Map, + views: Map, + cpu_views: Map, cpu_cores: Map>, core_bindings: Map>, } @@ -588,15 +606,20 @@ tracked struct SchedulerThreadViews { pub tracked struct TaskThreadView { scheduler: Ghost, task: Ghost, - thread_view: ThreadView, + thread_view: ThreadViewToken, } impl TaskThreadView { - proof fn new(scheduler: Loc, task: Loc, tracked thread_view: ThreadView) -> (tracked res: Self) + proof fn new( + scheduler: Loc, + task: Loc, + tracked thread_view: ThreadViewToken, + ) -> (tracked res: Self) ensures res.scheduler() == scheduler, res.task() == task, res.view() == thread_view@, + res.irc11_view() == thread_view@, { TaskThreadView { scheduler: Ghost(scheduler), task: Ghost(task), thread_view } } @@ -609,10 +632,15 @@ impl TaskThreadView { self.task@ } - pub closed spec fn view(self) -> WmView { + pub closed spec fn view(self) -> Irc11ThreadView { self.thread_view@ } + /// Transitional alias for callers already written against the native view. + pub open spec fn irc11_view(self) -> Irc11ThreadView { + self.view() + } + /// Connects the checked-out token to the scheduler view that owns it. /// /// The token's linear `ThreadView` must agree with both the checked-out @@ -646,12 +674,27 @@ impl TaskThreadView { /// After the borrow mutates the view, the caller must use /// `update_checked_out_task_view` on the scheduler view before relying on /// `TaskThreadView::wf` again. - pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: &mut ThreadView) + pub proof fn tracked_borrow_thread_view_mut(tracked &mut self) -> (tracked tv: + &mut ThreadViewToken) ensures (*tv)@ == old(self).view(), final(self).task() == old(self).task(), final(self).scheduler() == old(self).scheduler(), final(self).view() == (*final(tv))@, + final(self).irc11_view() == (*final(tv))@, + { + &mut self.thread_view + } + + /// Borrows the native IRC11 view carried by this task. + pub proof fn tracked_borrow_irc11_view_mut(tracked &mut self) -> (tracked view: + &mut ThreadViewToken) + ensures + (*view)@ == old(self).irc11_view(), + final(self).task() == old(self).task(), + final(self).scheduler() == old(self).scheduler(), + final(self).view() == (*final(view))@, + final(self).irc11_view() == (*final(view))@, { &mut self.thread_view } @@ -661,13 +704,13 @@ impl SchedulerThreadViews { proof fn empty(scheduler: Loc) -> (tracked res: Self) ensures res.scheduler() == scheduler, - res.view() == Map::::empty(), - res.cpu_view_map() == Map::::empty(), + res.view() == Map::::empty(), + res.cpu_view_map() == Map::::empty(), res.core_registration_map() == Map::::empty(), res.binding_registration_map() == Map::::empty(), { - let tracked views = Map::::tracked_empty(); - let tracked cpu_views = Map::::tracked_empty(); + let tracked views = Map::::tracked_empty(); + let tracked cpu_views = Map::::tracked_empty(); let tracked cpu_cores = Map::>::tracked_empty(); let tracked core_bindings = Map::< CpuId, @@ -686,11 +729,11 @@ impl SchedulerThreadViews { self.scheduler@ } - pub closed spec fn view(self) -> Map { + pub closed spec fn view(self) -> Map { Map::new(self.views.dom(), |task: Loc| self.views[task]@) } - pub closed spec fn cpu_view_map(self) -> Map { + pub closed spec fn cpu_view_map(self) -> Map { Map::new(self.cpu_views.dom(), |cpu: CpuId| self.cpu_views[cpu]@) } @@ -713,24 +756,38 @@ impl SchedulerThreadViews { self.views.contains_key(task) } - pub closed spec fn thread_view(self, task: Loc) -> WmView + pub closed spec fn thread_view(self, task: Loc) -> Irc11ThreadView recommends self.contains(task), { self.views[task]@ } + pub closed spec fn irc11_thread_view(self, task: Loc) -> Irc11ThreadView + recommends + self.contains(task), + { + self.thread_view(task) + } + pub closed spec fn contains_cpu(self, cpu: CpuId) -> bool { self.cpu_views.contains_key(cpu) } - pub closed spec fn cpu_thread_view(self, cpu: CpuId) -> WmView + pub closed spec fn cpu_thread_view(self, cpu: CpuId) -> Irc11ThreadView recommends self.contains_cpu(cpu), { self.cpu_views[cpu]@ } + pub closed spec fn irc11_cpu_thread_view(self, cpu: CpuId) -> Irc11ThreadView + recommends + self.contains_cpu(cpu), + { + self.cpu_thread_view(cpu) + } + closed spec fn cpu_cores_wf(self, sched_view: SchedulerView) -> bool { &&& self.cpu_cores.dom() == sched_view.stored_cpu_cores &&& forall|cpu: CpuId| #[trigger] @@ -802,7 +859,7 @@ impl SchedulerThreadViews { final(self).cpu_view_map() == sched_view.register_cpu(cpu, registration).cpu_views, final(self).wf(sched_view.register_cpu(cpu, registration)), final(self).contains_cpu(cpu), - final(self).cpu_thread_view(cpu) == WmView::empty(), + final(self).cpu_thread_view(cpu) == Irc11ThreadView::empty(), final(self).core_registration_map().contains_key(cpu), final(self).core_registration_map()[cpu] == registration, final(self).binding_registration_map() == sched_view.register_cpu( @@ -812,8 +869,8 @@ impl SchedulerThreadViews { final(identity).id() == old(identity).id(), final(identity)@ == sched_view.register_cpu(cpu, registration).cpu_core_registrations, { - let tracked cpu_view = ThreadView::new(); - let tracked participant = CpuRcuParticipant::new(cpu, WmView::empty()); + let tracked cpu_view = ThreadViewToken::new(); + let tracked participant = CpuRcuParticipant::new(cpu, Irc11ThreadView::empty()); participant.lemma_cpu_core_local_state(); let ghost participant_id = participant.id(); let tracked core = CpuCoreOwner::new(cpu, participant); @@ -874,10 +931,10 @@ impl SchedulerThreadViews { final(self).cpu_view_map() == old(self).cpu_view_map(), final(self).wf(sched_view.register_task(task)), final(self).contains(task), - final(self).thread_view(task) == WmView::empty(), + final(self).thread_view(task) == Irc11ThreadView::empty(), { sched_view.lemma_register_task_preserves_wf(task); - let tracked thread_view = ThreadView::new(); + let tracked thread_view = ThreadViewToken::new(); let tracked token = TaskThreadView::new(self.scheduler(), task, thread_view); self.lemma_cpu_cores_frame(sched_view, sched_view.register_task(task)); self.tracked_insert_initial_thread_view(token, sched_view.register_task(task)); @@ -923,7 +980,7 @@ impl SchedulerThreadViews { res.2.id() == sched_view.cpu_rcu_participant_id(cpu), res.2.cpu() == cpu, res.2.fraction() == 1real, - res.2.view().spec_le(res.0.view()), + res.2.view().spec_le(res.0.irc11_view()), res.2.wf(), res.3.registry() == sched_view.id, res.3.cpu() == cpu, @@ -947,8 +1004,14 @@ impl SchedulerThreadViews { core.tracked_schedule_in(task); assert(core.registration() == sched_view.cpu_core_registration(cpu)); let tracked (core_handle, participant) = core.tracked_open(); + let ghost participant_view = participant.view(); + let ghost task_irc11_view = thread_view@; + let ghost cpu_irc11_view = cpu_view@; let tracked binding = self.core_bindings.tracked_borrow(cpu).tracked_duplicate(); thread_view.tracked_join(cpu_view); + assert(participant_view.spec_le(cpu_irc11_view)); + task_irc11_view.lemma_join_right(cpu_irc11_view); + participant_view.lemma_spec_le_transitive(cpu_irc11_view, thread_view@); let tracked token = TaskThreadView { scheduler: Ghost(self.scheduler()), task: Ghost(task), @@ -1051,6 +1114,7 @@ impl SchedulerThreadViews { { let ghost task = context.task(); let ghost view = context.view(); + let ghost irc11_context_view = context.irc11_view(); let ghost cpu = context.cpu(); let ghost expected_registration = CpuCoreRegistration { owner_id: context.core_owner_id(), @@ -1094,9 +1158,9 @@ impl SchedulerThreadViews { checked.lemma_publish_cpu_view_preserves_wf(cpu, view); let ghost next = checked.publish_cpu_view(cpu, view); assert(next.wf()); - assert(participant_view.spec_le(view)); - old(self).cpu_thread_view(cpu).lemma_join_right(view); - participant_view.lemma_spec_le_transitive(view, self.cpu_views[cpu]@); + assert(participant_view.spec_le(irc11_context_view)); + old(self).cpu_thread_view(cpu).lemma_join_right(irc11_context_view); + participant_view.lemma_spec_le_transitive(irc11_context_view, self.cpu_views[cpu]@); assert(self.cpu_cores.dom() == next.stored_cpu_cores); assert forall|stored_cpu: CpuId| #[trigger] self.cpu_cores.contains_key(stored_cpu) implies { @@ -1108,7 +1172,9 @@ impl SchedulerThreadViews { == next.cpu_core_registrations[stored_cpu].locals_key &&& self.cpu_cores[stored_cpu].registration() == next.cpu_core_registrations[stored_cpu] &&& self.cpu_cores[stored_cpu].locals().fraction() == 1real - &&& self.cpu_cores[stored_cpu].locals().view().spec_le(self.cpu_views[stored_cpu]@) + &&& self.cpu_cores[stored_cpu].locals().view().spec_le( + self.cpu_views[stored_cpu]@, + ) } by { assert(next.cpu_core_registrations.contains_key(stored_cpu)); if stored_cpu == cpu { @@ -1227,7 +1293,7 @@ impl SchedulerGhostState { final(self).view().cpu_core_registration(cpu), ), final(self).view().cpu_has_thread_view(cpu), - final(self).view().cpu_thread_view(cpu) == WmView::empty(), + final(self).view().cpu_thread_view(cpu) == Irc11ThreadView::empty(), final(self).view().cpu_rcu_participant_is_stored(cpu), { let ghost old_view = self.view@; @@ -1251,7 +1317,7 @@ impl SchedulerGhostState { final(self).view() == old(self).view().register_task(task), final(self).view().state[task] is New, final(self).view().task_view_is_stored(task), - final(self).view().task_thread_view(task) == WmView::empty(), + final(self).view().task_thread_view(task) == Irc11ThreadView::empty(), { let ghost old_view = self.view@; self.thread_views.tracked_register_task(old_view, task); @@ -1360,7 +1426,7 @@ pub open spec fn can_enqueue(view: SchedulerView, task: Loc, flags: EnqueueFlags &&& view.state.contains_key(task) &&& view.state[task] is New &&& view.task_view_is_stored(task) - &&& view.task_thread_view(task) == WmView::empty() + &&& view.task_thread_view(task) == Irc11ThreadView::empty() }, EnqueueFlags::Wake => view.state.contains_key(task) && !(view.state[task] is Exited), } diff --git a/tools/patches/verus-irc11-vstd.patch b/tools/patches/verus-irc11-vstd.patch new file mode 100644 index 000000000..147cbe0d4 --- /dev/null +++ b/tools/patches/verus-irc11-vstd.patch @@ -0,0 +1,200 @@ +diff --git a/source/vstd/Cargo.toml b/source/vstd/Cargo.toml +index 1c106c81..ef05ea7e 100644 +--- a/source/vstd/Cargo.toml ++++ b/source/vstd/Cargo.toml +@@ -33,6 +33,7 @@ allocator = [] + strict_provenance_atomic_ptr = [] + allow_panic = [] # code is allowed to panic. + nonzero_internals = [] ++weak-memory = [] + + + [package.metadata.verus] +diff --git a/source/vstd/atomic.rs b/source/vstd/atomic.rs +index 8bb30533..14de2bd9 100644 +--- a/source/vstd/atomic.rs ++++ b/source/vstd/atomic.rs +@@ -5,10 +5,8 @@ use super::prelude::*; + use super::raw_ptr::PointsTo; + use super::view::*; + +-#[cfg(not(feature = "weak-memory"))] + pub use sc_atomic_types::*; + +-#[cfg(not(feature = "weak-memory"))] + mod sc_atomic_types { + + use core::sync::atomic::{ +@@ -22,6 +20,8 @@ mod sc_atomic_types { + use super::super::modes::*; + use super::super::pervasive::*; + use super::super::prelude::*; ++ use super::super::raw_ptr::PointsTo; ++ use super::super::thread_view::Objective; + use super::super::view::*; + use super::super::wrapping::*; + +@@ -73,6 +73,9 @@ mod sc_atomic_types { + unused: $value_ty, + } + ++ #[cfg(verus_keep_ghost)] ++ unsafe impl Objective for $p_ident {} ++ + pub ghost struct $p_data_ident { + pub patomic: int, + pub value: $value_ty, +@@ -122,6 +125,9 @@ mod sc_atomic_types { + unusued: $value_ty, + } + ++ #[cfg(verus_keep_ghost)] ++ unsafe impl Objective for $p_ident {} ++ + #[verifier::accept_recursive_types(T)] + pub ghost struct $p_data_ident { + pub patomic: int, +diff --git a/source/vstd/atomic_ghost.rs b/source/vstd/atomic_ghost.rs +index 0d6a3b8a..4b456bfb 100644 +--- a/source/vstd/atomic_ghost.rs ++++ b/source/vstd/atomic_ghost.rs +@@ -2,16 +2,15 @@ + //! See the [`atomic_with_ghost!`] documentation for more information. + #![allow(unused_imports)] + +-#[cfg(not(feature = "weak-memory"))] + pub use atomic_ghost::*; + +-#[cfg(not(feature = "weak-memory"))] + mod atomic_ghost { + + use super::super::atomic::*; + use super::super::invariant::*; + use super::super::modes::*; + use super::super::prelude::*; ++ use super::super::thread_view::Objective; + + verus! { + +@@ -26,7 +25,7 @@ pub trait AtomicInvariantPredicate { + + pub struct $atomic_pred_ty { p: Pred } + +- impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty ++ impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty + where Pred: AtomicInvariantPredicate + { + open spec fn inv(k_loc: (K, int), perm_g: ($perm_ty, G)) -> bool { +@@ -46,7 +45,7 @@ pub trait AtomicInvariantPredicate { + /// + /// See the [`atomic_with_ghost!`] documentation for usage information. + +- pub struct $at_ident ++ pub struct $at_ident + //where Pred: AtomicInvariantPredicate + { + #[doc(hidden)] +@@ -56,7 +55,7 @@ pub trait AtomicInvariantPredicate { + pub atomic_inv: Tracked>>, + } + +- impl $at_ident { ++ impl $at_ident { + pub open spec fn constant(&self) -> K { + self.atomic_inv@.constant().0 + } +@@ -66,7 +65,7 @@ pub trait AtomicInvariantPredicate { + } + } + +- impl $at_ident ++ impl $at_ident + where Pred: AtomicInvariantPredicate + { + #[inline(always)] +@@ -117,7 +116,7 @@ pub trait AtomicInvariantPredicate { + + pub struct $atomic_pred_ty { t: T, p: Pred } + +- impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty ++ impl InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty + where Pred: AtomicInvariantPredicate + { + open spec fn inv(k_loc: (K, int), perm_g: ($perm_ty, G)) -> bool { +@@ -137,7 +136,7 @@ pub trait AtomicInvariantPredicate { + /// + /// See the [`atomic_with_ghost!`] documentation for usage information. + +- pub struct $at_ident ++ pub struct $at_ident + //where Pred: AtomicInvariantPredicate + { + #[doc(hidden)] +@@ -147,7 +146,7 @@ pub trait AtomicInvariantPredicate { + pub atomic_inv: Tracked>>, + } + +- impl $at_ident ++ impl $at_ident + where Pred: AtomicInvariantPredicate + { + pub open spec fn well_formed(&self) -> bool { +diff --git a/source/vstd/thread_view.rs b/source/vstd/thread_view.rs +index bfb1c293..2a2ad51b 100644 +--- a/source/vstd/thread_view.rs ++++ b/source/vstd/thread_view.rs +@@ -223,6 +223,34 @@ unsafe impl Objective for algebra::Resource { + + } + ++// Memory ownership permissions are objective independently of the type stored ++// at the location. Their views describe global memory, not one thread's ++// subjective weak-memory observations. ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::cell::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::cell::pcell::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::cell::pcell_maybe_uninit::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::raw_ptr::PointsTo { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for super::simple_pptr::PointsTo { ++ ++} ++ + // primitive types are objective because they do not hold permissions + macro_rules! declare_primitive_is_objective { + ($($a:ty),*) => { +@@ -237,6 +265,19 @@ macro_rules! declare_primitive_is_objective { + + declare_primitive_is_objective!(bool, char, (), u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, int, nat, str); + ++// Verus does not currently derive the auto-trait bound for a tuple when one of ++// its elements is a type parameter. State the binary case explicitly since ++// invariants commonly package a permission and user ghost state as a pair. ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for (A, B) { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for Option { ++ ++} ++ + // note: the fact that tuples are Objective (above) suffices for OBJMOD-SEP + // OBJ with wand update + #[cfg(verus_keep_ghost)] diff --git a/verified_libs/vstd_extra/src/atomic_irc11.rs b/verified_libs/vstd_extra/src/atomic_irc11.rs new file mode 100644 index 000000000..0eaf68ba0 --- /dev/null +++ b/verified_libs/vstd_extra/src/atomic_irc11.rs @@ -0,0 +1,693 @@ +//! Adapters for Verus' native IRC11 weak-memory model. +//! +//! This module is intentionally thin. It exposes the native subjective thread +//! views, per-location histories, and points-to resources without translating +//! them into the older `atomic_weak` model. In particular, native histories are +//! finite maps with abstract natural-number timestamps rather than contiguous +//! sequences. +//! +//! Verus does not yet provide a weak-memory `AtomicPtr`. This module therefore +//! supplies only that executable wrapper, specified directly in terms of +//! [`AtomicPointsTo`] and the native load/store/update relations. +use core::sync::atomic::{AtomicPtr, Ordering}; + +pub use vstd::atomic_weak::{ + AtomicHistory, AtomicPointsTo, LoadData, PAtomicWeakBool, PAtomicWeakI8, PAtomicWeakI16, + PAtomicWeakI32, PAtomicWeakIsize, PAtomicWeakU8, PAtomicWeakU16, PAtomicWeakU32, + PAtomicWeakUsize, StoreData, UpdateData, fence_acquire, fence_release, + history_get_contains_timestamp, load_acquire, load_reads_from_history, load_relaxed, + load_timestamp_in_view, load_view_nondecreasing, store_insert_history, store_relaxed, + store_release, store_timestamp_in_view, store_view_increasing, +}; +#[cfg(target_has_atomic = "64")] +pub use vstd::atomic_weak::{PAtomicWeakI64, PAtomicWeakU64}; +pub use vstd::cell::CellId as AtomicId; +use vstd::prelude::*; +pub use vstd::thread_view::{ + AcquireViewSeen, Objective, ReleaseViewSeen, ThreadView, ViewAt, ViewSeen, +}; + +verus! { + +/// Logical timestamp used by one native atomic history. +pub type Timestamp = nat; + +/// Compatibility vocabulary for ordering native subjective views. +/// +/// `old.spec_le(new)` is only notation for the native relation +/// `new.contains(old)`; it introduces no second view model. +pub trait ThreadViewOrder { + spec fn spec_le(self, newer: Self) -> bool; + + spec fn view_join(self, other: Self) -> Self; + + proof fn lemma_spec_le_transitive(self, middle: Self, newer: Self) + requires + self.spec_le(middle), + middle.spec_le(newer), + ensures + self.spec_le(newer), + ; + + proof fn lemma_join_left(self, other: Self) + ensures + self.spec_le(self.view_join(other)), + ; + + proof fn lemma_join_right(self, other: Self) + ensures + other.spec_le(self.view_join(other)), + ; +} + +impl ThreadViewOrder for ThreadView { + open spec fn spec_le(self, newer: Self) -> bool { + newer.contains(self) + } + + open spec fn view_join(self, other: Self) -> Self { + self.join(other) + } + + proof fn lemma_spec_le_transitive(self, middle: Self, newer: Self) { + ThreadView::contains_trans(newer, middle, self); + } + + proof fn lemma_join_left(self, other: Self) { + ThreadView::join_contains(self, other); + } + + proof fn lemma_join_right(self, other: Self) { + ThreadView::join_comm(self, other); + ThreadView::join_contains(other, self); + } +} + +/// Scheduler-owned native subjective view. +/// +/// OSTD should create one token when it registers an execution participant, +/// move that token through schedule-in/schedule-out, and borrow it for native +/// weak atomic operations. The wrapper deliberately exposes no operation that +/// can manufacture an arbitrary non-empty view. +pub tracked struct ThreadViewToken { + view_seen: ViewSeen, +} + +impl View for ThreadViewToken { + type V = ThreadView; + + closed spec fn view(&self) -> ThreadView { + self.view_seen@ + } +} + +impl ThreadViewToken { + /// Creates the empty view used when registering a task or CPU. + pub proof fn new() -> (tracked res: Self) + ensures + res@ == ThreadView::empty(), + { + let tracked view_seen = ViewSeen::new(); + ThreadViewToken { view_seen } + } + + /// Wraps a native view returned by atomic or synchronization setup. + pub proof fn from_view_seen(tracked view_seen: ViewSeen) -> (tracked res: Self) + ensures + res@ == view_seen@, + { + ThreadViewToken { view_seen } + } + + /// Removes the scheduler wrapper without changing the represented view. + pub proof fn into_view_seen(tracked self) -> (tracked res: ViewSeen) + ensures + res@ == self@, + { + self.view_seen + } + + /// Borrows the native token for one atomic operation. + pub proof fn tracked_borrow_mut(tracked &mut self) -> (tracked res: &mut ViewSeen) + ensures + (*res)@ == old(self)@, + final(self)@ == (*final(res))@, + { + &mut self.view_seen + } + + /// Imports observations held by another execution participant. + /// + /// `ViewSeen` is persistent knowledge, so Verus' native model permits + /// copying it before the join. The source token consequently remains + /// available to its CPU or task owner. + pub proof fn tracked_join(tracked &mut self, tracked other: &Self) + ensures + final(self)@ == old(self)@.join(other@), + { + let tracked other_view = other.view_seen; + let tracked old_view = self.view_seen; + self.view_seen = old_view.join(other_view); + } + + /// Imports this stored lower bound into an executing thread's native view. + pub proof fn tracked_join_into_view_seen(tracked &self, tracked target: &mut ViewSeen) + ensures + final(target)@ == old(target)@.join(self@), + { + let tracked source = self.view_seen; + let tracked old_target = *target; + *target = old_target.join(source); + } + + /// Publishes an executing thread's current native view into this token. + pub proof fn tracked_join_view_seen(tracked &mut self, tracked source: &ViewSeen) + ensures + final(self)@ == old(self)@.join(source@), + { + let tracked source = *source; + let tracked old_view = self.view_seen; + self.view_seen = old_view.join(source); + } + + /// Returns a joined token without mutating the stored source token. + pub proof fn tracked_joined_view_seen(tracked &self, tracked source: &ViewSeen) -> (tracked res: + Self) + ensures + res@ == self@.join(source@), + { + let tracked source = *source; + let tracked stored = self.view_seen; + let tracked view_seen = stored.join(source); + ThreadViewToken { view_seen } + } +} + +/// IRC11 wrapper around Rust's sized-pointer atomic. +/// +/// The permission describes only the atomic location's modification history; +/// ownership of the pointee remains in the client invariant. +#[repr(transparent)] +#[verifier::accept_recursive_types(T)] +#[verifier::external_body] +pub struct PAtomicWeakPtr { + value: AtomicPtr, +} + +impl PAtomicWeakPtr { + pub uninterp spec fn loc(&self) -> AtomicId; + + #[inline(always)] + #[verifier::external_body] + pub const fn new(value: *mut T) -> ((atomic, points_to, view, timestamp): ( + Self, + Tracked>, + Tracked, + Ghost, + )) + ensures + atomic.loc() == points_to@.loc(), + points_to@.hist().is_singleton(timestamp@, (value, view@@)), + points_to@.get_timestamp(view@@) == Some(timestamp@), + { + ( + Self { value: AtomicPtr::new(value) }, + Tracked::assume_new(), + Tracked::assume_new(), + Ghost::assume_new(), + ) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn load( + &self, + order: Ordering, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(points_to): Tracked<&AtomicPointsTo<*mut T>>, + ) -> ((value, acquire_view, load): (*mut T, Tracked, Ghost)) + requires + self.loc() == points_to.loc(), + order matches Ordering::Acquire + || order matches Ordering::Relaxed, + ensures + match order { + Ordering::Acquire => load_acquire( + *points_to, + old(view)@, + final(view)@, + value, + load@.timestamp, + load@.message_view, + ), + Ordering::Relaxed => load_relaxed( + *points_to, + old(view)@, + final(view)@, + acquire_view@@, + value, + load@.timestamp, + load@.message_view, + ), + }, + opens_invariants none + no_unwind + { + (self.value.load(order), Tracked::assume_new(), Ghost::assume_new()) + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn store( + &self, + value: *mut T, + order: Ordering, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(release_view): Tracked, + Tracked(points_to): Tracked<&mut AtomicPointsTo<*mut T>>, + ) -> (store: Ghost) + requires + self.loc() == old(points_to).loc(), + order matches Ordering::Release + || order matches Ordering::Relaxed, + ensures + forall|observed_view: ThreadView| #[trigger] + old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( + observed_view, + ), + match order { + Ordering::Release => store_release( + *old(points_to), + *final(points_to), + old(view)@, + final(view)@, + value, + store@.timestamp, + store@.message_view, + ), + Ordering::Relaxed => store_relaxed( + *old(points_to), + *final(points_to), + old(view)@, + final(view)@, + release_view@, + value, + store@.timestamp, + store@.message_view, + ), + }, + opens_invariants none + no_unwind + { + self.value.store(value, order); + Ghost::assume_new() + } + + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn compare_exchange( + &self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(release_view): Tracked, + Tracked(points_to): Tracked<&mut AtomicPointsTo<*mut T>>, + ) -> ((result, acquire_view, update): ( + Result<*mut T, *mut T>, + Tracked, + Ghost, + )) + requires + self.loc() == old(points_to).loc(), + success matches Ordering::AcqRel + || success matches Ordering::Acquire + || success matches Ordering::Release + || success matches Ordering::Relaxed, + failure matches Ordering::Acquire + || failure matches Ordering::Relaxed, + ensures + result is Ok ==> old(points_to).hist().is_max_timestamp(update@.load_timestamp), + forall|observed_view: ThreadView| #[trigger] + old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( + observed_view, + ), + match result { + Ok(value) => { + &&& current.addr() == value.addr() + &&& update@.store_message_view.contains_strict(update@.load_message_view) + &&& match success { + Ordering::AcqRel => { + &&& load_acquire( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_release( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + Ordering::Acquire => { + &&& load_acquire( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_relaxed( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + release_view@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + Ordering::Release => { + &&& load_relaxed( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + acquire_view@@, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_release( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + Ordering::Relaxed => { + &&& load_relaxed( + *old(points_to), + old(view)@, + update@.intermediate_thread_view, + acquire_view@@, + value, + update@.load_timestamp, + update@.load_message_view, + ) + &&& store_relaxed( + *old(points_to), + *final(points_to), + update@.intermediate_thread_view, + final(view)@, + release_view@, + new, + update@.load_timestamp + 1, + update@.store_message_view, + ) + }, + } + }, + Err(value) => { + &&& current.addr() != value.addr() + &&& *final(points_to) == *old(points_to) + &&& match failure { + Ordering::Acquire => load_acquire( + *old(points_to), + old(view)@, + final(view)@, + value, + update@.load_timestamp, + update@.load_message_view, + ), + Ordering::Relaxed => load_relaxed( + *old(points_to), + old(view)@, + final(view)@, + acquire_view@@, + value, + update@.load_timestamp, + update@.load_message_view, + ), + } + }, + }, + opens_invariants none + no_unwind + { + ( + self.value.compare_exchange(current, new, success, failure), + Tracked::assume_new(), + Ghost::assume_new(), + ) + } + + /// Release swap reads the latest modification and immediately appends its + /// replacement in the same modification order. + #[inline(always)] + #[verifier::external_body] + #[verifier::atomic] + pub fn swap_release( + &self, + value: *mut T, + Tracked(view): Tracked<&mut ViewSeen>, + Tracked(points_to): Tracked<&mut AtomicPointsTo<*mut T>>, + ) -> ((old_value, acquire_view, swap): (*mut T, Tracked, Ghost)) + requires + self.loc() == old(points_to).loc(), + ensures + old(points_to).hist().is_max_timestamp(swap@.load_timestamp), + forall|observed_view: ThreadView| #[trigger] + old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( + observed_view, + ), + load_relaxed( + *old(points_to), + old(view)@, + swap@.intermediate_thread_view, + acquire_view@@, + old_value, + swap@.load_timestamp, + swap@.load_message_view, + ), + store_release( + *old(points_to), + *final(points_to), + swap@.intermediate_thread_view, + final(view)@, + value, + swap@.load_timestamp + 1, + swap@.store_message_view, + ), + opens_invariants none + no_unwind + { + (self.value.swap(value, Ordering::Release), Tracked::assume_new(), Ghost::assume_new()) + } +} + +/// The partial order used for weak-memory views, written from old to new. +pub open spec fn view_le(old_view: ThreadView, new_view: ThreadView) -> bool { + new_view.contains(old_view) +} + +/// View joins monotonically include both operands. +pub proof fn lemma_join_upper_bound(left: ThreadView, right: ThreadView) + ensures + view_le(left, left.join(right)), + view_le(right, left.join(right)), +{ + ThreadView::join_contains(left, right); + ThreadView::join_comm(left, right); + ThreadView::join_contains(right, left); +} + +// These executable examples are part of verification. They ensure that the +// native tokens can be threaded through the operation shapes needed by OSTD. +fn test_native_acquire_load() { + let (atomic, Tracked(pt), Tracked(mut view), Ghost(initial_ts)) = PAtomicWeakUsize::new(7); + let ghost old_view = view@; + assert(pt.hist().is_singleton(initial_ts, (7, old_view))); + let (value, Tracked(_acquire_view), Ghost(load)) = atomic.load( + Ordering::Acquire, + Tracked(&mut view), + Tracked(&pt), + ); + + proof { + assert(pt.hist().get(load.timestamp) == Some((value, load.message_view))); + history_get_contains_timestamp(pt.hist(), load.timestamp); + assert(pt.hist().contains_timestamp(load.timestamp)); + assert(load.timestamp == initial_ts); + assert(pt.hist().get(load.timestamp) == Some((7, old_view))); + assert(value == 7); + assert(view@.contains(old_view)); + assert(pt.get_timestamp(view@) == Some(load.timestamp)); + } +} + +fn test_native_release_store() { + let (atomic, Tracked(mut pt), Tracked(mut view), Ghost(_initial_ts)) = PAtomicWeakBool::new( + false, + ); + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let ghost old_history = pt.hist(); + let Ghost(store) = atomic.store( + true, + Ordering::Release, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut pt), + ); + + proof { + assert(pt.hist() == old_history.insert(store.timestamp, true, store.message_view)); + assert(pt.get_timestamp(view@) == Some(store.timestamp)); + assert(store.message_view == view@); + } +} + +fn test_native_compare_exchange() { + let (atomic, Tracked(mut pt), Tracked(mut view), Ghost(_initial_ts)) = PAtomicWeakUsize::new(0); + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let (result, Tracked(_acquire_view), Ghost(update)) = atomic.compare_exchange( + 0, + 1, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut pt), + ); + + proof { + match result { + Ok(value) => { + assert(value == 0); + assert(pt.hist().get_value(update.load_timestamp + 1) == Some(1)); + }, + Err(value) => { + assert(value != 0); + }, + } + } +} + +fn test_native_pointer_operations() { + let first = core::ptr::null_mut::(); + let second = core::ptr::null_mut::(); + let (atomic, Tracked(mut points_to), Tracked(mut view), Ghost(_initial_ts)) = + PAtomicWeakPtr::new(first); + + let (_loaded, Tracked(_acquire_view), Ghost(_load)) = atomic.load( + Ordering::Acquire, + Tracked(&mut view), + Tracked(&points_to), + ); + + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let Ghost(_store) = atomic.store( + second, + Ordering::Release, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut points_to), + ); + + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let (_result, Tracked(_acquire_view), Ghost(_update)) = atomic.compare_exchange( + second, + first, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(&mut view), + Tracked(release_view), + Tracked(&mut points_to), + ); + + let (_old, Tracked(_acquire_view), Ghost(swap)) = atomic.swap_release( + second, + Tracked(&mut view), + Tracked(&mut points_to), + ); + proof { + assert(points_to.get_timestamp(view@) == Some(swap.load_timestamp + 1)); + } +} + +fn test_scheduler_owned_view_token() { + let (atomic, Tracked(mut points_to), Tracked(view_seen), Ghost(initial_ts)) = + PAtomicWeakUsize::new(0); + proof_decl! { + let tracked mut token = ThreadViewToken::from_view_seen(view_seen); + } + let ghost initial_view = token@; + proof { + assert(points_to.hist().is_singleton(initial_ts, (0, initial_view))); + } + + let (value, Tracked(_acquire_view), Ghost(load)) = atomic.load( + Ordering::Acquire, + Tracked(token.tracked_borrow_mut()), + Tracked(&points_to), + ); + proof { + assert(points_to.get_timestamp(token@) == Some(load.timestamp)); + assert(points_to.hist().get(load.timestamp) == Some((value, load.message_view))); + history_get_contains_timestamp(points_to.hist(), load.timestamp); + assert(load.timestamp == initial_ts); + assert(points_to.hist().get(load.timestamp) == Some((0, initial_view))); + assert(value == 0); + } + + proof_decl! { + let tracked release_view = ReleaseViewSeen::new(); + } + let Ghost(store) = atomic.store( + 1, + Ordering::Release, + Tracked(token.tracked_borrow_mut()), + Tracked(release_view), + Tracked(&mut points_to), + ); + proof { + assert(points_to.get_timestamp(token@) == Some(store.timestamp)); + assert(points_to.hist().get_value(store.timestamp) == Some(1)); + } + + proof_decl! { + let tracked cpu_token = ThreadViewToken::new(); + } + proof { + let ghost before = token@; + token.tracked_join(&cpu_token); + assert(token@ == before.join(cpu_token@)); + ThreadView::join_contains(before, cpu_token@); + assert(token@.contains(before)); + } +} + +} // verus! diff --git a/verified_libs/vstd_extra/src/atomic_weak.rs b/verified_libs/vstd_extra/src/atomic_weak.rs index 0794f2ebc..d33a199e5 100644 --- a/verified_libs/vstd_extra/src/atomic_weak.rs +++ b/verified_libs/vstd_extra/src/atomic_weak.rs @@ -28,6 +28,7 @@ use vstd::prelude::*; use vstd::resource::Loc; use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; use vstd::seq::Seq; +use vstd::thread_view::Objective; verus! { @@ -192,6 +193,12 @@ pub tracked struct HistAuth { ghost len: nat, } +// The authoritative map and its logical length are global facts about one +// atomic location. They contain no thread-subjective view permission. +unsafe impl Objective for HistAuth { + +} + proof fn lemma_timestamp_range_insert_last(hi: Timestamp) ensures Set::range(0nat, hi).insert(hi) == Set::range(0nat, hi + 1), @@ -370,7 +377,7 @@ macro_rules! declare_weak_atomic_type { p: Pred, } - impl InvariantPredicate<(K, AtomicId), (HistAuth<$value_ty>, G)> for $pred_adapter< + impl InvariantPredicate<(K, AtomicId), (HistAuth<$value_ty>, G)> for $pred_adapter< Pred, > where Pred: WeakAtomicInvariantPredicate { open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<$value_ty>, G)) -> bool { @@ -390,7 +397,7 @@ macro_rules! declare_weak_atomic_type { /// executable atomic id. As in `vstd::atomic_ghost`, outer data /// structures put this predicate in their own /// `#[verifier::type_invariant]`. - pub struct $weak_atomic { + pub struct $weak_atomic { #[doc(hidden)] atomic: $raw_atomic, #[doc(hidden)] @@ -399,7 +406,7 @@ macro_rules! declare_weak_atomic_type { >, } - impl $weak_atomic { + impl $weak_atomic { pub closed spec fn constant(&self) -> K { self.atomic_inv@.constant().0 } @@ -451,7 +458,7 @@ macro_rules! declare_weak_atomic_type { } } - impl $weak_atomic where + impl $weak_atomic where Pred: WeakAtomicInvariantPredicate, { #[inline(always)] @@ -558,10 +565,10 @@ pub struct WeakAtomicPredPtr { p: Pred, } -impl InvariantPredicate<(K, AtomicId), (HistAuth<*mut T>, G)> for WeakAtomicPredPtr< - T, - Pred, -> where Pred: WeakAtomicInvariantPredicate { +impl InvariantPredicate< + (K, AtomicId), + (HistAuth<*mut T>, G), +> for WeakAtomicPredPtr where Pred: WeakAtomicInvariantPredicate { open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<*mut T>, G)) -> bool { let (k, id) = k_id; let (hist, g) = hist_g; @@ -578,7 +585,7 @@ impl InvariantPredicate<(K, AtomicId), (HistAuth<*mut T>, G)> for /// any ownership or validity claim about the pointed-to allocation belongs in /// the user-supplied ghost state `G` and invariant predicate. #[verifier::accept_recursive_types(T)] -pub struct WeakAtomicPtr { +pub struct WeakAtomicPtr { #[doc(hidden)] atomic: AtomicPtrW, #[doc(hidden)] @@ -587,7 +594,7 @@ pub struct WeakAtomicPtr { >, } -impl WeakAtomicPtr { +impl WeakAtomicPtr { pub closed spec fn constant(&self) -> K { self.atomic_inv@.constant().0 } @@ -634,7 +641,7 @@ impl WeakAtomicPtr { } } -impl WeakAtomicPtr where +impl WeakAtomicPtr where Pred: WeakAtomicInvariantPredicate, { #[inline(always)] diff --git a/verified_libs/vstd_extra/src/external/smart_ptr.rs b/verified_libs/vstd_extra/src/external/smart_ptr.rs index e205c935a..9e5140b3c 100644 --- a/verified_libs/vstd_extra/src/external/smart_ptr.rs +++ b/verified_libs/vstd_extra/src/external/smart_ptr.rs @@ -4,6 +4,7 @@ use alloc::sync::Arc; use vstd::layout::valid_layout; use vstd::prelude::*; use vstd::raw_ptr::*; +use vstd::thread_view::Objective; // A unified interface for the raw ptr permission returned by `into_raw` methods of smart pointers like `Box` and `Arc`. verus! { @@ -62,6 +63,16 @@ pub tracked struct ArcPointsTo { pub perm: &'static PointsTo, } +// Raw-memory ownership is global state and does not depend on a thread's +// subjective weak-memory view. +unsafe impl Objective for BoxPointsTo { + +} + +unsafe impl Objective for ArcPointsTo { + +} + impl BoxPointsTo { pub open spec fn perm(self) -> PointsTowithDealloc { self.perm diff --git a/verified_libs/vstd_extra/src/lib.rs b/verified_libs/vstd_extra/src/lib.rs index 1f8100ca0..9e67b975f 100644 --- a/verified_libs/vstd_extra/src/lib.rs +++ b/verified_libs/vstd_extra/src/lib.rs @@ -16,6 +16,7 @@ extern crate alloc; pub mod arithmetic; pub mod array_ptr; +pub mod atomic_irc11; pub mod atomic_weak; pub mod auxiliary; pub mod cast_ptr; diff --git a/verified_libs/vstd_extra/src/resource/ghost_resource/count.rs b/verified_libs/vstd_extra/src/resource/ghost_resource/count.rs index 9bf793db6..b2ebaf2ce 100644 --- a/verified_libs/vstd_extra/src/resource/ghost_resource/count.rs +++ b/verified_libs/vstd_extra/src/resource/ghost_resource/count.rs @@ -6,6 +6,7 @@ use vstd::resource::Loc; use vstd::resource::algebra::ResourceAlgebra; use vstd::resource::pcm::{PCM, Resource}; use vstd::resource::storage_protocol::*; +use vstd::thread_view::Objective; verus! { @@ -295,6 +296,14 @@ pub tracked struct EmptyCount { r: StorageResource<(), T, FractionalCarrierOpt>, } +unsafe impl Objective for Count { + +} + +unsafe impl Objective for EmptyCount { + +} + impl Count { #[verifier::type_invariant] spec fn inv(self) -> bool { diff --git a/verified_libs/vstd_extra/src/sum.rs b/verified_libs/vstd_extra/src/sum.rs index bb3096b8a..fcbe83e96 100644 --- a/verified_libs/vstd_extra/src/sum.rs +++ b/verified_libs/vstd_extra/src/sum.rs @@ -1,6 +1,7 @@ use crate::ownership::Inv; use vstd::modes::tracked_swap; use vstd::prelude::*; +use vstd::thread_view::Objective; verus! { @@ -10,6 +11,10 @@ pub tracked enum Sum { Right(R), } +unsafe impl Objective for Sum { + +} + impl Sum { pub open spec fn left(self) -> L { self->Left_0 From 3b8c4929eb12570b737eab7557cebd84e05213c5 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 4 Aug 2026 05:24:08 -0400 Subject: [PATCH 35/47] format --- ostd/specs/sync/rcu.rs | 17 +++++---------- ostd/specs/sync/weak_memory.rs | 23 +++++--------------- ostd/src/task/scheduler/mod.rs | 11 +++------- verified_libs/vstd_extra/src/atomic_irc11.rs | 12 +++++----- 4 files changed, 20 insertions(+), 43 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 1c2503e4f..f7305c9fe 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -1503,9 +1503,7 @@ unsafe impl Objective for RcuMonitorFlagGhost { impl RcuMonitorFlagGhost { pub open spec fn initial(timestamp: nat) -> Self { - RcuMonitorFlagGhost { - states: Map::empty().insert(timestamp, MonitorStateView::initial()), - } + RcuMonitorFlagGhost { states: Map::empty().insert(timestamp, MonitorStateView::initial()) } } /// Proof-mode constructor for the tracked ghost state stored inside the @@ -1514,9 +1512,7 @@ impl RcuMonitorFlagGhost { ensures res == Self::initial(timestamp), { - RcuMonitorFlagGhost { - states: Map::empty().insert(timestamp, MonitorStateView::initial()), - } + RcuMonitorFlagGhost { states: Map::empty().insert(timestamp, MonitorStateView::initial()) } } pub open spec fn insert(self, timestamp: nat, state: MonitorStateView) -> Self { @@ -1593,10 +1589,7 @@ pub proof fn rcu_monitor_flag_initial_inv( requires history.is_singleton(timestamp, (false, message_view)), ensures - rcu_monitor_flag_history_inv( - history, - RcuMonitorFlagGhost::initial(timestamp), - ), + rcu_monitor_flag_history_inv(history, RcuMonitorFlagGhost::initial(timestamp)), { assert(history.dom() == Set::empty().insert(timestamp)) by { assert forall|ts: nat| @@ -1637,8 +1630,8 @@ pub proof fn preserve_rcu_monitor_flag_inv_on_insert( rcu_monitor_flag_history_inv(next, next_ghost), { assert(next_ghost.states.dom() == next.dom()); - assert forall|ts: nat| - next.contains_timestamp(ts) implies (#[trigger] next_ghost.states[ts]).wf() by { + assert forall|ts: nat| next.contains_timestamp(ts) implies ( + #[trigger] next_ghost.states[ts]).wf() by { if ts == timestamp { assert(next_ghost.states[ts] == state); } else { diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index cce6ab7cd..4376694dd 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -4,7 +4,6 @@ //! This module contains only transitions coupled to the RCU root and monitor //! ghost state. Generic native primitives are re-exported by //! [`vstd_extra::atomic_irc11`]. - use core::sync::atomic::Ordering; use super::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec}; @@ -13,9 +12,9 @@ use vstd::prelude::*; use vstd::resource::Loc; use vstd::thread_view::Objective; use vstd_extra::atomic_irc11::{ - AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, - PAtomicWeakPtr, ReleaseViewSeen, ThreadView as Irc11ThreadView, - ThreadViewOrder as Irc11ThreadViewOrder, Timestamp, ViewSeen, + AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, PAtomicWeakPtr, + ReleaseViewSeen, ThreadView as Irc11ThreadView, ThreadViewOrder as Irc11ThreadViewOrder, + Timestamp, ViewSeen, }; verus! { @@ -965,25 +964,15 @@ impl RcuMonitorWeakAtomicBool { Irc11AtomicBool::new(false); let tracked flag_ghost = rcu_spec::RcuMonitorFlagGhost::tracked_initial(timestamp); proof { - rcu_spec::rcu_monitor_flag_initial_inv( - points_to.hist(), - timestamp, - initial_view@, - ); - assert(rcu_spec::RcuMonitorFlagInv::inv( - atomic.loc(), - (points_to, flag_ghost), - )); + rcu_spec::rcu_monitor_flag_initial_inv(points_to.hist(), timestamp, initial_view@); + assert(rcu_spec::RcuMonitorFlagInv::inv(atomic.loc(), (points_to, flag_ghost))); } let tracked pair = (points_to, flag_ghost); let tracked atomic_inv = AtomicInvariant::new(atomic.loc(), pair, 0); Self { atomic, tracked_atomic_inv: Tracked(atomic_inv) } } - pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( - bool, - Ghost, - )) + pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: (bool, Ghost)) requires self.well_formed(), ensures diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 8f6a42964..87b1aefee 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -610,11 +610,8 @@ pub tracked struct TaskThreadView { } impl TaskThreadView { - proof fn new( - scheduler: Loc, - task: Loc, - tracked thread_view: ThreadViewToken, - ) -> (tracked res: Self) + proof fn new(scheduler: Loc, task: Loc, tracked thread_view: ThreadViewToken) -> (tracked res: + Self) ensures res.scheduler() == scheduler, res.task() == task, @@ -1172,9 +1169,7 @@ impl SchedulerThreadViews { == next.cpu_core_registrations[stored_cpu].locals_key &&& self.cpu_cores[stored_cpu].registration() == next.cpu_core_registrations[stored_cpu] &&& self.cpu_cores[stored_cpu].locals().fraction() == 1real - &&& self.cpu_cores[stored_cpu].locals().view().spec_le( - self.cpu_views[stored_cpu]@, - ) + &&& self.cpu_cores[stored_cpu].locals().view().spec_le(self.cpu_views[stored_cpu]@) } by { assert(next.cpu_core_registrations.contains_key(stored_cpu)); if stored_cpu == cpu { diff --git a/verified_libs/vstd_extra/src/atomic_irc11.rs b/verified_libs/vstd_extra/src/atomic_irc11.rs index 0eaf68ba0..9e5a14409 100644 --- a/verified_libs/vstd_extra/src/atomic_irc11.rs +++ b/verified_libs/vstd_extra/src/atomic_irc11.rs @@ -230,7 +230,7 @@ impl PAtomicWeakPtr { requires self.loc() == points_to.loc(), order matches Ordering::Acquire - || order matches Ordering::Relaxed, + | | order matches Ordering::Relaxed, ensures match order { Ordering::Acquire => load_acquire( @@ -271,7 +271,7 @@ impl PAtomicWeakPtr { requires self.loc() == old(points_to).loc(), order matches Ordering::Release - || order matches Ordering::Relaxed, + | | order matches Ordering::Relaxed, ensures forall|observed_view: ThreadView| #[trigger] old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( @@ -325,11 +325,11 @@ impl PAtomicWeakPtr { requires self.loc() == old(points_to).loc(), success matches Ordering::AcqRel - || success matches Ordering::Acquire - || success matches Ordering::Release - || success matches Ordering::Relaxed, + | | success matches Ordering::Acquire + | | success matches Ordering::Release + | | success matches Ordering::Relaxed, failure matches Ordering::Acquire - || failure matches Ordering::Relaxed, + | | failure matches Ordering::Relaxed, ensures result is Ok ==> old(points_to).hist().is_max_timestamp(update@.load_timestamp), forall|observed_view: ThreadView| #[trigger] From a20bdc6b50c14b512978e54ea9c22d74f6d70c75 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 4 Aug 2026 05:43:32 -0400 Subject: [PATCH 36/47] Stabilize IRC11 CI toolchain --- .github/workflows/ci-macos.yml | 29 +- .github/workflows/ci-upstream-verus.yml | 17 +- .github/workflows/ci.yml | 33 +- .github/workflows/doc.yml | 33 +- tools/patches/verus-irc11-vstd.patch | 5 +- tools/patches/verus-irc11.patch | 3053 ++++++++++++++++++ verified_libs/vstd_extra/src/atomic_irc11.rs | 15 +- 7 files changed, 3123 insertions(+), 62 deletions(-) create mode 100644 tools/patches/verus-irc11.patch diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index cfd2612be..5679dc9f1 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -14,8 +14,9 @@ jobs: runs-on: macos-14 env: CARGO_TERM_COLOR: always - VERUS_REPOSITORY: https://github.com/verus-lang/verus.git - VERUS_BRANCH: irc11 + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: @@ -55,17 +56,11 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Get Verus commit + - name: Record toolchain revisions id: verus shell: bash run: | - VERUS_COMMIT=$(git ls-remote "$VERUS_REPOSITORY" "refs/heads/$VERUS_BRANCH" | cut -f1) - if [ -z "$VERUS_COMMIT" ]; then - echo "Failed to resolve $VERUS_REPOSITORY branch $VERUS_BRANCH" - exit 1 - fi - echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus $VERUS_BRANCH commit: $VERUS_COMMIT" + echo "Using pinned Asterinas Verus base: $VERUS_BASE_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -81,7 +76,7 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11-vstd.patch') }} + key: ${{ runner.os }}-verus-irc11-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Bootstrap Verus (if needed) shell: bash @@ -89,11 +84,17 @@ jobs: if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus $VERUS_BRANCH..." + echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + git clone --no-checkout "$VERUS_REPOSITORY" tools/verus + git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + cargo dv bootstrap fi - test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_COMMIT" + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + test -f tools/verus/source/vstd/atomic_weak.rs + test -f tools/verus/source/vstd/thread_view.rs - name: Enable IRC11 alongside existing SC atomics shell: bash diff --git a/.github/workflows/ci-upstream-verus.yml b/.github/workflows/ci-upstream-verus.yml index 681ea7c8e..2bc4fa156 100644 --- a/.github/workflows/ci-upstream-verus.yml +++ b/.github/workflows/ci-upstream-verus.yml @@ -1,4 +1,4 @@ -name: Verify VOSTD with verus-lang/verus IRC11 +name: Verify VOSTD with rebased Verus IRC11 on: push: @@ -23,7 +23,9 @@ jobs: runs-on: ubuntu-24.04 env: CARGO_TERM_COLOR: always - VERUS_BRANCH: irc11 + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: @@ -70,8 +72,15 @@ jobs: sudo apt update -qq sudo apt install -y build-essential unzip pkg-config libssl-dev llvm - - name: Bootstrap upstream Verus IRC11 - run: cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + - name: Bootstrap rebased Verus IRC11 + run: | + rm -rf tools/verus + git clone --no-checkout "$VERUS_REPOSITORY" tools/verus + git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + cargo dv bootstrap + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" - name: Enable IRC11 alongside existing SC atomics run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7099d2856..38b8b6d4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,9 @@ jobs: runs-on: ubuntu-24.04 env: CARGO_TERM_COLOR: always - VERUS_REPOSITORY: https://github.com/verus-lang/verus.git - VERUS_BRANCH: irc11 + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: @@ -64,16 +65,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Get Verus commit + - name: Record toolchain revisions id: verus run: | - VERUS_COMMIT=$(git ls-remote "$VERUS_REPOSITORY" "refs/heads/$VERUS_BRANCH" | cut -f1) - if [ -z "$VERUS_COMMIT" ]; then - echo "Failed to resolve $VERUS_REPOSITORY branch $VERUS_BRANCH" - exit 1 - fi - echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus $VERUS_BRANCH commit: $VERUS_COMMIT" + echo "Using pinned Asterinas Verus base: $VERUS_BASE_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -89,30 +84,36 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11-vstd.patch') }} + key: ${{ runner.os }}-verus-irc11-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt uses: actions/cache@v6 with: path: ~/.cargo/bin/verusfmt - key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BASE_COMMIT }} - name: Bootstrap Verus (if needed) run: | if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus $VERUS_BRANCH..." + echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + git clone --no-checkout "$VERUS_REPOSITORY" tools/verus + git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + cargo dv bootstrap fi if ! command -v verusfmt >/dev/null 2>&1; then echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + cargo dv bootstrap fi - test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_COMMIT" + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + test -f tools/verus/source/vstd/atomic_weak.rs + test -f tools/verus/source/vstd/thread_view.rs verusfmt --version - name: Enable IRC11 alongside existing SC atomics diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index e3fd65177..292cdfed0 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -22,8 +22,9 @@ jobs: build: runs-on: ubuntu-latest env: - VERUS_REPOSITORY: https://github.com/verus-lang/verus.git - VERUS_BRANCH: irc11 + VERUS_REPOSITORY: https://github.com/asterinas/verus.git + VERUS_BASE_COMMIT: bb61343fc97e4b3a97b1029f385ffb6bbb291da2 + VERUS_IRC11_PATCH: tools/patches/verus-irc11.patch VERUS_PATCH: tools/patches/verus-irc11-vstd.patch steps: - name: Checkout repository @@ -66,16 +67,10 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: Get Verus commit + - name: Record toolchain revisions id: verus run: | - VERUS_COMMIT=$(git ls-remote "$VERUS_REPOSITORY" "refs/heads/$VERUS_BRANCH" | cut -f1) - if [ -z "$VERUS_COMMIT" ]; then - echo "Failed to resolve $VERUS_REPOSITORY branch $VERUS_BRANCH" - exit 1 - fi - echo "VERUS_COMMIT=$VERUS_COMMIT" >> "$GITHUB_ENV" - echo "Using Verus $VERUS_BRANCH commit: $VERUS_COMMIT" + echo "Using pinned Asterinas Verus base: $VERUS_BASE_COMMIT" DV_COMMIT=$(git rev-parse HEAD:dv) echo "DV_COMMIT=$DV_COMMIT" >> "$GITHUB_ENV" echo "Using dv commit: $DV_COMMIT" @@ -91,30 +86,36 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11-vstd.patch') }} + key: ${{ runner.os }}-verus-irc11-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt uses: actions/cache@v6 with: path: ~/.cargo/bin/verusfmt - key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BRANCH }}-${{ env.VERUS_COMMIT }} + key: ${{ runner.os }}-verusfmt-${{ env.VERUS_BASE_COMMIT }} - name: Bootstrap Verus (if needed) run: | if [ "${{ steps.cache-verus.outputs.cache-hit }}" = "true" ]; then echo "Using cached Verus" else - echo "Cache miss, bootstrapping Verus $VERUS_BRANCH..." + echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + git clone --no-checkout "$VERUS_REPOSITORY" tools/verus + git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" + cargo dv bootstrap fi if ! command -v verusfmt >/dev/null 2>&1; then echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap --upstream-verus --branch "$VERUS_BRANCH" + cargo dv bootstrap fi - test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_COMMIT" + test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" + test -f tools/verus/source/vstd/atomic_weak.rs + test -f tools/verus/source/vstd/thread_view.rs verusfmt --version - name: Enable IRC11 alongside existing SC atomics diff --git a/tools/patches/verus-irc11-vstd.patch b/tools/patches/verus-irc11-vstd.patch index 147cbe0d4..f002105b8 100644 --- a/tools/patches/verus-irc11-vstd.patch +++ b/tools/patches/verus-irc11-vstd.patch @@ -11,7 +11,6 @@ index 1c106c81..ef05ea7e 100644 [package.metadata.verus] diff --git a/source/vstd/atomic.rs b/source/vstd/atomic.rs -index 8bb30533..14de2bd9 100644 --- a/source/vstd/atomic.rs +++ b/source/vstd/atomic.rs @@ -5,10 +5,8 @@ use super::prelude::*; @@ -25,11 +24,11 @@ index 8bb30533..14de2bd9 100644 mod sc_atomic_types { use core::sync::atomic::{ -@@ -22,6 +20,8 @@ mod sc_atomic_types { +@@ -22,7 +20,8 @@ mod sc_atomic_types { use super::super::modes::*; use super::super::pervasive::*; use super::super::prelude::*; -+ use super::super::raw_ptr::PointsTo; + use super::super::raw_ptr::PointsTo; + use super::super::thread_view::Objective; use super::super::view::*; use super::super::wrapping::*; diff --git a/tools/patches/verus-irc11.patch b/tools/patches/verus-irc11.patch new file mode 100644 index 000000000..e22bfedf5 --- /dev/null +++ b/tools/patches/verus-irc11.patch @@ -0,0 +1,3053 @@ +diff --git a/source/rust_verify_test/tests/core_special_setup.rs b/source/rust_verify_test/tests/core_special_setup.rs +index d9f26d56..7dcb1fbf 100644 +--- a/source/rust_verify_test/tests/core_special_setup.rs ++++ b/source/rust_verify_test/tests/core_special_setup.rs +@@ -28,6 +28,7 @@ test_verify_one_file_with_options! { + verus_keep_ghost, + feature(fn_traits), + )] ++ #![cfg_attr(verus_keep_ghost, feature(auto_traits))] + #![cfg_attr(verus_keep_ghost, verifier::exec_allows_no_decreases_clause)] + + #[verifier::external] +diff --git a/source/vstd/atomic.rs b/source/vstd/atomic.rs +index bcc34e13..8bb30533 100644 +--- a/source/vstd/atomic.rs ++++ b/source/vstd/atomic.rs +@@ -1,56 +1,67 @@ + #![allow(unused_imports)] + +-use core::sync::atomic::{ +- AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, +- AtomicU32, AtomicUsize, Ordering, +-}; +- +-#[cfg(target_has_atomic = "64")] +-use core::sync::atomic::{AtomicI64, AtomicU64}; +- +-use super::modes::*; + use super::pervasive::*; + use super::prelude::*; + use super::raw_ptr::PointsTo; + use super::view::*; +-use super::wrapping::*; +- +-macro_rules! make_unsigned_integer_atomic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { +- atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); +- #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] +- impl $at_ident { +- atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); +- atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); +- } +- }; +-} + +-macro_rules! make_signed_integer_atomic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { +- atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); +- #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] +- impl $at_ident { +- atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); +- atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); +- } +- }; +-} ++#[cfg(not(feature = "weak-memory"))] ++pub use sc_atomic_types::*; + +-macro_rules! make_bool_atomic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); +- #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] +- impl $at_ident { +- atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); +- atomic_bool_methods!($at_ident, $p_ident, $rust_ty, $value_ty); +- } ++#[cfg(not(feature = "weak-memory"))] ++mod sc_atomic_types { ++ ++ use core::sync::atomic::{ ++ AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, ++ AtomicU32, AtomicUsize, Ordering, + }; +-} + +-macro_rules! atomic_types { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- verus! { ++ #[cfg(target_has_atomic = "64")] ++ use core::sync::atomic::{AtomicI64, AtomicU64}; ++ ++ use super::super::modes::*; ++ use super::super::pervasive::*; ++ use super::super::prelude::*; ++ use super::super::raw_ptr::PointsTo; ++ use super::super::view::*; ++ use super::super::wrapping::*; ++ ++ macro_rules! make_unsigned_integer_atomic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++ } ++ ++ macro_rules! make_signed_integer_atomic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $p_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++ } ++ ++ macro_rules! make_bool_atomic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ atomic_types!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $p_ident, $p_data_ident, $rust_ty, $value_ty, []); ++ atomic_bool_methods!($at_ident, $p_ident, $rust_ty, $value_ty); ++ } ++ }; ++ } ++ ++ macro_rules! atomic_types { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus! { + + #[verifier::external_body] /* vattr */ + pub struct $at_ident { +@@ -92,12 +102,12 @@ macro_rules! atomic_types { + } + + } +- }; +-} ++ }; ++ } + +-macro_rules! atomic_types_generic { +- ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- verus! { ++ macro_rules! atomic_types_generic { ++ ($at_ident:ident, $p_ident:ident, $p_data_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus! { + + #[verifier::accept_recursive_types(T)] + #[verifier::external_body] /* vattr */ +@@ -142,12 +152,12 @@ macro_rules! atomic_types_generic { + } + + } +- }; +-} ++ }; ++ } + +-pub type AtomicCellId = int; ++ pub type AtomicCellId = int; + +-macro_rules! atomic_common_methods { ++ macro_rules! atomic_common_methods { + ($at_ident: ty, $p_ident: ty, $p_data_ident: ty, $rust_ty: ty, $value_ty: ty, [ $($addr:tt)* ]) => { + verus_impl!{ + +@@ -268,7 +278,7 @@ macro_rules! atomic_common_methods { + }; + } + +-macro_rules! atomic_integer_methods { ++ macro_rules! atomic_integer_methods { + ($at_ident:ident, $p_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { + verus_impl!{ + +@@ -436,9 +446,9 @@ macro_rules! atomic_integer_methods { + }; + } + +-macro_rules! atomic_bool_methods { +- ($at_ident:ident, $p_ident:ident, $rust_ty: ty, $value_ty: ty) => { +- verus!{ ++ macro_rules! atomic_bool_methods { ++ ($at_ident:ident, $p_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus!{ + + #[inline(always)] + #[verifier::external_body] /* vattr */ +@@ -505,8 +515,8 @@ macro_rules! atomic_bool_methods { + } + + } +- }; +-} ++ }; ++ } + + macro_rules! ptr_atomic_methods { + ($at_ty: ty, $rust_ty: ty, $value_ty: ty) => { +@@ -593,90 +603,163 @@ ptr_atomic_methods!(PAtomicIsize, AtomicIsize, isize); + + make_bool_atomic!(PAtomicBool, PermissionBool, PermissionDataBool, AtomicBool, bool); + +-make_unsigned_integer_atomic!(PAtomicU8, PermissionU8, PermissionDataU8, AtomicU8, u8, u8_specs); +-make_unsigned_integer_atomic!( +- PAtomicU16, +- PermissionU16, +- PermissionDataU16, +- AtomicU16, +- u16, +- u16_specs +-); +-make_unsigned_integer_atomic!( +- PAtomicU32, +- PermissionU32, +- PermissionDataU32, +- AtomicU32, +- u32, +- u32_specs +-); ++ make_unsigned_integer_atomic!( ++ PAtomicU8, ++ PermissionU8, ++ PermissionDataU8, ++ AtomicU8, ++ u8, ++ u8_specs ++ ); ++ make_unsigned_integer_atomic!( ++ PAtomicU16, ++ PermissionU16, ++ PermissionDataU16, ++ AtomicU16, ++ u16, ++ u16_specs ++ ); ++ make_unsigned_integer_atomic!( ++ PAtomicU32, ++ PermissionU32, ++ PermissionDataU32, ++ AtomicU32, ++ u32, ++ u32_specs ++ ); + +-#[cfg(target_has_atomic = "64")] +-make_unsigned_integer_atomic!( +- PAtomicU64, +- PermissionU64, +- PermissionDataU64, +- AtomicU64, +- u64, +- u64_specs +-); +-make_unsigned_integer_atomic!( +- PAtomicUsize, +- PermissionUsize, +- PermissionDataUsize, +- AtomicUsize, +- usize, +- usize_specs +-); +- +-make_signed_integer_atomic!(PAtomicI8, PermissionI8, PermissionDataI8, AtomicI8, i8, i8_specs); +-make_signed_integer_atomic!( +- PAtomicI16, +- PermissionI16, +- PermissionDataI16, +- AtomicI16, +- i16, +- i16_specs +-); +-make_signed_integer_atomic!( +- PAtomicI32, +- PermissionI32, +- PermissionDataI32, +- AtomicI32, +- i32, +- i32_specs +-); ++ #[cfg(target_has_atomic = "64")] ++ make_unsigned_integer_atomic!( ++ PAtomicU64, ++ PermissionU64, ++ PermissionDataU64, ++ AtomicU64, ++ u64, ++ u64_specs ++ ); ++ make_unsigned_integer_atomic!( ++ PAtomicUsize, ++ PermissionUsize, ++ PermissionDataUsize, ++ AtomicUsize, ++ usize, ++ usize_specs ++ ); + +-#[cfg(target_has_atomic = "64")] +-make_signed_integer_atomic!( +- PAtomicI64, +- PermissionI64, +- PermissionDataI64, +- AtomicI64, +- i64, +- i64_specs +-); +-make_signed_integer_atomic!( +- PAtomicIsize, +- PermissionIsize, +- PermissionDataIsize, +- AtomicIsize, +- isize, +- isize_specs +-); +- +-atomic_types_generic!(PAtomicPtr, PermissionPtr, PermissionDataPtr, AtomicPtr, *mut T); +- +-#[cfg_attr(verus_keep_ghost, verifier::verus_macro)] +-impl PAtomicPtr { +- atomic_common_methods!( +- PAtomicPtr::, +- PermissionPtr::, +- PermissionDataPtr::, +- AtomicPtr::, +- *mut T, +- [ .view().addr ] ++ make_signed_integer_atomic!(PAtomicI8, PermissionI8, PermissionDataI8, AtomicI8, i8, i8_specs); ++ make_signed_integer_atomic!( ++ PAtomicI16, ++ PermissionI16, ++ PermissionDataI16, ++ AtomicI16, ++ i16, ++ i16_specs ++ ); ++ make_signed_integer_atomic!( ++ PAtomicI32, ++ PermissionI32, ++ PermissionDataI32, ++ AtomicI32, ++ i32, ++ i32_specs ++ ); ++ ++ #[cfg(target_has_atomic = "64")] ++ make_signed_integer_atomic!( ++ PAtomicI64, ++ PermissionI64, ++ PermissionDataI64, ++ AtomicI64, ++ i64, ++ i64_specs ++ ); ++ make_signed_integer_atomic!( ++ PAtomicIsize, ++ PermissionIsize, ++ PermissionDataIsize, ++ AtomicIsize, ++ isize, ++ isize_specs + ); ++ ++ atomic_types_generic!(PAtomicPtr, PermissionPtr, PermissionDataPtr, AtomicPtr, *mut T); ++ ++ #[cfg_attr(verus_keep_ghost, verifier::verus_macro)] ++ impl PAtomicPtr { ++ atomic_common_methods!( ++ PAtomicPtr::, ++ PermissionPtr::, ++ PermissionDataPtr::, ++ AtomicPtr::, ++ *mut T, ++ [ .view().addr ] ++ ); ++ } ++ ++ impl PAtomicPtr { ++ verus_impl! { ++ ++ #[inline(always)] ++ #[verifier::external_body] /* vattr */ ++ #[verifier::atomic] /* vattr */ ++ #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] ++ pub fn fetch_and(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: ++ *mut T) ++ requires ++ equal(self.id(), old(perm).view().patomic), ++ ensures ++ equal(old(perm).view().value, ret), ++ final(perm).view().patomic == old(perm).view().patomic, ++ final(perm).view().value@.addr == (old(perm).view().value@.addr & n), ++ final(perm).view().value@.provenance == old(perm).view().value@.provenance, ++ final(perm).view().value@.metadata == old(perm).view().value@.metadata, ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.fetch_and(n, Ordering::SeqCst) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] /* vattr */ ++ #[verifier::atomic] /* vattr */ ++ #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] ++ pub fn fetch_xor(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: ++ *mut T) ++ requires ++ equal(self.id(), old(perm).view().patomic), ++ ensures ++ equal(old(perm).view().value, ret), ++ final(perm).view().patomic == old(perm).view().patomic, ++ final(perm).view().value@.addr == (old(perm).view().value@.addr ^ n), ++ final(perm).view().value@.provenance == old(perm).view().value@.provenance, ++ final(perm).view().value@.metadata == old(perm).view().value@.metadata, ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.fetch_xor(n, Ordering::SeqCst) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] /* vattr */ ++ #[verifier::atomic] /* vattr */ ++ #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] ++ pub fn fetch_or(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: *mut T) ++ requires ++ equal(self.id(), old(perm).view().patomic), ++ ensures ++ equal(old(perm).view().value, ret), ++ final(perm).view().patomic == old(perm).view().patomic, ++ final(perm).view().value@.addr == (old(perm).view().value@.addr | n), ++ final(perm).view().value@.provenance == old(perm).view().value@.provenance, ++ final(perm).view().value@.metadata == old(perm).view().value@.metadata, ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.fetch_or(n, Ordering::SeqCst) ++ } ++ ++ } ++ } + } + + impl core::fmt::Debug for AtomicUpdate { +@@ -1125,65 +1208,4 @@ pub use { + try_open_atomic_update_in_proof, + }; + +-impl PAtomicPtr { +- #[inline(always)] +- #[verifier::external_body] /* vattr */ +- #[verifier::atomic] /* vattr */ +- #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] +- pub fn fetch_and(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: +- *mut T) +- requires +- equal(self.id(), old(perm).view().patomic), +- ensures +- equal(old(perm).view().value, ret), +- final(perm).view().patomic == old(perm).view().patomic, +- final(perm).view().value@.addr == (old(perm).view().value@.addr & n), +- final(perm).view().value@.provenance == old(perm).view().value@.provenance, +- final(perm).view().value@.metadata == old(perm).view().value@.metadata, +- opens_invariants none +- no_unwind +- { +- self.ato.fetch_and(n, Ordering::SeqCst) +- } +- +- #[inline(always)] +- #[verifier::external_body] /* vattr */ +- #[verifier::atomic] /* vattr */ +- #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] +- pub fn fetch_xor(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: +- *mut T) +- requires +- equal(self.id(), old(perm).view().patomic), +- ensures +- equal(old(perm).view().value, ret), +- final(perm).view().patomic == old(perm).view().patomic, +- final(perm).view().value@.addr == (old(perm).view().value@.addr ^ n), +- final(perm).view().value@.provenance == old(perm).view().value@.provenance, +- final(perm).view().value@.metadata == old(perm).view().value@.metadata, +- opens_invariants none +- no_unwind +- { +- self.ato.fetch_xor(n, Ordering::SeqCst) +- } +- +- #[inline(always)] +- #[verifier::external_body] /* vattr */ +- #[verifier::atomic] /* vattr */ +- #[cfg(any(verus_keep_ghost, feature = "strict_provenance_atomic_ptr"))] +- pub fn fetch_or(&self, Tracked(perm): Tracked<&mut PermissionPtr>, n: usize) -> (ret: *mut T) +- requires +- equal(self.id(), old(perm).view().patomic), +- ensures +- equal(old(perm).view().value, ret), +- final(perm).view().patomic == old(perm).view().patomic, +- final(perm).view().value@.addr == (old(perm).view().value@.addr | n), +- final(perm).view().value@.provenance == old(perm).view().value@.provenance, +- final(perm).view().value@.metadata == old(perm).view().value@.metadata, +- opens_invariants none +- no_unwind +- { +- self.ato.fetch_or(n, Ordering::SeqCst) +- } +-} +- + } // verus! +diff --git a/source/vstd/atomic_ghost.rs b/source/vstd/atomic_ghost.rs +index 317e8c52..0d6a3b8a 100644 +--- a/source/vstd/atomic_ghost.rs ++++ b/source/vstd/atomic_ghost.rs +@@ -2,21 +2,27 @@ + //! See the [`atomic_with_ghost!`] documentation for more information. + #![allow(unused_imports)] + +-use super::atomic::*; +-use super::invariant::*; +-use super::modes::*; +-use super::prelude::*; ++#[cfg(not(feature = "weak-memory"))] ++pub use atomic_ghost::*; + +-verus! { ++#[cfg(not(feature = "weak-memory"))] ++mod atomic_ghost { ++ ++ use super::super::atomic::*; ++ use super::super::invariant::*; ++ use super::super::modes::*; ++ use super::super::prelude::*; ++ ++ verus! { + + pub trait AtomicInvariantPredicate { + spec fn atomic_inv(k: K, v: V, g: G) -> bool; + } + + } // verus! +-macro_rules! declare_atomic_type { +- ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { +- verus!{ ++ macro_rules! declare_atomic_type { ++ ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { ++ verus!{ + + pub struct $atomic_pred_ty { p: Pred } + +@@ -103,11 +109,11 @@ macro_rules! declare_atomic_type { + } + + } +- }; +-} +-macro_rules! declare_atomic_type_generic { +- ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { +- verus!{ ++ }; ++ } ++ macro_rules! declare_atomic_type_generic { ++ ($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => { ++ verus!{ + + pub struct $atomic_pred_ty { t: T, p: Pred } + +@@ -190,142 +196,142 @@ macro_rules! declare_atomic_type_generic { + } + + } +- }; +-} ++ }; ++ } + +-#[cfg(target_has_atomic = "64")] +-declare_atomic_type!(AtomicU64, PAtomicU64, PermissionU64, u64, AtomicPredU64); +- +-declare_atomic_type!(AtomicU32, PAtomicU32, PermissionU32, u32, AtomicPredU32); +-declare_atomic_type!(AtomicU16, PAtomicU16, PermissionU16, u16, AtomicPredU16); +-declare_atomic_type!(AtomicU8, PAtomicU8, PermissionU8, u8, AtomicPredU8); +-declare_atomic_type!(AtomicUsize, PAtomicUsize, PermissionUsize, usize, AtomicPredUsize); +- +-#[cfg(target_has_atomic = "64")] +-declare_atomic_type!(AtomicI64, PAtomicI64, PermissionI64, i64, AtomicPredI64); +- +-declare_atomic_type!(AtomicI32, PAtomicI32, PermissionI32, i32, AtomicPredI32); +-declare_atomic_type!(AtomicI16, PAtomicI16, PermissionI16, i16, AtomicPredI16); +-declare_atomic_type!(AtomicI8, PAtomicI8, PermissionI8, i8, AtomicPredI8); +-declare_atomic_type!(AtomicIsize, PAtomicIsize, PermissionIsize, isize, AtomicPredIsize); +- +-declare_atomic_type!(AtomicBool, PAtomicBool, PermissionBool, bool, AtomicPredBool); +- +-declare_atomic_type_generic!(AtomicPtr, PAtomicPtr, PermissionPtr, *mut T, AtomicPredPtr); +- +-/// Performs a given atomic operation on a given atomic +-/// while providing access to its ghost state. +-/// +-/// `atomic_with_ghost!` supports the types +-/// [`AtomicU64`] [`AtomicU32`], [`AtomicU16`], [`AtomicU8`], +-/// [`AtomicI64`], [`AtomicI32`], [`AtomicI16`], [`AtomicI8`], and [`AtomicBool`]. +-/// +-/// For each type, it supports all applicable atomic operations among +-/// `load`, `store`, `swap`, `compare_exchange`, `compare_exchange_weak`, +-/// `fetch_add`, `fetch_add_wrapping`, `fetch_sub`, `fetch_sub_wrapping`, +-/// `fetch_or`, `fetch_and`, `fetch_xor`, `fetch_nand`, `fetch_max`, and `fetch_min`. +-/// +-/// Naturally, `AtomicBool` does not support the arithmetic-specific operations. +-/// +-/// In general, the syntax is: +-/// +-/// let result = atomic_with_ghost!( +-/// $atomic => $operation_name($operands...); +-/// update $prev -> $next; // `update` line is optional +-/// returning $ret; // `returning` line is optional +-/// ghost $g => { +-/// /* Proof code with access to `tracked` variable `g: G` */ +-/// } +-/// ); +-/// +-/// Here, the `$operation_name` is one of `load`, `store`, etc. Meanwhile, +-/// `$prev`, `$next`, and `$ret` are all identifiers which +-/// will be available as spec variable inside the block to describe the +-/// atomic action which is performed. +-/// +-/// For example, suppose the user performs `fetch_add(1)`. The atomic +-/// operation might load the value 5, add 1, store the value 6, +-/// and return the original value, 5. In that case, we would have +-/// `prev == 5`, `next == 6`, and `ret == 5`. +-/// +-/// The specification for a given operation is given as a relation between +-/// `prev`, `next`, and `ret`; that is, at the beginning of the proof block, +-/// the user may assume the given specification holds: +-/// +-/// | operation | specification | +-/// |-------------------------------|----------------------------------------------------------------------------------------------------------------------------| +-/// | `load()` | `next == prev && rev == prev` | +-/// | `store(x)` | `next == x && ret == ()` | +-/// | `swap(x)` | `next == x && ret == prev` | +-/// | `compare_exchange(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`prev != x && next == prev && ret == Err(prev)` ("failure") | +-/// | `compare_exchange_weak(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`next == prev && ret == Err(prev)` ("failure") | +-/// | `fetch_add(x)` (*) | `next == prev + x && ret == prev` | +-/// | `fetch_add_wrapping(x)` | `next == wrapping_add(prev, x) && ret == prev` | +-/// | `fetch_sub(x)` (*) | `next == prev - x && ret == prev` | +-/// | `fetch_sub_wrapping(x)` | `next == wrapping_sub(prev, x) && ret == prev` | +-/// | `fetch_or(x)` | next == prev \| x && ret == prev | +-/// | `fetch_and(x)` | `next == prev & x && ret == prev` | +-/// | `fetch_xor(x)` | `next == prev ^ x && ret == prev` | +-/// | `fetch_nand(x)` | `next == !(prev & x) && ret == prev` | +-/// | `fetch_max(x)` | `next == max(prev, x) && ret == prev` | +-/// | `fetch_min(x)` | `next == max(prev, x) && ret == prev` | +-/// | `no_op()` (**) | `next == prev && ret == ()` | +-/// +-/// (*) Note that `fetch_add` and `fetch_sub` do not specify +-/// wrapping-on-overflow; instead, they require the user to +-/// prove that overflow _does not occur_, i.e., the user must show +-/// that `next` is in bounds for the integer type in question. +-/// Furthermore, for `fetch_add` and `fetch_sub`, the spec values of +-/// `prev`, `next`, and `ret` are all given with type `int`, so the +-/// user may reason about boundedness within the proof block. +-/// +-/// (As executable code, `fetch_add` is equivalent to `fetch_add_wrapping`, +-/// and likewise for `fetch_sub` and `fetch_sub_wrapping`. +-/// We have both because it's frequently the case that the user needs to verify +-/// lack-of-overflow _anyway_, and having it as an explicit precondition by default +-/// then makes verification errors easier to diagnose. Furthermore, when overflow is +-/// intended, the wrapping operations document that intent.) +-/// +-/// (**) `no_op` is entirely a ghost operation and doesn't emit any actual instruction. +-/// This allows the user to access the ghost state and the stored value (as `spec` data) +-/// without actually performing a load. +-/// +-/// --- +-/// +-/// At the beginning of the proof block, the user may assume, in addition +-/// to the specified relation between `prev`, `next`, and `ret`, that +-/// `atomic.inv(prev, g)` holds. The user is required to update `g` such that +-/// `atomic.inv(next, g)` holds at the end of the block. +-/// In other words, the ghost block has the implicit pre- and post-conditions: +-/// +-/// let result = atomic_with_ghost!( +-/// $atomic => $operation_name($operands...); +-/// update $prev -> $next; +-/// returning $ret; +-/// ghost $g => { +-/// assume(specified relation on (prev, next, ret)); +-/// assume(atomic.inv(prev, g)); +-/// +-/// // User code here; may update variable `g` with full +-/// // access to variables in the outer context. +-/// +-/// assert(atomic.inv(next, g)); +-/// } +-/// ); +-/// +-/// Note that the necessary action on ghost state might depend +-/// on the result of the operation; for example, if the user performs a +-/// compare-and-swap, then the ghost action that they then need to do +-/// will probably depend on whether the operation succeeded or not. +-/// +-/// The value returned by the `atomic_with_ghost!(...)` expression will be equal +-/// to `ret`, although the return value is an `exec` value (the actual result of +-/// the operation) while `ret` is a `spec` value. +-/// +-/// ### Example (TODO) +- +-#[macro_export] +-macro_rules! atomic_with_ghost { ++ #[cfg(target_has_atomic = "64")] ++ declare_atomic_type!(AtomicU64, PAtomicU64, PermissionU64, u64, AtomicPredU64); ++ ++ declare_atomic_type!(AtomicU32, PAtomicU32, PermissionU32, u32, AtomicPredU32); ++ declare_atomic_type!(AtomicU16, PAtomicU16, PermissionU16, u16, AtomicPredU16); ++ declare_atomic_type!(AtomicU8, PAtomicU8, PermissionU8, u8, AtomicPredU8); ++ declare_atomic_type!(AtomicUsize, PAtomicUsize, PermissionUsize, usize, AtomicPredUsize); ++ ++ #[cfg(target_has_atomic = "64")] ++ declare_atomic_type!(AtomicI64, PAtomicI64, PermissionI64, i64, AtomicPredI64); ++ ++ declare_atomic_type!(AtomicI32, PAtomicI32, PermissionI32, i32, AtomicPredI32); ++ declare_atomic_type!(AtomicI16, PAtomicI16, PermissionI16, i16, AtomicPredI16); ++ declare_atomic_type!(AtomicI8, PAtomicI8, PermissionI8, i8, AtomicPredI8); ++ declare_atomic_type!(AtomicIsize, PAtomicIsize, PermissionIsize, isize, AtomicPredIsize); ++ ++ declare_atomic_type!(AtomicBool, PAtomicBool, PermissionBool, bool, AtomicPredBool); ++ ++ declare_atomic_type_generic!(AtomicPtr, PAtomicPtr, PermissionPtr, *mut T, AtomicPredPtr); ++ ++ /// Performs a given atomic operation on a given atomic ++ /// while providing access to its ghost state. ++ /// ++ /// `atomic_with_ghost!` supports the types ++ /// [`AtomicU64`] [`AtomicU32`], [`AtomicU16`], [`AtomicU8`], ++ /// [`AtomicI64`], [`AtomicI32`], [`AtomicI16`], [`AtomicI8`], and [`AtomicBool`]. ++ /// ++ /// For each type, it supports all applicable atomic operations among ++ /// `load`, `store`, `swap`, `compare_exchange`, `compare_exchange_weak`, ++ /// `fetch_add`, `fetch_add_wrapping`, `fetch_sub`, `fetch_sub_wrapping`, ++ /// `fetch_or`, `fetch_and`, `fetch_xor`, `fetch_nand`, `fetch_max`, and `fetch_min`. ++ /// ++ /// Naturally, `AtomicBool` does not support the arithmetic-specific operations. ++ /// ++ /// In general, the syntax is: ++ /// ++ /// let result = atomic_with_ghost!( ++ /// $atomic => $operation_name($operands...); ++ /// update $prev -> $next; // `update` line is optional ++ /// returning $ret; // `returning` line is optional ++ /// ghost $g => { ++ /// /* Proof code with access to `tracked` variable `g: G` */ ++ /// } ++ /// ); ++ /// ++ /// Here, the `$operation_name` is one of `load`, `store`, etc. Meanwhile, ++ /// `$prev`, `$next`, and `$ret` are all identifiers which ++ /// will be available as spec variable inside the block to describe the ++ /// atomic action which is performed. ++ /// ++ /// For example, suppose the user performs `fetch_add(1)`. The atomic ++ /// operation might load the value 5, add 1, store the value 6, ++ /// and return the original value, 5. In that case, we would have ++ /// `prev == 5`, `next == 6`, and `ret == 5`. ++ /// ++ /// The specification for a given operation is given as a relation between ++ /// `prev`, `next`, and `ret`; that is, at the beginning of the proof block, ++ /// the user may assume the given specification holds: ++ /// ++ /// | operation | specification | ++ /// |-------------------------------|----------------------------------------------------------------------------------------------------------------------------| ++ /// | `load()` | `next == prev && rev == prev` | ++ /// | `store(x)` | `next == x && ret == ()` | ++ /// | `swap(x)` | `next == x && ret == prev` | ++ /// | `compare_exchange(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`prev != x && next == prev && ret == Err(prev)` ("failure") | ++ /// | `compare_exchange_weak(x, y)` | `prev == x && next == y && ret == Ok(prev)` ("success") OR
`next == prev && ret == Err(prev)` ("failure") | ++ /// | `fetch_add(x)` (*) | `next == prev + x && ret == prev` | ++ /// | `fetch_add_wrapping(x)` | `next == wrapping_add(prev, x) && ret == prev` | ++ /// | `fetch_sub(x)` (*) | `next == prev - x && ret == prev` | ++ /// | `fetch_sub_wrapping(x)` | `next == wrapping_sub(prev, x) && ret == prev` | ++ /// | `fetch_or(x)` | next == prev \| x && ret == prev | ++ /// | `fetch_and(x)` | `next == prev & x && ret == prev` | ++ /// | `fetch_xor(x)` | `next == prev ^ x && ret == prev` | ++ /// | `fetch_nand(x)` | `next == !(prev & x) && ret == prev` | ++ /// | `fetch_max(x)` | `next == max(prev, x) && ret == prev` | ++ /// | `fetch_min(x)` | `next == max(prev, x) && ret == prev` | ++ /// | `no_op()` (**) | `next == prev && ret == ()` | ++ /// ++ /// (*) Note that `fetch_add` and `fetch_sub` do not specify ++ /// wrapping-on-overflow; instead, they require the user to ++ /// prove that overflow _does not occur_, i.e., the user must show ++ /// that `next` is in bounds for the integer type in question. ++ /// Furthermore, for `fetch_add` and `fetch_sub`, the spec values of ++ /// `prev`, `next`, and `ret` are all given with type `int`, so the ++ /// user may reason about boundedness within the proof block. ++ /// ++ /// (As executable code, `fetch_add` is equivalent to `fetch_add_wrapping`, ++ /// and likewise for `fetch_sub` and `fetch_sub_wrapping`. ++ /// We have both because it's frequently the case that the user needs to verify ++ /// lack-of-overflow _anyway_, and having it as an explicit precondition by default ++ /// then makes verification errors easier to diagnose. Furthermore, when overflow is ++ /// intended, the wrapping operations document that intent.) ++ /// ++ /// (**) `no_op` is entirely a ghost operation and doesn't emit any actual instruction. ++ /// This allows the user to access the ghost state and the stored value (as `spec` data) ++ /// without actually performing a load. ++ /// ++ /// --- ++ /// ++ /// At the beginning of the proof block, the user may assume, in addition ++ /// to the specified relation between `prev`, `next`, and `ret`, that ++ /// `atomic.inv(prev, g)` holds. The user is required to update `g` such that ++ /// `atomic.inv(next, g)` holds at the end of the block. ++ /// In other words, the ghost block has the implicit pre- and post-conditions: ++ /// ++ /// let result = atomic_with_ghost!( ++ /// $atomic => $operation_name($operands...); ++ /// update $prev -> $next; ++ /// returning $ret; ++ /// ghost $g => { ++ /// assume(specified relation on (prev, next, ret)); ++ /// assume(atomic.inv(prev, g)); ++ /// ++ /// // User code here; may update variable `g` with full ++ /// // access to variables in the outer context. ++ /// ++ /// assert(atomic.inv(next, g)); ++ /// } ++ /// ); ++ /// ++ /// Note that the necessary action on ghost state might depend ++ /// on the result of the operation; for example, if the user performs a ++ /// compare-and-swap, then the ghost action that they then need to do ++ /// will probably depend on whether the operation succeeded or not. ++ /// ++ /// The value returned by the `atomic_with_ghost!(...)` expression will be equal ++ /// to `ret`, although the return value is an `exec` value (the actual result of ++ /// the operation) while `ret` is a `spec` value. ++ /// ++ /// ### Example (TODO) ++ ++ #[macro_export] ++ macro_rules! atomic_with_ghost { + ($($tokens:tt)*) => { + // The helper is used to parse things using Verus syntax + // The helper then calls atomic_with_ghost_inner, below: +@@ -335,127 +341,127 @@ macro_rules! atomic_with_ghost { + } + } + +-pub use atomic_with_ghost; +- +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_inner { +- (load, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_load!($e, $prev, $next, $ret, $g, $b) +- }; +- (store, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_store!( +- $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (swap, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- swap, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- +- (fetch_or, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_or, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_and, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_and, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_xor, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_xor, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_nand, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_nand, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_max, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_max, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_min, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_min, $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_add_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_add_wrapping, +- $e, +- $operand, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- (fetch_sub_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( +- fetch_sub_wrapping, +- $e, +- $operand, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- +- (fetch_add, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_add!( +- $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- (fetch_sub, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_sub!( +- $e, $operand, $prev, $next, $ret, $g, $b +- ) +- }; +- +- (compare_exchange, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( +- compare_exchange, +- $e, +- $operand1, +- $operand2, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- (compare_exchange_weak, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( +- compare_exchange_weak, +- $e, +- $operand1, +- $operand2, +- $prev, +- $next, +- $ret, +- $g, +- $b +- ) +- }; +- (no_op, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { +- $crate::vstd::atomic_ghost::atomic_with_ghost_no_op!($e, $prev, $next, $ret, $g, $b) +- }; +-} ++ pub use atomic_with_ghost; ++ ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_inner { ++ (load, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_load!($e, $prev, $next, $ret, $g, $b) ++ }; ++ (store, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_store!( ++ $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (swap, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ swap, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ ++ (fetch_or, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_or, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_and, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_and, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_xor, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_xor, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_nand, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_nand, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_max, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_max, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_min, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_min, $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_add_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_add_wrapping, ++ $e, ++ $operand, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ (fetch_sub_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!( ++ fetch_sub_wrapping, ++ $e, ++ $operand, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ ++ (fetch_add, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_add!( ++ $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ (fetch_sub, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_sub!( ++ $e, $operand, $prev, $next, $ret, $g, $b ++ ) ++ }; ++ ++ (compare_exchange, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( ++ compare_exchange, ++ $e, ++ $operand1, ++ $operand2, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ (compare_exchange_weak, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!( ++ compare_exchange_weak, ++ $e, ++ $operand1, ++ $operand2, ++ $prev, ++ $next, ++ $ret, ++ $g, ++ $b ++ ) ++ }; ++ (no_op, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => { ++ $crate::vstd::atomic_ghost::atomic_with_ghost_no_op!($e, $prev, $next, $ret, $g, $b) ++ }; ++ } + +-pub use atomic_with_ghost_inner; ++ pub use atomic_with_ghost_inner; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_store { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_store { + ($e:expr, $operand:expr, $prev:pat, $next:pat, $res:pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let atomic = &($e); +@@ -474,11 +480,11 @@ macro_rules! atomic_with_ghost_store { + } } + }; + } +-pub use atomic_with_ghost_store; ++ pub use atomic_with_ghost_store; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_load { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_load { + ($e:expr, $prev:pat, $next: pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -500,11 +506,11 @@ macro_rules! atomic_with_ghost_load { + }; + } + +-pub use atomic_with_ghost_load; ++ pub use atomic_with_ghost_load; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_no_op { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_no_op { + ($e:expr, $prev:pat, $next: pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let atomic = &($e); +@@ -524,11 +530,11 @@ macro_rules! atomic_with_ghost_no_op { + }; + } + +-pub use atomic_with_ghost_no_op; ++ pub use atomic_with_ghost_no_op; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_with_1_operand { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_with_1_operand { + ($name:ident, $e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -551,11 +557,11 @@ macro_rules! atomic_with_ghost_update_with_1_operand { + }; + } + +-pub use atomic_with_ghost_update_with_1_operand; ++ pub use atomic_with_ghost_update_with_1_operand; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_with_2_operand { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_with_2_operand { + ($name:ident, $e:expr, $operand1:expr, $operand2:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -579,11 +585,11 @@ macro_rules! atomic_with_ghost_update_with_2_operand { + }; + } + +-pub use atomic_with_ghost_update_with_2_operand; ++ pub use atomic_with_ghost_update_with_2_operand; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_fetch_add { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_fetch_add { + ($e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + ($crate::vstd::prelude::verus_exec_expr!( { + let result; +@@ -610,11 +616,11 @@ macro_rules! atomic_with_ghost_update_fetch_add { + } + } + +-pub use atomic_with_ghost_update_fetch_add; ++ pub use atomic_with_ghost_update_fetch_add; + +-#[doc(hidden)] +-#[macro_export] +-macro_rules! atomic_with_ghost_update_fetch_sub { ++ #[doc(hidden)] ++ #[macro_export] ++ macro_rules! atomic_with_ghost_update_fetch_sub { + ($e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => { + $crate::vstd::prelude::verus_exec_expr! { { + let result; +@@ -641,4 +647,5 @@ macro_rules! atomic_with_ghost_update_fetch_sub { + }; + } + +-pub use atomic_with_ghost_update_fetch_sub; ++ pub use atomic_with_ghost_update_fetch_sub; ++} +diff --git a/source/vstd/atomic_weak.rs b/source/vstd/atomic_weak.rs +new file mode 100644 +index 00000000..644ac089 +--- /dev/null ++++ b/source/vstd/atomic_weak.rs +@@ -0,0 +1,743 @@ ++#[cfg(feature = "weak-memory")] ++pub use weak_atomic_types::*; ++ ++#[cfg(feature = "weak-memory")] ++mod weak_atomic_types { ++ ++ use core::sync::atomic::{ ++ AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, ++ AtomicU32, AtomicUsize, Ordering, ++ }; ++ ++ #[cfg(target_has_atomic = "64")] ++ use core::sync::atomic::{AtomicI64, AtomicU64}; ++ ++ use super::super::cell::CellId; ++ use super::super::prelude::*; ++ use super::super::thread_view::*; ++ use super::super::wrapping::*; ++ ++ verus! { ++ ++broadcast use crate::group_vstd_default; ++ ++#[verifier::external_body] ++pub fn fence_release(Tracked(vs): Tracked) -> (rel_vs: Tracked) ++ ensures ++ vs.view() == rel_vs@.view(), ++ opens_invariants none ++ no_unwind ++{ ++ core::sync::atomic::fence(Ordering::Release); ++ Tracked::assume_new() ++} ++ ++#[verifier::external_body] ++pub fn fence_acquire(Tracked(acq_vs): Tracked) -> (vs: Tracked) ++ ensures ++ acq_vs.view() == vs@.view(), ++ opens_invariants none ++ no_unwind ++{ ++ core::sync::atomic::fence(Ordering::Acquire); ++ Tracked::assume_new() ++} ++ ++pub ghost struct AtomicHistory(pub Map); ++ ++impl AtomicHistory { ++ pub open spec fn dom(&self) -> Set { ++ self.0.dom() ++ } ++ ++ pub open spec fn contains_timestamp(&self, timestamp: nat) -> bool { ++ self.0.dom().contains(timestamp) ++ } ++ ++ pub open spec fn index(&self, timestamp: nat) -> (T, ThreadView) ++ recommends ++ self.contains_timestamp(timestamp), ++ { ++ self.0.index(timestamp) ++ } ++ ++ pub open spec fn value(&self, timestamp: nat) -> T ++ recommends ++ self.contains_timestamp(timestamp), ++ { ++ self.0.index(timestamp).0 ++ } ++ ++ pub open spec fn thread_view(&self, timestamp: nat) -> ThreadView ++ recommends ++ self.contains_timestamp(timestamp), ++ { ++ self.0.index(timestamp).1 ++ } ++ ++ pub open spec fn get(&self, timestamp: nat) -> Option<(T, ThreadView)> { ++ self.0.get(timestamp) ++ } ++ ++ pub open spec fn get_value(&self, timestamp: nat) -> Option { ++ match self.get(timestamp) { ++ Some((val, _)) => Some(val), ++ None => None, ++ } ++ } ++ ++ pub open spec fn get_thread_view(&self, timestamp: nat) -> Option { ++ match self.get(timestamp) { ++ Some((_, view)) => Some(view), ++ None => None, ++ } ++ } ++ ++ pub open spec fn insert(&self, timestamp: nat, val: T, view: ThreadView) -> Self ++ recommends ++ !self.contains_timestamp(timestamp), ++ { ++ AtomicHistory(self.0.insert(timestamp, (val, view))) ++ } ++ ++ pub open spec fn remove(&self, timestamp: nat) -> Self { ++ AtomicHistory(self.0.remove(timestamp)) ++ } ++ ++ pub open spec fn is_singleton(&self, timestamp: nat, val: (T, ThreadView)) -> bool { ++ &&& self.contains_timestamp(timestamp) ++ &&& forall|ts| #[trigger] ++ self.contains_timestamp(ts) ==> ts == timestamp && self.get(ts) == Some(val) ++ } ++ ++ pub open spec fn is_max_timestamp(&self, timestamp: nat) -> bool { ++ &&& self.contains_timestamp(timestamp) ++ &&& forall|ts| #[trigger] self.contains_timestamp(ts) ==> ts <= timestamp ++ } ++} ++ ++pub broadcast proof fn history_insert_contains_timestamp_cases( ++ h: AtomicHistory, ++ t: nat, ++ v: T, ++ o: ThreadView, ++ t2: nat, ++) ++ requires ++ #[trigger] h.insert(t, v, o).contains_timestamp(t2), ++ ensures ++ t == t2 || h.contains_timestamp(t2), ++{ ++} ++ ++pub broadcast proof fn history_insert_contains_inserted_timestamp( ++ h: AtomicHistory, ++ t: nat, ++ v: T, ++ o: ThreadView, ++) ++ ensures ++ (#[trigger] h.insert(t, v, o)).contains_timestamp(t), ++{ ++} ++ ++pub broadcast proof fn history_get_contains_timestamp(h: AtomicHistory, t: nat) ++ requires ++ (#[trigger] h.get(t)).is_some(), ++ ensures ++ h.contains_timestamp(t), ++{ ++} ++ ++pub broadcast proof fn history_singleton_dom_singleton( ++ h: AtomicHistory, ++ ts: nat, ++ val: (T, ThreadView), ++) ++ requires ++ #[trigger] h.is_singleton(ts, val), ++ ensures ++ h.0.dom().is_singleton(), ++{ ++ assert(forall|ts1| #[trigger] h.0.dom().contains(ts1) ==> h.contains_timestamp(ts1)); ++ assert(forall|ts1| #[trigger] h.0.dom().contains(ts1) ==> ts1 == ts); ++} ++ ++pub broadcast group group_view_history { ++ group_thread_view_axioms, ++ history_insert_contains_inserted_timestamp, ++ history_insert_contains_timestamp_cases, ++ history_get_contains_timestamp, ++} ++ ++#[verifier::external_body] ++#[verifier::accept_recursive_types(T)] ++pub tracked struct AtomicPointsTo { ++ no_copy: NoCopy, ++ unused: T, ++} ++ ++unsafe impl Objective for AtomicPointsTo { ++ ++} ++ ++impl AtomicPointsTo { ++ pub uninterp spec fn loc(&self) -> CellId; ++ ++ pub uninterp spec fn hist(&self) -> AtomicHistory; ++ ++ pub uninterp spec fn get_timestamp(&self, view: ThreadView) -> Option; ++ ++ pub axiom fn get_timestamp_monotonic(tracked &self, v1: ThreadView, v2: ThreadView) ++ requires ++ v1.contains(v2), ++ ensures ++ self.get_timestamp(v2).is_some() ==> { ++ &&& self.get_timestamp(v1).is_some() ++ &&& self.get_timestamp(v2).unwrap() <= self.get_timestamp(v1).unwrap() ++ }, ++ ; ++ ++ pub axiom fn disjoint(tracked &mut self, tracked other: &Self) ++ ensures ++ final(self).loc() != other.loc(), ++ ; ++} ++ ++/// On a load, the thread must read a timestamp no smaller than that in its old view. ++/// After a load, the thread's new view will contain the timestamp that was read. ++pub open spec fn load_timestamp_in_view( ++ pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ timestamp: nat, ++) -> bool { ++ &&& pt.get_timestamp(old_view).is_none() || pt.get_timestamp(old_view).unwrap() <= timestamp ++ &&& pt.get_timestamp(new_view) == Some(timestamp) ++} ++ ++/// On a load, the location's AtomicHistory must have included [timestamp -> (val, message_view)]. ++pub open spec fn load_reads_from_history( ++ hist: AtomicHistory, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ hist.get(timestamp) == Some((val, message_view)) ++} ++ ++/// After a load, the thread's new view will contain the old view. ++pub open spec fn load_view_nondecreasing(old_view: ThreadView, new_view: ThreadView) -> bool { ++ new_view.contains(old_view) ++} ++ ++pub open spec fn load_acquire( ++ pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& load_timestamp_in_view(pt, old_view, new_view, timestamp) ++ &&& load_reads_from_history(pt.hist(), val, timestamp, message_view) ++ &&& load_view_nondecreasing( ++ old_view, ++ new_view, ++ ) ++ // because this is an acquire load, the message view is joined to the thread's current view ++ &&& new_view.contains(message_view) ++} ++ ++pub open spec fn load_relaxed( ++ pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ acquire_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& load_timestamp_in_view(pt, old_view, new_view, timestamp) ++ &&& load_reads_from_history(pt.hist(), val, timestamp, message_view) ++ &&& load_view_nondecreasing( ++ old_view, ++ new_view, ++ ) ++ // because this is a relaxed load, the message view is joined to the thread's acquire view ++ &&& acquire_view.contains(message_view) ++} ++ ++/// On a store, the store's timestamp must be greater than that in the thread's old view. ++/// After a store, the thread's new view will contain the timestamp of the store. ++/// The message view for the store will also contain the timestamp of the store. ++pub open spec fn store_timestamp_in_view( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ message_view: ThreadView, ++ timestamp: nat, ++) -> bool { ++ &&& old_pt.get_timestamp(old_view).is_none() || old_pt.get_timestamp(old_view).unwrap() ++ < timestamp ++ &&& new_pt.get_timestamp(new_view) == Some(timestamp) ++ &&& new_pt.get_timestamp(message_view) == Some(timestamp) ++} ++ ++/// After a store, the thread's new view will strictly contain its old view. ++/// This is a strict containment because the new view will contain the timestamp of the store. ++pub open spec fn store_view_increasing(old_view: ThreadView, new_view: ThreadView) -> bool { ++ &&& new_view.contains_strict(old_view) ++} ++ ++/// After a store, the locations's AtomicHistory is updated to contain the store. ++/// The timestamp of the store must not have previously been an entry in the location's AtomicHistory. ++pub open spec fn store_insert_history( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& !old_pt.hist().contains_timestamp(timestamp) ++ &&& new_pt.loc() == old_pt.loc() ++ &&& new_pt.hist() == old_pt.hist().insert(timestamp, val, message_view) ++} ++ ++pub open spec fn store_release( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_insert_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a release store, the message view is the thread's current view ++ &&& message_view == new_view ++} ++ ++pub open spec fn store_relaxed( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ release_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_insert_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a relaxed store, the message view contains the release view ++ &&& message_view.contains( ++ release_view, ++ ) ++ // and the thread's current view will now contain the message view ++ &&& new_view.contains(message_view) ++} ++ ++/// After a store_mut, the locations's AtomicHistory is updated to be a singleton containing only the new store. ++/// The timestamp of the store must not have previously been an entry in the location's AtomicHistory. ++pub open spec fn store_mut_truncate_history( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& !old_pt.hist().contains_timestamp(timestamp) ++ &&& new_pt.loc() == old_pt.loc() ++ &&& new_pt.hist().is_singleton(timestamp, (val, message_view)) ++} ++ ++pub open spec fn store_mut_release( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_mut_truncate_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a release store, the message view is the thread's current view ++ &&& message_view == new_view ++} ++ ++pub open spec fn store_mut_relaxed( ++ old_pt: AtomicPointsTo, ++ new_pt: AtomicPointsTo, ++ old_view: ThreadView, ++ new_view: ThreadView, ++ release_view: ThreadView, ++ val: T, ++ timestamp: nat, ++ message_view: ThreadView, ++) -> bool { ++ &&& store_timestamp_in_view(old_pt, new_pt, old_view, new_view, message_view, timestamp) ++ &&& store_view_increasing(old_view, new_view) ++ &&& store_mut_truncate_history( ++ old_pt, ++ new_pt, ++ val, ++ timestamp, ++ message_view, ++ ) ++ // because this is a relaxed store, the message view contains the release view ++ &&& message_view.contains( ++ release_view, ++ ) ++ // and the thread's current view will now contain the message view ++ &&& new_view.contains(message_view) ++} ++ ++pub ghost struct LoadData { ++ pub timestamp: nat, ++ pub message_view: ThreadView, ++} ++ ++pub ghost struct StoreData { ++ pub timestamp: nat, ++ pub message_view: ThreadView, ++} ++ ++pub ghost struct UpdateData { ++ pub load_timestamp: nat, ++ pub load_message_view: ThreadView, ++ pub store_message_view: ThreadView, ++ pub intermediate_thread_view: ThreadView, ++} ++ ++macro_rules! make_unsigned_integer_atomic { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++} ++ ++macro_rules! make_signed_integer_atomic { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ atomic_types!($at_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $rust_ty, $value_ty, []); ++ atomic_integer_methods!($at_ident, $rust_ty, $value_ty, $modname); ++ } ++ }; ++} ++ ++macro_rules! make_bool_atomic { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ atomic_types!($at_ident, $rust_ty, $value_ty); ++ #[cfg_attr(verus_keep_ghost, verus::internal(verus_macro))] ++ impl $at_ident { ++ atomic_common_methods!($at_ident, $rust_ty, $value_ty, []); ++ atomic_bool_methods!($at_ident, $rust_ty, $value_ty); ++ } ++ }; ++} ++ ++macro_rules! atomic_types { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus! { ++ ++ #[verifier::external_body] ++ pub struct $at_ident { ++ ato: $rust_ty, ++ } ++ ++ } ++ }; ++} ++ ++macro_rules! atomic_common_methods { ++ ($at_ident: ty, $rust_ty: ty, $value_ty: ty, [ $($addr:tt)* ]) => { ++ verus_impl!{ ++ ++ pub uninterp spec fn loc(&self) -> CellId; ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ pub const fn new(i: $value_ty) -> ((ato, pt, vs, ts): ( ++ Self, ++ Tracked>, ++ Tracked, ++ Ghost, ++ )) ++ ensures ++ ato.loc() == pt@.loc(), ++ pt@.hist().is_singleton(ts@, (i, vs@@)), ++ pt@.get_timestamp(vs@@) == Some(ts@) ++ { ++ let p = $at_ident { ato: $rust_ty::new(i) }; ++ (p, Tracked::assume_new(), Tracked::assume_new(), Ghost::assume_new()) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ pub const fn new_incl(i: $value_ty, Tracked(vs0) : Tracked) -> ((ato, pt, vs, ts): ( ++ Self, ++ Tracked>, ++ Tracked, ++ Ghost, ++ )) ++ ensures ++ ato.loc() == pt@.loc(), ++ pt@.hist().is_singleton(ts@, (i, vs@@)), ++ pt@.get_timestamp(vs@@) == Some(ts@), ++ vs@@.contains(vs0@) ++ { ++ let p = $at_ident { ato: $rust_ty::new(i) }; ++ (p, Tracked::assume_new(), Tracked::assume_new(), Ghost::assume_new()) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn load( ++ &self, ++ order: Ordering, ++ Tracked(vs): Tracked<&mut ViewSeen>, ++ Tracked(pt): Tracked<&AtomicPointsTo<$value_ty>>, ++ ) -> ((val, acq_vs, ld): ($value_ty, Tracked, Ghost)) ++ requires ++ self.loc() == pt.loc(), ++ order matches Ordering::Acquire || order matches Ordering::Relaxed ++ ensures ++ match order { ++ Ordering::Acquire => load_acquire(*pt, old(vs)@, final(vs)@, val, ld@.timestamp, ld@.message_view), ++ Ordering::Relaxed => load_relaxed(*pt, old(vs)@, final(vs)@, acq_vs@@, val, ld@.timestamp, ld@.message_view) ++ } ++ opens_invariants none ++ no_unwind ++ { ++ return (self.ato.load(order), Tracked::assume_new(), Ghost::assume_new()); ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn store( ++ &self, ++ v: $value_ty, ++ order: Ordering, ++ Tracked(vs): Tracked<&mut ViewSeen>, ++ Tracked(rel_vs): Tracked, ++ Tracked(pt): Tracked<&mut AtomicPointsTo<$value_ty>>, ++ ) -> (st: (Ghost)) ++ requires ++ self.loc() == old(pt).loc(), ++ order matches Ordering::Release || order matches Ordering::Relaxed ++ ensures ++ match order { ++ Ordering::Release => store_release(*old(pt), *final(pt), old(vs)@, final(vs)@, v, st@.timestamp, st@.message_view), ++ Ordering::Relaxed => store_relaxed(*old(pt), *final(pt), old(vs)@, final(vs)@, rel_vs@, v, st@.timestamp, st@.message_view) ++ } ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.store(v, order); ++ (Ghost::assume_new()) ++ } ++ ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn store_mut( ++ &mut self, ++ v: $value_ty, ++ order: Ordering, ++ Tracked(v_sn): Tracked<&mut ViewSeen>, ++ Tracked(rel_v_sn): Tracked, ++ Tracked(pt): Tracked<&mut AtomicPointsTo<$value_ty>>, ++ ) -> (st: (Ghost)) ++ requires ++ old(self).loc() == old(pt).loc(), ++ order matches Ordering::Release || order matches Ordering::Relaxed ++ ensures ++ match order { ++ Ordering::Release => store_mut_release(*old(pt), *final(pt), old(v_sn)@, final(v_sn)@, v, st@.timestamp, st@.message_view), ++ Ordering::Relaxed => store_mut_relaxed(*old(pt), *final(pt), old(v_sn)@, final(v_sn)@, rel_v_sn@, v, st@.timestamp, st@.message_view) ++ }, ++ final(self).loc() == old(self).loc() ++ opens_invariants none ++ no_unwind ++ { ++ self.ato.store(v, order); ++ (Ghost::assume_new()) ++ } ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ #[verifier::atomic] ++ pub fn compare_exchange( ++ &self, ++ current: $value_ty, ++ new: $value_ty, ++ success: Ordering, ++ failure: Ordering, ++ Tracked(vs): Tracked<&mut ViewSeen>, ++ Tracked(rel_vs): Tracked, ++ Tracked(pt): Tracked<&mut AtomicPointsTo<$value_ty>>, ++ ) -> ((res, acq_vs, up): (Result<$value_ty, $value_ty>, Tracked, Ghost)) ++ requires ++ self.loc() == old(pt).loc(), ++ success matches Ordering::AcqRel || success matches Ordering::Acquire || success matches Ordering::Release || success matches Ordering::Relaxed, ++ failure matches Ordering::Acquire || failure matches Ordering::Relaxed ++ ensures ++ match res { ++ Ok(v) => { ++ &&& current == v ++ &&& up@.store_message_view.contains_strict(up@.load_message_view) ++ &&& match success { ++ Ordering::AcqRel => { ++ &&& load_acquire(*old(pt), old(vs)@, up@.intermediate_thread_view, current, up@.load_timestamp, up@.load_message_view) ++ &&& store_release(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, new, up@.load_timestamp + 1, up@.store_message_view) ++ }, ++ Ordering::Acquire => { ++ &&& load_acquire(*old(pt), old(vs)@, up@.intermediate_thread_view, current, up@.load_timestamp, up@.load_message_view) ++ &&& store_relaxed(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, rel_vs@, new, up@.load_timestamp + 1, up@.store_message_view) ++ }, ++ Ordering::Release => { ++ &&& load_relaxed(*old(pt), old(vs)@, up@.intermediate_thread_view, acq_vs@@, v, up@.load_timestamp, up@.load_message_view) ++ &&& store_release(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, new, up@.load_timestamp + 1, up@.store_message_view) ++ }, ++ Ordering::Relaxed => { ++ &&& load_relaxed(*old(pt), old(vs)@, up@.intermediate_thread_view, acq_vs@@, v, up@.load_timestamp, up@.load_message_view) ++ &&& store_relaxed(*old(pt), *final(pt), up@.intermediate_thread_view, final(vs)@, rel_vs@, new, up@.load_timestamp + 1, up@.store_message_view) ++ } ++ } ++ }, ++ Err(v) => { ++ &&& current != v ++ &&& *final(pt) == *old(pt) ++ &&& match failure { ++ Ordering::Acquire => load_acquire(*old(pt), old(vs)@, final(vs)@, v, up@.load_timestamp, up@.load_message_view), ++ Ordering::Relaxed => load_relaxed(*old(pt), old(vs)@, final(vs)@, acq_vs@@, v, up@.load_timestamp, up@.load_message_view) ++ } ++ } ++ } ++ opens_invariants none ++ no_unwind ++ { ++ return (self.ato.compare_exchange(current, new, success, failure), Tracked::assume_new(), Ghost::assume_new()); ++ } ++ ++ // TODO - compare_exchange_weak, swap ++ ++ #[inline(always)] ++ pub axiom fn truncate_history(tracked &mut self, tracked pt: &mut AtomicPointsTo<$value_ty>, tracked vs: &mut ViewSeen) -> (ts: nat) ++ requires ++ old(self).loc() == old(pt).loc() ++ ensures ++ *final(self) == *old(self), ++ final(pt).loc() == old(pt).loc(), ++ old(pt).hist().is_max_timestamp(ts), ++ final(pt).hist().is_singleton(ts, old(pt).hist().get(ts).unwrap()), ++ final(vs)@.contains(old(vs)@), ++ final(pt).get_timestamp(final(vs)@) == Some(ts), ++ forall |t| #[trigger] old(pt).hist().contains_timestamp(t) ==> final(vs)@.contains(old(pt).hist().thread_view(t)) ++ ++ opens_invariants none ++ ; ++ ++ #[inline(always)] ++ #[verifier::external_body] ++ pub const fn into_inner(self, Tracked(pt): Tracked>) -> ((val, vs, ts): ($value_ty, Tracked, Ghost)) ++ requires ++ self.loc() == pt.loc(), ++ ensures ++ pt.hist().is_max_timestamp(ts@), ++ val == pt.hist().value(ts@), ++ pt.get_timestamp(vs@@) == Some(ts@), ++ forall |t| #[trigger] pt.hist().contains_timestamp(t) ==> vs@@.contains(pt.hist().thread_view(t)) ++ opens_invariants none ++ no_unwind ++ { ++ (self.ato.into_inner(), Tracked::assume_new(), Ghost::assume_new()) ++ } ++ ++ } ++ }; ++} ++ ++macro_rules! atomic_integer_methods { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty, $modname:ident) => { ++ verus_impl!{ ++ ++ // this macro is currently a stub for the functions we plan to implement: ++ // TODO - fetch_add_wrapping, fetch_sub_wrapping, fetch_add, fetch_sub, fetch_and, fetch_or, fetch_xor, fetch_nand, fetch_max, fetch_min ++ ++ } ++ }; ++} ++ ++macro_rules! atomic_bool_methods { ++ ($at_ident:ident, $rust_ty: ty, $value_ty: ty) => { ++ verus!{ ++ ++ // this macro is currently a stub for the functions we plan to implement: ++ // TODO - fetch_and, fetch_or, fetch_xor, fetch_nand ++ ++ } ++ }; ++} ++ ++make_bool_atomic!(PAtomicWeakBool, AtomicBool, bool); ++ ++make_unsigned_integer_atomic!(PAtomicWeakU8, AtomicU8, u8, u8_specs); ++ ++make_unsigned_integer_atomic!(PAtomicWeakU16, AtomicU16, u16, u16_specs); ++ ++make_unsigned_integer_atomic!(PAtomicWeakU32, AtomicU32, u32, u32_specs); ++ ++#[cfg(target_has_atomic = "64")] ++make_unsigned_integer_atomic!(PAtomicWeakU64, AtomicU64, u64, u64_specs); ++ ++make_unsigned_integer_atomic!(PAtomicWeakUsize, AtomicUsize, usize, usize_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakI8, AtomicI8, i8, i8_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakI16, AtomicI16, i16, i16_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakI32, AtomicI32, i32, i32_specs); ++ ++#[cfg(target_has_atomic = "64")] ++make_signed_integer_atomic!(PAtomicWeakI64, AtomicI64, i64,i64_specs); ++ ++make_signed_integer_atomic!(PAtomicWeakIsize, AtomicIsize, isize, isize_specs); ++ ++// TODO - AtomicPtr ++} // verus! ++} +diff --git a/source/vstd/invariant.rs b/source/vstd/invariant.rs +index 927650bf..b5f0f058 100644 +--- a/source/vstd/invariant.rs ++++ b/source/vstd/invariant.rs +@@ -1,7 +1,8 @@ + #[allow(unused_imports)] + use super::pervasive::*; +-#[allow(unused_imports)] + use super::prelude::*; ++#[cfg(verus_keep_ghost)] ++use super::thread_view::Objective; + + // TODO: + // * utility for conveniently creating unique namespaces +@@ -114,6 +115,7 @@ pub trait InvariantPredicate { + /// + /// **Note:** Rather than using `AtomicInvariant` directly, we generally recommend + /// using the [`atomic_ghost` APIs](crate::atomic_ghost). ++#[cfg(not(feature = "weak-memory"))] + #[cfg_attr(verus_keep_ghost, verifier::proof)] + #[cfg_attr(verus_keep_ghost, verifier::external_body)] /* vattr */ + #[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(K))] +@@ -124,6 +126,20 @@ pub struct AtomicInvariant { + dummy1: super::prelude::AlwaysSyncSend<(K, Pred, *mut V)>, + } + ++// TODO - document ++// TODO - can we fold this into a single AtomicInvariant definition without the Objective trait bound? ++// Creusot only implements Sync on AtomicInvariant when T: Send + Objective ++#[cfg(feature = "weak-memory")] ++#[cfg_attr(verus_keep_ghost, verifier::proof)] ++#[cfg_attr(verus_keep_ghost, verifier::external_body)] /* vattr */ ++#[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(K))] ++#[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(V))] ++#[cfg_attr(verus_keep_ghost, verifier::accept_recursive_types(Pred))] ++pub struct AtomicInvariant { ++ dummy: super::prelude::SyncSendIfSend, ++ dummy1: super::prelude::AlwaysSyncSend<(K, Pred, *mut V)>, ++} ++ + /// A `LocalInvariant` is a ghost object that provides "interior mutability" + /// for ghost objects, specifically, for `tracked` ghost objects. + /// A reference `&LocalInvariant` may be shared between clients. +@@ -267,7 +283,109 @@ macro_rules! declare_invariant_impl { + }; + } + ++#[cfg(feature = "weak-memory")] ++macro_rules! declare_invariant_impl_objective { ++ ($invariant:ident => $selfid:ident => $($into_inner_clause:tt)*) => { ++ // note the path names of `inv` and `namespace` are harcoded into the VIR crate. ++ ++ verus!{ ++ ++ impl $invariant { ++ /// The constant specified upon the initialization of this ` ++ #[doc = stringify!($invariant)] ++ ///`. ++ pub uninterp spec fn constant(&self) -> K; ++ ++ /// Namespace the invariant was declared in. ++ #[rustc_diagnostic_item = concat!("verus::vstd::invariant::", stringify!($invariant), "::namespace")] ++ pub uninterp spec fn namespace(&self) -> int; ++ } ++ ++ impl> $invariant { ++ /// Returns `true` if it is possible to store the value `v` into the ` ++ #[doc = stringify!($invariant)] ++ ///`. ++ /// ++ /// This is equivalent to `Pred::inv(self.constant(), v)`. ++ ++ #[rustc_diagnostic_item = concat!("verus::vstd::invariant::", stringify!($invariant), "::inv")] ++ pub open spec fn inv(&self, v: V) -> bool { ++ Pred::inv(self.constant(), v) ++ } ++ ++ /// Initialize a new ` ++ #[doc = stringify!($invariant)] ++ ///` with constant `k`. initial stored (tracked) value `v`, ++ /// and in the namespace `ns`. ++ ++ pub axiom fn new(k: K, tracked v: V, ns: int) -> (tracked i: $invariant) ++ requires ++ Pred::inv(k, v), ++ ensures ++ i.constant() == k, ++ i.namespace() == ns; ++ ++ // Q. Why does AtomicInvariant::into_inner have an opens_invariant clause ++ // while LocalInvariant::into_inner doesn't? ++ // ++ // A. It has to do with the way we prevent double-opening via into_inner, ++ // i.e., how we prevent the user from calling into_inner on an already-open ++ // invariant: ++ // ++ // open_{atomic|local}_invariant!(&inv => i => { ++ // inv.into_inner(); // this should error ++ // } ++ // ++ // There are two broad approaches: ++ // 1. Use the mask-checking, treating into_inner the same as an invariant-open ++ // 2. Use lifetimes, ensuring the borrow used to open the block extends through ++ // the entire block. ++ // ++ // Approach (1) is easier to use, but (1) is not sound for LocalInvariants. ++ // Thus, we use approach (1) for AtomicInvariants and (2) for LocalInvariants. ++ // ++ // Q. Why is approach (1) easier for AtomicInvariants? ++ // ++ // A. This makes it easier to perform an atomic operation that relinquishes ++ // the permission to access the same atomic. ++ // ++ // Q. Why is approach (1) unsound for LocalInvariants? ++ // ++ // A. Because into_inner is not the only problem. We want to implement ++ // the bound `impl => LocalInvariant: Send`. ++ // Furthermore, moving a local invariant to another thread basically means moving ++ // the invariant from one thread's invariant pool to another, which basically ++ // needs the same restrictions as `into_inner`. ++ // ++ // open_{atomic|local}_invariant!(&inv => i => { ++ // send_to_another_thread(inv); // this must be disallowed, too ++ // } ++ // ++ // However, we cannot (easily) put a mask bound on the send operation. ++ // ++ // Q. How are the lifetime restrictions implemented for LocalInvariant? ++ // The short answer is that we use the lifetime argument of the InvariantBlockGuard ++ // and force it to remain alive for the duration of the open_local_invariant! block. ++ // However, this requires special support from Verus in the lifetime-erasure system. ++ // See rustc_mir_build_additional_files/verus_builder.rs for more information. ++ ++ /// Destroys the ` ++ #[doc = stringify!($invariant)] ++ ///`, returning the tracked value contained within. ++ ++ pub axiom fn into_inner(tracked $selfid) -> (tracked v: V) ++ ensures $selfid.inv(v), ++ $($into_inner_clause)* ; ++ } ++ ++ } ++ }; ++} ++ ++#[cfg(not(feature = "weak-memory"))] + declare_invariant_impl!(AtomicInvariant => self => opens_invariants [ self.namespace() ] ); ++#[cfg(feature = "weak-memory")] ++declare_invariant_impl_objective!(AtomicInvariant => self => opens_invariants [ self.namespace() ] ); + declare_invariant_impl!(LocalInvariant => self => ); + + #[doc(hidden)] +@@ -365,7 +483,7 @@ pub fn spend_open_invariant_credit( + // Why does this use the 'static type param for the InvariantBlockGuard? This is because + // AtomicInvariant doesn't need the lifetime-checking like LocalInvariant does. See the explanation + // over `into_inner`. +-#[cfg(verus_keep_ghost)] ++#[cfg(all(verus_keep_ghost, not(feature = "weak-memory")))] + #[rustc_diagnostic_item = "verus::vstd::invariant::open_atomic_invariant_begin"] + #[doc(hidden)] + #[verifier::external] /* vattr */ +@@ -375,6 +493,16 @@ pub fn open_atomic_invariant_begin<'a, K, V, Pred: InvariantPredicate>( + unimplemented!(); + } + ++#[cfg(all(verus_keep_ghost, feature = "weak-memory"))] ++#[rustc_diagnostic_item = "verus::vstd::invariant::open_atomic_invariant_begin"] ++#[doc(hidden)] ++#[verifier::external] /* vattr */ ++pub fn open_atomic_invariant_begin<'a, K, V: Objective, Pred: InvariantPredicate>( ++ _inv: &'a AtomicInvariant, ++) -> (InvariantBlockGuard<'static>, V) { ++ unimplemented!(); ++} ++ + #[cfg(verus_keep_ghost)] + #[rustc_diagnostic_item = "verus::vstd::invariant::open_local_invariant_begin"] + #[doc(hidden)] +diff --git a/source/vstd/rwlock.rs b/source/vstd/rwlock.rs +index ba7ed260..8b7ce233 100644 +--- a/source/vstd/rwlock.rs ++++ b/source/vstd/rwlock.rs +@@ -2,239 +2,245 @@ + #![allow(unused_imports)] + #![allow(non_shorthand_field_patterns)] + +-use super::atomic_ghost::*; +-use super::cell::CellId; +-use super::cell::pcell_maybe_uninit as un; +-use super::invariant::InvariantPredicate; +-use super::modes::*; +-use super::multiset::*; +-use super::prelude::*; +-use super::set::*; +-use core::marker::PhantomData; +-use verus_state_machines_macros::tokenized_state_machine_vstd; +- +-tokenized_state_machine_vstd!( +-RwLockToks> { +- fields { +- #[sharding(constant)] +- pub k: K, +- +- #[sharding(constant)] +- pub pred: PhantomData, +- +- #[sharding(variable)] +- pub flag_exc: bool, +- +- #[sharding(variable)] +- pub flag_rc: nat, +- +- #[sharding(storage_option)] +- pub storage: Option, +- +- #[sharding(option)] +- pub pending_writer: Option<()>, +- +- #[sharding(option)] +- pub writer: Option<()>, +- +- #[sharding(multiset)] +- pub pending_reader: Multiset<()>, +- +- #[sharding(multiset)] +- pub reader: Multiset, +- } ++#[cfg(not(feature = "weak-memory"))] ++pub use rwlock::*; ++ ++#[cfg(not(feature = "weak-memory"))] ++mod rwlock { ++ ++ use super::super::atomic_ghost::*; ++ use super::super::cell::CellId; ++ use super::super::cell::pcell_maybe_uninit as un; ++ use super::super::invariant::InvariantPredicate; ++ use super::super::modes::*; ++ use super::super::multiset::*; ++ use super::super::prelude::*; ++ use super::super::set::*; ++ use core::marker::PhantomData; ++ use verus_state_machines_macros::tokenized_state_machine_vstd; ++ ++ tokenized_state_machine_vstd!( ++ RwLockToks> { ++ fields { ++ #[sharding(constant)] ++ pub k: K, ++ ++ #[sharding(constant)] ++ pub pred: PhantomData, ++ ++ #[sharding(variable)] ++ pub flag_exc: bool, ++ ++ #[sharding(variable)] ++ pub flag_rc: nat, ++ ++ #[sharding(storage_option)] ++ pub storage: Option, + +- init!{ +- initialize_full(k: K, t: V) { +- require Pred::inv(k, t); +- init k = k; +- init pred = PhantomData; +- init flag_exc = false; +- init flag_rc = 0; +- init storage = Option::Some(t); +- init pending_writer = Option::None; +- init writer = Option::None; +- init pending_reader = Multiset::empty(); +- init reader = Multiset::empty(); ++ #[sharding(option)] ++ pub pending_writer: Option<()>, ++ ++ #[sharding(option)] ++ pub writer: Option<()>, ++ ++ #[sharding(multiset)] ++ pub pending_reader: Multiset<()>, ++ ++ #[sharding(multiset)] ++ pub reader: Multiset, + } +- } + +- #[inductive(initialize_full)] +- fn initialize_full_inductive(post: Self, k: K, t: V) { +- broadcast use group_multiset_axioms; +- } ++ init!{ ++ initialize_full(k: K, t: V) { ++ require Pred::inv(k, t); ++ init k = k; ++ init pred = PhantomData; ++ init flag_exc = false; ++ init flag_rc = 0; ++ init storage = Option::Some(t); ++ init pending_writer = Option::None; ++ init writer = Option::None; ++ init pending_reader = Multiset::empty(); ++ init reader = Multiset::empty(); ++ } ++ } + +- /// Increment the 'rc' counter, obtain a pending_reader +- transition!{ +- acquire_read_start() { +- update flag_rc = pre.flag_rc + 1; +- add pending_reader += {()}; ++ #[inductive(initialize_full)] ++ fn initialize_full_inductive(post: Self, k: K, t: V) { ++ broadcast use group_multiset_axioms; ++ } ++ ++ /// Increment the 'rc' counter, obtain a pending_reader ++ transition!{ ++ acquire_read_start() { ++ update flag_rc = pre.flag_rc + 1; ++ add pending_reader += {()}; ++ } + } +- } + +- /// Exchange the pending_reader for a reader by checking +- /// that the 'exc' bit is 0 +- transition!{ +- acquire_read_end() { +- require(pre.flag_exc == false); ++ /// Exchange the pending_reader for a reader by checking ++ /// that the 'exc' bit is 0 ++ transition!{ ++ acquire_read_end() { ++ require(pre.flag_exc == false); + +- remove pending_reader -= {()}; ++ remove pending_reader -= {()}; + +- birds_eye let x: V = pre.storage->0; +- add reader += {x}; ++ birds_eye let x: V = pre.storage->0; ++ add reader += {x}; + +- assert Pred::inv(pre.k, x); ++ assert Pred::inv(pre.k, x); ++ } + } +- } + +- /// Decrement the 'rc' counter, abandon the attempt to gain +- /// the 'read' lock. +- transition!{ +- acquire_read_abandon() { +- remove pending_reader -= {()}; +- assert(pre.flag_rc >= 1); +- update flag_rc = (pre.flag_rc - 1) as nat; ++ /// Decrement the 'rc' counter, abandon the attempt to gain ++ /// the 'read' lock. ++ transition!{ ++ acquire_read_abandon() { ++ remove pending_reader -= {()}; ++ assert(pre.flag_rc >= 1); ++ update flag_rc = (pre.flag_rc - 1) as nat; ++ } + } +- } + +- /// Atomically set 'exc' bit from 'false' to 'true' +- /// Obtain a pending_writer +- transition!{ +- acquire_exc_start() { +- require(pre.flag_exc == false); +- update flag_exc = true; +- add pending_writer += Some(()); ++ /// Atomically set 'exc' bit from 'false' to 'true' ++ /// Obtain a pending_writer ++ transition!{ ++ acquire_exc_start() { ++ require(pre.flag_exc == false); ++ update flag_exc = true; ++ add pending_writer += Some(()); ++ } + } +- } + +- /// Finish obtaining the write lock by checking that 'rc' is 0. +- /// Exchange the pending_writer for a writer and withdraw the +- /// stored object. +- transition!{ +- acquire_exc_end() { +- require(pre.flag_rc == 0); ++ /// Finish obtaining the write lock by checking that 'rc' is 0. ++ /// Exchange the pending_writer for a writer and withdraw the ++ /// stored object. ++ transition!{ ++ acquire_exc_end() { ++ require(pre.flag_rc == 0); + +- remove pending_writer -= Some(()); ++ remove pending_writer -= Some(()); + +- add writer += Some(()); ++ add writer += Some(()); + +- birds_eye let x = pre.storage->0; +- withdraw storage -= Some(x); ++ birds_eye let x = pre.storage->0; ++ withdraw storage -= Some(x); + +- assert Pred::inv(pre.k, x); ++ assert Pred::inv(pre.k, x); ++ } + } +- } + +- /// Release the write-lock. Update the 'exc' bit back to 'false'. +- /// Return the 'writer' and also deposit an object back into storage. +- transition!{ +- release_exc(x: V) { +- require Pred::inv(pre.k, x); +- remove writer -= Some(()); ++ /// Release the write-lock. Update the 'exc' bit back to 'false'. ++ /// Return the 'writer' and also deposit an object back into storage. ++ transition!{ ++ release_exc(x: V) { ++ require Pred::inv(pre.k, x); ++ remove writer -= Some(()); + +- update flag_exc = false; ++ update flag_exc = false; + +- deposit storage += Some(x); ++ deposit storage += Some(x); ++ } + } +- } + +- /// Check that the 'reader' is actually a guard for the given object. +- property!{ +- read_guard(x: V) { +- have reader >= {x}; +- guard storage >= Some(x); ++ /// Check that the 'reader' is actually a guard for the given object. ++ property!{ ++ read_guard(x: V) { ++ have reader >= {x}; ++ guard storage >= Some(x); ++ } + } +- } + +- property!{ +- read_match(x: V, y: V) { +- have reader >= {x}; +- have reader >= {y}; +- assert(equal(x, y)); ++ property!{ ++ read_match(x: V, y: V) { ++ have reader >= {x}; ++ have reader >= {y}; ++ assert(equal(x, y)); ++ } + } +- } + +- /// Release the reader-lock. Decrement 'rc' and return the 'reader' object. +- #[transition] +- transition!{ +- release_shared(x: V) { +- remove reader -= {x}; ++ /// Release the reader-lock. Decrement 'rc' and return the 'reader' object. ++ #[transition] ++ transition!{ ++ release_shared(x: V) { ++ remove reader -= {x}; + +- assert(pre.flag_rc >= 1) by { +- //assert(pre.reader.count(x) >= 1); +- assert(equal(pre.storage, Option::Some(x))); +- //assert(equal(x, pre.storage->0)); +- }; +- update flag_rc = (pre.flag_rc - 1) as nat; ++ assert(pre.flag_rc >= 1) by { ++ //assert(pre.reader.count(x) >= 1); ++ assert(equal(pre.storage, Option::Some(x))); ++ //assert(equal(x, pre.storage->0)); ++ }; ++ update flag_rc = (pre.flag_rc - 1) as nat; ++ } + } +- } + +- #[invariant] +- pub fn exc_bit_matches(&self) -> bool { +- (if self.flag_exc { 1 } else { 0 as int }) == +- (if self.pending_writer is Some { 1 } else { 0 as int }) as int +- + (if self.writer is Some { 1 } else { 0 as int }) as int +- } ++ #[invariant] ++ pub fn exc_bit_matches(&self) -> bool { ++ (if self.flag_exc { 1 } else { 0 as int }) == ++ (if self.pending_writer is Some { 1 } else { 0 as int }) as int ++ + (if self.writer is Some { 1 } else { 0 as int }) as int ++ } + +- #[invariant] +- pub fn count_matches(&self) -> bool { +- self.flag_rc == self.pending_reader.count(()) +- + self.reader.count(self.storage->0) +- } ++ #[invariant] ++ pub fn count_matches(&self) -> bool { ++ self.flag_rc == self.pending_reader.count(()) ++ + self.reader.count(self.storage->0) ++ } + +- #[invariant] +- pub fn reader_agrees_storage(&self) -> bool { +- forall |t: V| imply(#[trigger] self.reader.count(t) > 0, +- equal(self.storage, Option::Some(t))) +- } ++ #[invariant] ++ pub fn reader_agrees_storage(&self) -> bool { ++ forall |t: V| imply(#[trigger] self.reader.count(t) > 0, ++ equal(self.storage, Option::Some(t))) ++ } + +- #[invariant] +- pub fn writer_agrees_storage(&self) -> bool { +- imply(self.writer is Some, self.storage is None) +- } ++ #[invariant] ++ pub fn writer_agrees_storage(&self) -> bool { ++ imply(self.writer is Some, self.storage is None) ++ } + +- #[invariant] +- pub fn writer_agrees_storage_rev(&self) -> bool { +- imply(self.storage is None, self.writer is Some) +- } ++ #[invariant] ++ pub fn writer_agrees_storage_rev(&self) -> bool { ++ imply(self.storage is None, self.writer is Some) ++ } + +- #[invariant] +- pub fn sto_user_inv(&self) -> bool { +- self.storage.is_some() ==> Pred::inv(self.k, self.storage.unwrap()) +- } ++ #[invariant] ++ pub fn sto_user_inv(&self) -> bool { ++ self.storage.is_some() ==> Pred::inv(self.k, self.storage.unwrap()) ++ } + +- #[inductive(acquire_read_start)] +- fn acquire_read_start_inductive(pre: Self, post: Self) { +- broadcast use group_multiset_axioms; +- } ++ #[inductive(acquire_read_start)] ++ fn acquire_read_start_inductive(pre: Self, post: Self) { ++ broadcast use group_multiset_axioms; ++ } + +- #[inductive(acquire_read_end)] +- fn acquire_read_end_inductive(pre: Self, post: Self) { +- broadcast use group_multiset_axioms; +- } ++ #[inductive(acquire_read_end)] ++ fn acquire_read_end_inductive(pre: Self, post: Self) { ++ broadcast use group_multiset_axioms; ++ } + +- #[inductive(acquire_read_abandon)] +- fn acquire_read_abandon_inductive(pre: Self, post: Self) { +- broadcast use group_multiset_axioms; +- } ++ #[inductive(acquire_read_abandon)] ++ fn acquire_read_abandon_inductive(pre: Self, post: Self) { ++ broadcast use group_multiset_axioms; ++ } + +- #[inductive(acquire_exc_start)] +- fn acquire_exc_start_inductive(pre: Self, post: Self) { } ++ #[inductive(acquire_exc_start)] ++ fn acquire_exc_start_inductive(pre: Self, post: Self) { } + +- #[inductive(acquire_exc_end)] +- fn acquire_exc_end_inductive(pre: Self, post: Self) { } ++ #[inductive(acquire_exc_end)] ++ fn acquire_exc_end_inductive(pre: Self, post: Self) { } + +- #[inductive(release_exc)] +- fn release_exc_inductive(pre: Self, post: Self, x: V) { } ++ #[inductive(release_exc)] ++ fn release_exc_inductive(pre: Self, post: Self, x: V) { } + +- #[inductive(release_shared)] +- fn release_shared_inductive(pre: Self, post: Self, x: V) { +- broadcast use group_multiset_axioms; +- assert(equal(pre.storage, Option::Some(x))); +- } +-}); ++ #[inductive(release_shared)] ++ fn release_shared_inductive(pre: Self, post: Self, x: V) { ++ broadcast use group_multiset_axioms; ++ assert(equal(pre.storage, Option::Some(x))); ++ } ++ }); + +-verus! { ++ verus! { + + pub trait RwLockPredicate: Sized { + spec fn inv(self, v: V) -> bool; +@@ -709,3 +715,4 @@ impl> RwLock { + } + + } // verus! ++} +diff --git a/source/vstd/thread_view.rs b/source/vstd/thread_view.rs +new file mode 100644 +index 00000000..bfb1c293 +--- /dev/null ++++ b/source/vstd/thread_view.rs +@@ -0,0 +1,396 @@ ++use super::cell::CellId; ++use super::prelude::*; ++use super::resource::algebra; ++use super::resource::pcm; ++use super::view::*; ++ ++/// A type that implements `Objective` cannot carry permissions that depend on a thread's subjective view of memory, ++/// as determined by Rust's weak memory model. This trait is only relevant in the weak memory setting. ++// todo - add tests to ensure the (non-)implementations of this marker trait are as expected ++#[cfg(verus_keep_ghost)] ++pub unsafe auto trait Objective {} ++ ++verus! { ++ ++#[cfg(verus_keep_ghost)] ++#[verifier::external_trait_specification] ++pub trait ExObjective: core::marker::PointeeSized { ++ type ExternalTraitSpecificationFor: Objective; ++} ++ ++/// Represents a thread's subjective view of memory. ++/// For non-atomic memory locations, the relationship between two views is modeled abstractly (i.e., no timestamps are compared). ++/// For atomic memory locations, we can reason about the per-location timestamp contained in a view. ++// This type will be defined with uninterp spec fns, so it should be treated abstractly by the verifier. ++#[verifier::external_body] ++pub ghost struct ThreadView; ++ ++impl ThreadView { ++ /// The empty view. ++ pub uninterp spec fn empty() -> Self; ++ ++ /// True when `other` is contained in `self`. ++ pub uninterp spec fn contains(self, other: Self) -> bool; ++ ++ /// True when `other` is contained in `self` and `other` is not equal to `self`. ++ pub uninterp spec fn contains_strict(self, other: Self) -> bool; ++ ++ /// Returns the union of `self` and `other`. ++ pub uninterp spec fn join(self, other: Self) -> Self; ++ ++ /// View containment is reflexive. ++ pub broadcast axiom fn contains_refl(v: Self) ++ ensures ++ #[trigger] v.contains(v), ++ ; ++ ++ /// View containment is anti-symmetric. ++ pub broadcast axiom fn contains_anti_sym(v1: Self, v2: Self) ++ requires ++ #[trigger] v1.contains(v2), ++ v1 != v2, ++ ensures ++ !(#[trigger] v2.contains(v1)), ++ ; ++ ++ /// View containment is transitive. ++ pub broadcast axiom fn contains_trans(v1: Self, v2: Self, v3: Self) ++ requires ++ #[trigger] v1.contains(v2), ++ #[trigger] v2.contains(v3), ++ ensures ++ #[trigger] v1.contains(v3), ++ ; ++ ++ pub broadcast axiom fn contains_strict_contains(v1: Self, v2: Self) ++ requires ++ #[trigger] v1.contains_strict(v2), ++ ensures ++ v1.contains(v2), ++ ; ++ ++ /// Joining of views is associative. ++ pub broadcast axiom fn join_assoc(v1: Self, v2: Self, v3: Self) ++ ensures ++ #[trigger] v1.join(v2.join(v3)) =~= #[trigger] v1.join(v2).join(v3), ++ ; ++ ++ /// Joining of views is commutative. ++ pub broadcast axiom fn join_comm(v1: Self, v2: Self) ++ ensures ++ #[trigger] v1.join(v2) =~= v2.join(v1), ++ ; ++ ++ /// Joining a view with itself results in the same view. ++ pub broadcast axiom fn join_identity(v: Self) ++ ensures ++ #[trigger] v.join(v) =~= v, ++ ; ++ ++ /// The result of joining a view with another view contains the original view. ++ pub broadcast axiom fn join_contains(v1: Self, v2: Self) ++ ensures ++ #[trigger] v1.join(v2).contains(v1), ++ ; ++} ++ ++pub broadcast group group_thread_view_axioms { ++ ThreadView::contains_refl, ++ ThreadView::contains_anti_sym, ++ ThreadView::contains_trans, ++ ThreadView::contains_strict_contains, ++ ThreadView::join_assoc, ++ ThreadView::join_comm, ++ ThreadView::join_identity, ++ ThreadView::join_contains, ++} ++ ++/// Resource representing a thread's subjective view of memory. ++/// Owning a `ViewSeen` provides a lower-bound on the thread's current view. ++#[derive(Clone, Copy)] ++#[verifier::external_body] ++pub tracked struct ViewSeen; ++ ++impl View for ViewSeen { ++ type V = ThreadView; ++ ++ open spec fn view(&self) -> ThreadView { ++ self.thread_view() ++ } ++} ++ ++impl ViewSeen { ++ /// The view that this permission represents. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// Creates a [`ViewSeen`] permission corresponding to the empty view. ++ pub axiom fn new() -> (tracked out: ViewSeen) ++ ensures ++ out@ == ThreadView::empty(), ++ ; ++ ++ /// Joins this [`ViewSeen`] permission with another [`ViewSeen`] to create a new [`ViewSeen`], ++ /// representing the join of the two views. ++ pub axiom fn join(tracked self, tracked other: Self) -> (tracked out: Self) ++ ensures ++ out@ == self@.join(other@), ++ ; ++ ++ /// Creates a new [`ViewSeen`] representing a view which is contained in the view corresponding to the original [`ViewSeen`]. ++ pub axiom fn weaken(tracked self, v: ThreadView) -> (tracked out: Self) ++ requires ++ self@.contains(v), ++ ensures ++ out@ == v, ++ ; ++} ++ ++/// Resource representing the ``release view" in a thread's subjective view of memory, according to Rust's weak memory model. ++/// If a thread holds a [`ReleaseViewSeen`], then that view that was held by a thread at some point that it performed a release fence in the past. ++#[derive(Clone, Copy)] ++#[verifier::external_body] ++pub tracked struct ReleaseViewSeen; ++ ++impl View for ReleaseViewSeen { ++ type V = ThreadView; ++ ++ open spec fn view(&self) -> ThreadView { ++ self.thread_view() ++ } ++} ++ ++impl ReleaseViewSeen { ++ /// The view that this permission represents. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// Creates a new permission corresponding to the empty view. ++ pub axiom fn new() -> (tracked out: Self) ++ ensures ++ out@ == ThreadView::empty(), ++ ; ++} ++ ++/// Resource representing the ``acquire view" in a thread's subjective view of memory, according to Rust's weak memory model. ++/// If a thread holds an [`AcquireViewSeen`], then that permission represents a view that would be held by a thread ++/// if it were to perform an acquire fence in the future. ++#[derive(Clone, Copy)] ++#[verifier::external_body] ++pub tracked struct AcquireViewSeen; ++ ++impl View for AcquireViewSeen { ++ type V = ThreadView; ++ ++ open spec fn view(&self) -> ThreadView { ++ self.thread_view() ++ } ++} ++ ++impl AcquireViewSeen { ++ /// The view that this permission represents. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// Creates a new permission corresponding to the empty view. ++ pub axiom fn new() -> (tracked out: Self) ++ ensures ++ out@ == ThreadView::empty(), ++ ; ++} ++ ++// ViewSeen permissions are not objective as they represent a thread's subjective view of memory. ++#[cfg(verus_keep_ghost)] ++impl !Objective for ViewSeen { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++impl !Objective for AcquireViewSeen { ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++impl !Objective for ReleaseViewSeen { ++ ++} ++ ++// PCMs and RAs are objective ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for pcm::Resource

{ ++ ++} ++ ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for algebra::Resource { ++ ++} ++ ++// primitive types are objective because they do not hold permissions ++macro_rules! declare_primitive_is_objective { ++ ($($a:ty),*) => { ++ verus! { ++ $( ++ #[cfg(verus_keep_ghost)] ++ unsafe impl Objective for $a {} ++ )* ++ } ++ } ++} ++ ++declare_primitive_is_objective!(bool, char, (), u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, int, nat, str); ++ ++// note: the fact that tuples are Objective (above) suffices for OBJMOD-SEP ++// OBJ with wand update ++#[cfg(verus_keep_ghost)] ++unsafe impl<'a, P: Objective, Q: Objective, F: ProofFnOnce> Objective for proof_fn<'a, F>( ++ tracked p: P, ++) -> tracked Q { ++ ++} ++ ++/// Represents a permission of type `T` which is safe for a thread to own, provided that this thread has ++/// seen a particular view. ++#[derive(Copy)] ++#[verifier::external_body] ++#[verifier::accept_recursive_types(T)] ++pub tracked struct ViewAt { ++ _dummy: core::marker::PhantomData, ++} ++ ++impl Clone for ViewAt { ++ #[verifier::external_body] ++ fn clone(&self) -> Self { ++ unimplemented!() ++ } ++} ++ ++// ViewAt is objective, because it does not give direct access to memory permissions themselves ++#[cfg(verus_keep_ghost)] ++unsafe impl Objective for ViewAt { ++ ++} ++ ++impl ViewAt { ++ /// View that a thread must synchronize with in order to safely start using the inner permission. ++ pub uninterp spec fn thread_view(&self) -> ThreadView; ++ ++ /// The inner permission represented by this [`ViewAt`]. ++ pub uninterp spec fn value(&self) -> T; ++ ++ /// Creates a new [`ViewAt`] from the given permission. ++ /// This permission will be safe to start using at an arbitrary view, ++ /// represented by the [`ViewSeen`] returned by this operation. ++ pub axiom fn new(tracked t: T) -> (tracked (va, vs): (Self, ViewSeen)) ++ ensures ++ va.value() == t, ++ va.thread_view() == vs@, ++ ; ++ ++ /// Creates a new [`ViewAt`] from the given permission and lower bound on the synchronizing view. ++ /// This permission will be safe to start using at some view that is larger than the given view `sn`, ++ /// represented by the [`ViewSeen`] returned by this operation. ++ pub axiom fn new_incl(tracked t: T, tracked vs_0: ViewSeen) -> (tracked (va, vs): ( ++ Self, ++ ViewSeen, ++ )) ++ ensures ++ va.value() == t, ++ va.thread_view() == vs@, ++ va.thread_view().contains(vs_0@), ++ ; ++ ++ // Weaker version of `join_tup`. ++ axiom fn join_tup_inner(tracked v0: ViewAt, tracked v1: ViewAt) -> (tracked out: ++ ViewAt<(T, U)>) ++ requires ++ v0.thread_view() == v1.thread_view(), ++ ensures ++ out.thread_view() == v0.thread_view(), ++ out.value().0 == v0.value(), ++ out.value().1 == v1.value(), ++ ; ++ ++ /// Given two [`ViewAt`] permissions, they can be joined into a single [`ViewAt`] permission, ++ /// whose inner permission a tuple of the original inner permissions, ++ /// and whose synchronizing view is the join of the original synchronizing views. ++ pub proof fn join_tup(tracked v0: ViewAt, tracked v1: ViewAt) -> (tracked out: ViewAt< ++ (T, U), ++ >) ++ ensures ++ out.thread_view() == v0.thread_view().join(v1.thread_view()), ++ out.value().0 == v0.value(), ++ out.value().1 == v1.value(), ++ { ++ let view0 = v0.thread_view(); ++ let view1 = v1.thread_view(); ++ let view_join = view0.join(view1); ++ assert(view_join.contains(view0)) by { ++ ThreadView::join_contains(view0, view1); ++ } ++ assert(view_join.contains(view1)) by { ++ ThreadView::join_comm(view0, view1); ++ ThreadView::join_contains(view1, view0); ++ } ++ let tracked v0 = v0.weaken(view_join); ++ let tracked v1 = v1.weaken(view_join); ++ ViewAt::join_tup_inner(v0, v1) ++ } ++ ++ /// Given a [`ViewAt`] permission, its synchronizing view can be weakened to a larger view. ++ pub axiom fn weaken(tracked self, v: ThreadView) -> (tracked out: Self) ++ requires ++ v.contains(self.thread_view()), ++ ensures ++ out.thread_view() == v, ++ out.value() == self.value(), ++ ; ++ ++ /// Returns the inner permission, provided that the calling thread has obtained the synchronizing view `self.thread_view()`. ++ pub axiom fn into_inner(tracked self, tracked sn: ViewSeen) -> (tracked out: T) ++ requires ++ sn@.contains(self.thread_view()), ++ ensures ++ out == self.value(), ++ ; ++ ++ /// Weaker version of `apply_fn`. ++ axiom fn apply_fn_inner( ++ tracked self, ++ tracked f: ViewAt tracked U>, ++ ) -> (tracked out: ViewAt) ++ requires ++ f.value().requires((self.value(),)), ++ f.thread_view() == self.thread_view(), ++ ensures ++ f.value().ensures((self.value(),), out.value()), ++ out.thread_view() == self.thread_view(), ++ ; ++ ++ /// Given a proof closure `f`, it can be applied to a resource `self` which is ``under" a [`ViewAt`]. ++ /// The resulting resource will be returned under a [`ViewAt`] at some larger view than the original resource. ++ pub proof fn apply_fn( ++ tracked self, ++ tracked f: proof_fn[Once](tracked v1: T) -> tracked U, ++ ) -> (tracked out: ViewAt) ++ requires ++ f.requires((self.value(),)), ++ ensures ++ f.ensures((self.value(),), out.value()), ++ out.thread_view().contains(self.thread_view()), ++ { ++ let tracked va_f = ViewAt::new(f).0; ++ let view1 = va_f.thread_view(); ++ let view2 = self.thread_view(); ++ let view_join = view1.join(view2); ++ assert(view_join.contains(view1)) by { ++ ThreadView::join_contains(view1, view2); ++ } ++ assert(view_join.contains(view2)) by { ++ ThreadView::join_comm(view1, view2); ++ ThreadView::join_contains(view2, view1); ++ } ++ let tracked va_f = va_f.weaken(view_join); ++ let tracked va_t = self.weaken(view_join); ++ va_t.apply_fn_inner(va_f) ++ } ++} ++ ++} // verus! +diff --git a/source/vstd/vstd.rs b/source/vstd/vstd.rs +index 24ef737c..900e9a6c 100644 +--- a/source/vstd/vstd.rs ++++ b/source/vstd/vstd.rs +@@ -22,6 +22,7 @@ + #![cfg_attr(verus_keep_ghost, feature(slice_index_methods))] + #![cfg_attr(all(feature = "alloc", verus_keep_ghost), feature(liballoc_internals))] + #![cfg_attr(verus_keep_ghost, feature(nonzero_internals))] ++#![cfg_attr(verus_keep_ghost, feature(auto_traits))] + + #[cfg(feature = "alloc")] + extern crate alloc; +@@ -30,6 +31,7 @@ pub mod arithmetic; + pub mod array; + pub mod atomic; + pub mod atomic_ghost; ++pub mod atomic_weak; + pub mod bits; + pub mod bytes; + pub mod calc_macro; +@@ -81,6 +83,7 @@ pub mod state_machine_internal; + pub mod string; + #[cfg(feature = "std")] + pub mod thread; ++pub mod thread_view; + pub mod tokens; + pub mod utf8; + pub mod view; +diff --git a/source/vstd_build/src/main.rs b/source/vstd_build/src/main.rs +index 6b849bf2..2250d553 100644 +--- a/source/vstd_build/src/main.rs ++++ b/source/vstd_build/src/main.rs +@@ -37,6 +37,7 @@ fn main() { + let mut no_lifetime = false; + let mut expand_errors = false; + let mut no_solver_version_check = false; ++ let mut weak_memory = false; + for arg in args { + if arg == "--release" { + release = true; +@@ -58,6 +59,8 @@ fn main() { + expand_errors = true; + } else if arg == "--no-solver-version-check" { + no_solver_version_check = true; ++ } else if arg == "--weak-memory" { ++ weak_memory = true; + } else { + panic!("unexpected argument: {:}", arg) + } +@@ -142,6 +145,10 @@ fn main() { + child_args.push("--cfg".to_string()); + child_args.push("feature=\"alloc\"".to_string()); + } ++ if weak_memory { ++ child_args.push("--cfg".to_string()); ++ child_args.push("feature=\"weak-memory\"".to_string()); ++ } + child_args.push("--cfg".to_string()); + child_args.push("feature=\"nonzero_internals\"".to_string()); + child_args.push(VSTD_RS_PATH.to_string()); +diff --git a/tools/vargo/src/cli.rs b/tools/vargo/src/cli.rs +index 47afffb8..f4f2883c 100644 +--- a/tools/vargo/src/cli.rs ++++ b/tools/vargo/src/cli.rs +@@ -66,6 +66,10 @@ pub struct BuildOptions { + /// Turn expand errors on when building vstd + #[arg(long)] + pub vstd_expand_errors: bool, ++ ++ /// Enable use of weak memory ++ #[arg(long)] ++ pub vstd_weak_memory: bool, + } + + #[derive(Clone, Debug, Args, PartialEq, Eq)] +diff --git a/tools/vargo/src/commands/build.rs b/tools/vargo/src/commands/build.rs +index 5afacfe5..affd0a32 100644 +--- a/tools/vargo/src/commands/build.rs ++++ b/tools/vargo/src/commands/build.rs +@@ -490,6 +490,9 @@ fn rebuild_vstd( + if vargo_cmd.build_options.vstd_expand_errors { + vstd_build.arg("--expand-errors"); + } ++ if vargo_cmd.build_options.vstd_weak_memory { ++ vstd_build.arg("--weak-memory"); ++ } + if options.vargo_verbose { + vstd_build.arg("--verbose"); + } diff --git a/verified_libs/vstd_extra/src/atomic_irc11.rs b/verified_libs/vstd_extra/src/atomic_irc11.rs index 9e5a14409..31d5afd1c 100644 --- a/verified_libs/vstd_extra/src/atomic_irc11.rs +++ b/verified_libs/vstd_extra/src/atomic_irc11.rs @@ -229,8 +229,7 @@ impl PAtomicWeakPtr { ) -> ((value, acquire_view, load): (*mut T, Tracked, Ghost)) requires self.loc() == points_to.loc(), - order matches Ordering::Acquire - | | order matches Ordering::Relaxed, + order matches Ordering::Acquire || order matches Ordering::Relaxed, ensures match order { Ordering::Acquire => load_acquire( @@ -270,8 +269,7 @@ impl PAtomicWeakPtr { ) -> (store: Ghost) requires self.loc() == old(points_to).loc(), - order matches Ordering::Release - | | order matches Ordering::Relaxed, + order matches Ordering::Release || order matches Ordering::Relaxed, ensures forall|observed_view: ThreadView| #[trigger] old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( @@ -325,11 +323,10 @@ impl PAtomicWeakPtr { requires self.loc() == old(points_to).loc(), success matches Ordering::AcqRel - | | success matches Ordering::Acquire - | | success matches Ordering::Release - | | success matches Ordering::Relaxed, - failure matches Ordering::Acquire - | | failure matches Ordering::Relaxed, + || success matches Ordering::Acquire + || success matches Ordering::Release + || success matches Ordering::Relaxed, + failure matches Ordering::Acquire || failure matches Ordering::Relaxed, ensures result is Ok ==> old(points_to).hist().is_max_timestamp(update@.load_timestamp), forall|observed_view: ThreadView| #[trigger] From 5903d21d536e60b0c6a1f5a63656c46c0d514461 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 4 Aug 2026 05:48:44 -0400 Subject: [PATCH 37/47] Fix pinned Verus checkout in CI --- .github/workflows/ci-macos.yml | 6 ++++-- .github/workflows/ci-upstream-verus.yml | 6 ++++-- .github/workflows/ci.yml | 6 ++++-- .github/workflows/doc.yml | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 5679dc9f1..d5cba5ac9 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -86,8 +86,10 @@ jobs: else echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - git clone --no-checkout "$VERUS_REPOSITORY" tools/verus - git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" cargo dv bootstrap diff --git a/.github/workflows/ci-upstream-verus.yml b/.github/workflows/ci-upstream-verus.yml index 2bc4fa156..36f338f20 100644 --- a/.github/workflows/ci-upstream-verus.yml +++ b/.github/workflows/ci-upstream-verus.yml @@ -75,8 +75,10 @@ jobs: - name: Bootstrap rebased Verus IRC11 run: | rm -rf tools/verus - git clone --no-checkout "$VERUS_REPOSITORY" tools/verus - git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" cargo dv bootstrap test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38b8b6d4f..2871da9b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,8 +100,10 @@ jobs: else echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - git clone --no-checkout "$VERUS_REPOSITORY" tools/verus - git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" cargo dv bootstrap diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 292cdfed0..dc0b38588 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -102,8 +102,10 @@ jobs: else echo "Cache miss, bootstrapping rebased Verus IRC11..." rm -rf tools/verus - git clone --no-checkout "$VERUS_REPOSITORY" tools/verus - git -C tools/verus checkout --detach "$VERUS_BASE_COMMIT" + git init tools/verus + git -C tools/verus remote add origin "$VERUS_REPOSITORY" + git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" + git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" cargo dv bootstrap From 4df23988f140b671516f773cec1d7cb022df5042 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Tue, 4 Aug 2026 06:03:45 -0400 Subject: [PATCH 38/47] Fix IRC11 atomic ordering predicates --- verified_libs/vstd_extra/src/atomic_irc11.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/verified_libs/vstd_extra/src/atomic_irc11.rs b/verified_libs/vstd_extra/src/atomic_irc11.rs index 31d5afd1c..582c640c3 100644 --- a/verified_libs/vstd_extra/src/atomic_irc11.rs +++ b/verified_libs/vstd_extra/src/atomic_irc11.rs @@ -229,7 +229,7 @@ impl PAtomicWeakPtr { ) -> ((value, acquire_view, load): (*mut T, Tracked, Ghost)) requires self.loc() == points_to.loc(), - order matches Ordering::Acquire || order matches Ordering::Relaxed, + order == Ordering::Acquire || order == Ordering::Relaxed, ensures match order { Ordering::Acquire => load_acquire( @@ -269,7 +269,7 @@ impl PAtomicWeakPtr { ) -> (store: Ghost) requires self.loc() == old(points_to).loc(), - order matches Ordering::Release || order matches Ordering::Relaxed, + order == Ordering::Release || order == Ordering::Relaxed, ensures forall|observed_view: ThreadView| #[trigger] old(points_to).get_timestamp(observed_view) == final(points_to).get_timestamp( @@ -322,11 +322,9 @@ impl PAtomicWeakPtr { )) requires self.loc() == old(points_to).loc(), - success matches Ordering::AcqRel - || success matches Ordering::Acquire - || success matches Ordering::Release - || success matches Ordering::Relaxed, - failure matches Ordering::Acquire || failure matches Ordering::Relaxed, + success == Ordering::AcqRel || success == Ordering::Acquire || success + == Ordering::Release || success == Ordering::Relaxed, + failure == Ordering::Acquire || failure == Ordering::Relaxed, ensures result is Ok ==> old(points_to).hist().is_max_timestamp(update@.load_timestamp), forall|observed_view: ThreadView| #[trigger] From 97b630e2a1fadac64539bd9f802435982b58184e Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 5 Aug 2026 04:03:47 -0400 Subject: [PATCH 39/47] clean up weak memory and add glues --- ostd/specs/sync/rcu.rs | 118 + ostd/specs/sync/rcu_cpu.rs | 1745 +++++++++++- ostd/specs/sync/weak_memory.rs | 1404 ++++++++-- ostd/src/sync/rcu/mod.rs | 737 ++++- ostd/src/sync/rcu/monitor.rs | 230 +- ostd/src/task/preempt/guard.rs | 12 + ostd/src/task/scheduler/mod.rs | 4 +- verified_libs/vstd_extra/src/atomic_weak.rs | 2417 +---------------- verified_libs/vstd_extra/src/raw_callback.rs | 99 + verified_libs/vstd_extra/src/rcu_read_pool.rs | 86 + 10 files changed, 4080 insertions(+), 2772 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index f7305c9fe..dabc6ddff 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -617,6 +617,12 @@ pub open spec fn rcu_history_inv(nullable: bool, history: Irc11History<*mut T history.contains_timestamp(ts) ==> #[trigger] history.value(ts).addr() != 0 } +/// Scheduler registry used by the kernel's singleton RCU domain. +/// +/// Scheduler integration must establish that every `RunningTaskContext` +/// passed to the global RCU API is checked out from this registry. +pub uninterp spec fn rcu_scheduler() -> Loc; + /// Immutable identity carried by an executable RCU root atomic. /// /// Besides nullability, the key records the two resource locations needed to @@ -624,9 +630,13 @@ pub open spec fn rcu_history_inv(nullable: bool, history: Irc11History<*mut T /// invariant has been closed. pub ghost struct RcuRootKey { pub nullable: bool, + /// Scheduler registry whose canonical CPU participants protect this root. + pub scheduler: Loc, pub domain: Loc, pub reader_registry: Loc, pub retire_observation_registry: Loc, + pub reclaim_registry: Loc, + pub active_lease_registry: Loc, } /// Typed ownership state paired with one executable RCU root atomic. @@ -670,6 +680,21 @@ impl RcuRootOwnedGhost { self.root().domain_auth().retire_observation_registry() } + /// Agrees a persistent callback retirement fact with this root's + /// authoritative removal map. + pub proof fn lemma_retired_fact_agrees(tracked &self, tracked fact: &RcuRetiredFact) + requires + self.root().domain_wf(), + self.removals() == self.root().domain_auth().retire_observations(), + fact.wf(), + fact.domain() == self.domain(), + fact.retire_observation_registry() == self.retire_observation_registry(), + ensures + self.removals().contains_pair(fact.obj(), fact.removal()), + { + fact.lemma_observation_agrees(&self.root.domain); + } + pub open spec fn published_at(self, ts: nat) -> Option recommends self.publications().contains_key(ts), @@ -714,6 +739,13 @@ impl RcuRootOwnedGhost { /// `BlockInfo` available to justify stale weak-memory history reads. pub open spec fn infos_wf(self) -> bool { &&& self.infos().dom() == self.root().objects().dom() + &&& match self.current_owned() { + Some(owned) => { + &&& self.infos().contains_key(owned.block_info().obj()) + &&& equal(self.infos()[owned.block_info().obj()].ptr(), owned.block_info().ptr()) + }, + None => true, + } &&& forall|obj: nat| self.infos().contains_key(obj) ==> { let info = #[trigger] self.infos()[obj]; @@ -783,11 +815,13 @@ impl RcuRootOwnedGhost { (Some(object), Some(info)) => { &&& object.domain == self.domain() &&& object.addr == history.value(ts).addr() + &&& self.infos().contains_key(info.obj()) &&& info.wf() &&& info.domain() == object.domain &&& info.obj() == object.obj &&& info.addr() == object.addr &&& equal(info.ptr(), history.value(ts)) + &&& equal(info.ptr(), self.infos()[info.obj()].ptr()) }, _ => false, }, @@ -914,6 +948,7 @@ impl RcuRootOwnedGhost { final(self).reader_registry() == old(self).reader_registry(), final(self).retire_observation_registry() == old(self).retire_observation_registry(), final(self).current_owned() == old(self).current_owned(), + final(self).current_registration() == old(self).current_registration(), final(self).publications() == old(self).publications(), final(self).infos() == old(self).infos(), final(self).removals() == old(self).removals(), @@ -958,6 +993,10 @@ impl RcuRootOwnedGhost { == ptr, res.current_ownership() == ownership, res.removals() == Map::::empty(), + res.infos().dom() == match res.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }, match res.current_owned() { Some(owned) => { &&& ptr.addr() != 0 @@ -1000,6 +1039,10 @@ impl RcuRootOwnedGhost { }, }; let tracked res = RcuRootOwnedGhost { root, current, infos, removals: Map::empty() }; + assert(res.infos().dom() == match res.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); assert(res.infos_wf()); assert(res.removals_wf(history)); assert(res.removals() == res.root().domain_auth().retire_observations()); @@ -1041,6 +1084,9 @@ impl RcuRootOwnedGhost { Some(detached) => { &&& old(self).current_registration() is Some &&& detached.object() == old(self).current_registration()->Some_0.0 + &&& detached.object().domain() == old(self).domain() + &&& detached.obj() == old(self).current_registration()->Some_0.0.obj() + &&& equal(detached.ptr(), old(self).infos()[detached.obj()].ptr()) &&& detached.retired().domain() == detached.domain() &&& detached.retired().obj() == detached.obj() &&& detached.retired().ptr() == detached.ptr() @@ -1049,6 +1095,9 @@ impl RcuRootOwnedGhost { timestamp: new_timestamp, message_view, }) + &&& detached.retired().retire_observation_registry() == old( + self, + ).retire_observation_registry() &&& equal(detached.ptr(), prev.value(old_timestamp)) &&& old(self).current_ownership() == Some(detached.ownership()) &&& OwnPred::owns(detached.ptr(), detached.ownership()) @@ -1059,6 +1108,19 @@ impl RcuRootOwnedGhost { final(self).current_registration() is Some ==> final(self).current_registration()->Some_0.0.ptr() == value, final(self).current_ownership() == ownership, + match final(self).current_registration() { + Some(registration) => { + &&& !old(self).infos().contains_key(registration.0.obj()) + &&& final(self).infos().dom() == old(self).infos().dom().insert( + registration.0.obj(), + ) + &&& forall|obj: nat| #[trigger] + old(self).infos().contains_key(obj) ==> final(self).infos()[obj] == old( + self, + ).infos()[obj] + }, + None => final(self).infos() == old(self).infos(), + }, final(self).removals() == match detached { Some(detached) => old(self).removals().insert( detached.obj(), @@ -1142,6 +1204,13 @@ impl RcuRootOwnedGhost { None => None, }; self.current = new_current; + assert(match self.current_registration() { + Some(registration) => { + &&& !old(self).infos().contains_key(registration.0.obj()) + &&& self.infos().dom() == old(self).infos().dom().insert(registration.0.obj()) + }, + None => self.infos() == old(self).infos(), + }); self.removals = match removed_obj { Some(obj) => self.removals.insert(obj, removal), None => self.removals, @@ -2085,6 +2154,7 @@ impl RcuDomainAuth { res.obj() == retire.obj(), res.ptr() == retire.ptr(), res.removal() == removal, + res.retire_observation_registry() == final(self).retire_observation_registry(), res.wf(), { let ghost domain = retire.domain(); @@ -2334,6 +2404,16 @@ impl RcuBlockInfo { &&& self.ptr().addr() != 0 } + /// Opens the address facts hidden by the block-info abstraction. + pub proof fn lemma_wf_facts(tracked &self) + requires + self.wf(), + ensures + self.addr() == self.ptr().addr(), + self.ptr().addr() != 0, + { + } + /// Persistent block information can be retained by both the client and /// every historical atomic message that mentions the allocation. pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) @@ -2945,6 +3025,11 @@ impl RcuRetired { self.fact.removal() } + /// Authoritative observation registry that issued this retirement fact. + pub closed spec fn retire_observation_registry(self) -> Loc { + self.fact.retire_observation_registry() + } + pub closed spec fn wf(self) -> bool { &&& self.fact.wf() &&& self.addr() == self.ptr().addr() @@ -2958,6 +3043,7 @@ impl RcuRetired { res.obj() == self.obj(), res.addr() == self.ptr().addr(), res.removal() == self.removal(), + res.retire_observation_registry() == self.retire_observation_registry(), { self.fact } @@ -3136,6 +3222,7 @@ pub proof fn certify_callback_from_retired( cert.domain() == retired.domain(), cert.obj() == object.obj(), cert.removal() == retired.removal(), + cert.retire_observation_registry() == retired.retire_observation_registry(), callback_safety_from_traversal(cert, *object), { use_type_invariant(&retired); @@ -3381,6 +3468,37 @@ impl RcuProtectedPtr { &&& !self.seen_removed().removed.contains(self.obj()) &&& guard.protects(self.ptr().addr(), self.obj()) } + + /// Materializes the linear protection witness for a pointer already + /// installed in a direct-root guard's protection map. + /// + /// Generic traversal clients should normally use [`protect_root`] or + /// [`protect_next`]. This constructor is for the executable `Rcu

` + /// adapter, whose guarded atomic load performs the root protection + /// transition before returning to the caller. + pub proof fn tracked_from_guard( + tracked guard: &RcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: Self) + requires + guard.wf(), + info.wf(), + info.domain() == guard.domain(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + ensures + res.domain() == info.domain(), + res.obj() == info.obj(), + res.ptr() == info.ptr(), + res.protected_by(*guard), + { + RcuProtectedPtr { + domain: info.domain(), + obj: info.obj(), + ptr: info.ptr(), + seen_removed: guard.seen_removed(), + } + } } /// Traversal specification for an RCU-protected data structure. diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs index abed5904e..02360faaa 100644 --- a/ostd/specs/sync/rcu_cpu.rs +++ b/ostd/specs/sync/rcu_cpu.rs @@ -46,7 +46,7 @@ //! counter and is not an authority for this persistent CPU generation. Reader //! contexts obtain their CPU generation from [`CpuRcuReaderFragment`]. use crate::specs::{ - mm::cpu::CpuId, + mm::cpu::{CpuId, online_cpus}, task::cpu_core::{CpuCoreLocalState, CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreRegistration}, }; use vstd::{ @@ -57,7 +57,7 @@ use vstd::{ agree::AgreementRA, algebra::{Resource, ResourceAlgebra}, frac::FractionRA, - map::GhostMapAuth, + map::{GhostMapAuth, GhostPointsTo}, product::ProductRA, relations::frame_preserving_update_opt, }, @@ -65,9 +65,10 @@ use vstd::{ use super::rcu::{ RcuBlockInfo, RcuInactive, RcuProtectedPtr, RcuReadGuardToken, RcuReaderContext, - RcuRetiredFacts, RcuRetiredRecord, RcuSeenRemoved, + RcuRetiredFact, RcuRetiredFacts, RcuRetiredRecord, RcuSeenRemoved, }; use vstd_extra::atomic_irc11::{ThreadView as Irc11ThreadView, ThreadViewOrder}; +use vstd_extra::rcu_read_pool::{RcuTrackedReadLease, RcuTrackedReadPoolRegistry}; verus! { @@ -295,6 +296,102 @@ pub tracked struct CpuRcuReadGuardToken { binding: CpuRcuCoreBinding, } +/// CPU-generation witness retained beside one active physical read lease. +/// +/// The executable guard keeps the other half of `reader`. The ghost snapshot +/// records which abstract guard protected the allocation without duplicating +/// that guard's linear `Guard(tid, X, G)` resource. +#[verifier::reject_recursive_types(T)] +pub tracked struct CpuRcuReadLeaseWitness { + reader: CpuRcuReaderFragment, + ghost paper_guard: RcuReadGuardToken, + binding: CpuRcuCoreBinding, + protected: RcuProtectedPtr, +} + +/// Physical-permission pools associated with one RCU root. +/// +/// A pool is indexed by the allocation identity from [`RcuBlockInfo`], rather +/// than by its address. Pools therefore survive root replacement and remain +/// distinguishable when a reclaimed address is later reused. Every active +/// lease record retains the CPU-generation witness needed by the monitor to +/// rule it out after a completed grace period. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRootPermissionState { + registry: RcuTrackedReadPoolRegistry>, + active_leases: GhostMapAuth, + reclaim_state: GhostMapAuth>, + unretired_claims: Map>>, + reclaimed: Map, + ghost scheduler: Loc, + ghost domain: Loc, + ghost root: Loc, + ghost retire_observation_registry: Loc, +} + +/// Authoritative metadata for one lease currently registered at an RCU root. +/// +/// The matching [`RcuRootReadLease`] owns the linear map points-to token. Its +/// agreement with the authority lets a later invariant opening recover the +/// exact active record created by the guarded load. +pub ghost struct RcuActiveLeaseBinding { + key: nat, + pool_id: Loc, + fraction: real, + participant_id: Loc, + reader_fraction: real, + domain: Loc, + root: Loc, + reader: RcuReaderContext, + start_view: Irc11ThreadView, + protected_addr: usize, +} + +/// Physical read lease together with proof that its active record still +/// belongs to the same RCU root. +#[verifier::reject_recursive_types(O)] +pub tracked struct RcuRootReadLease { + lease: RcuTrackedReadLease, + active: GhostPointsTo, +} + +/// Unique right to reclaim one retired allocation's physical permission. +/// +/// The matching authority remains in [`RcuRootPermissionState`]. Validation +/// against that authority proves that the allocation has not already been +/// reclaimed; consuming the claim changes its authoritative state exactly +/// once. +pub tracked struct RcuReclaimClaim { + points_to: GhostPointsTo>, +} + +/// Persistent grace-period evidence retained after one allocation is reclaimed. +/// +/// A later weak load may still select the allocation's old atomic-history +/// message. The closed generation for its CPU proves that such a coexisting +/// reader is newer and already carries `record` in its retired set. +pub tracked struct RcuReclaimedWitness { + ghost record: RcuRetiredRecord, + ghost scheduler: Loc, + retired: RcuRetiredFact, + closed_generations: Map, +} + +unsafe impl< + T, + O: vstd::thread_view::Objective, +> vstd::thread_view::Objective for RcuRootPermissionState { + +} + +unsafe impl vstd::thread_view::Objective for RcuReclaimClaim { + +} + +unsafe impl vstd::thread_view::Objective for RcuReclaimedWitness { + +} + proof fn lemma_choose_singleton_report(report: CpuRcuReportView) ensures (choose|candidate: CpuRcuReportView| Set::empty().insert(report).contains(candidate)) @@ -919,6 +1016,90 @@ impl CpuRcuReaderFragment { pub open spec fn wf(self) -> bool { 0real < self.fraction() <= 1real } + + /// Splits this live-reader authority into two equal fragments. + /// + /// RCU uses the second fragment as the active-lease witness retained by + /// the root invariant. The first remains in the executable read guard. + /// Neither fragment alone permits the CPU participant to report a + /// quiescent state. + pub proof fn tracked_split(tracked self) -> (tracked res: (Self, Self)) + requires + self.wf(), + ensures + res.0.wf(), + res.1.wf(), + res.0.participant_id() == self.participant_id(), + res.1.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.1.cpu() == self.cpu(), + res.0.generation() == self.generation(), + res.1.generation() == self.generation(), + res.0.participant_view() == self.participant_view(), + res.1.participant_view() == self.participant_view(), + res.0.known_retired() == self.known_retired(), + res.1.known_retired() == self.known_retired(), + res.0.fraction() == self.fraction() / 2real, + res.1.fraction() == self.fraction() / 2real, + { + use_type_invariant(&self); + let ghost half = self.fraction() / 2real; + let ghost carrier = CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.participant_view(), + self.known_retired(), + half, + ); + assert(0real < half <= 1real); + assert(FractionRA::op(FractionRA::Frac(half), FractionRA::Frac(half)) == FractionRA::Frac( + self.fraction(), + )); + assert(AgreementRA::op( + AgreementRA::Agree(self.resource.value().state_view()), + AgreementRA::Agree(self.resource.value().state_view()), + ) == AgreementRA::Agree(self.resource.value().state_view())); + assert(self.resource.value() == CpuRcuCarrier::state( + self.cpu(), + self.generation(), + self.participant_view(), + self.known_retired(), + self.fraction(), + )); + assert(Option::::op(carrier.state, carrier.state) + == self.resource.value().state); + assert(carrier.closed.union(carrier.closed) == self.resource.value().closed); + assert(self.resource.value() == CpuRcuCarrier::op(carrier, carrier)); + let tracked duplicate_known_retired = self.known_retired.tracked_duplicate(); + let tracked (left, right) = self.resource.split(carrier, carrier); + ( + CpuRcuReaderFragment { resource: left, known_retired: self.known_retired }, + CpuRcuReaderFragment { resource: right, known_retired: duplicate_known_retired }, + ) + } + + /// Recombines two fragments from the same CPU participant generation. + pub proof fn tracked_join(tracked self, tracked other: Self) -> (tracked res: Self) + requires + self.wf(), + other.wf(), + self.participant_id() == other.participant_id(), + ensures + res.wf(), + res.participant_id() == self.participant_id(), + res.cpu() == self.cpu(), + res.generation() == self.generation(), + res.participant_view() == self.participant_view(), + res.known_retired() == self.known_retired(), + res.fraction() == self.fraction() + other.fraction(), + { + use_type_invariant(&self); + use_type_invariant(&other); + let tracked mut resource = self.resource; + resource.validate_2(&other.resource); + let tracked resource = resource.join(other.resource); + CpuRcuReaderFragment { resource, known_retired: self.known_retired } + } } impl CpuRcuReadGuardToken { @@ -1042,6 +1223,7 @@ impl CpuRcuReadGuardToken { pub open spec fn wf(self) -> bool { &&& self.paper_guard().wf() &&& self.reader_fragment().wf() + &&& self.binding().locals_key().len() == 1 &&& self.binding().single_local_id() == self.participant_id() &&& self.binding().cpu() == self.cpu() &&& self.scheduler() == self.reader_context().scheduler @@ -1068,6 +1250,7 @@ impl CpuRcuReadGuardToken { paper_guard.reader().generation == reader.generation(), binding.registry() == paper_guard.reader().scheduler, binding.cpu() == reader.cpu(), + binding.locals_key().len() == 1, binding.single_local_id() == reader.participant_id(), reader.participant_view().spec_le(paper_guard.start_view()), forall|record: RcuRetiredRecord| #[trigger] @@ -1081,6 +1264,7 @@ impl CpuRcuReadGuardToken { res.reader_fragment() == reader, res.reader_context() == paper_guard.reader(), res.scheduler() == binding.registry(), + res.binding() == binding, res.binding().cpu() == binding.cpu(), res.binding().single_local_id() == binding.single_local_id(), res.participant_id() == reader.participant_id(), @@ -1094,6 +1278,7 @@ impl CpuRcuReadGuardToken { res.reader_registry() == paper_guard.reader_registry(), res.retire_observation_registry() == paper_guard.retire_observation_registry(), res.expired() == paper_guard.expired(), + res.seen_removed() == paper_guard.seen_removed(), res.protected() == paper_guard.protected(), { CpuRcuReadGuardToken { paper_guard, reader, binding } @@ -1116,6 +1301,7 @@ impl CpuRcuReadGuardToken { res.1 == self.reader_fragment(), res.2.registry() == self.scheduler(), res.2.cpu() == self.cpu(), + res.2.locals_key().len() == 1, res.2.single_local_id() == self.participant_id(), res.0.wf(), res.1.wf(), @@ -1173,6 +1359,108 @@ impl CpuRcuReadGuardToken { (inactive, reader) } + /// Splits out the CPU fragment retained with an active physical read + /// lease, while preserving a valid guard for the executable reader. + pub proof fn tracked_split_lease_fragment(tracked self) -> (tracked res: ( + Self, + CpuRcuReaderFragment, + )) + requires + self.wf(), + ensures + res.0.wf(), + res.0.paper_guard() == self.paper_guard(), + res.0.binding() == self.binding(), + res.0.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.0.generation() == self.generation(), + res.0.participant_view() == self.participant_view(), + res.0.known_retired() == self.known_retired(), + res.0.reader_fragment().fraction() == self.reader_fragment().fraction() / 2real, + res.1.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + res.1.participant_view() == self.participant_view(), + res.1.known_retired() == self.known_retired(), + res.1.fraction() == self.reader_fragment().fraction() / 2real, + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + let tracked CpuRcuReadGuardToken { paper_guard, reader, binding } = self; + let tracked (reader, lease_reader) = reader.tracked_split(); + assert forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry == paper_guard.retire_observation_registry() + && record.removal.root == paper_guard.root() implies paper_guard.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + let tracked guard = CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding); + (guard, lease_reader) + } + + /// Returns an active lease's CPU fragment to its executable guard before + /// the normal `Guard -> Inactive` transition. + pub proof fn tracked_join_lease_fragment( + tracked self, + tracked lease_reader: CpuRcuReaderFragment, + ) -> (tracked res: Self) + requires + self.wf(), + lease_reader.wf(), + self.participant_id() == lease_reader.participant_id(), + ensures + res.wf(), + res.paper_guard() == self.paper_guard(), + res.binding() == self.binding(), + res.participant_id() == self.participant_id(), + res.cpu() == self.cpu(), + res.generation() == self.generation(), + res.participant_view() == self.participant_view(), + res.known_retired() == self.known_retired(), + res.domain() == self.domain(), + res.root() == self.root(), + res.reader_registry() == self.reader_registry(), + res.retire_observation_registry() == self.retire_observation_registry(), + res.reader_context() == self.reader_context(), + res.start_view() == self.start_view(), + res.expired() == self.expired(), + res.seen_removed() == self.seen_removed(), + res.protected() == self.protected(), + res.reader_fragment().fraction() == self.reader_fragment().fraction() + + lease_reader.fraction(), + { + let ghost known_retired = self.known_retired(); + let ghost domain = self.domain(); + let ghost retire_observation_registry = self.retire_observation_registry(); + let ghost root = self.root(); + let ghost expired = self.expired(); + assert forall|record: RcuRetiredRecord| #[trigger] + known_retired.contains(record) && record.domain == domain + && record.retire_observation_registry == retire_observation_registry + && record.removal.root == root implies expired.contains(record.obj) by {}; + let tracked CpuRcuReadGuardToken { paper_guard, reader, binding } = self; + let tracked reader = reader.tracked_join(lease_reader); + assert forall|record: RcuRetiredRecord| #[trigger] + reader.known_retired().contains(record) && record.domain == paper_guard.domain() + && record.retire_observation_registry == paper_guard.retire_observation_registry() + && record.removal.root == paper_guard.root() implies paper_guard.expired().contains( + record.obj, + ) by { + assert(known_retired.contains(record)); + }; + CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding) + } + /// Applies the paper's `Guard-protect` rule without changing CPU /// participation or the captured start view. pub proof fn tracked_protect(tracked &mut self, tracked info: &RcuBlockInfo) @@ -1220,79 +1508,1351 @@ impl CpuRcuReadGuardToken { } } -impl CpuRcuClosedGeneration { - #[verifier::type_invariant] - closed spec fn type_inv(self) -> bool { - &&& self.resource.value().state is None - &&& self.resource.value().closed =~= Set::empty().insert(self.report()) - &&& self.report().known_retired == self.known_retired.records() - &&& self.binding.single_local_id() == self.resource.loc() - &&& self.binding.cpu() == self.report().cpu - &&& CpuRcuCarrier::records_observed(self.report().known_retired, self.report().view) - } - - closed spec fn report(self) -> CpuRcuReportView { - choose|report: CpuRcuReportView| self.resource.value().closed.contains(report) +impl CpuRcuReadLeaseWitness { + pub closed spec fn reader(self) -> CpuRcuReaderFragment { + self.reader } - pub closed spec fn participant_id(self) -> Loc { - self.resource.loc() + pub closed spec fn paper_guard(self) -> RcuReadGuardToken { + self.paper_guard } + /// Persistent scheduler registration for the CPU participant retained by + /// this lease. This survives type erasure in the root registry so a grace + /// period report can recover the canonical participant for the same CPU. pub closed spec fn binding(self) -> CpuRcuCoreBinding { self.binding } - pub closed spec fn scheduler(self) -> Loc { - self.binding().registry() + pub closed spec fn protected(self) -> RcuProtectedPtr { + self.protected } - pub closed spec fn cpu(self) -> CpuId { - self.report().cpu + pub open spec fn wf(self) -> bool { + &&& self.reader().wf() + &&& self.paper_guard().wf() + &&& online_cpus().contains(self.reader().cpu()) + &&& self.binding().registry() == self.paper_guard().reader().scheduler + &&& self.binding().cpu() == self.reader().cpu() + &&& self.binding().locals_key().len() == 1 + &&& self.binding().single_local_id() == self.reader().participant_id() + &&& self.paper_guard().reader().cpu == self.reader().cpu() + &&& self.paper_guard().reader().generation == self.reader().generation() + &&& self.reader().participant_view().spec_le(self.paper_guard().start_view()) + &&& self.paper_guard().expired().subset_of(self.paper_guard().seen_removed().removed) + &&& forall|record: RcuRetiredRecord| #[trigger] + self.reader().known_retired().contains(record) && record.domain + == self.paper_guard().domain() && record.retire_observation_registry + == self.paper_guard().retire_observation_registry() && record.removal.root + == self.paper_guard().root() ==> self.paper_guard().expired().contains(record.obj) + &&& self.protected().protected_by(self.paper_guard()) } - pub closed spec fn closed_generation(self) -> nat { - self.report().generation + /// Relates this lease to the canonical participant closed by a grace-period + /// report for the same scheduler and CPU. + pub proof fn lemma_same_participant_as_closed( + tracked &self, + tracked closed: &CpuRcuClosedGeneration, + ) + requires + self.wf(), + closed.wf(), + closed.scheduler() == self.binding().registry(), + closed.cpu() == self.reader().cpu(), + ensures + self.reader().participant_id() == closed.participant_id(), + { + closed.lemma_same_participant_as_binding(&self.binding); } - pub closed spec fn view(self) -> Irc11ThreadView { - self.report().view + /// Splits the registry fragment from a live executable guard. + pub proof fn tracked_from_guard( + tracked guard: CpuRcuReadGuardToken, + tracked protected: RcuProtectedPtr, + ) -> (tracked res: (CpuRcuReadGuardToken, Self)) + requires + guard.wf(), + protected.protected_by(guard.paper_guard()), + online_cpus().contains(guard.cpu()), + ensures + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.participant_id() == guard.participant_id(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.known_retired() == guard.known_retired(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.wf(), + res.1.reader().participant_id() == guard.participant_id(), + res.1.reader().cpu() == guard.cpu(), + res.1.reader().generation() == guard.generation(), + res.1.reader().participant_view() == guard.participant_view(), + res.1.reader().known_retired() == guard.known_retired(), + res.1.reader().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.paper_guard() == guard.paper_guard(), + res.1.binding().registry() == guard.scheduler(), + res.1.binding().cpu() == guard.cpu(), + res.1.binding().locals_key() == guard.binding().locals_key(), + res.1.binding().single_local_id() == guard.participant_id(), + res.1.protected() == protected, + { + guard.lemma_expired_is_removed(); + let ghost paper_guard = guard.paper_guard(); + let tracked binding = guard.binding.tracked_duplicate(); + let tracked (guard, reader) = guard.tracked_split_lease_fragment(); + let tracked witness = CpuRcuReadLeaseWitness { reader, paper_guard, binding, protected }; + (guard, witness) } - pub closed spec fn known_retired(self) -> Set { - self.known_retired.records() + /// Builds the direct-root protection witness returned by a guarded atomic + /// load and splits off the registry's CPU-reader fragment in one step. + pub proof fn tracked_from_loaded_guard( + tracked guard: CpuRcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: (CpuRcuReadGuardToken, Self)) + requires + guard.wf(), + info.wf(), + info.domain() == guard.domain(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + online_cpus().contains(guard.cpu()), + ensures + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.wf(), + res.1.reader().participant_id() == guard.participant_id(), + res.1.reader().cpu() == guard.cpu(), + res.1.reader().generation() == guard.generation(), + res.1.reader().participant_view() == guard.participant_view(), + res.1.reader().known_retired() == guard.known_retired(), + res.1.reader().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.paper_guard() == guard.paper_guard(), + res.1.binding().registry() == guard.scheduler(), + res.1.binding().cpu() == guard.cpu(), + res.1.binding().locals_key() == guard.binding().locals_key(), + res.1.binding().single_local_id() == guard.participant_id(), + res.1.protected().domain() == info.domain(), + res.1.protected().obj() == info.obj(), + res.1.protected().ptr() == info.ptr(), + { + let tracked protected = RcuProtectedPtr::tracked_from_guard(&guard.paper_guard, info); + Self::tracked_from_guard(guard, protected) } - pub closed spec fn wf(self) -> bool { - &&& self.resource.value() == CpuRcuCarrier::closed(self.report()) - &&& self.binding().single_local_id() == self.participant_id() - &&& self.binding().cpu() == self.cpu() + /// Returns the registry fragment to the corresponding executable guard. + pub proof fn tracked_return_to_guard( + tracked self, + tracked guard: CpuRcuReadGuardToken, + ) -> (tracked res: CpuRcuReadGuardToken) + requires + self.wf(), + guard.wf(), + self.reader().participant_id() == guard.participant_id(), + ensures + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.binding() == guard.binding(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.participant_view() == guard.participant_view(), + res.known_retired() == guard.known_retired(), + res.reader_fragment().fraction() == guard.reader_fragment().fraction() + + self.reader().fraction(), + { + guard.tracked_join_lease_fragment(self.reader) + } +} + +impl RcuReclaimClaim { + pub closed spec fn registry(self) -> Loc { + self.points_to.id() } - /// Splits the idempotent closed-generation fact. - pub proof fn tracked_duplicate(tracked self) -> (tracked res: ( - CpuRcuClosedGeneration, - CpuRcuClosedGeneration, - )) + pub closed spec fn obj(self) -> nat { + self.points_to.key() + } + + pub closed spec fn is_pending(self) -> bool { + self.points_to.value() is Some + } + + pub closed spec fn ptr(self) -> *mut T + recommends + self.is_pending(), + { + self.points_to.value()->Some_0 + } +} + +impl RcuReclaimedWitness { + pub closed spec fn record(self) -> RcuRetiredRecord { + self.record + } + + pub closed spec fn closed_generations(self) -> Map { + self.closed_generations + } + + pub closed spec fn scheduler(self) -> Loc { + self.scheduler + } + + /// Persistent base-retirement fact retained after physical reclamation. + pub closed spec fn retired_fact(self) -> RcuRetiredFact { + self.retired + } + + /// Persistent retirement fact used to agree the callback summary with the + /// root domain's authoritative removal observation. + pub proof fn tracked_retired_fact(tracked &self) -> (tracked res: &RcuRetiredFact) requires self.wf(), ensures - res.0.participant_id() == self.participant_id(), - res.0.cpu() == self.cpu(), - res.0.closed_generation() == self.closed_generation(), - res.0.view() == self.view(), - res.0.known_retired() == self.known_retired(), - res.0.scheduler() == self.scheduler(), - res.0.wf(), - res.1.participant_id() == self.participant_id(), - res.1.cpu() == self.cpu(), - res.1.closed_generation() == self.closed_generation(), - res.1.view() == self.view(), - res.1.known_retired() == self.known_retired(), - res.1.scheduler() == self.scheduler(), - res.1.wf(), + res.wf(), + res.record() == self.record(), { - use_type_invariant(&self); + &self.retired + } + + pub proof fn tracked_closed_generation(tracked &self, cpu: CpuId) -> (tracked res: + &CpuRcuClosedGeneration) + requires + self.wf(), + online_cpus().contains(cpu), + ensures + *res == self.closed_generations()[cpu], + res.wf(), + res.cpu() == cpu, + res.scheduler() == self.scheduler(), + { + self.closed_generations.tracked_borrow(cpu) + } + + pub open spec fn wf(self) -> bool { + &&& self.retired_fact().wf() + &&& self.retired_fact().record() == self.record() + &&& self.closed_generations().dom() == online_cpus() + &&& forall|cpu: CpuId| #[trigger] + self.closed_generations().contains_key(cpu) ==> { + let closed = self.closed_generations()[cpu]; + &&& closed.wf() + &&& closed.cpu() == cpu + &&& closed.scheduler() == self.scheduler() + &&& closed.known_retired().contains(self.record()) + } + } + + pub proof fn tracked_new( + scheduler: Loc, + tracked retired: RcuRetiredFact, + tracked closed_generations: Map, + ) -> (tracked res: Self) + requires + retired.wf(), + closed_generations.dom() == online_cpus(), + forall|cpu: CpuId| #[trigger] + closed_generations.contains_key(cpu) ==> { + let closed = closed_generations[cpu]; + &&& closed.wf() + &&& closed.cpu() == cpu + &&& closed.scheduler() == scheduler + &&& closed.known_retired().contains(retired.record()) + }, + ensures + res.wf(), + res.record() == retired.record(), + res.scheduler() == scheduler, + { + let ghost record = retired.record(); + Self { record, scheduler, retired, closed_generations } + } + + /// Classifies a coexisting reader as later than the completed grace period. + pub proof fn tracked_later_reader( + tracked &self, + tracked reader: CpuRcuReaderFragment, + ) -> (tracked res: CpuRcuReaderFragment) + requires + self.wf(), + reader.wf(), + online_cpus().contains(reader.cpu()), + self.closed_generations()[reader.cpu()].participant_id() == reader.participant_id(), + ensures + res == reader, + res.wf(), + res.known_retired().contains(self.record()), + { + let tracked closed = self.closed_generations.tracked_borrow(reader.cpu()); + let tracked reader = closed.lemma_later_reader(reader); + assert(closed.known_retired().contains(self.record())); + reader + } +} + +impl RcuActiveLeaseBinding { + pub closed spec fn from_record( + record: vstd_extra::rcu_read_pool::RcuReadLeaseRecord>, + ) -> Self { + Self { + key: record.key(), + pool_id: record.pool_id(), + fraction: record.fraction(), + participant_id: record.witness().reader().participant_id(), + reader_fraction: record.witness().reader().fraction(), + domain: record.witness().paper_guard().domain(), + root: record.witness().paper_guard().root(), + reader: record.witness().paper_guard().reader(), + start_view: record.witness().paper_guard().start_view(), + protected_addr: record.witness().protected().ptr().addr(), + } + } +} + +impl RcuRootReadLease { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.active.key() == self.lease.lease_id() + &&& self.active.value().key == self.lease.key() + &&& self.active.value().pool_id == self.lease.pool_id() + &&& self.active.value().fraction == self.lease.fraction() + } + + pub closed spec fn lease_id(self) -> nat { + self.lease.lease_id() + } + + pub closed spec fn key(self) -> nat { + self.lease.key() + } + + pub closed spec fn pool_id(self) -> Loc { + self.lease.pool_id() + } + + pub closed spec fn fraction(self) -> real { + self.lease.fraction() + } + + pub closed spec fn resource(self) -> O { + self.lease.resource() + } + + pub closed spec fn active_registry(self) -> Loc { + self.active.id() + } + + pub closed spec fn participant_id(self) -> Loc { + self.active.value().participant_id + } + + pub closed spec fn reader_fraction(self) -> real { + self.active.value().reader_fraction + } + + pub closed spec fn domain(self) -> Loc { + self.active.value().domain + } + + pub closed spec fn root(self) -> Loc { + self.active.value().root + } + + pub closed spec fn reader_context(self) -> RcuReaderContext { + self.active.value().reader + } + + pub closed spec fn start_view(self) -> Irc11ThreadView { + self.active.value().start_view + } + + pub closed spec fn protected_addr(self) -> usize { + self.active.value().protected_addr + } + + pub proof fn borrow(tracked &self) -> (tracked resource: &O) + ensures + *resource == self.resource(), + { + self.lease.borrow() + } +} + +impl RcuRootPermissionState { + /// Creates an empty permission registry for one RCU root. + pub proof fn empty( + scheduler: Loc, + domain: Loc, + root: Loc, + retire_observation_registry: Loc, + ) -> (tracked res: Self) + ensures + res.wf(), + res.scheduler() == scheduler, + res.domain() == domain, + res.root() == root, + res.retire_observation_registry() == retire_observation_registry, + res.keys() == Set::::empty(), + res.allocations() == Set::::empty(), + res.reclaimed() == Map::::empty(), + res.active_ids() == Set::::empty(), + { + let tracked registry = RcuTrackedReadPoolRegistry::empty(); + let tracked (active_leases, _active) = GhostMapAuth::new(Map::empty()); + let tracked (reclaim_state, _claims) = GhostMapAuth::new(Map::empty()); + let tracked res = RcuRootPermissionState { + registry, + active_leases, + reclaim_state, + unretired_claims: Map::tracked_empty(), + reclaimed: Map::tracked_empty(), + scheduler, + domain, + root, + retire_observation_registry, + }; + assert(res.allocations() == Set::::empty()); + assert(res.unretired_claims().dom() == Set::::empty()); + assert(res.reclaimed().dom() == Set::::empty()); + assert(res.active_lease_bindings().dom() == Set::::empty()); + assert(res.reclaimed().dom() == res.allocations().difference(res.keys())); + assert forall|obj: nat| #[trigger] res.reclaimed().contains_key(obj) implies { + let completed = res.reclaimed()[obj]; + &&& completed.wf() + &&& completed.record().domain == res.domain() + &&& completed.record().obj == obj + &&& completed.record().retire_observation_registry == res.retire_observation_registry() + &&& completed.record().removal.root == res.root() + } by {}; + assert forall|obj: nat| #[trigger] res.allocations().contains(obj) implies { + res.keys().contains(obj) <==> res.reclaim_states()[obj] is Some + } by {}; + res + } + + pub closed spec fn registry(self) -> RcuTrackedReadPoolRegistry< + nat, + O, + CpuRcuReadLeaseWitness, + > { + self.registry + } + + pub proof fn tracked_registry_mut(tracked &mut self) -> (tracked res: + &mut RcuTrackedReadPoolRegistry>) + ensures + *res == old(self).registry(), + final(self).registry() == *final(res), + *final(res) == old(self).registry() ==> *final(self) == *old(self), + { + &mut self.registry + } + + pub closed spec fn domain(self) -> Loc { + self.domain + } + + pub closed spec fn scheduler(self) -> Loc { + self.scheduler + } + + pub closed spec fn root(self) -> Loc { + self.root + } + + pub closed spec fn retire_observation_registry(self) -> Loc { + self.retire_observation_registry + } + + pub closed spec fn keys(self) -> Set { + self.registry().keys() + } + + pub closed spec fn contains(self, obj: nat) -> bool { + self.registry().contains(obj) + } + + pub proof fn lemma_contains_iff_key(tracked &self, obj: nat) + ensures + self.contains(obj) <==> self.keys().contains(obj), + { + self.registry.lemma_contains_iff_key(obj); + } + + /// Opens the allocation-state facts associated with a live permission pool. + pub proof fn lemma_live_reclaim_state(tracked &self, obj: nat) + requires + self.wf(), + self.contains(obj), + ensures + self.keys().contains(obj), + self.allocations().contains(obj), + self.reclaim_states().dom().contains(obj), + self.reclaim_states()[obj] is Some, + { + self.registry.lemma_contains_iff_key(obj); + assert(self.keys().contains(obj)); + assert(self.keys().subset_of(self.allocations())); + assert(self.allocations().contains(obj)); + } + + /// An unretired root claim always names a live permission pool. + pub proof fn lemma_unretired_is_live(tracked &self, obj: nat) + requires + self.wf(), + self.has_unretired_claim(obj), + ensures + self.contains(obj), + self.keys().contains(obj), + self.allocations().contains(obj), + self.reclaim_states().dom().contains(obj), + self.reclaim_states()[obj] is Some, + { + assert(self.unretired_claims().dom().contains(obj)); + assert(self.keys().contains(obj)); + self.registry.lemma_contains_iff_key(obj); + self.lemma_live_reclaim_state(obj); + } + + /// Opens the live allocation facts for every key in one quantifier. + pub proof fn lemma_all_live_reclaim_states(tracked &self) + requires + self.wf(), + ensures + forall|obj: nat| #[trigger] + self.keys().contains(obj) ==> { + &&& self.contains(obj) + &&& self.allocations().contains(obj) + &&& self.reclaim_states().dom().contains(obj) + &&& self.reclaim_states()[obj] is Some + }, + { + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + self.registry.lemma_all_contains_iff_keys(); + assert forall|obj: nat| #[trigger] self.keys().contains(obj) implies { + &&& self.contains(obj) + &&& self.allocations().contains(obj) + &&& self.reclaim_states().dom().contains(obj) + &&& self.reclaim_states()[obj] is Some + } by { + assert(self.contains(obj)); + assert(self.allocations().contains(obj)); + }; + } + + /// Every allocation identity ever registered by this root. + pub closed spec fn allocations(self) -> Set { + self.reclaim_state.dom() + } + + pub closed spec fn reclaim_states(self) -> Map> { + self.reclaim_state@ + } + + pub closed spec fn unretired_claims(self) -> Map>> { + self.unretired_claims + } + + pub closed spec fn reclaimed(self) -> Map { + self.reclaimed + } + + pub closed spec fn reclaim_registry(self) -> Loc { + self.reclaim_state.id() + } + + pub closed spec fn ownership(self, obj: nat) -> O + recommends + self.contains(obj), + { + self.registry().pool(obj).resource() + } + + pub closed spec fn has_unretired_claim(self, obj: nat) -> bool { + self.unretired_claims().contains_key(obj) + } + + /// Relates the claim predicate to membership in the claim map domain. + pub proof fn lemma_has_unretired_iff_domain(tracked &self, obj: nat) + ensures + self.has_unretired_claim(obj) <==> self.unretired_claims().dom().contains(obj), + { + } + + /// Opens claim-map membership for all allocation identities. + pub proof fn lemma_all_unretired_domains(tracked &self) + ensures + forall|obj: nat| #[trigger] + self.has_unretired_claim(obj) <==> self.unretired_claims().dom().contains(obj), + { + } + + pub closed spec fn active_ids(self) -> Set { + self.registry().active_ids() + } + + pub closed spec fn active_lease_bindings(self) -> Map { + self.active_leases@ + } + + pub closed spec fn active_lease_registry(self) -> Loc { + self.active_leases.id() + } + + pub open spec fn has_active(self, obj: nat) -> bool { + self.registry().has_active(obj) + } + + pub closed spec fn active_record( + self, + lease_id: nat, + ) -> vstd_extra::rcu_read_pool::RcuReadLeaseRecord> + recommends + self.active_ids().contains(lease_id), + { + self.registry().active_record(lease_id) + } + + /// Exposes the active-lease registry projections without revealing the + /// rest of the root permission state's representation. + pub proof fn lemma_active_registry_projection(tracked &self) + ensures + self.active_ids() == self.registry().active_ids(), + forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> self.active_record(lease_id) + == self.registry().active_record(lease_id), + { + } + + /// The registry accounting is valid and every active witness belongs to + /// this exact RCU root. + pub open spec fn wf(self) -> bool { + &&& self.registry().wf() + &&& self.active_lease_bindings().dom() == self.active_ids() + &&& forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> self.active_lease_bindings()[lease_id] + == RcuActiveLeaseBinding::from_record(self.active_record(lease_id)) + &&& self.registry().keys().subset_of(self.allocations()) + &&& forall|obj: nat| #[trigger] + self.allocations().contains(obj) ==> { + self.keys().contains(obj) <==> self.reclaim_states()[obj] is Some + } + &&& self.unretired_claims().dom().subset_of(self.registry().keys()) + &&& self.reclaimed().dom() == self.allocations().difference(self.keys()) + &&& forall|obj: nat| #[trigger] + self.reclaimed().contains_key(obj) ==> { + let completed = self.reclaimed()[obj]; + &&& completed.wf() + &&& completed.scheduler() == self.scheduler() + &&& completed.record().domain == self.domain() + &&& completed.record().obj == obj + &&& completed.record().retire_observation_registry + == self.retire_observation_registry() + &&& completed.record().removal.root == self.root() + } + &&& forall|obj: nat| #[trigger] + self.unretired_claims().contains_key(obj) ==> { + let claim = self.unretired_claims()[obj]; + &&& claim.id() == self.reclaim_registry() + &&& claim.key() == obj + &&& claim.value() == Some(self.reclaim_states()[obj]->Some_0) + } + &&& forall|lease_id: nat| #[trigger] + self.active_ids().contains(lease_id) ==> { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& witness.binding().registry() == self.scheduler() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } + } + + pub proof fn tracked_reclaimed(tracked &self, obj: nat) -> (tracked res: &RcuReclaimedWitness) + requires + self.wf(), + self.allocations().contains(obj), + !self.contains(obj), + ensures + self.reclaimed().contains_key(obj), + *res == self.reclaimed()[obj], + res.wf(), + res.record().domain == self.domain(), + res.record().obj == obj, + res.record().retire_observation_registry == self.retire_observation_registry(), + res.record().removal.root == self.root(), + { + self.registry.lemma_contains_iff_key(obj); + assert(self.reclaimed().contains_key(obj)); + self.reclaimed.tracked_borrow(obj) + } + + /// Registers the complete physical ownership of one published allocation. + pub proof fn tracked_insert( + tracked &mut self, + tracked info: &RcuBlockInfo, + tracked ownership: O, + ) + requires + old(self).wf(), + info.wf(), + info.domain() == old(self).domain(), + !old(self).allocations().contains(info.obj()), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys().insert(info.obj()), + final(self).allocations() == old(self).allocations().insert(info.obj()), + final(self).reclaim_states() == old(self).reclaim_states().insert( + info.obj(), + Some(info.ptr()), + ), + final(self).reclaimed() == old(self).reclaimed(), + final(self).has_unretired_claim(info.obj()), + final(self).unretired_claims().dom() == old(self).unretired_claims().dom().insert( + info.obj(), + ), + final(self).contains(info.obj()), + final(self).active_ids() == old(self).active_ids(), + final(self).registry().pool(info.obj()).resource() == ownership, + final(self).ownership(info.obj()) == ownership, + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + { + let ghost old_active = self.active_ids(); + let ghost new_obj = info.obj(); + self.registry.lemma_contains_iff_key(new_obj); + assert(!self.contains(new_obj)) by { + if self.contains(new_obj) { + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + } + }; + self.registry.insert(info.obj(), ownership); + let tracked claim = self.reclaim_state.insert(info.obj(), Some(info.ptr())); + self.unretired_claims.tracked_insert(info.obj(), claim); + assert(self.registry().keys().subset_of(self.allocations())) by { + assert forall|obj: nat| #[trigger] + self.registry().keys().contains(obj) implies self.allocations().contains(obj) by { + if obj != new_obj { + assert(old(self).keys().contains(obj)); + assert(old(self).allocations().contains(obj)); + } + }; + }; + assert forall|obj: nat| #[trigger] self.allocations().contains(obj) implies { + self.keys().contains(obj) <==> self.reclaim_states()[obj] is Some + } by { + if obj == new_obj { + assert(self.keys().contains(obj)); + assert(self.reclaim_states()[obj] == Some(info.ptr())); + } else { + assert(old(self).allocations().contains(obj)); + assert(self.reclaim_states()[obj] == old(self).reclaim_states()[obj]); + assert(self.keys().contains(obj) == old(self).keys().contains(obj)); + } + }; + assert(self.unretired_claims().dom().subset_of(self.registry().keys())); + assert(self.reclaimed().dom() == self.allocations().difference(self.keys())) by { + assert(self.reclaimed() == old(self).reclaimed()); + assert(old(self).reclaimed().dom() == old(self).allocations().difference( + old(self).keys(), + )); + }; + assert forall|obj: nat| #[trigger] self.reclaimed().contains_key(obj) implies { + let completed = self.reclaimed()[obj]; + &&& completed.wf() + &&& completed.record().domain == self.domain() + &&& completed.record().obj == obj + &&& completed.record().retire_observation_registry == self.retire_observation_registry() + &&& completed.record().removal.root == self.root() + } by { + assert(old(self).reclaimed().contains_key(obj)); + assert(self.reclaimed()[obj] == old(self).reclaimed()[obj]); + }; + assert forall|obj: nat| #[trigger] self.unretired_claims().contains_key(obj) implies { + let saved = self.unretired_claims()[obj]; + &&& saved.id() == self.reclaim_registry() + &&& saved.key() == obj + &&& saved.value() == Some(self.reclaim_states()[obj]->Some_0) + } by { + if obj == new_obj { + assert(self.unretired_claims()[obj] == claim); + } else { + assert(old(self).unretired_claims().contains_key(obj)); + assert(self.unretired_claims()[obj] == old(self).unretired_claims()[obj]); + } + }; + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(old_active.contains(lease_id)); + assert(old(self).active_ids().contains(lease_id)); + assert(self.registry().active_record(lease_id) == old(self).registry().active_record( + lease_id, + )); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + }; + } + + /// Moves the unique reclaim claim out when the root publication is retired. + pub proof fn tracked_retire(tracked &mut self, obj: nat) -> (tracked claim: RcuReclaimClaim) + requires + old(self).wf(), + old(self).has_unretired_claim(obj), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).registry() == old(self).registry(), + forall|candidate: nat| #[trigger] + old(self).contains(candidate) ==> final(self).ownership(candidate) == old( + self, + ).ownership(candidate), + final(self).active_ids() == old(self).active_ids(), + !final(self).has_unretired_claim(obj), + final(self).unretired_claims().dom() == old(self).unretired_claims().dom().remove(obj), + claim.registry() == old(self).reclaim_registry(), + claim.obj() == obj, + claim.is_pending(), + claim.ptr() == old(self).reclaim_states()[obj]->Some_0, + { + let tracked points_to = self.unretired_claims.tracked_remove(obj); + assert(self.registry() == old(self).registry()); + assert(self.reclaim_states() == old(self).reclaim_states()); + assert(self.reclaim_registry() == old(self).reclaim_registry()); + assert(self.unretired_claims().dom().subset_of(self.registry().keys())); + assert forall|other: nat| #[trigger] self.unretired_claims.contains_key(other) implies { + let claim = self.unretired_claims[other]; + &&& claim.id() == self.reclaim_state.id() + &&& claim.key() == other + &&& claim.value() == Some(self.reclaim_states()[other]->Some_0) + } by { + assert(old(self).unretired_claims.contains_key(other)); + assert(self.unretired_claims[other] == old(self).unretired_claims[other]); + }; + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + assert(self.domain() == old(self).domain()); + assert(self.root() == old(self).root()); + assert(self.retire_observation_registry() == old(self).retire_observation_registry()); + }; + RcuReclaimClaim { points_to } + } + + /// Splits a physical read lease for the object selected by a guarded load. + /// + /// The CPU reader fragment is split at the same time and its matching half + /// is retained in the active registry record. + pub proof fn tracked_split_loaded( + tracked &mut self, + tracked guard: CpuRcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: (CpuRcuReadGuardToken, RcuRootReadLease)) + requires + old(self).wf(), + old(self).contains(info.obj()), + guard.wf(), + guard.scheduler() == old(self).scheduler(), + info.wf(), + info.domain() == old(self).domain(), + guard.domain() == old(self).domain(), + guard.root() == old(self).root(), + guard.retire_observation_registry() == old(self).retire_observation_registry(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + online_cpus().contains(guard.cpu()), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|candidate: nat| #[trigger] + final(self).has_unretired_claim(candidate) == old(self).has_unretired_claim( + candidate, + ), + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.scheduler() == guard.scheduler(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.domain() == guard.domain(), + res.0.root() == guard.root(), + res.0.reader_registry() == guard.reader_registry(), + res.0.retire_observation_registry() == guard.retire_observation_registry(), + res.0.reader_context() == guard.reader_context(), + res.0.start_view() == guard.start_view(), + res.0.expired() == guard.expired(), + res.0.seen_removed() == guard.seen_removed(), + res.0.protected() == guard.protected(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.key() == info.obj(), + res.1.resource() == old(self).ownership(info.obj()), + res.1.active_registry() == old(self).active_lease_registry(), + res.1.participant_id() == guard.participant_id(), + res.1.reader_fraction() == res.0.reader_fragment().fraction(), + res.1.domain() == guard.domain(), + res.1.root() == guard.root(), + res.1.reader_context() == guard.reader_context(), + res.1.start_view() == guard.start_view(), + res.1.protected_addr() == info.addr(), + final(self).active_ids() == old(self).active_ids().insert(res.1.lease_id()), + final(self).active_record(res.1.lease_id()).witness().paper_guard() + == guard.paper_guard(), + final(self).active_record(res.1.lease_id()).witness().protected().obj() == info.obj(), + { + let tracked (guard, witness) = CpuRcuReadLeaseWitness::tracked_from_loaded_guard( + guard, + info, + ); + let tracked lease = self.registry.split_lease(info.obj(), witness); + let ghost binding = RcuActiveLeaseBinding::from_record( + self.active_record(lease.lease_id()), + ); + reveal(RcuActiveLeaseBinding::from_record); + assert(self.active_record(lease.lease_id()).witness() == witness); + assert(self.active_record(lease.lease_id()).witness().protected().ptr() == info.ptr()); + info.lemma_wf_facts(); + assert(info.addr() == info.ptr().addr()); + assert(binding.protected_addr == info.addr()); + let tracked active = self.active_leases.insert(lease.lease_id(), binding); + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + if lease_id == lease.lease_id() { + assert(self.active_record(lease_id).witness() == witness); + assert(self.active_record(lease_id).key() == info.obj()); + } else { + assert(old(self).active_ids().contains(lease_id)); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + } + }; + let tracked lease = RcuRootReadLease { lease, active }; + (guard, lease) + } + + /// Returns one physical lease and rejoins its CPU fragment with the + /// executable guard that originally issued it. + pub proof fn tracked_return_loaded( + tracked &mut self, + tracked lease: RcuRootReadLease, + tracked guard: CpuRcuReadGuardToken, + ) -> (tracked res: CpuRcuReadGuardToken) + requires + old(self).wf(), + lease.active_registry() == old(self).active_lease_registry(), + lease.participant_id() == guard.participant_id(), + lease.reader_fraction() == guard.reader_fragment().fraction(), + lease.domain() == guard.domain(), + lease.root() == guard.root(), + lease.reader_context() == guard.reader_context(), + lease.start_view() == guard.start_view(), + guard.protects(lease.protected_addr(), lease.key()), + guard.wf(), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|obj: nat| #[trigger] + final(self).has_unretired_claim(obj) == old(self).has_unretired_claim(obj), + forall|obj: nat| #[trigger] final(self).contains(obj) == old(self).contains(obj), + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + final(self).active_ids() == old(self).active_ids().remove(lease.lease_id()), + res.wf(), + res.paper_guard() == guard.paper_guard(), + res.binding() == guard.binding(), + res.participant_id() == guard.participant_id(), + res.cpu() == guard.cpu(), + res.generation() == guard.generation(), + res.participant_view() == guard.participant_view(), + res.known_retired() == guard.known_retired(), + res.domain() == guard.domain(), + res.root() == guard.root(), + res.reader_registry() == guard.reader_registry(), + res.retire_observation_registry() == guard.retire_observation_registry(), + res.reader_context() == guard.reader_context(), + res.start_view() == guard.start_view(), + res.expired() == guard.expired(), + res.seen_removed() == guard.seen_removed(), + res.protected() == guard.protected(), + res.reader_fragment().fraction() == guard.reader_fragment().fraction() + old( + self, + ).active_record(lease.lease_id()).witness().reader().fraction(), + res.reader_fragment().fraction() == guard.reader_fragment().fraction() * 2real, + { + use_type_invariant(&lease); + let ghost protected_addr = lease.protected_addr(); + let tracked RcuRootReadLease { lease, active } = lease; + active.agree(&self.active_leases); + assert(self.active_ids().contains(lease.lease_id())); + assert(self.active_lease_bindings()[lease.lease_id()] == RcuActiveLeaseBinding::from_record( + self.active_record(lease.lease_id()), + )); + assert(self.active_record(lease.lease_id()).key() == lease.key()); + assert(self.active_record(lease.lease_id()).pool_id() == lease.pool_id()); + assert(self.active_record(lease.lease_id()).fraction() == lease.fraction()); + assert(self.active_record(lease.lease_id()).witness().reader().participant_id() + == guard.participant_id()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().domain() + == guard.domain()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().root() == guard.root()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().reader() + == guard.reader_context()); + assert(self.active_record(lease.lease_id()).witness().paper_guard().start_view() + == guard.start_view()); + assert(self.active_record(lease.lease_id()).witness().protected().ptr().addr() + == protected_addr); + let ghost returned_id = lease.lease_id(); + let ghost returned_key = lease.key(); + self.active_leases.delete_points_to(active); + let tracked witness = self.registry.return_lease(lease); + let tracked guard = witness.tracked_return_to_guard(guard); + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(lease_id != returned_id); + assert(old(self).active_ids().contains(lease_id)); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + }; + assert forall|obj: nat| #[trigger] old(self).contains(obj) implies self.ownership(obj) + == old(self).ownership(obj) by { + if obj == returned_key { + assert(self.registry().pool(obj).resource() == old(self).registry().pool( + obj, + ).resource()); + } else { + assert(self.registry().pool(obj) == old(self).registry().pool(obj)); + } + }; + assert forall|obj: nat| #[trigger] + self.has_unretired_claim(obj) == old(self).has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] self.contains(obj) == old(self).contains(obj) by {}; + guard + } + + /// Recovers one allocation after completion has ruled out every active + /// lease for its identity. + pub proof fn tracked_reclaim( + tracked &mut self, + tracked claim: RcuReclaimClaim, + tracked completed: RcuReclaimedWitness, + ) -> (tracked ownership: O) + requires + old(self).wf(), + claim.registry() == old(self).reclaim_registry(), + claim.is_pending(), + !old(self).has_active(claim.obj()), + completed.wf(), + completed.scheduler() == old(self).scheduler(), + completed.record().domain == old(self).domain(), + completed.record().obj == claim.obj(), + completed.record().retire_observation_registry == old( + self, + ).retire_observation_registry(), + completed.record().removal.root == old(self).root(), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys().remove(claim.obj()), + final(self).allocations() == old(self).allocations(), + final(self).active_ids() == old(self).active_ids(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|candidate: nat| #[trigger] + final(self).has_unretired_claim(candidate) == old(self).has_unretired_claim( + candidate, + ), + final(self).reclaimed() == old(self).reclaimed().insert(claim.obj(), completed), + forall|candidate: nat| #[trigger] + final(self).keys().contains(candidate) ==> { + &&& old(self).keys().contains(candidate) + &&& old(self).contains(candidate) + &&& final(self).ownership(candidate) == old(self).ownership(candidate) + }, + forall|candidate: nat| #[trigger] + old(self).allocations().contains(candidate) && candidate != claim.obj() + ==> final(self).reclaim_states()[candidate] == old( + self, + ).reclaim_states()[candidate], + old(self).contains(claim.obj()), + claim.ptr() == old(self).reclaim_states()[claim.obj()]->Some_0, + ownership == old(self).registry().pool(claim.obj()).resource(), + ownership == old(self).ownership(claim.obj()), + final(self).reclaimed().contains_pair(claim.obj(), completed), + { + let ghost obj = claim.obj(); + self.registry.lemma_all_contains_iff_keys(); + let tracked RcuReclaimClaim { mut points_to } = claim; + points_to.agree(&self.reclaim_state); + assert(self.reclaim_state@[obj] == Some(points_to.value()->Some_0)); + assert(self.allocations().contains(obj)); + assert(self.keys().contains(obj)); + self.registry.lemma_contains_iff_key(obj); + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + assert(self.contains(obj)); + if self.unretired_claims.contains_key(obj) { + let tracked existing = self.unretired_claims.tracked_borrow_mut(obj); + points_to.disjoint(existing); + assert(false); + } + let ghost old_reclaim_states = self.reclaim_states(); + points_to.update(&mut self.reclaim_state, None); + let tracked ownership = self.registry.reclaim(obj); + self.reclaimed.tracked_insert(obj, completed); + assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { + let record = self.active_record(lease_id); + let witness = record.witness(); + &&& witness.wf() + &&& witness.binding().registry() == self.scheduler() + &&& record.key() == witness.protected().obj() + &&& witness.protected().domain() == self.domain() + &&& witness.paper_guard().domain() == self.domain() + &&& witness.paper_guard().root() == self.root() + &&& witness.paper_guard().retire_observation_registry() + == self.retire_observation_registry() + } by { + assert(old(self).active_ids().contains(lease_id)); + assert(old(self).active_record(lease_id).key() != obj); + assert(self.registry().active_records() == old(self).registry().active_records()); + assert(self.registry().active_record(lease_id) == old(self).registry().active_record( + lease_id, + )); + assert(self.active_record(lease_id) == old(self).active_record(lease_id)); + }; + assert(self.registry().keys().subset_of(self.allocations())); + assert forall|candidate: nat| #[trigger] self.allocations().contains(candidate) implies { + self.keys().contains(candidate) <==> self.reclaim_states()[candidate] is Some + } by { + if candidate == obj { + assert(!self.keys().contains(candidate)); + assert(self.reclaim_states()[candidate] is None); + } else { + assert(self.reclaim_states()[candidate] == old_reclaim_states[candidate]); + assert(self.keys().contains(candidate) == old(self).keys().contains(candidate)); + } + }; + assert(self.unretired_claims().dom().subset_of(self.registry().keys())); + assert(self.reclaimed().dom() == self.allocations().difference(self.keys())) by { + assert(old(self).reclaimed().dom() == old(self).allocations().difference( + old(self).keys(), + )); + }; + assert forall|candidate: nat| #[trigger] self.reclaimed().contains_key(candidate) implies { + let saved = self.reclaimed()[candidate]; + &&& saved.wf() + &&& saved.scheduler() == self.scheduler() + &&& saved.record().domain == self.domain() + &&& saved.record().obj == candidate + &&& saved.record().retire_observation_registry == self.retire_observation_registry() + &&& saved.record().removal.root == self.root() + } by { + if candidate == obj { + assert(self.reclaimed()[candidate] == completed); + } else { + assert(old(self).reclaimed().contains_key(candidate)); + assert(self.reclaimed()[candidate] == old(self).reclaimed()[candidate]); + } + }; + self.registry.lemma_all_contains_iff_keys(); + assert forall|candidate: nat| #[trigger] self.keys().contains(candidate) implies { + &&& old(self).keys().contains(candidate) + &&& old(self).contains(candidate) + &&& self.ownership(candidate) == old(self).ownership(candidate) + } by { + assert(candidate != obj); + assert(self.contains(candidate)); + assert(old(self).keys().contains(candidate)); + assert(old(self).contains(candidate)); + assert(self.registry().pool(candidate) == old(self).registry().pool(candidate)); + }; + ownership + } +} + +impl CpuRcuClosedGeneration { + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.resource.value().state is None + &&& self.resource.value().closed =~= Set::empty().insert(self.report()) + &&& self.report().known_retired == self.known_retired.records() + &&& self.binding.single_local_id() == self.resource.loc() + &&& self.binding.cpu() == self.report().cpu + &&& CpuRcuCarrier::records_observed(self.report().known_retired, self.report().view) + } + + closed spec fn report(self) -> CpuRcuReportView { + choose|report: CpuRcuReportView| self.resource.value().closed.contains(report) + } + + pub closed spec fn participant_id(self) -> Loc { + self.resource.loc() + } + + pub closed spec fn binding(self) -> CpuRcuCoreBinding { + self.binding + } + + pub closed spec fn scheduler(self) -> Loc { + self.binding().registry() + } + + pub closed spec fn cpu(self) -> CpuId { + self.report().cpu + } + + pub closed spec fn closed_generation(self) -> nat { + self.report().generation + } + + pub closed spec fn view(self) -> Irc11ThreadView { + self.report().view + } + + pub closed spec fn known_retired(self) -> Set { + self.known_retired.records() + } + + pub closed spec fn wf(self) -> bool { + &&& self.resource.value() == CpuRcuCarrier::closed(self.report()) + &&& self.binding().single_local_id() == self.participant_id() + &&& self.binding().cpu() == self.cpu() + } + + /// Relates this report to another client of the scheduler's canonical + /// CPU-local registration. Agreement of the registry entry rules out a + /// second RCU participant identity for the same CPU. + pub proof fn lemma_same_participant_as_binding(tracked &self, tracked other: &CpuRcuCoreBinding) + requires + self.wf(), + other.registry() == self.scheduler(), + other.cpu() == self.cpu(), + other.locals_key().len() == 1, + ensures + other.single_local_id() == self.participant_id(), + { + self.binding.lemma_same_cpu_agree(other); + } + + /// Splits the idempotent closed-generation fact. + pub proof fn tracked_duplicate(tracked self) -> (tracked res: ( + CpuRcuClosedGeneration, + CpuRcuClosedGeneration, + )) + requires + self.wf(), + ensures + res.0.participant_id() == self.participant_id(), + res.0.cpu() == self.cpu(), + res.0.closed_generation() == self.closed_generation(), + res.0.view() == self.view(), + res.0.known_retired() == self.known_retired(), + res.0.scheduler() == self.scheduler(), + res.0.wf(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.closed_generation() == self.closed_generation(), + res.1.view() == self.view(), + res.1.known_retired() == self.known_retired(), + res.1.scheduler() == self.scheduler(), + res.1.wf(), + { + use_type_invariant(&self); let ghost report = self.report(); let ghost records = self.known_retired.records(); let ghost carrier = CpuRcuCarrier::closed(self.report()); @@ -1420,6 +2980,67 @@ impl CpuRcuClosedGeneration { reader } + /// Reference-preserving form of [`Self::lemma_later_reader`]. + /// + /// This form is used when a reader fragment is retained as the witness of + /// an active read lease. Validating it against the persistent closed- + /// generation resource does not consume either token. + pub proof fn lemma_later_reader_ref(tracked &self, tracked reader: &mut CpuRcuReaderFragment) + requires + self.wf(), + old(reader).wf(), + self.participant_id() == old(reader).participant_id(), + ensures + *final(reader) == *old(reader), + self.closed_generation() < final(reader).generation(), + self.view().spec_le(final(reader).participant_view()), + self.known_retired().subset_of(final(reader).known_retired()), + { + use_type_invariant(&*reader); + use_type_invariant(self); + assert(reader.resource.value().state_view().known_retired + == reader.known_retired.records()); + assert(self.report().known_retired == self.known_retired.records()); + reader.resource.validate_2(&self.resource); + let ghost report = self.report(); + let ghost reader_state = reader.resource.value().state_view(); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()).valid()); + assert(Option::::op(reader.resource.value().state, None) + == reader.resource.value().state); + assert(reader.resource.value().closed.is_empty()); + assert(reader.resource.value().closed.union(Set::empty().insert(report)) + =~= Set::empty().insert(report)); + assert(CpuRcuCarrier::op(reader.resource.value(), self.resource.value()) == CpuRcuCarrier { + state: reader.resource.value().state, + closed: Set::empty().insert(report), + }); + assert(CpuRcuCarrier::reports_fit(reader_state, Set::empty().insert(report))); + assert(Set::empty().insert(report).contains(report)); + assert(report.generation < reader_state.generation); + assert(self.view().spec_le(reader.participant_view())); + assert(report.known_retired.subset_of(reader_state.known_retired)); + assert(self.known_retired().subset_of(reader.known_retired())); + } + + /// Classifies an active physical-lease witness as a later reader without + /// consuming the witness retained by the root registry. + pub proof fn lemma_later_lease_witness_ref( + tracked &self, + tracked witness: &mut CpuRcuReadLeaseWitness, + ) + requires + self.wf(), + old(witness).wf(), + self.participant_id() == old(witness).reader().participant_id(), + ensures + *final(witness) == *old(witness), + self.closed_generation() < final(witness).reader().generation(), + self.view().spec_le(final(witness).reader().participant_view()), + self.known_retired().subset_of(final(witness).reader().known_retired()), + { + self.lemma_later_reader_ref(&mut witness.reader); + } + /// Lifts [`Self::lemma_later_reader`] to the task view used to start a /// reader. /// @@ -1495,6 +3116,26 @@ impl CpuRcuClosedGeneration { CpuRcuReadGuardToken::tracked_new(paper_guard, reader, binding) } + /// Reference-preserving classification of a live refined guard. + pub proof fn lemma_later_guard_ref( + tracked &self, + tracked guard: &mut CpuRcuReadGuardToken, + ) + requires + self.wf(), + old(guard).wf(), + self.participant_id() == old(guard).participant_id(), + self.cpu() == old(guard).cpu(), + ensures + *final(guard) == *old(guard), + self.closed_generation() < final(guard).generation(), + self.view().spec_le(final(guard).start_view()), + self.known_retired().subset_of(final(guard).known_retired()), + { + self.lemma_later_reader_ref(&mut guard.reader); + self.view().lemma_spec_le_transitive(guard.start_view(), guard.start_view()); + } + /// The old-reader branch of the paper proof: a guard from a generation /// covered by this report cannot still own its CPU reader fragment. pub proof fn lemma_excludes_old_guard(tracked &self, tracked guard: CpuRcuReadGuardToken) diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 4376694dd..d333b0da6 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -4,10 +4,12 @@ //! This module contains only transitions coupled to the RCU root and monitor //! ghost state. Generic native primitives are re-exported by //! [`vstd_extra::atomic_irc11`]. -use core::sync::atomic::Ordering; +use core::{marker::PhantomData, sync::atomic::Ordering}; use super::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec}; +use crate::specs::mm::cpu::online_cpus; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; +use vstd::modes::tracked_static_ref; use vstd::prelude::*; use vstd::resource::Loc; use vstd::thread_view::Objective; @@ -21,24 +23,270 @@ verus! { broadcast use {vstd::atomic_weak::group_view_history, vstd::thread_view::group_thread_view_axioms}; +/// Complete ghost state protected by one RCU root atomic invariant. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRootAtomicState { + pub(crate) points_to: AtomicPointsTo<*mut T>, + pub(crate) root: rcu_spec::RcuRootOwnedGhost, + pub(crate) permissions: rcu_cpu_spec::RcuRootPermissionState, +} + +unsafe impl Objective for RcuRootAtomicState { + +} + +impl RcuRootAtomicState { + pub closed spec fn points_to(self) -> AtomicPointsTo<*mut T> { + self.points_to + } + + pub closed spec fn root(self) -> rcu_spec::RcuRootOwnedGhost { + self.root + } + + pub closed spec fn permissions(self) -> rcu_cpu_spec::RcuRootPermissionState { + self.permissions + } +} + +/// Invariant tying native IRC11 history to paper identities and physical +/// permissions for every allocation that has not yet been reclaimed. +pub struct RcuRootAtomicInv { + _marker: PhantomData, +} + +impl InvariantPredicate< + (rcu_spec::RcuRootKey, Irc11AtomicId), + RcuRootAtomicState, +> for RcuRootAtomicInv where OwnPred: rcu_spec::RcuRootOwnershipPredicate { + open spec fn inv( + key_loc: (rcu_spec::RcuRootKey, Irc11AtomicId), + state: RcuRootAtomicState, + ) -> bool { + let (key, loc) = key_loc; + let g = state.root(); + let permissions = state.permissions(); + &&& rcu_spec::RcuOwnedWeakAtomicInv::::inv( + key_loc, + (state.points_to(), g), + ) + &&& permissions.wf() + &&& permissions.scheduler() == key.scheduler + &&& permissions.domain() == key.domain + &&& permissions.root() == key.domain + &&& permissions.retire_observation_registry() == key.retire_observation_registry + &&& permissions.reclaim_registry() == key.reclaim_registry + &&& permissions.active_lease_registry() == key.active_lease_registry + &&& permissions.allocations() == g.infos().dom() + &&& forall|obj: nat| #[trigger] + permissions.keys().contains(obj) ==> { + &&& permissions.contains(obj) + &&& permissions.allocations().contains(obj) + &&& permissions.reclaim_states().dom().contains(obj) + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } + &&& permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + } + &&& forall|obj: nat| #[trigger] + g.removals().contains_key(obj) ==> !permissions.has_unretired_claim(obj) + &&& forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) ==> { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } + } +} + +pub type RcuRootAtomicInvariant = AtomicInvariant< + (rcu_spec::RcuRootKey, Irc11AtomicId), + RcuRootAtomicState, + RcuRootAtomicInv, +>; + +/// Exposes the permission-state facts carried by the RCU root invariant. +/// +/// Keeping this unfolding lemma beside [`RcuRootAtomicInv`] avoids making +/// executable callback code depend on the invariant's concrete conjunction. +pub(crate) proof fn lemma_root_atomic_permission_facts( + key_loc: (rcu_spec::RcuRootKey, Irc11AtomicId), + tracked state: &RcuRootAtomicState, +) where OwnPred: rcu_spec::RcuRootOwnershipPredicate + requires + RcuRootAtomicInv::::inv(key_loc, *state), + ensures + rcu_spec::RcuOwnedWeakAtomicInv::::inv( + key_loc, + (state.points_to, state.root), + ), + state.root.root().domain_wf(), + state.root.domain() == key_loc.0.domain, + state.root.retire_observation_registry() == key_loc.0.retire_observation_registry, + state.root.removals() == state.root.root().domain_auth().retire_observations(), + state.permissions.wf(), + state.permissions.scheduler() == key_loc.0.scheduler, + state.permissions.domain() == key_loc.0.domain, + state.permissions.root() == key_loc.0.domain, + state.permissions.retire_observation_registry() == key_loc.0.retire_observation_registry, + state.permissions.reclaim_registry() == key_loc.0.reclaim_registry, + state.permissions.active_lease_registry() == key_loc.0.active_lease_registry, + state.permissions.allocations() == state.root.infos().dom(), + state.permissions.unretired_claims().dom() == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }, + forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) ==> { + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states().dom().contains(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 == state.root.infos()[obj].ptr() + &&& OwnPred::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + }, + forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) ==> !state.permissions.has_unretired_claim(obj), + forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) ==> { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + }, +{ +} + +/// Re-folds the root invariant after a proof-only permission-state update. +pub(crate) proof fn lemma_build_root_atomic_inv( + key_loc: (rcu_spec::RcuRootKey, Irc11AtomicId), + tracked state: &RcuRootAtomicState, +) where OwnPred: rcu_spec::RcuRootOwnershipPredicate + requires + rcu_spec::RcuOwnedWeakAtomicInv::::inv( + key_loc, + (state.points_to, state.root), + ), + state.permissions.wf(), + state.permissions.scheduler() == key_loc.0.scheduler, + state.permissions.domain() == key_loc.0.domain, + state.permissions.root() == key_loc.0.domain, + state.permissions.retire_observation_registry() == key_loc.0.retire_observation_registry, + state.permissions.reclaim_registry() == key_loc.0.reclaim_registry, + state.permissions.active_lease_registry() == key_loc.0.active_lease_registry, + state.permissions.allocations() == state.root.infos().dom(), + forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) ==> { + &&& state.permissions.contains(obj) + &&& state.permissions.allocations().contains(obj) + &&& state.permissions.reclaim_states().dom().contains(obj) + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 == state.root.infos()[obj].ptr() + &&& OwnPred::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + }, + state.permissions.unretired_claims().dom() == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }, + forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) ==> !state.permissions.has_unretired_claim(obj), + forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) ==> { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + }, + ensures + RcuRootAtomicInv::::inv(key_loc, *state), +{ +} + +/// Retired root metadata paired with the unique claim for its permission pool. +#[verifier::reject_recursive_types(T)] +pub tracked struct RcuRetiredRootObject { + detached: rcu_spec::RcuRetiredOwnedObject, + claim: rcu_cpu_spec::RcuReclaimClaim, +} + +impl RcuRetiredRootObject { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + &&& self.detached.object().wf() + &&& self.claim.obj() == self.detached.object().obj() + &&& self.claim.is_pending() + &&& equal(self.claim.ptr(), self.detached.object().ptr()) + } + + pub closed spec fn object(self) -> rcu_spec::RcuObjectId { + self.detached.object() + } + + pub closed spec fn retired(self) -> rcu_spec::RcuRetired { + self.detached.retired() + } + + pub closed spec fn ptr(self) -> *mut T { + self.detached.ptr() + } + + pub closed spec fn obj(self) -> nat { + self.detached.obj() + } + + pub closed spec fn claim(self) -> rcu_cpu_spec::RcuReclaimClaim { + self.claim + } + + pub proof fn tracked_into_parts(tracked self) -> (tracked res: ( + rcu_spec::RcuObjectId, + rcu_spec::RcuRetired, + rcu_cpu_spec::RcuReclaimClaim, + )) + ensures + res.0 == self.object(), + res.1 == self.retired(), + res.2 == self.claim(), + res.0.domain() == res.1.domain(), + res.0.obj() == res.1.obj(), + res.0.ptr() == res.1.ptr(), + equal(res.0.ptr(), self.object().ptr()), + res.0.obj() == res.2.obj(), + res.0.wf(), + res.2.is_pending(), + equal(res.2.ptr(), res.0.ptr()), + { + use_type_invariant(&self); + assert(self.claim.obj() == self.detached.object().obj()); + let tracked (object, retired, _unit) = self.detached.tracked_into_parts(); + assert(object == self.detached.object()); + assert(self.claim.obj() == object.obj()); + (object, retired, self.claim) + } +} + /// OSTD's RCU-specific specialization of the generic weak pointer atomic. /// /// This is an RCU client of Verus' native IRC11 protocol. The only local TCB /// component is `PAtomicWeakPtr`, needed because upstream does not yet expose /// a native weak-memory `AtomicPtr`. #[verifier::reject_recursive_types(T)] -pub struct RcuWeakAtomicPtr { +pub struct RcuWeakAtomicPtr { atomic: PAtomicWeakPtr, - tracked_atomic_inv: Tracked< - AtomicInvariant< - (rcu_spec::RcuRootKey, Irc11AtomicId), - (AtomicPointsTo<*mut T>, rcu_spec::RcuRootOwnedGhost), - rcu_spec::RcuOwnedWeakAtomicInv, - >, - >, + tracked_atomic_inv: Tracked<&'static RcuRootAtomicInvariant>, } -impl RcuWeakAtomicPtr { +impl RcuWeakAtomicPtr { pub closed spec fn constant(&self) -> rcu_spec::RcuRootKey { self.tracked_atomic_inv@.constant().0 } @@ -61,11 +309,12 @@ impl RcuWeakAtomicPtr { } } -impl RcuWeakAtomicPtr where +impl RcuWeakAtomicPtr where OwnPred: rcu_spec::RcuRootOwnershipPredicate, { pub const fn new( Ghost(nullable): Ghost, + Ghost(scheduler): Ghost, init: *mut T, Tracked(ownership): Tracked>, ) -> (res: Self) @@ -76,54 +325,131 @@ impl RcuWeakAtomicPtr where ensures res.well_formed(), res.constant().nullable == nullable, + res.constant().scheduler == scheduler, { let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = PAtomicWeakPtr::new(init); + proof_decl! { + let tracked unit_ownership: Option<()>; + let tracked physical_ownership: Option; + } + proof { + match ownership { + Some(ownership) => { + unit_ownership = Some(()); + physical_ownership = Some(ownership); + }, + None => { + unit_ownership = None; + physical_ownership = None; + }, + } + } let tracked g = rcu_spec::RcuRootOwnedGhost::tracked_initial( init, - ownership, + unit_ownership, points_to.hist(), timestamp, initial_view@, ); + proof_decl! { + let tracked state: RcuRootAtomicState; + } + proof { + let tracked mut permissions = rcu_cpu_spec::RcuRootPermissionState::empty( + scheduler, + g.domain(), + g.domain(), + g.retire_observation_registry(), + ); + assert(permissions.allocations() == Set::::empty()); + if physical_ownership is Some { + let tracked info = g.tracked_info_at(points_to.hist(), timestamp).tracked_unwrap(); + let ghost initial_obj = info.obj(); + let ghost initial_ownership = physical_ownership->Some_0; + assert(!permissions.allocations().contains(info.obj())); + permissions.tracked_insert(&info, physical_ownership.tracked_unwrap()); + assert(g.current_registration() is Some); + assert(info.obj() == g.current_registration()->Some_0.0.obj()); + assert(permissions.has_unretired_claim(info.obj())); + assert(permissions.keys() == Set::::empty().insert(initial_obj)); + assert(equal(info.ptr(), init)); + assert(OwnPred::owns(info.ptr(), initial_ownership)); + permissions.lemma_live_reclaim_state(initial_obj); + assert forall|obj: nat| #[trigger] permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(obj == initial_obj); + assert(g.infos().contains_key(initial_obj)); + assert(equal(info.ptr(), g.infos()[initial_obj].ptr())); + assert(permissions.reclaim_states()[initial_obj] == Some(info.ptr())); + assert(permissions.ownership(initial_obj) == initial_ownership); + }; + } else { + assert(g.current_registration() is None); + assert(permissions.keys() == Set::::empty()); + assert forall|obj: nat| #[trigger] permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by {}; + } + assert(permissions.allocations() == g.infos().dom()); + assert(permissions.scheduler() == scheduler); + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by {}; + state = RcuRootAtomicState { points_to, root: g, permissions }; + } let ghost key = rcu_spec::RcuRootKey { nullable, - domain: g.domain(), - reader_registry: g.reader_registry(), - retire_observation_registry: g.retire_observation_registry(), + scheduler, + domain: state.root.domain(), + reader_registry: state.root.reader_registry(), + retire_observation_registry: state.root.retire_observation_registry(), + reclaim_registry: state.permissions.reclaim_registry(), + active_lease_registry: state.permissions.active_lease_registry(), }; - let tracked pair = (points_to, g); proof { - assert(rcu_spec::rcu_history_inv(nullable, pair.0.hist())) by { - assert(!pair.0.hist().dom().is_empty()); + assert(rcu_spec::rcu_history_inv(nullable, state.points_to.hist())) by { + assert(!state.points_to.hist().dom().is_empty()); if !nullable { assert forall|ts: nat| - pair.0.hist().contains_timestamp(ts) implies #[trigger] pair.0.hist().value( - ts, - ).addr() != 0 by { + state.points_to.hist().contains_timestamp( + ts, + ) implies #[trigger] state.points_to.hist().value(ts).addr() != 0 by { assert(ts == timestamp); - assert(equal(pair.0.hist().value(ts), init)); + assert(equal(state.points_to.hist().value(ts), init)); }; } }; - assert(rcu_spec::rcu_current_ownership_inv::(pair.1)) by { - match pair.1.current_owned() { - Some(owned) => { - assert(ownership == Some(owned.ownership())); - assert(equal(owned.block_info().ptr(), init)); - }, - None => {}, - } - }; - assert forall|obj: nat| pair.1.removals().contains_key(obj) implies { - let removal = #[trigger] pair.1.removals()[obj]; - pair.0.get_timestamp(removal.message_view) == Some(removal.timestamp) + assert forall|obj: nat| state.root.removals().contains_key(obj) implies { + let removal = #[trigger] state.root.removals()[obj]; + state.points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) } by { - assert(pair.1.removals() == Map::empty()); + assert(state.root.removals() == Map::empty()); }; - assert(rcu_spec::RcuOwnedWeakAtomicInv::::inv((key, atomic.loc()), pair)); + assert(RcuRootAtomicInv::::inv((key, atomic.loc()), state)); } - let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), pair, 0); + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); + let tracked atomic_inv = tracked_static_ref(atomic_inv); Self { atomic, tracked_atomic_inv: Tracked(atomic_inv) } } @@ -132,21 +458,23 @@ impl RcuWeakAtomicPtr where self.well_formed(), ensures res.loc() == self.native_loc(), + opens_invariants none + no_unwind { &self.atomic } - proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &vstd::invariant::AtomicInvariant< - (rcu_spec::RcuRootKey, Irc11AtomicId), - (AtomicPointsTo<*mut T>, rcu_spec::RcuRootOwnedGhost), - rcu_spec::RcuOwnedWeakAtomicInv, + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &'static RcuRootAtomicInvariant< + T, + O, + OwnPred, >) requires self.well_formed(), ensures res.constant() == (self.constant(), self.native_loc()), { - self.tracked_atomic_inv.borrow() + self.tracked_atomic_inv.get() } /// Acquire-load helper for RCU root pointers. @@ -182,8 +510,14 @@ impl RcuWeakAtomicPtr where use_type_invariant(self); } let raw_atomic = self.raw_atomic(); - vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (points_to, g) = pair; + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { points_to, root: g, permissions } = state; proof { assert(points_to.loc() == self.native_loc()); } @@ -219,7 +553,11 @@ impl RcuWeakAtomicPtr where } result = (loaded.0, Ghost(timestamp), Ghost(published), Tracked(loaded_info)); proof { - pair = (points_to, g); + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } }); result @@ -273,6 +611,7 @@ impl RcuWeakAtomicPtr where &&& info.addr() == object.addr &&& equal(info.ptr(), res.0) &&& !res.4@.expired().contains(info.obj()) + &&& !res.4@.seen_removed().removed.contains(info.obj()) &&& res.4@.protects(info.addr(), info.obj()) }, _ => false, @@ -284,12 +623,21 @@ impl RcuWeakAtomicPtr where } let ghost start_view = tv@; let raw_atomic = self.raw_atomic(); - vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (points_to, mut g) = pair; + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { points_to, root: mut g, permissions } = state; + let ghost root_before_reader = g; proof { assert(points_to.loc() == self.native_loc()); assert(g.retire_observation_registry() == self.constant().retire_observation_registry); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); } proof_decl! { let tracked base_guard = @@ -298,7 +646,7 @@ impl RcuWeakAtomicPtr where proof { g.lemma_retired_facts_observed( points_to.hist(), - retired_facts, + &retired_facts, self.id(), start_view, ); @@ -346,7 +694,6 @@ impl RcuWeakAtomicPtr where == self.constant().retire_observation_registry); assert(base_guard.retire_observation_registry() == self.constant().retire_observation_registry); - assert(rcu_spec::rcu_current_ownership_inv::(g)); } proof_decl! { let tracked mut guard = @@ -412,7 +759,60 @@ impl RcuWeakAtomicPtr where Tracked(guard), ); proof { - pair = (points_to, g); + assert(g.current_owned() == root_before_reader.current_owned()); + assert(g.domain() == root_before_reader.domain()); + assert(g.reader_registry() == root_before_reader.reader_registry()); + assert(g.retire_observation_registry() + == root_before_reader.retire_observation_registry()); + assert(g.publications() == root_before_reader.publications()); + assert(g.infos() == root_before_reader.infos()); + assert(g.removals() == root_before_reader.removals()); + assert(rcu_spec::rcu_current_ownership_inv::< + T, + (), + rcu_spec::UnitRcuRootOwnership, + >(g)); + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + assert(root_before_reader.removals().contains_key(obj)); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + assert(permissions.allocations() == g.infos().dom()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by {}; + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by {}; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } }); result @@ -459,6 +859,7 @@ impl RcuWeakAtomicPtr where &&& info.addr() == object.addr &&& equal(info.ptr(), res.0) &&& !res.4@.expired().contains(info.obj()) + &&& !res.4@.seen_removed().removed.contains(info.obj()) &&& res.4@.protects(info.addr(), info.obj()) }, _ => false, @@ -494,6 +895,7 @@ impl RcuWeakAtomicPtr where Ghost>, Tracked>>, Tracked>, + Tracked>>, )) requires self.well_formed(), @@ -501,8 +903,11 @@ impl RcuWeakAtomicPtr where reader.cpu == cpu_reader.cpu(), reader.generation == cpu_reader.generation(), binding.registry() == reader.scheduler, + reader.scheduler == self.constant().scheduler, binding.cpu() == cpu_reader.cpu(), + binding.locals_key().len() == 1, binding.single_local_id() == cpu_reader.participant_id(), + online_cpus().contains(cpu_reader.cpu()), cpu_reader.participant_view().spec_le(old(tv)@), ensures old(tv)@.spec_le(final(tv)@), @@ -512,7 +917,6 @@ impl RcuWeakAtomicPtr where res.4@.cpu() == cpu_reader.cpu(), res.4@.generation() == cpu_reader.generation(), res.4@.participant_view() == cpu_reader.participant_view(), - res.4@.reader_fragment() == cpu_reader, res.4@.scheduler() == binding.registry(), res.4@.domain() == self.constant().domain, res.4@.reader_registry() == self.constant().reader_registry, @@ -520,9 +924,12 @@ impl RcuWeakAtomicPtr where res.4@.reader_context() == reader, res.4@.root() == self.id(), res.4@.start_view() == old(tv)@, - match (res.2@, res.3@) { - (None, None) => res.0.addr() == 0, - (Some(object), Some(info)) => { + match (res.2@, res.3@, res.5@) { + (None, None, None) => { + &&& res.0.addr() == 0 + &&& res.4@.reader_fragment() == cpu_reader + }, + (Some(object), Some(info), Some(lease)) => { &&& res.0.addr() != 0 &&& object.addr == res.0.addr() &&& info.wf() @@ -532,72 +939,448 @@ impl RcuWeakAtomicPtr where &&& info.addr() == object.addr &&& equal(info.ptr(), res.0) &&& !res.4@.expired().contains(info.obj()) + &&& !res.4@.seen_removed().removed.contains(info.obj()) &&& res.4@.protects(info.addr(), info.obj()) + &&& res.4@.reader_fragment().fraction() == cpu_reader.fraction() / 2real + &&& lease.key() == info.obj() + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == res.4@.participant_id() + &&& lease.reader_fraction() == res.4@.reader_fragment().fraction() + &&& lease.domain() == res.4@.domain() + &&& lease.root() == res.4@.root() + &&& lease.reader_context() == res.4@.reader_context() + &&& lease.start_view() == res.4@.start_view() + &&& lease.protected_addr() == info.addr() + &&& OwnPred::owns(res.0, lease.resource()) }, _ => false, }, { - let loaded = { + let result; + proof { + use_type_invariant(self); + } + let ghost start_view = tv@; + let ghost cpu_reader_at_entry = cpu_reader; + proof_decl! { + let tracked retired_facts_ref = + cpu_reader.tracked_retired_facts_observed_by(start_view); + let tracked retired_facts = retired_facts_ref.tracked_duplicate(); + let tracked mut cpu_reader = cpu_reader; + } + proof { + assert(retired_facts.observed_by(start_view)); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { + points_to, + root: mut g, + permissions: mut permissions, + } = state; + let ghost root_before_cpu_reader = g; + let ghost permissions_before_cpu_reader = permissions; + proof { + assert(points_to.loc() == self.native_loc()); + assert(g.retire_observation_registry() + == self.constant().retire_observation_registry); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + } proof_decl! { - let tracked retired_facts = - cpu_reader.tracked_retired_facts_observed_by(tv@); + let tracked base_guard = + g.tracked_start_reader(points_to.hist(), self.id(), start_view, reader); + } + proof { + g.lemma_retired_facts_observed( + points_to.hist(), + &retired_facts, + self.id(), + start_view, + ); + } + let loaded = raw_atomic.load(Ordering::Acquire, Tracked(tv), Tracked(&points_to)); + let ghost timestamp = loaded.2@.timestamp; + proof_decl! { + let tracked loaded_info; + } + proof { + assert(rcu_spec::rcu_owned_root_history_inv(points_to.hist(), g)); + loaded_info = g.tracked_info_at(points_to.hist(), timestamp); + assert(g.publications().contains_key(timestamp)); + } + proof_decl! { + let ghost published = g.published_at(timestamp); + } + proof { + match (published, &loaded_info) { + (Some(object), Some(info)) => { + assert(equal(points_to.hist().value(timestamp), loaded.0)); + assert(equal(info.ptr(), loaded.0)); + assert(info.domain() == g.domain()); + assert(info.domain() == base_guard.domain()); + }, + (None, None) => assert(loaded.0.addr() == 0), + _ => assert(false), + } + if !self.constant().nullable { + rcu_spec::rcu_history_inv_read_nonnull::(points_to.hist(), timestamp); + assert(!loaded.0.is_null()); + } + assert(base_guard.domain() == self.constant().domain); + assert(base_guard.reader_registry() == self.constant().reader_registry); + assert(base_guard.retire_observation_registry() + == self.constant().retire_observation_registry); + } + proof_decl! { + let tracked mut paper_guard = + rcu_spec::RcuReadGuardToken::tracked_from_base(base_guard); + } + proof { + assert(paper_guard.expired() + == g.root().domain_auth().observed_retired(self.id(), start_view)); + match &loaded_info { + Some(info) => { + assert(g.infos().contains_key(info.obj())); + assert(permissions.allocations().contains(info.obj())); + if !permissions.contains(info.obj()) { + let tracked completed = permissions.tracked_reclaimed(info.obj()); + assert(permissions.reclaimed().contains_key(info.obj())); + assert(g.removals().contains_key(info.obj())); + assert(completed.record().removal == g.removals()[info.obj()]); + let tracked closed = completed.tracked_closed_generation( + cpu_reader.cpu(), + ); + assert(closed.scheduler() == permissions.scheduler()); + assert(permissions.scheduler() == self.constant().scheduler); + assert(binding.registry() == self.constant().scheduler); + closed.lemma_same_participant_as_binding(&binding); + assert(completed.closed_generations()[cpu_reader.cpu()].participant_id() + == cpu_reader.participant_id()); + cpu_reader = completed.tracked_later_reader(cpu_reader); + assert(retired_facts.records().contains(completed.record())); + } + if paper_guard.expired().contains(info.obj()) { + assert(g.root().domain_auth().observed_retired( + self.id(), + start_view, + ).contains(info.obj())); + g.lemma_observed_retired( + points_to.hist(), + self.id(), + start_view, + info.obj(), + ); + let ghost removal = g.removals()[info.obj()]; + assert(removal.root == self.id()); + assert(removal.observed_by(start_view)); + assert(points_to.get_timestamp(removal.message_view) + == Some(removal.timestamp)); + points_to.get_timestamp_monotonic(start_view, removal.message_view); + assert(points_to.get_timestamp(start_view) is Some); + assert(removal.timestamp + <= points_to.get_timestamp(start_view)->Some_0); + assert(points_to.get_timestamp(start_view)->Some_0 <= timestamp); + assert(removal.timestamp <= timestamp); + assert(g.removals_wf(points_to.hist())); + assert(g.publications()[timestamp] != Some(info.obj())); + assert(published == Some(rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + })); + g.lemma_published_object_id( + points_to.hist(), + timestamp, + rcu_spec::RcuPublishedObject { + domain: info.domain(), + obj: info.obj(), + addr: info.addr(), + }, + ); + assert(g.publications()[timestamp] == Some(info.obj())); + assert(false); + } + assert(permissions.contains(info.obj())); + assert(paper_guard.can_protect(*info)); + paper_guard.tracked_protect(info); + }, + None => {}, + } + assert(cpu_reader == cpu_reader_at_entry); + } + proof_decl! { + let tracked cpu_guard = rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new( + paper_guard, + cpu_reader, + binding, + ); + let tracked final_guard; + let tracked lease; } - let loaded = self.load_acquire_rcu_guarded_with_retired( - Ghost(reader), - Tracked(retired_facts), - Tracked(tv), - ); proof { - assert forall|record: rcu_spec::RcuRetiredRecord| #[trigger] - cpu_reader.known_retired().contains(record) && record.domain - == loaded.4@.domain() && record.retire_observation_registry - == loaded.4@.retire_observation_registry() && record.removal.root - == loaded.4@.root() implies loaded.4@.expired().contains(record.obj) by { - assert(retired_facts.records().contains(record)); + assert(cpu_guard.reader_context() == reader); + match &loaded_info { + Some(info) => { + assert(permissions.contains(info.obj())); + permissions.lemma_live_reclaim_state(info.obj()); + assert(g.infos().contains_key(info.obj())); + let ghost loaded_ownership = permissions.ownership(info.obj()); + assert(permissions.reclaim_states()[info.obj()] is Some); + assert(permissions.reclaim_states()[info.obj()]->Some_0 == info.ptr()); + assert(equal(info.ptr(), loaded.0)); + assert(OwnPred::owns(loaded.0, loaded_ownership)); + let tracked split = permissions.tracked_split_loaded( + cpu_guard, + info, + ); + final_guard = split.0; + lease = Some(split.1); + assert(split.1.resource() == loaded_ownership); + assert(OwnPred::owns(loaded.0, split.1.resource())); + assert(final_guard.scheduler() == binding.registry()); + assert(final_guard.domain() == self.constant().domain); + }, + None => { + final_guard = cpu_guard; + lease = None; + assert(final_guard.scheduler() == binding.registry()); + assert(final_guard.domain() == self.constant().domain); + }, + } + assert(final_guard.reader_context() == reader); + assert(final_guard.start_view() == start_view); + match (&loaded_info, &lease) { + (None, None) => { + assert(final_guard.reader_fragment() == cpu_reader_at_entry); + }, + (Some(info), Some(lease)) => { + assert(lease.key() == info.obj()); + assert(final_guard.reader_fragment().fraction() + == cpu_reader_at_entry.fraction() / 2real); + }, + _ => assert(false), + } + assert(g.current_owned() == root_before_cpu_reader.current_owned()); + assert(g.domain() == root_before_cpu_reader.domain()); + assert(g.reader_registry() == root_before_cpu_reader.reader_registry()); + assert(g.retire_observation_registry() + == root_before_cpu_reader.retire_observation_registry()); + assert(g.publications() == root_before_cpu_reader.publications()); + assert(g.infos() == root_before_cpu_reader.infos()); + assert(g.removals() == root_before_cpu_reader.removals()); + assert(rcu_spec::rcu_current_ownership_inv::< + T, + (), + rcu_spec::UnitRcuRootOwnership, + >(g)); + assert forall|obj: nat| g.removals().contains_key(obj) implies { + let removal = #[trigger] g.removals()[obj]; + points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) + } by { + assert(root_before_cpu_reader.removals().contains_key(obj)); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + assert(permissions.allocations() + == permissions_before_cpu_reader.allocations()); + assert(permissions.reclaim_states() + == permissions_before_cpu_reader.reclaim_states()); + assert(permissions.reclaimed() == permissions_before_cpu_reader.reclaimed()); + assert(permissions.unretired_claims() + == permissions_before_cpu_reader.unretired_claims()); + assert(permissions.wf()); + assert(permissions.domain() == self.constant().domain); + assert(permissions.root() == self.constant().domain); + assert(permissions.retire_observation_registry() + == self.constant().retire_observation_registry); + assert(permissions.reclaim_registry() == self.constant().reclaim_registry); + assert(permissions.allocations() == g.infos().dom()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(permissions_before_cpu_reader.keys().contains(obj)); + assert(permissions.ownership(obj) + == permissions_before_cpu_reader.ownership(obj)); }; + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by {}; + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by {}; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } - loaded - }; - proof { - assert(loaded.4@.reader() == reader); - assert(match (loaded.2@, loaded.3@) { - (None, None) => loaded.0.addr() == 0, - (Some(object), Some(info)) => { - &&& loaded.0.addr() != 0 - &&& object.addr == loaded.0.addr() - &&& info.wf() - &&& info.domain() == object.domain - &&& info.domain() == loaded.4@.domain() - &&& info.obj() == object.obj - &&& info.addr() == object.addr - &&& equal(info.ptr(), loaded.0) - &&& !loaded.4@.expired().contains(info.obj()) - &&& loaded.4@.protects(info.addr(), info.obj()) + result = ( + loaded.0, + Ghost(timestamp), + Ghost(published), + Tracked(loaded_info), + Tracked(final_guard), + Tracked(lease), + ); + }); + result + } + + /// Return a guarded load's physical lease to this root. + /// + /// The lease's linear membership receipt identifies the active registry + /// entry after the atomic invariant is reopened. Returning it also rejoins + /// the CPU fragment retained by that entry with the executable guard. + #[verifier::atomic] + pub fn return_cpu_rcu_read_lease( + &self, + Tracked(lease): Tracked>>, + Tracked(guard): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked>) + requires + self.well_formed(), + match lease { + None => true, + Some(lease) => { + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) }, - _ => false, - }); - } - let (ptr, timestamp, published, info, Tracked(paper_guard)) = loaded; + }, + guard.wf(), + guard.domain() == self.constant().domain, + guard.root() == self.id(), + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + old(tv)@.spec_le(final(tv)@), + res@.wf(), + res@.paper_guard() == guard.paper_guard(), + res@.binding() == guard.binding(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + res@.participant_view() == guard.participant_view(), + res@.known_retired() == guard.known_retired(), + res@.domain() == guard.domain(), + res@.root() == guard.root(), + res@.reader_registry() == guard.reader_registry(), + res@.retire_observation_registry() == guard.retire_observation_registry(), + res@.reader_context() == guard.reader_context(), + res@.start_view() == guard.start_view(), + res@.expired() == guard.expired(), + res@.seen_removed() == guard.seen_removed(), + res@.protected() == guard.protected(), + res@.reader_fragment().fraction() == match lease { + None => guard.reader_fragment().fraction(), + Some(_) => guard.reader_fragment().fraction() * 2real, + }, + no_unwind + { + let raw_atomic = &self.atomic; proof_decl! { - let tracked guard = - rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new(paper_guard, cpu_reader, binding); + let tracked final_guard; } - proof { - assert(guard.reader_context() == reader); - assert(guard.reader_fragment() == cpu_reader); - match (&published@, &info@) { - (Some(object), Some(info)) => { - assert(info.domain() == guard.domain()); - assert(!guard.expired().contains(info.obj())); - assert(guard.protects(info.addr(), info.obj())); - }, - (None, None) => { - assert(ptr.addr() == 0); - }, - _ => assert(false), + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.permissions.active_lease_registry() + == self.constant().active_lease_registry); + match lease { + None => { + final_guard = guard; + }, + Some(lease) => { + let ghost permissions_before = state.permissions; + final_guard = state.permissions.tracked_return_loaded(lease, guard); + assert(state.permissions.allocations() + == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() + == permissions_before.reclaim_states()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + assert(state.permissions.reclaimed() + == permissions_before.reclaimed()); + assert forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) implies { + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 + == state.root.infos()[obj].ptr() + &&& OwnPred::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + } by { + assert(permissions_before.keys().contains(obj)); + assert(permissions_before.contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.allocations().contains(obj)); + assert(state.permissions.reclaim_states().dom().contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + }; + assert forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) implies + !state.permissions.has_unretired_claim(obj) by { + assert(!permissions_before.has_unretired_claim(obj)); + }; + assert forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) implies { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + } by { + assert(permissions_before.reclaimed().contains_key(obj)); + }; + }, + } + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } - } - (ptr, timestamp, published, info, Tracked(guard)) + }); + Tracked(final_guard) } /// End a paper read-side guard without executing another atomic operation. @@ -665,7 +1448,7 @@ impl RcuWeakAtomicPtr where value: *mut T, Tracked(ownership): Tracked>, Tracked(tv): Tracked<&mut ViewSeen>, - ) -> (res: (*mut T, Tracked>>)) + ) -> (res: (*mut T, Tracked>>)) requires self.well_formed(), self.constant().nullable || !value.is_null(), @@ -680,25 +1463,65 @@ impl RcuWeakAtomicPtr where old(tv)@.spec_le(final(tv)@), (res.1@ is Some) == !res.0.is_null(), res.1@ is Some ==> res.1@->Some_0.object().wf(), + res.1@ is Some ==> res.1@->Some_0.object().domain() == self.constant().domain, + res.1@ is Some ==> equal(res.1@->Some_0.object().ptr(), res.0), res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), res.1@ is Some ==> res.1@->Some_0.retired().obj() == res.1@->Some_0.obj(), res.1@ is Some ==> res.1@->Some_0.retired().removal().root == self.id(), + res.1@ is Some ==> res.1@->Some_0.retired().removal().root == self.constant().domain, + res.1@ is Some ==> res.1@->Some_0.retired().retire_observation_registry() + == self.constant().retire_observation_registry, res.1@ is Some ==> res.1@->Some_0.retired().removal().observed_by(final(tv)@), - res.1@ is Some ==> OwnPred::owns(res.0, res.1@->Some_0.ownership()), + res.1@ is Some ==> res.1@->Some_0.claim().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> res.1@->Some_0.claim().registry() + == self.constant().reclaim_registry, { let result; let ghost start_view = tv@; proof_decl! { let tracked retired_ownership; + let tracked unit_ownership: Option<()>; + let tracked physical_ownership: Option; } proof { use_type_invariant(self); + match ownership { + Some(ownership) => { + unit_ownership = Some(()); + physical_ownership = Some(ownership); + }, + None => { + unit_ownership = None; + physical_ownership = None; + }, + } } let raw_atomic = self.raw_atomic(); - vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (mut points_to, mut g) = pair; + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { + points_to: mut points_to, + root: mut g, + permissions: mut permissions, + } = state; + let ghost root_before_update = g; + let ghost permissions_before_update = permissions; proof { assert(points_to.loc() == self.native_loc()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + match g.current_registration() { + Some(registration) => { + assert(permissions.has_unretired_claim(registration.0.obj())); + permissions.lemma_unretired_is_live(registration.0.obj()); + }, + None => {}, + } } let ghost prev = points_to.hist(); let ghost previous_removals = g.removals(); @@ -708,8 +1531,6 @@ impl RcuWeakAtomicPtr where let ghost next = points_to.hist(); proof { assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); - assert(rcu_spec::rcu_current_ownership_inv::(g)); - rcu_spec::lemma_current_owned_resources::(prev, &g); if !self.constant().nullable { assert(!value.is_null()); } @@ -721,7 +1542,7 @@ impl RcuWeakAtomicPtr where value, update.store_message_view, ); - let tracked detached = g.tracked_push_fresh::( + let tracked detached = g.tracked_push_fresh::( prev, next, update.load_timestamp, @@ -729,14 +1550,10 @@ impl RcuWeakAtomicPtr where value, update.store_message_view, self.id(), - ownership, + unit_ownership, ); assert(detached is Some ==> detached->Some_0.object().wf()); assert(detached is Some ==> equal(detached->Some_0.ptr(), result)); - assert(detached is Some ==> OwnPred::owns( - result, - detached->Some_0.ownership(), - )); assert(detached is Some ==> detached->Some_0.retired().removal().root == self.id()); assert(detached is Some ==> detached->Some_0.retired().removal().timestamp @@ -744,15 +1561,14 @@ impl RcuWeakAtomicPtr where assert(detached is Some ==> detached->Some_0.retired().removal().observed_by( tv@, )); - assert(rcu_spec::rcu_current_ownership_inv::(g)) by { - match g.current_owned() { - Some(owned) => { - assert(ownership == Some(owned.ownership())); - assert(equal(owned.block_info().ptr(), value)); - }, - None => {}, - } - }; + assert(match detached { + Some(detached) => { + &&& root_before_update.current_registration() is Some + &&& detached.object() + == root_before_update.current_registration()->Some_0.0 + }, + None => root_before_update.current_registration() is None, + }); assert forall|obj: nat| g.removals().contains_key(obj) implies { let removal = #[trigger] g.removals()[obj]; points_to.get_timestamp(removal.message_view) == Some(removal.timestamp) @@ -772,8 +1588,123 @@ impl RcuWeakAtomicPtr where }, } }; - retired_ownership = detached; - pair = (points_to, g); + retired_ownership = match detached { + Some(detached) => { + assert(permissions.has_unretired_claim(detached.obj())); + assert(permissions.keys().contains(detached.obj())); + assert(permissions.reclaim_states()[detached.obj()] is Some); + assert(permissions.reclaim_states()[detached.obj()]->Some_0 + == root_before_update.infos()[detached.obj()].ptr()); + assert(equal( + root_before_update.infos()[detached.obj()].ptr(), + detached.object().ptr(), + )); + let tracked claim = permissions.tracked_retire(detached.obj()); + assert(claim.obj() == detached.object().obj()); + assert(equal(claim.ptr(), detached.object().ptr())); + Some(RcuRetiredRootObject { detached, claim }) + }, + None => None, + }; + permissions.lemma_all_live_reclaim_states(); + let ghost permissions_after_retire = permissions; + if physical_ownership is Some { + let tracked info = g.tracked_info_at( + points_to.hist(), + update.load_timestamp + 1, + ).tracked_unwrap(); + let ghost inserted_obj = info.obj(); + let ghost inserted_ownership = physical_ownership->Some_0; + permissions.tracked_insert(&info, physical_ownership.tracked_unwrap()); + assert(permissions.contains(inserted_obj)); + permissions.lemma_live_reclaim_state(inserted_obj); + permissions.lemma_all_live_reclaim_states(); + assert(permissions.ownership(inserted_obj) == inserted_ownership); + assert(equal(info.ptr(), value)); + assert(OwnPred::owns(info.ptr(), inserted_ownership)); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + if obj == inserted_obj { + assert(equal(info.ptr(), g.infos()[obj].ptr())); + } else { + assert(permissions_before_update.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(permissions_after_retire.contains(obj)); + assert(root_before_update.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_update.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_update.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_update.ownership(obj)); + } + }; + } else { + permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(permissions_before_update.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(root_before_update.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_update.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_update.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_update.ownership(obj)); + }; + } + assert(permissions.allocations() == g.infos().dom()); + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by { + assert(permissions_before_update.reclaimed().contains_key(obj)); + assert(root_before_update.removals().contains_key(obj)); + assert(g.removals()[obj] == root_before_update.removals()[obj]); + }; + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + permissions.lemma_all_unretired_domains(); + assert(permissions.unretired_claims().dom() == match g.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim(obj) by { + if !root_before_update.removals().contains_key(obj) { + assert(retired_ownership is Some); + assert(retired_ownership->Some_0.obj() == obj); + } + }; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } }); (result, Tracked(retired_ownership)) @@ -794,7 +1725,7 @@ impl RcuWeakAtomicPtr where ) -> (res: ( Result<*mut T, *mut T>, Ghost, - Tracked<(Option>, Option)>, + Tracked<(Option>, Option)>, )) requires self.well_formed(), @@ -813,25 +1744,66 @@ impl RcuWeakAtomicPtr where res.0 is Ok ==> res.2@.1 is None, res.0 is Ok ==> ((res.2@.0 is Some) == !res.0->Ok_0.is_null()), res.2@.0 is Some ==> res.2@.0->Some_0.object().wf(), + res.2@.0 is Some ==> res.2@.0->Some_0.object().domain() == self.constant().domain, + res.2@.0 is Some ==> equal(res.2@.0->Some_0.object().ptr(), res.0->Ok_0), res.2@.0 is Some ==> equal(res.2@.0->Some_0.ptr(), res.0->Ok_0), res.2@.0 is Some ==> res.2@.0->Some_0.retired().obj() == res.2@.0->Some_0.obj(), res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().root == self.id(), + res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().root + == self.constant().domain, + res.2@.0 is Some ==> res.2@.0->Some_0.retired().retire_observation_registry() + == self.constant().retire_observation_registry, res.2@.0 is Some ==> res.2@.0->Some_0.retired().removal().observed_by(final(tv)@), - res.2@.0 is Some ==> OwnPred::owns(res.0->Ok_0, res.2@.0->Some_0.ownership()), + res.2@.0 is Some ==> res.2@.0->Some_0.claim().obj() == res.2@.0->Some_0.obj(), + res.2@.0 is Some ==> res.2@.0->Some_0.claim().registry() + == self.constant().reclaim_registry, { let result; let ghost start_view = tv@; proof_decl! { let tracked retired_ownership; + let tracked unit_ownership: Option<()>; + let tracked physical_ownership: Option; } proof { use_type_invariant(self); + match new_ownership { + Some(ownership) => { + unit_ownership = Some(()); + physical_ownership = Some(ownership); + }, + None => { + unit_ownership = None; + physical_ownership = None; + }, + } } let raw_atomic = self.raw_atomic(); - vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => pair => { - let tracked (mut points_to, mut g) = pair; + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + let tracked RcuRootAtomicState { + points_to: mut points_to, + root: mut g, + permissions: mut permissions, + } = state; + let ghost root_before_cas = g; + let ghost permissions_before_cas = permissions; proof { assert(points_to.loc() == self.native_loc()); + permissions.lemma_all_live_reclaim_states(); + permissions.lemma_all_unretired_domains(); + match g.current_registration() { + Some(registration) => { + assert(permissions.has_unretired_claim(registration.0.obj())); + permissions.lemma_unretired_is_live(registration.0.obj()); + }, + None => {}, + } } let ghost prev = points_to.hist(); let ghost previous_removals = g.removals(); @@ -852,8 +1824,6 @@ impl RcuWeakAtomicPtr where let ghost next = points_to.hist(); proof { assert(rcu_spec::rcu_owned_root_history_inv(prev, g)); - assert(rcu_spec::rcu_current_ownership_inv::(g)); - rcu_spec::lemma_current_owned_resources::(prev, &g); match cas_result.0 { Result::Ok(_) => { rcu_spec::preserve_rcu_history_inv_on_push( @@ -864,7 +1834,9 @@ impl RcuWeakAtomicPtr where new, update.store_message_view, ); - let tracked detached = g.tracked_push_fresh::( + let tracked detached = g.tracked_push_fresh::< + rcu_spec::UnitRcuRootOwnership, + >( prev, next, update.load_timestamp, @@ -872,30 +1844,25 @@ impl RcuWeakAtomicPtr where new, update.store_message_view, self.id(), - new_ownership, + unit_ownership, ); assert(detached is Some ==> detached->Some_0.object().wf()); assert(detached is Some ==> equal( detached->Some_0.ptr(), cas_result.0->Ok_0, )); - assert(detached is Some ==> OwnPred::owns( - cas_result.0->Ok_0, - detached->Some_0.ownership(), - )); assert(detached is Some ==> detached->Some_0.retired().removal().root == self.id()); assert(detached is Some ==> detached->Some_0.retired().removal().observed_by(tv@)); - assert(rcu_spec::rcu_current_ownership_inv::(g)) by { - match g.current_owned() { - Some(owned) => { - assert(new_ownership == Some(owned.ownership())); - assert(equal(owned.block_info().ptr(), new)); - }, - None => {}, - } - }; + assert(match detached { + Some(detached) => { + &&& root_before_cas.current_registration() is Some + &&& detached.object() + == root_before_cas.current_registration()->Some_0.0 + }, + None => root_before_cas.current_registration() is None, + }); assert forall|obj: nat| g.removals().contains_key(obj) implies { let removal = #[trigger] g.removals()[obj]; points_to.get_timestamp(removal.message_view) @@ -916,14 +1883,147 @@ impl RcuWeakAtomicPtr where }, } }; + let tracked detached = match detached { + Some(detached) => { + assert(permissions.has_unretired_claim(detached.obj())); + assert(permissions.keys().contains(detached.obj())); + assert(permissions.reclaim_states()[detached.obj()] is Some); + assert(permissions.reclaim_states()[detached.obj()]->Some_0 + == root_before_cas.infos()[detached.obj()].ptr()); + assert(equal( + root_before_cas.infos()[detached.obj()].ptr(), + detached.object().ptr(), + )); + let tracked claim = permissions.tracked_retire(detached.obj()); + assert(claim.obj() == detached.object().obj()); + assert(equal(claim.ptr(), detached.object().ptr())); + Some(RcuRetiredRootObject { detached, claim }) + }, + None => None, + }; + permissions.lemma_all_live_reclaim_states(); + let ghost permissions_after_retire = permissions; + if physical_ownership is Some { + let tracked info = g.tracked_info_at( + points_to.hist(), + update.load_timestamp + 1, + ).tracked_unwrap(); + let ghost inserted_obj = info.obj(); + let ghost inserted_ownership = physical_ownership->Some_0; + permissions.tracked_insert( + &info, + physical_ownership.tracked_unwrap(), + ); + assert(permissions.contains(inserted_obj)); + permissions.lemma_live_reclaim_state(inserted_obj); + permissions.lemma_all_live_reclaim_states(); + assert(permissions.ownership(inserted_obj) == inserted_ownership); + assert(equal(info.ptr(), new)); + assert(OwnPred::owns(info.ptr(), inserted_ownership)); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 + == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + if obj == inserted_obj { + assert(equal(info.ptr(), g.infos()[obj].ptr())); + } else { + assert(permissions_before_cas.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(permissions_after_retire.contains(obj)); + assert(root_before_cas.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_cas.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_cas.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_cas.ownership(obj)); + } + }; + } else { + permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + permissions.keys().contains(obj) implies { + &&& g.infos().contains_key(obj) + &&& permissions.reclaim_states()[obj] is Some + &&& permissions.reclaim_states()[obj]->Some_0 + == g.infos()[obj].ptr() + &&& OwnPred::owns( + permissions.reclaim_states()[obj]->Some_0, + permissions.ownership(obj), + ) + } by { + assert(permissions_before_cas.keys().contains(obj)); + assert(permissions_after_retire.keys().contains(obj)); + assert(root_before_cas.infos().contains_key(obj)); + assert(g.infos()[obj] == root_before_cas.infos()[obj]); + assert(permissions.reclaim_states()[obj] + == permissions_before_cas.reclaim_states()[obj]); + assert(permissions.ownership(obj) + == permissions_after_retire.ownership(obj)); + assert(permissions_after_retire.ownership(obj) + == permissions_before_cas.ownership(obj)); + }; + } + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv( + (self.constant(), self.native_loc()), + (points_to, g), + )); + permissions.lemma_all_unretired_domains(); + assert(permissions.unretired_claims().dom() + == match g.current_registration() { + Some(registration) => { + Set::empty().insert(registration.0.obj()) + }, + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + g.removals().contains_key(obj) implies !permissions.has_unretired_claim( + obj, + ) by { + if !root_before_cas.removals().contains_key(obj) { + assert(detached is Some); + assert(detached->Some_0.obj() == obj); + } + }; retired_ownership = (detached, None); }, Result::Err(_) => { - retired_ownership = (None, new_ownership); + retired_ownership = (None, physical_ownership); assert(next == prev); + assert(permissions == permissions_before_cas); + assert(g == root_before_cas); }, } - pair = (points_to, g); + assert(permissions.allocations() == g.infos().dom()); + assert forall|obj: nat| #[trigger] + permissions.reclaimed().contains_key(obj) implies { + &&& g.removals().contains_key(obj) + &&& permissions.reclaimed()[obj].record().removal == g.removals()[obj] + } by { + assert(permissions_before_cas.reclaimed().contains_key(obj)); + match cas_result.0 { + Result::Ok(_) => { + assert(root_before_cas.removals().contains_key(obj)); + assert(g.removals()[obj] == root_before_cas.removals()[obj]); + }, + Result::Err(_) => {}, + } + }; + state = RcuRootAtomicState { points_to, root: g, permissions }; + assert(RcuRootAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } }); (result.0, result.1, Tracked(retired_ownership)) diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 09e161216..4423faaee 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -49,12 +49,15 @@ //! //! # Callback boundary //! -//! Executable callbacks are represented by `vstd_extra::raw_callback::RawCallback`. -//! `RawCallback` is proof-opaque: it only stores a thin data pointer plus a -//! monomorphized runner pointer. The RCU monitor wraps it in `monitor::RcuCallback`, -//! which can only be constructed from a `RcuCallbackSafety` certificate. This -//! prevents the proof layer from treating an arbitrary type-erased callback as a -//! safe reclamation callback. +//! Executable callbacks use +//! `vstd_extra::raw_callback::RawCallbackWithProof`. The raw +//! representation stores a thin data pointer plus a monomorphized runner +//! pointer, while its type requires the monitor's linear reclaim permit at +//! invocation. The RCU monitor wraps it in `monitor::RcuCallback`, which can +//! only be constructed from a `RcuCallbackSafety` certificate. This prevents +//! the proof layer from treating an arbitrary type-erased callback as a safe +//! reclamation callback or dropping the completion proof at the erasure +//! boundary. //! //! The monitor also has a weak-memory `is_monitoring` flag with an RCU-specific //! invariant: every flag-history message records a snapshot of the @@ -86,8 +89,8 @@ //! guard destruction reverses both changes. The scheduler can check the //! updated view back in only after the context is quiescent. //! -//! Delayed reclamation is still being wired into the weak-memory proof. The -//! weak atomic invariant retains the current registration together with +//! Delayed reclamation is connected to the weak-memory proof. The weak atomic +//! invariant retains the current registration together with //! `P::Permission`; release swap and successful CAS establish root removal, //! return the old raw pointer and matching ownership, and route a certified //! callback into the monitor. Scheduler handoff now preserves a per-CPU @@ -106,10 +109,9 @@ //! guard's protection map. The proof derives the guard's expired set from the //! entering task's view and the recorded root-removal observations. If the //! loaded AId were expired, weak-memory coherence and the root history's -//! removal invariant would contradict the load timestamp. The remaining -//! traversal boundary is converting that abstract protection into the client -//! pointer's physical reference permission; `assume_shared_ref` still stands -//! in for that final argument. +//! removal invariant would contradict the load timestamp. The guarded load +//! also splits a physical read lease from `P::Permission`; `get()` borrows that +//! lease to derive `P::RefPermission`, with no pointer-permission assumption. //! //! The proof-only `rcu_cpu` module now defines the required persistent //! `CpuRcuParticipant`: a reader splits a fractional fragment, and a quiescent @@ -123,21 +125,33 @@ //! expired objects in `SeenRemoved`, a callback permit and a guard-protected //! pointer to the same object are proved mutually exclusive. //! -//! The remaining end-to-end boundary is physical reference ownership. Guarded -//! loads must split an `RcuReadLease`, guard destruction must -//! return it, and reclamation must recover the whole pool before invoking the -//! callback. Until that is connected, `assume_shared_ref` remains the explicit -//! reference-permission bypass. +//! Before invoking a callback, its reclaim permit excludes every active lease +//! for the retired allocation. Reclamation then recovers the complete +//! `P::Permission` from the root invariant and passes it to the typed callback. +//! Two language-integration boundaries remain explicit: `read_with()` uses +//! `assume_shared_ref` until external atomic-mode guards expose the same lease +//! protocol, and verified callers use the consuming guard `drop()` method +//! because Verus cannot yet attach this invariant-opening transition to Rust's +//! implicit `Drop::drop(&mut self)`. Runtime destruction still restores the +//! executable preemption counter through `DisabledPreemptGuard`. use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; +use vstd::invariant::InvariantPredicate; use vstd::prelude::*; +use vstd::resource::Loc; use vstd_extra::prelude::*; -use vstd_extra::raw_callback::{RawCallback, RawCallbackContext}; -use vstd_extra::rcu_read_pool::RcuReadLease; +use vstd_extra::raw_callback::{RawCallbackContextWithProof, RawCallbackWithProof}; use crate::{ specs::{ - sync::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, weak_memory::RcuWeakAtomicPtr}, + mm::cpu::online_cpus, + sync::{ + rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, + weak_memory::{ + RcuRetiredRootObject, RcuRootAtomicInv, RcuRootAtomicInvariant, RcuRootAtomicState, + RcuWeakAtomicPtr, + }, + }, task::InAtomicMode, }, sync::Once, @@ -146,6 +160,7 @@ use crate::{ use vstd_extra::atomic_irc11::{ThreadViewOrder, ViewSeen}; use non_null::{NonNullPtr, NonNullPtrRef}; +use rcu_spec::RcuRootOwnershipPredicate; pub mod monitor; pub mod non_null; @@ -226,6 +241,7 @@ struct RcuReadGuardInner<'a, P: NonNullPtr> { _inner_guard: DisabledPreemptGuard, tracked_info: Tracked::Target>>>, tracked_guard: Tracked::Target>>>, + tracked_lease: Tracked::Permission>>>, tracked_session: Tracked>, } @@ -233,7 +249,18 @@ struct RcuReadGuardInner<'a, P: NonNullPtr> { /// RCU object until the monitor executes its callback. struct RcuDropCallbackContext { pointer: NonNull<

::Target>, - permission: Tracked<

::Permission>, + tracked_object: Tracked::Target>>, + tracked_claim: Tracked::Target>>, + ghost_removal: Ghost, + ghost_retire_observation_registry: Ghost, + ghost_scheduler: Ghost, + tracked_root_inv: Tracked< + &'static RcuRootAtomicInvariant< +

::Target, +

::Permission, + RcuPointerOwnership

, + >, + >, } // SAFETY: the callback consumes the same owning pointer type `P` that was @@ -243,23 +270,248 @@ unsafe impl Send for RcuDropCallbackContext

{ } -impl RawCallbackContext for RcuDropCallbackContext

{ - fn run(self) { +impl RawCallbackContextWithProof< + monitor::RcuReclaimPermit, +> for RcuDropCallbackContext

{ + open spec fn call_requires(&self, permit: monitor::RcuReclaimPermit) -> bool { + self.permit_matches(permit) + } + + fn run(self, Tracked(permit): Tracked) { + let pointer = self.pointer; + let Tracked(credit) = vstd::invariant::create_open_invariant_credit(); + proof_decl! { + let tracked permission; + } proof { use_type_invariant(&self); + use_type_invariant(&permit); + reveal(RcuDropCallbackContext::type_inv); + permit.lemma_authorizes_callback(); + let tracked root_inv = self.tracked_root_inv.get(); + let ghost callback = permit.callback(); + assert(self.permit_matches(permit)); + assert(permit.authorizes(callback)); + vstd::invariant::open_atomic_invariant_in_proof!(credit => root_inv => state => { + assert(RcuRootAtomicInv::>::inv( + root_inv.constant(), + state, + )); + crate::specs::sync::weak_memory::lemma_root_atomic_permission_facts::< +

::Target, +

::Permission, + RcuPointerOwnership

, + >(root_inv.constant(), &state); + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv(root_inv.constant(), (state.points_to, state.root))); + assert(state.permissions.wf()); + assert(callback.scheduler == state.permissions.scheduler()); + assert(callback.domain == state.permissions.domain()); + assert(callback.retire_observation_registry + == state.permissions.retire_observation_registry()); + assert(callback.removal.root == state.permissions.root()); + state.permissions.lemma_active_registry_projection(); + let ghost permissions_before_exclusion = state.permissions; + let ghost registry_before_exclusion = state.permissions.registry(); + assert forall|lease_id: nat| + state.permissions.active_ids().contains(lease_id) + && state.permissions.active_record(lease_id).key() == callback.obj + implies { + let witness = state.permissions.active_record(lease_id).witness(); + &&& witness.wf() + &&& witness.protected().obj() == callback.obj + &&& witness.reader().cpu() == witness.paper_guard().reader().cpu + &&& permit.reports().contains_key(witness.reader().cpu()) + &&& callback.scheduler == witness.binding().registry() + &&& callback.domain == witness.paper_guard().domain() + &&& callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry() + &&& callback.removal.root == witness.paper_guard().root() + } by { + assert(state.permissions.active_ids().contains(lease_id)); + assert(state.permissions.wf()); + let witness = state.permissions.active_record(lease_id).witness(); + assert(witness.wf()); + assert(witness.reader().cpu() == witness.paper_guard().reader().cpu); + assert(online_cpus().contains(witness.reader().cpu())); + assert(permit.reports().dom() == online_cpus()); + assert(permit.reports().contains_key(witness.reader().cpu())); + assert(state.permissions.active_record(lease_id).key() + == witness.protected().obj()); + assert(witness.protected().obj() == callback.obj); + assert(callback.scheduler == witness.binding().registry()); + assert(callback.domain == witness.paper_guard().domain()); + assert(callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry()); + assert(callback.removal.root == witness.paper_guard().root()); + }; + { + let tracked registry = state.permissions.tracked_registry_mut(); + assert(*registry == registry_before_exclusion); + assert forall|lease_id: nat| + (*registry).active_ids().contains(lease_id) + && (*registry).active_record(lease_id).key() == callback.obj + implies { + let witness = (*registry).active_record(lease_id).witness(); + &&& witness.wf() + &&& witness.protected().obj() == callback.obj + &&& witness.reader().cpu() == witness.paper_guard().reader().cpu + &&& permit.reports().contains_key(witness.reader().cpu()) + &&& callback.scheduler == witness.binding().registry() + &&& callback.domain == witness.paper_guard().domain() + &&& callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry() + &&& callback.removal.root == witness.paper_guard().root() + } by {}; + permit.tracked_excludes_active_leases(callback, registry); + } + assert(state.permissions == permissions_before_exclusion); + assert(state.permissions.wf()); + let tracked completed = permit.tracked_into_reclaimed_witness(callback); + assert(completed.scheduler() == state.permissions.scheduler()); + assert(completed.record() == callback.retired_record()); + assert(completed.record().domain == state.permissions.domain()); + assert(completed.record().retire_observation_registry + == state.permissions.retire_observation_registry()); + assert(completed.record().removal.root == state.permissions.root()); + { + let tracked retired_fact = completed.tracked_retired_fact(); + state.root.lemma_retired_fact_agrees(retired_fact); + } + assert(state.root.removals().contains_pair( + callback.obj, + callback.removal, + )); + state.permissions.lemma_all_unretired_domains(); + let ghost before = state.permissions; + assert(before.reclaim_registry() == root_inv.constant().0.reclaim_registry); + assert(before.unretired_claims().dom() + == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) implies + !before.has_unretired_claim(obj) by {}; + state.permissions.lemma_contains_iff_key(self.tracked_claim@.obj()); + permission = state.permissions.tracked_reclaim( + self.tracked_claim.get(), + completed, + ); + assert(before.contains(callback.obj)); + assert(before.keys().contains(callback.obj)); + assert(before.reclaim_states()[callback.obj] is Some); + assert(before.reclaim_states()[callback.obj]->Some_0 == pointer.as_ptr()); + assert(RcuPointerOwnership::

::owns(pointer.as_ptr(), permission)); + assert(P::ptr_perm_match(pointer.as_ptr(), permission)); + assert(permission.inv()); + state.permissions.lemma_all_live_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert(state.permissions.allocations() == state.root.infos().dom()); + assert forall|obj: nat| #[trigger] + state.permissions.keys().contains(obj) implies { + &&& state.permissions.contains(obj) + &&& state.permissions.allocations().contains(obj) + &&& state.permissions.reclaim_states().dom().contains(obj) + &&& state.root.infos().contains_key(obj) + &&& state.permissions.reclaim_states()[obj] is Some + &&& state.permissions.reclaim_states()[obj]->Some_0 + == state.root.infos()[obj].ptr() + &&& RcuPointerOwnership::

::owns( + state.permissions.reclaim_states()[obj]->Some_0, + state.permissions.ownership(obj), + ) + } by { + assert(obj != callback.obj); + assert(before.keys().contains(obj)); + assert(before.contains(obj)); + assert(state.permissions.ownership(obj) == before.ownership(obj)); + assert(state.permissions.reclaim_states()[obj] + == before.reclaim_states()[obj]); + }; + assert(state.permissions.unretired_claims() == before.unretired_claims()); + assert(state.permissions.unretired_claims().dom() + == match state.root.current_registration() { + Some(registration) => Set::empty().insert(registration.0.obj()), + None => Set::empty(), + }); + assert forall|obj: nat| #[trigger] + state.root.removals().contains_key(obj) implies + !state.permissions.has_unretired_claim(obj) by { + assert(!before.has_unretired_claim(obj)); + assert(before.has_unretired_claim(obj) + == before.unretired_claims().dom().contains(obj)); + assert(state.permissions.has_unretired_claim(obj) + == state.permissions.unretired_claims().dom().contains(obj)); + assert(state.permissions.has_unretired_claim(obj) + == before.has_unretired_claim(obj)); + }; + assert forall|obj: nat| #[trigger] + state.permissions.reclaimed().contains_key(obj) implies { + &&& state.root.removals().contains_key(obj) + &&& state.permissions.reclaimed()[obj].record().removal + == state.root.removals()[obj] + } by { + if obj == callback.obj { + assert(state.permissions.reclaimed()[obj].record() + == callback.retired_record()); + assert(state.root.removals()[obj] == callback.removal); + } else { + assert(before.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed()[obj] == before.reclaimed()[obj]); + } + }; + assert(state.permissions.scheduler() == root_inv.constant().0.scheduler); + assert(state.permissions.domain() == root_inv.constant().0.domain); + assert(state.permissions.root() == root_inv.constant().0.domain); + assert(state.permissions.retire_observation_registry() + == root_inv.constant().0.retire_observation_registry); + assert(state.permissions.reclaim_registry() == before.reclaim_registry()); + assert(state.permissions.reclaim_registry() + == root_inv.constant().0.reclaim_registry); + assert(state.permissions.active_lease_registry() + == root_inv.constant().0.active_lease_registry); + assert(rcu_spec::RcuOwnedWeakAtomicInv::< + rcu_spec::UnitRcuRootOwnership, + >::inv(root_inv.constant(), (state.points_to, state.root))); + crate::specs::sync::weak_memory::lemma_build_root_atomic_inv::< +

::Target, +

::Permission, + RcuPointerOwnership

, + >( + root_inv.constant(), + &state, + ); + }); } - proof_decl! { - let tracked permission = self.permission.get(); - } - let _pointer = unsafe { P::from_raw(self.pointer, Tracked(permission)) }; + let _pointer = unsafe { P::from_raw(pointer, Tracked(permission)) }; } } impl RcuDropCallbackContext

{ + pub closed spec fn permit_matches(&self, permit: monitor::RcuReclaimPermit) -> bool { + &&& permit.wf() + &&& permit.callback().domain == self.tracked_object@.domain() + &&& permit.callback().obj == self.tracked_object@.obj() + &&& permit.callback().removal == self.ghost_removal@ + &&& permit.callback().retire_observation_registry == self.ghost_retire_observation_registry@ + &&& permit.callback().scheduler == self.ghost_scheduler@ + } + #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { - &&& P::ptr_perm_match(self.pointer.as_ptr(), self.permission@) - &&& self.permission@.inv() + &&& self.tracked_object@.wf() + &&& equal(self.tracked_object@.ptr(), self.pointer.view_ptr_mut()) + &&& self.tracked_claim@.obj() == self.tracked_object@.obj() + &&& self.tracked_claim@.is_pending() + &&& equal(self.tracked_claim@.ptr(), self.pointer.view_ptr_mut()) + &&& self.tracked_object@.domain() == self.tracked_root_inv@.constant().0.domain + &&& self.tracked_claim@.registry() == self.tracked_root_inv@.constant().0.reclaim_registry + &&& self.ghost_scheduler@ == self.tracked_root_inv@.constant().0.scheduler + &&& self.ghost_removal@.root == self.tracked_root_inv@.constant().0.domain + &&& self.ghost_retire_observation_registry@ + == self.tracked_root_inv@.constant().0.retire_observation_registry } } @@ -269,31 +521,74 @@ impl RcuDropCallbackContext

{ /// still require `RcuRetired` and a monitor grace-period certificate. fn callback_from_detached( pointer: *mut

::Target, - Tracked(owned): Tracked< - rcu_spec::RcuRetiredOwnedObject<

::Target,

::Permission>, + Tracked(owned): Tracked::Target>>, + Tracked(root_inv): Tracked< + &'static RcuRootAtomicInvariant< +

::Target, +

::Permission, + RcuPointerOwnership

, + >, >, -) -> (res: (RawCallback, Tracked)) +) -> (res: (RawCallbackWithProof, Tracked)) requires !pointer.is_null(), equal(owned.ptr(), pointer), - P::ptr_perm_match(pointer, owned.ownership()), - owned.ownership().inv(), + equal(owned.object().ptr(), pointer), + owned.object().domain() == root_inv.constant().0.domain, + owned.claim().registry() == root_inv.constant().0.reclaim_registry, + owned.retired().removal().root == root_inv.constant().0.domain, + owned.retired().retire_observation_registry() + == root_inv.constant().0.retire_observation_registry, ensures res.1@.removal() == owned.retired().removal(), + forall|permit: monitor::RcuReclaimPermit| + permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj + == res.1@.obj() && permit.callback().removal == res.1@.removal() + && permit.callback().retire_observation_registry + == res.1@.retire_observation_registry() && permit.callback().scheduler + == root_inv.constant().0.scheduler ==> res.0.call_requires(permit), { proof { use_type_invariant(&owned); } proof_decl! { - let tracked (object, retired, permission) = owned.tracked_into_parts(); + let tracked (object, retired, claim) = owned.tracked_into_parts(); let tracked cert = rcu_spec::certify_callback_from_retired(&object, retired); } + proof { + assert(object.domain() == root_inv.constant().0.domain); + assert(claim.registry() == root_inv.constant().0.reclaim_registry); + assert(cert.removal().root == root_inv.constant().0.domain); + assert(cert.retire_observation_registry() + == root_inv.constant().0.retire_observation_registry); + } let pointer = unsafe { NonNull::new_unchecked(pointer) }; - let context = RcuDropCallbackContext::

{ pointer, permission: Tracked(permission) }; + proof { + assert(object.wf()); + assert(equal(object.ptr(), pointer.view_ptr_mut())); + assert(claim.obj() == object.obj()); + assert(claim.is_pending()); + assert(equal(claim.ptr(), pointer.view_ptr_mut())); + assert(object.domain() == root_inv.constant().0.domain); + assert(claim.registry() == root_inv.constant().0.reclaim_registry); + assert(root_inv.constant().0.scheduler == root_inv.constant().0.scheduler); + assert(cert.removal().root == root_inv.constant().0.domain); + assert(cert.retire_observation_registry() + == root_inv.constant().0.retire_observation_registry); + } + let context = RcuDropCallbackContext::

{ + pointer, + tracked_object: Tracked(object), + tracked_claim: Tracked(claim), + ghost_removal: Ghost(cert.removal()), + ghost_retire_observation_registry: Ghost(cert.retire_observation_registry()), + ghost_scheduler: Ghost(root_inv.constant().0.scheduler), + tracked_root_inv: Tracked(root_inv), + }; proof { use_type_invariant(&context); } - (RawCallback::new(context), Tracked(cert)) + (RawCallbackWithProof::new(context), Tracked(cert)) } impl RcuInner

{ @@ -304,6 +599,7 @@ impl RcuInner

{ closed spec fn wf(self) -> bool { &&& self.ptr.well_formed() &&& self.ptr.constant().nullable == self.ghost_nullable@ + &&& self.ptr.constant().scheduler == rcu_spec::rcu_scheduler() } } @@ -328,8 +624,14 @@ impl RcuInner

{ ensures res.type_inv(), res.is_nullable(), + res.ptr.constant().scheduler == rcu_spec::rcu_scheduler(), { - let ptr = RcuAtomicPtr::

::new(Ghost(true), core::ptr::null_mut(), Tracked(None)); + let ptr = RcuAtomicPtr::

::new( + Ghost(true), + Ghost(rcu_spec::rcu_scheduler()), + core::ptr::null_mut(), + Tracked(None), + ); Self { ptr, ghost_nullable: Ghost(true), @@ -344,6 +646,7 @@ impl RcuInner

{ ensures res.type_inv(), res.is_nullable() == nullable, + res.ptr.constant().scheduler == rcu_spec::rcu_scheduler(), )] fn new(pointer: P) -> Self { let (raw, Tracked(perm)) = P::into_raw(pointer); @@ -351,7 +654,12 @@ impl RcuInner

{ proof { assert(!raw_ptr.is_null()); } - let ptr = RcuAtomicPtr::

::new(Ghost(nullable), raw_ptr, Tracked(Some(perm))); + let ptr = RcuAtomicPtr::

::new( + Ghost(nullable), + Ghost(rcu_spec::rcu_scheduler()), + raw_ptr, + Tracked(Some(perm)), + ); Self { ptr, ghost_nullable: Ghost(nullable), @@ -402,14 +710,18 @@ impl RcuInner

{ *mut

::Target, Tracked::Target>>>, Tracked::Target>>, + Tracked::Permission>>>, )) requires self.type_inv(), cpu_reader.wf(), + online_cpus().contains(cpu_reader.cpu()), reader.cpu == cpu_reader.cpu(), reader.generation == cpu_reader.generation(), binding.registry() == reader.scheduler, + reader.scheduler == self.ptr.constant().scheduler, binding.cpu() == cpu_reader.cpu(), + binding.locals_key().len() == 1, binding.single_local_id() == cpu_reader.participant_id(), cpu_reader.participant_view().spec_le(old(tv)@), ensures @@ -420,23 +732,38 @@ impl RcuInner

{ res.2@.cpu() == cpu_reader.cpu(), res.2@.generation() == cpu_reader.generation(), res.2@.participant_view() == cpu_reader.participant_view(), - res.2@.reader_fragment() == cpu_reader, res.2@.scheduler() == binding.registry(), res.2@.domain() == self.ptr.constant().domain, res.2@.reader_registry() == self.ptr.constant().reader_registry, res.2@.retire_observation_registry() == self.ptr.constant().retire_observation_registry, res.2@.root() == self.ptr.id(), res.2@.reader_context() == reader, - match res.1@ { - None => res.0.is_null(), - Some(info) => { + match (res.1@, res.3@) { + (None, None) => { + &&& res.0.is_null() + &&& res.2@.reader_fragment() == cpu_reader + }, + (Some(info), Some(lease)) => { &&& !res.0.is_null() &&& info.wf() &&& info.domain() == res.2@.domain() &&& equal(info.ptr(), res.0) &&& !res.2@.expired().contains(info.obj()) + &&& !res.2@.seen_removed().removed.contains(info.obj()) &&& res.2@.protects(info.addr(), info.obj()) + &&& res.2@.reader_fragment().fraction() == cpu_reader.fraction() / 2real + &&& lease.key() == info.obj() + &&& lease.active_registry() == self.ptr.constant().active_lease_registry + &&& lease.participant_id() == res.2@.participant_id() + &&& lease.reader_fraction() == res.2@.reader_fragment().fraction() + &&& lease.domain() == res.2@.domain() + &&& lease.root() == res.2@.root() + &&& lease.reader_context() == res.2@.reader_context() + &&& lease.start_view() == res.2@.start_view() + &&& lease.protected_addr() == info.addr() + &&& RcuPointerOwnership::

::owns(res.0, lease.resource()) }, + _ => false, }, { proof { @@ -454,7 +781,7 @@ impl RcuInner

{ assert(!res.0.is_null()); } } - (res.0, res.3, res.4) + (res.0, res.3, res.4, res.5) } #[inline(always)] @@ -465,14 +792,7 @@ impl RcuInner

{ Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( *mut

::Target, - Tracked< - Option< - rcu_spec::RcuRetiredOwnedObject< -

::Target, -

::Permission, - >, - >, - >, + Tracked::Target>>>, )) requires self.type_inv(), @@ -489,16 +809,30 @@ impl RcuInner

{ old(tv)@.spec_le(final(tv)@), (res.1@ is Some) == !res.0.is_null(), res.1@ is Some ==> res.1@->Some_0.object().wf(), + res.1@ is Some ==> res.1@->Some_0.object().domain() == self.ptr.constant().domain, + res.1@ is Some ==> equal(res.1@->Some_0.object().ptr(), res.0), res.1@ is Some ==> equal(res.1@->Some_0.ptr(), res.0), res.1@ is Some ==> res.1@->Some_0.retired().removal().observed_by(final(tv)@), - res.1@ is Some ==> P::ptr_perm_match(res.0, res.1@->Some_0.ownership()), - res.1@ is Some ==> res.1@->Some_0.ownership().inv(), + res.1@ is Some ==> res.1@->Some_0.claim().obj() == res.1@->Some_0.obj(), + res.1@ is Some ==> res.1@->Some_0.claim().registry() + == self.ptr.constant().reclaim_registry, + res.1@ is Some ==> res.1@->Some_0.retired().removal().root + == self.ptr.constant().domain, + res.1@ is Some ==> res.1@->Some_0.retired().retire_observation_registry() + == self.ptr.constant().retire_observation_registry, { proof { assert(self.ptr.constant().nullable == self.is_nullable()); assert(self.ptr.constant().nullable || !new_ptr.is_null()); } - self.ptr.swap_release_rcu(new_ptr, Tracked(ownership), Tracked(tv)) + let res = self.ptr.swap_release_rcu(new_ptr, Tracked(ownership), Tracked(tv)); + proof { + if res.1@ is Some { + assert(res.1@->Some_0.object().domain() == self.ptr.constant().domain); + assert(res.1@->Some_0.retired().removal().root == self.ptr.constant().domain); + } + } + res } fn update(&self, new_ptr: Option

, Tracked(session): Tracked<&mut RunningTaskContext>) @@ -506,6 +840,7 @@ impl RcuInner

{ self.type_inv(), self.is_nullable() || new_ptr is Some, old(session).wf(), + old(session).scheduler() == self.ptr.constant().scheduler, ensures final(session).wf(), final(session).task() == old(session).task(), @@ -541,8 +876,13 @@ impl RcuInner

{ if !old_raw.is_null() { proof_decl! { let tracked detached = detached.tracked_unwrap(); + let tracked root_inv = self.ptr.tracked_atomic_inv(); } - let (callback, cert) = callback_from_detached::

(old_raw, Tracked(detached)); + let (callback, cert) = callback_from_detached::

( + old_raw, + Tracked(detached), + Tracked(root_inv), + ); if let Some(monitor) = RCU_MONITOR.get() { #[verus_spec(with Tracked(session))] monitor.after_grace_period(callback, cert); @@ -555,6 +895,7 @@ impl RcuInner

{ requires self.type_inv(), old(session).wf(), + old(session).scheduler() == self.ptr.constant().scheduler, old(session).available_fractions() > 1, ensures res.type_inv(), @@ -574,6 +915,7 @@ impl RcuInner

{ } proof { inner_guard.lemma_matches_context_preserved(context_before_reader, session); + session.lemma_cpu_online(); assert(session.rcu_participant_id() == context_before_disable.rcu_participant_id()); assert(session.rcu_generation() == context_before_disable.rcu_generation()); assert(session.rcu_participant_view() == context_before_disable.rcu_participant_view()); @@ -599,7 +941,7 @@ impl RcuInner

{ &inner_guard, ); } - let (obj_ptr, tracked_info, tracked_guard) = self.load_ptr_acquire_guarded( + let (obj_ptr, tracked_info, tracked_guard, tracked_lease) = self.load_ptr_acquire_guarded( Ghost(reader), Tracked(cpu_reader), Tracked(rcu_binding), @@ -628,8 +970,6 @@ impl RcuInner

{ assert(context_before_load.rcu_fraction() == context_before_reader.rcu_fraction() / 2real); assert(session.rcu_fraction() == context_before_load.rcu_fraction()); - assert(tracked_guard@.reader_fragment().fraction() == cpu_reader.fraction()); - assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction()); assert(tracked_guard@.reader_context() == (rcu_spec::RcuReaderContext { scheduler: session.scheduler(), task: session.task(), @@ -637,16 +977,33 @@ impl RcuInner

{ cpu: session.cpu(), generation: session.rcu_generation(), })); - match tracked_info@ { - None => assert(obj_ptr.is_null()), - Some(info) => { + match (tracked_info@, tracked_lease@) { + (None, None) => { + assert(obj_ptr.is_null()); + assert(tracked_guard@.reader_fragment() == cpu_reader); + assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction()); + }, + (Some(info), Some(lease)) => { assert(!obj_ptr.is_null()); assert(info.wf()); assert(info.domain() == tracked_guard@.domain()); assert(equal(info.ptr(), obj_ptr)); assert(!tracked_guard@.expired().contains(info.obj())); + assert(!tracked_guard@.seen_removed().removed.contains(info.obj())); assert(tracked_guard@.protects(info.addr(), info.obj())); + assert(tracked_guard@.reader_fragment().fraction() == session.rcu_fraction() + / 2real); + assert(lease.key() == info.obj()); + assert(lease.participant_id() == tracked_guard@.participant_id()); + assert(lease.reader_fraction() == tracked_guard@.reader_fragment().fraction()); + assert(lease.domain() == tracked_guard@.domain()); + assert(lease.root() == tracked_guard@.root()); + assert(lease.reader_context() == tracked_guard@.reader_context()); + assert(lease.start_view() == tracked_guard@.start_view()); + assert(lease.protected_addr() == info.addr()); + assert(RcuPointerOwnership::

::owns(obj_ptr, lease.resource())); }, + _ => assert(false), } } let res = RcuReadGuardInner { @@ -656,6 +1013,7 @@ impl RcuInner

{ _inner_guard: inner_guard, tracked_info, tracked_guard: Tracked(Some(tracked_guard.get())), + tracked_lease, tracked_session: Tracked(Some(session)), }; proof { @@ -664,7 +1022,12 @@ impl RcuInner

{ assert(res.guard_token().participant_id() == stored_context.rcu_participant_id()); assert(res.guard_token().cpu() == stored_context.cpu()); assert(res.guard_token().generation() == stored_context.rcu_generation()); - assert(res.guard_token().reader_fragment().fraction() == stored_context.rcu_fraction()); + match res.tracked_info@ { + None => assert(res.guard_token().reader_fragment().fraction() + == stored_context.rcu_fraction()), + Some(_) => assert(res.guard_token().reader_fragment().fraction() * 2real + == stored_context.rcu_fraction()), + } assert(res.guard_token().reader_context() == reader); assert(res.matches_context(stored_context)); } @@ -678,11 +1041,18 @@ impl RcuInner

{ /// The surrounding guard enters a private transitional state that still owns /// the preemption resource. Normal completion returns the updated session /// before the executable guard can be observed again. -fn take_reader_state<'a, T>( +fn take_reader_state<'a, T, O>( proof_active: &mut bool, Tracked(guard_slot): Tracked<&mut Tracked>>>, + Tracked(lease_slot): Tracked<&mut Tracked>>>, Tracked(session_slot): Tracked<&mut Tracked>>, -) -> (res: Tracked<(rcu_cpu_spec::CpuRcuReadGuardToken, &'a mut RunningTaskContext)>) +) -> (res: Tracked< + ( + rcu_cpu_spec::CpuRcuReadGuardToken, + Option>, + &'a mut RunningTaskContext, + ), +>) requires *old(proof_active), old(guard_slot)@ is Some, @@ -690,18 +1060,22 @@ fn take_reader_state<'a, T>( ensures !*final(proof_active), final(guard_slot)@ is None, + final(lease_slot)@ is None, final(session_slot)@ is None, res@.0 == old(guard_slot)@->Some_0, - equal(*res@.1, *old(session_slot)@->Some_0), + res@.1 == old(lease_slot)@, + equal(*res@.2, *old(session_slot)@->Some_0), opens_invariants none no_unwind { proof_decl! { let tracked guard = guard_slot.borrow_mut().tracked_take(); + let tracked mut lease = None; + vstd::modes::tracked_swap(lease_slot.borrow_mut(), &mut lease); let tracked session = session_slot.borrow_mut().tracked_take(); } * proof_active = false; - Tracked((guard, session)) + Tracked((guard, lease, session)) } /// Completes `Guard -> Inactive` and returns both reader fractions. @@ -709,6 +1083,7 @@ fn finish_reader_state<'a, P: NonNullPtr>( rcu: &RcuInner

, inner_guard: &mut DisabledPreemptGuard, Tracked(guard): Tracked::Target>>, + Tracked(lease): Tracked::Permission>>>, Tracked(session): Tracked<&'a mut RunningTaskContext>, ) -> (res: Tracked<&'a mut RunningTaskContext>) requires @@ -720,14 +1095,27 @@ fn finish_reader_state<'a, P: NonNullPtr>( guard.root() == rcu.ptr.id(), guard.retire_observation_registry() == rcu.ptr.constant().retire_observation_registry, guard.participant_id() == old(session).rcu_participant_id(), - guard.reader_fragment().fraction() == old(session).rcu_fraction(), + match lease { + None => guard.reader_fragment().fraction() == old(session).rcu_fraction(), + Some(lease) => { + &&& lease.active_registry() == rcu.ptr.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + &&& guard.reader_fragment().fraction() * 2real == old(session).rcu_fraction() + }, + }, ensures !final(inner_guard).has_resource(), (*res@).wf(), (*res@).task() == old(session).task(), (*res@).scheduler() == old(session).scheduler(), (*res@).cpu() == old(session).cpu(), - (*res@).view() == old(session).view(), + old(session).view().spec_le((*res@).view()), (*res@).session_id() == old(session).session_id(), (*res@).quiescent_generation() == old(session).quiescent_generation(), (*res@).available_fractions() == old(session).available_fractions() + 1, @@ -736,18 +1124,40 @@ fn finish_reader_state<'a, P: NonNullPtr>( (*res@).rcu_generation() == old(session).rcu_generation(), (*res@).rcu_participant_view() == old(session).rcu_participant_view(), (*res@).rcu_fraction() == old(session).rcu_fraction() * 2real, - opens_invariants none no_unwind { + let ghost context_at_entry = *session; + proof_decl! { + let tracked tv = DisabledPreemptGuard::tracked_borrow_irc11_view_mut_from_context( + session, + inner_guard, + ); + } + let Tracked(guard) = rcu.ptr.return_cpu_rcu_read_lease( + Tracked(lease), + Tracked(guard), + Tracked(tv), + ); + proof { + assert(guard.reader_fragment().fraction() == session.rcu_fraction()); + assert(context_at_entry.view().spec_le(session.view())); + } let ghost context_before_stop = *session; let Tracked(cpu_reader) = rcu.ptr.stop_cpu_rcu_reader(Tracked(guard)); proof { inner_guard.lemma_matches_context_depth(session); session.tracked_stop_rcu_reader(cpu_reader); + assert(session.view() == context_before_stop.view()); inner_guard.lemma_matches_context_preserved(context_before_stop, session); inner_guard.lemma_matches_context_depth(session); } + let ghost context_before_release = *session; inner_guard.release_in_place_to_context(Tracked(session)); + proof { + assert(context_before_release.view() == context_before_stop.view()); + assert(session.view() == context_before_release.view()); + assert(context_at_entry.view().spec_le(session.view())); + } Tracked(session) } @@ -811,19 +1221,52 @@ impl RcuInner

{ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { #[inline] #[verus_spec(res => + requires + self.proof_active, ensures !self.rcu.is_nullable() ==> res is Some, )] fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> { - let res = NonNull::new(self.obj_ptr).map(|ptr| unsafe { assume_shared_ref::

(ptr) }); proof { use_type_invariant(self); + reveal(RcuReadGuardInner::type_inv); + if !self.obj_ptr.is_null() { + match self.tracked_info@ { + None => assert(false), + Some(info) => { + assert(self.tracked_lease@ is Some); + assert(RcuPointerOwnership::

::owns( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + )); + assert(P::ptr_perm_match( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + )); + assert(self.tracked_lease@->Some_0.resource().inv()); + }, + } + } + } + let res = NonNull::new(self.obj_ptr).map( + |ptr| + requires + self.tracked_lease@ is Some, + P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_lease@->Some_0.resource()), + { + proof_decl! { + let tracked lease = self.tracked_lease.tracked_borrow(); + let tracked ref_perm = P::borrow_perm_as_ref_perm(lease.borrow()); + } + unsafe { P::raw_as_ref(ptr, Tracked(ref_perm)) } + }, + ); + proof { if !self.rcu.is_nullable() { assert(!self.obj_ptr.is_null()); assert(res is Some); } } - res } @@ -841,13 +1284,17 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { } let expected = this.obj_ptr; let rcu = this.rcu; - let tracked_state = take_reader_state::<

::Target>( + let tracked_state = take_reader_state::< +

::Target, +

::Permission, + >( &mut this.proof_active, Tracked(&mut this.tracked_guard), + Tracked(&mut this.tracked_lease), Tracked(&mut this.tracked_session), ); proof_decl! { - let tracked (guard, session) = tracked_state.get(); + let tracked (guard, lease, session) = tracked_state.get(); } let ghost context_at_entry = *session; proof_decl! { @@ -897,8 +1344,13 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { if !old_raw.is_null() { proof_decl! { let tracked detached = detached.tracked_unwrap(); + let tracked root_inv = rcu.ptr.tracked_atomic_inv(); } - let (callback, cert) = callback_from_detached::

(old_raw, Tracked(detached)); + let (callback, cert) = callback_from_detached::

( + old_raw, + Tracked(detached), + Tracked(root_inv), + ); if let Some(monitor) = RCU_MONITOR.get() { #[verus_spec(with Tracked(session))] monitor.after_grace_period(callback, cert); @@ -924,6 +1376,7 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { rcu, &mut this._inner_guard, Tracked(guard), + Tracked(lease), Tracked(session), ); let ghost restored = *session; @@ -936,67 +1389,37 @@ impl<'a, P: NonNullPtr + Send> RcuReadGuardInner<'a, P> { } } -impl<'a, P: NonNullPtr> Drop for RcuReadGuardInner<'a, P> { - fn drop(&mut self) - ensures - !final(self).is_active(), - old(self).is_active() ==> { - &&& final(self).stored_context().wf() - &&& final(self).stored_context().task() == old(self).stored_context().task() - &&& final(self).stored_context().scheduler() == old( - self, - ).stored_context().scheduler() - &&& final(self).stored_context().cpu() == old(self).stored_context().cpu() - &&& final(self).stored_context().view() == old(self).stored_context().view() - &&& final(self).stored_context().session_id() == old( - self, - ).stored_context().session_id() - &&& final(self).stored_context().quiescent_generation() == old( - self, - ).stored_context().quiescent_generation() - &&& final(self).stored_context().available_fractions() == old( - self, - ).stored_context().available_fractions() + 1 - &&& final(self).stored_context().preempt_depth() + 1 == old( - self, - ).stored_context().preempt_depth() - &&& final(self).stored_context().rcu_participant_id() == old( - self, - ).stored_context().rcu_participant_id() - &&& final(self).stored_context().rcu_generation() == old( - self, - ).stored_context().rcu_generation() - &&& final(self).stored_context().rcu_participant_view() == old( - self, - ).stored_context().rcu_participant_view() - &&& final(self).stored_context().rcu_fraction() == old( - self, - ).stored_context().rcu_fraction() * 2real - }, - opens_invariants none +impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { + fn finish(self) no_unwind { + let mut this = self; proof { - use_type_invariant(&*self); + use_type_invariant(&this); } - if self.proof_active { - let tracked_state = take_reader_state::<

::Target>( - &mut self.proof_active, - Tracked(&mut self.tracked_guard), - Tracked(&mut self.tracked_session), + if this.proof_active { + let tracked_state = take_reader_state::< +

::Target, +

::Permission, + >( + &mut this.proof_active, + Tracked(&mut this.tracked_guard), + Tracked(&mut this.tracked_lease), + Tracked(&mut this.tracked_session), ); proof_decl! { - let tracked (guard, session) = tracked_state.get(); + let tracked (guard, lease, session) = tracked_state.get(); } let Tracked(session) = finish_reader_state( - self.rcu, - &mut self._inner_guard, + this.rcu, + &mut this._inner_guard, Tracked(guard), + Tracked(lease), Tracked(session), ); let ghost restored = *session; restore_reader_session( - Tracked(&mut self.tracked_session), + Tracked(&mut this.tracked_session), Tracked(session), Ghost(restored), ); @@ -1012,23 +1435,6 @@ unsafe fn assume_shared_ref<'a, P: NonNullPtrRef<'a>>(ptr: NonNull) - unsafe { P::raw_as_ref(ptr, Tracked(perm)) } } -/// Converts an RCU storage-protocol lease into the pointer implementation's -/// reusable shared-reference permission. -/// -/// Once guarded loads carry this lease, `RcuReadGuardInner::get` can pass the -/// result directly to `P::raw_as_ref` without manufacturing a permission. -proof fn borrow_lease_as_ref_permission<'a, P: NonNullPtrRef<'a>>( - tracked lease: &'a RcuReadLease, -) -> (tracked res: P::RefPermission) - requires - lease.resource().inv(), - ensures - res.inv(), - P::ref_perm_view_permission(res) == lease.resource(), -{ - P::borrow_perm_as_ref_perm(lease.borrow()) -} - #[verus_verify] impl Rcu

{ /// Creates a new RCU primitive with the given pointer. @@ -1047,6 +1453,7 @@ impl Rcu

{ Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), @@ -1069,6 +1476,7 @@ impl Rcu

{ Tracked(session): Tracked<&'a mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), old(session).available_fractions() > 1, )] pub fn read<'a>(&'a self) -> RcuReadGuard<'a, P> { @@ -1107,6 +1515,7 @@ impl RcuOption

{ Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), ensures final(session).wf(), final(session).scheduler() == old(session).scheduler(), @@ -1129,6 +1538,7 @@ impl RcuOption

{ Tracked(session): Tracked<&'a mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), old(session).available_fractions() > 1, )] pub fn read<'a>(&'a self) -> RcuOptionReadGuard<'a, P> { @@ -1166,6 +1576,7 @@ impl RcuOption

{ impl RcuReadGuard<'_, P> { #[inline] pub fn drop(self) { + self.0.finish(); } #[inline] @@ -1196,10 +1607,14 @@ impl RcuReadGuard<'_, P> { impl RcuOptionReadGuard<'_, P> { #[inline] pub fn drop(self) { + self.0.finish(); } #[inline] pub fn get<'a>(&'a self) -> Option<

>::Ref> where P: NonNullPtrRef<'a> { + proof { + use_type_invariant(self); + } self.0.get() } @@ -1270,6 +1685,7 @@ impl Deref for RcuDrop { Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), old(session).is_quiescent(), ensures final(session).wf(), @@ -1297,6 +1713,15 @@ pub fn init() { } } // verus! +// Verus requires trait destructors to open no invariants. The verified API uses +// the consuming `Rcu(Read|OptionRead)Guard::drop` methods above; runtime builds +// retain ordinary Rust destruction, after which the embedded preemption guard +// performs the executable counter decrement. +#[cfg(not(verus_keep_ghost))] +impl<'a, P: NonNullPtr> Drop for RcuReadGuardInner<'a, P> { + fn drop(&mut self) {} +} + verus! { impl RcuInner

{ @@ -1422,9 +1847,11 @@ impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { &&& !self.rcu.is_nullable() ==> !self.obj_ptr.is_null() &&& self.proof_active == (self.tracked_guard@ is Some) &&& self.proof_active ==> self.tracked_session@ is Some + &&& self.proof_active ==> ((self.tracked_info@ is Some) == (self.tracked_lease@ is Some)) &&& self.tracked_session@ is Some ==> self.stored_context().wf() &&& self.proof_active ==> { &&& self._inner_guard.has_resource() + &&& self.stored_context().scheduler() == self.rcu.ptr.constant().scheduler &&& self.guard_token().wf() &&& self.guard_token().domain() == self.rcu.ptr.constant().domain &&& self.guard_token().root() == self.rcu.ptr.id() @@ -1432,17 +1859,41 @@ impl<'a, P: NonNullPtr> RcuReadGuardInner<'a, P> { &&& self.guard_token().retire_observation_registry() == self.rcu.ptr.constant().retire_observation_registry &&& self.matches_context(self.stored_context()) - &&& self.guard_token().reader_fragment().fraction() - == self.stored_context().rcu_fraction() &&& match self.tracked_info@ { - None => self.obj_ptr.is_null(), + None => { + &&& self.obj_ptr.is_null() + &&& self.tracked_lease@ is None + &&& self.guard_token().reader_fragment().fraction() + == self.stored_context().rcu_fraction() + }, Some(info) => { + &&& self.tracked_lease@ is Some &&& !self.obj_ptr.is_null() &&& info.wf() &&& info.domain() == self.guard_token().domain() &&& equal(info.ptr(), self.obj_ptr) &&& !self.guard_token().expired().contains(info.obj()) + &&& !self.guard_token().seen_removed().removed.contains(info.obj()) &&& self.guard_token().protects(info.addr(), info.obj()) + &&& self.guard_token().reader_fragment().fraction() * 2real + == self.stored_context().rcu_fraction() + &&& self.tracked_lease@->Some_0.key() == info.obj() + &&& self.tracked_lease@->Some_0.active_registry() + == self.rcu.ptr.constant().active_lease_registry + &&& self.tracked_lease@->Some_0.participant_id() + == self.guard_token().participant_id() + &&& self.tracked_lease@->Some_0.reader_fraction() + == self.guard_token().reader_fragment().fraction() + &&& self.tracked_lease@->Some_0.domain() == self.guard_token().domain() + &&& self.tracked_lease@->Some_0.root() == self.guard_token().root() + &&& self.tracked_lease@->Some_0.reader_context() + == self.guard_token().reader_context() + &&& self.tracked_lease@->Some_0.start_view() == self.guard_token().start_view() + &&& self.tracked_lease@->Some_0.protected_addr() == info.addr() + &&& RcuPointerOwnership::

::owns( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + ) }, } } diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 604d2f2a2..5e7ac210f 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -3,7 +3,8 @@ use alloc::collections::VecDeque; use core::sync::atomic::Ordering; use vstd::{predicate::Predicate as DataPredicate, prelude::*, resource::Loc}; -use vstd_extra::raw_callback::RawCallback; +use vstd_extra::raw_callback::RawCallbackWithProof; +use vstd_extra::rcu_read_pool::RcuTrackedReadPoolRegistry; use crate::specs::{ mm::cpu::{AtomicCpuSet, AtomicCpuSetToken, CpuId, CpuSet, online_cpus}, @@ -110,7 +111,7 @@ impl RcuQuiescentContext { } /// Historical record of one quiescent context bound to a monitor generation. -ghost struct RcuCpuQuiescentReport { +pub(super) ghost struct RcuCpuQuiescentReport { cpu: CpuId, task: Loc, scheduler: Loc, @@ -188,12 +189,12 @@ proof fn duplicate_closed_generations( /// RCU-specific wrapper around a type-erased executable callback. /// -/// `RawCallback` is intentionally proof-opaque. The summary records the object -/// identity that the monitor invariant will use to decide when this callback is -/// safe to run after a grace period. +/// The executable callback is type-erased but records that invocation must +/// supply an [`RcuReclaimPermit`]. The summary records the object identity that +/// the monitor invariant uses to decide when the callback is safe to run. #[must_use] pub struct RcuCallback { - raw: RawCallback, + raw: RawCallbackWithProof, summary: Ghost, safety: Tracked, } @@ -211,7 +212,7 @@ impl RcuCallback { /// safe to run after a grace period. #[inline] fn from_raw( - raw: RawCallback, + raw: RawCallbackWithProof, Tracked(cert): Tracked, Ghost(retire_epoch): Ghost, Ghost(retire_view): Ghost, @@ -219,6 +220,12 @@ impl RcuCallback { ) -> (res: Self) requires cert.removal().observed_by(retire_view), + forall|permit: RcuReclaimPermit| + permit.wf() && permit.callback().domain == cert.domain() && permit.callback().obj + == cert.obj() && permit.callback().removal == cert.removal() + && permit.callback().retire_observation_registry + == cert.retire_observation_registry() && permit.callback().scheduler + == scheduler ==> raw.call_requires(permit), ensures res.wf(), res@ == (rcu_spec::RcuCallbackSummary { @@ -242,6 +249,11 @@ impl RcuCallback { }; proof { cert.lemma_matches(summary); + assert forall|permit: RcuReclaimPermit| + permit.authorizes(summary) implies raw.call_requires(permit) by { + assert(permit.callback() == summary); + assert(permit.wf()); + }; } Self { raw, summary: Ghost(summary), safety: Tracked(cert) } } @@ -249,26 +261,32 @@ impl RcuCallback { /// Runs the underlying callback once the monitor has completed the grace /// period that contained this callback's retire summary. /// - /// This remains an executable type-erasure boundary. `RcuReclaimPermit` - /// proves batch membership, weak-memory view coverage, and carries a - /// closed-generation resource for every online CPU. Recovering the - /// callback object's physical permission still depends on the separate - /// read-lease protocol. + /// `RcuReclaimPermit` proves batch membership, weak-memory view coverage, + /// and carries a closed-generation resource for every online CPU. The + /// proof token crosses the executable type-erasure boundary and is handed + /// to the concrete callback context. Recovering the callback object's + /// physical permission still depends on the separate read-lease protocol. #[inline] - #[verifier::external_body] unsafe fn call_once(self, Tracked(permit): Tracked) requires self.wf(), permit.authorizes(self@), { + proof { + use_type_invariant(&self); + use_type_invariant(&permit); + assert(self.raw.call_requires(permit)); + } unsafe { - self.raw.call_once(); + self.raw.call_once(Tracked(permit)); } } closed spec fn wf(self) -> bool { &&& self.safety@.matches(self@) &&& self@.removal.observed_by(self@.retire_view) + &&& forall|permit: RcuReclaimPermit| + permit.authorizes(self@) ==> self.raw.call_requires(permit) } /// Duplicates the persistent base-retirement fact retained by this @@ -316,7 +334,7 @@ tracked struct CompletedGracePeriod { /// Those resources classify every coexisting executable guard as a later /// reader whose start view observes the callback's removal. Physical /// permission recovery remains the responsibility of the read-lease layer. -tracked struct RcuReclaimPermit { +pub(super) tracked struct RcuReclaimPermit { summary: Ghost, retired: rcu_spec::RcuRetiredFact, reports: Ghost>, @@ -324,18 +342,32 @@ tracked struct RcuReclaimPermit { } impl RcuReclaimPermit { - closed spec fn closed_generations(self) -> Map { + pub closed spec fn callback(self) -> rcu_spec::RcuCallbackSummary { + self.summary@ + } + + pub closed spec fn reports(self) -> Map { + self.reports@ + } + + pub closed spec fn closed_generations(self) -> Map< + CpuId, + rcu_cpu_spec::CpuRcuClosedGeneration, + > { self.closed_generations } - closed spec fn authorizes(self, callback: rcu_spec::RcuCallbackSummary) -> bool { + pub closed spec fn authorizes(self, callback: rcu_spec::RcuCallbackSummary) -> bool { &&& self.summary@ == callback + &&& self.retired.wf() &&& self.retired.matches(callback) + &&& self.retired.record() == callback.retired_record() &&& self.reports@.dom() == online_cpus() &&& self.closed_generations().dom() == self.reports@.dom() &&& forall|cpu: CpuId| #[trigger] self.reports@.contains_key(cpu) ==> { &&& self.reports@[cpu].cpu == cpu + &&& self.reports@[cpu].scheduler == callback.scheduler &&& self.reports@[cpu].epoch == callback.retire_epoch &&& self.reports@[cpu].matches_closed(self.closed_generations()[cpu]) &&& callback.retire_view.spec_le(self.reports@[cpu].view) @@ -346,6 +378,15 @@ impl RcuReclaimPermit { } } + /// Exposes the callback-specific authorization carried by this permit. + pub(super) proof fn lemma_authorizes_callback(tracked &self) + ensures + self.authorizes(self.callback()), + self.reports().dom() == online_cpus(), + { + use_type_invariant(self); + } + /// Classifies any still-live guard on a reported CPU as a later reader. /// /// The old-reader branch is excluded by resource validity: a guard from @@ -446,6 +487,108 @@ impl RcuReclaimPermit { assert(!guard.expired().contains(protected.obj())); assert(false); } + + /// A completed callback excludes every active physical lease for its + /// retired allocation. + /// + /// The caller supplies the registry invariant connecting each matching + /// allocation key to the split CPU-reader fragment retained with that + /// lease. The conclusion is exactly the side condition required by + /// `RcuTrackedReadPoolRegistry::reclaim`. + pub(super) proof fn tracked_excludes_active_leases( + tracked &self, + callback: rcu_spec::RcuCallbackSummary, + tracked registry: &mut RcuTrackedReadPoolRegistry< + nat, + O, + rcu_cpu_spec::CpuRcuReadLeaseWitness, + >, + ) + requires + self.authorizes(callback), + old(registry).wf(), + forall|lease_id: nat| + old(registry).active_ids().contains(lease_id) && old(registry).active_record( + lease_id, + ).key() == callback.obj ==> { + let witness = old(registry).active_record(lease_id).witness(); + &&& witness.wf() + &&& witness.protected().obj() == callback.obj + &&& witness.reader().cpu() == witness.paper_guard().reader().cpu + &&& self.reports().contains_key(witness.reader().cpu()) + &&& callback.scheduler == witness.binding().registry() + &&& callback.domain == witness.paper_guard().domain() + &&& callback.retire_observation_registry + == witness.paper_guard().retire_observation_registry() + &&& callback.removal.root == witness.paper_guard().root() + }, + ensures + *final(registry) == *old(registry), + (*final(registry)).wf(), + !(*final(registry)).has_active(callback.obj), + { + if old(registry).has_active(callback.obj) { + let ghost lease_id = choose|lease_id: nat| + old(registry).active_ids().contains(lease_id) && old(registry).active_record( + lease_id, + ).key() == callback.obj; + let tracked witness = registry.tracked_borrow_active_witness_mut(lease_id); + assert(witness.wf()); + let ghost cpu = witness.reader().cpu(); + let tracked closed = self.closed_generations.tracked_borrow(cpu); + assert(self.reports@[cpu].matches_closed(*closed)); + assert(closed.scheduler() == callback.scheduler); + witness.lemma_same_participant_as_closed(closed); + assert(witness.reader().participant_id() == closed.participant_id()); + assert(closed.known_retired().contains(callback.retired_record())); + closed.lemma_later_lease_witness_ref(witness); + assert(witness.reader().known_retired().contains(callback.retired_record())); + assert(witness.paper_guard().expired().contains(callback.obj)); + assert(witness.paper_guard().expired().subset_of( + witness.paper_guard().seen_removed().removed, + )); + assert(witness.paper_guard().seen_removed().removed.contains(callback.obj)); + assert(!witness.protected().seen_removed().removed.contains(witness.protected().obj())); + assert(witness.protected().seen_removed() == witness.paper_guard().seen_removed()); + assert(false); + } + } + + /// Retains the completed per-CPU generations for future stale-history loads. + pub(super) proof fn tracked_into_reclaimed_witness( + tracked self, + callback: rcu_spec::RcuCallbackSummary, + ) -> (tracked res: rcu_cpu_spec::RcuReclaimedWitness) + requires + self.authorizes(callback), + ensures + res.wf(), + res.record() == callback.retired_record(), + res.scheduler() == callback.scheduler, + { + let tracked RcuReclaimPermit { summary: _, retired, reports: _, closed_generations } = self; + assert forall|cpu: CpuId| #[trigger] closed_generations.contains_key(cpu) implies { + let closed = closed_generations[cpu]; + &&& closed.wf() + &&& closed.cpu() == cpu + &&& closed.scheduler() == callback.scheduler + &&& closed.known_retired().contains(callback.retired_record()) + } by {}; + rcu_cpu_spec::RcuReclaimedWitness::tracked_new( + callback.scheduler, + retired, + closed_generations, + ) + } + + pub closed spec fn wf(self) -> bool { + self.authorizes(self.callback()) + } + + #[verifier::type_invariant] + pub(super) closed spec fn type_inv(self) -> bool { + self.wf() + } } impl CompletedGracePeriod { @@ -483,6 +626,7 @@ impl CompletedGracePeriod { closed spec fn callbacks_covered(self) -> bool { forall|i: int, cpu: CpuId| 0 <= i < self.callbacks().len() && #[trigger] self.reports().contains_key(cpu) ==> { + &&& self.reports()[cpu].scheduler == self.callbacks()[i].scheduler &&& (#[trigger] self.callbacks()[i]).retire_view.spec_le(self.reports()[cpu].view) &&& self.closed_generations()[cpu].known_retired().contains( self.callbacks()[i].retired_record(), @@ -497,6 +641,7 @@ impl CompletedGracePeriod { &&& self.reports_wf() &&& forall|cpu: CpuId| #[trigger] self.reports().contains_key(cpu) ==> { + &&& self.reports()[cpu].scheduler == callback.scheduler &&& callback.retire_view.spec_le(self.reports()[cpu].view) &&& self.closed_generations()[cpu].known_retired().contains( callback.retired_record(), @@ -737,6 +882,10 @@ impl GracePeriod { requires old(self).wf(), callback_summaries(callbacks).len() > 0, + forall|i: int| + 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( + callbacks, + )[i]).scheduler == rcu_spec::rcu_scheduler(), forall|i: int| 0 <= i < callback_summaries(callbacks).len() ==> (#[trigger] callback_summaries( callbacks, @@ -778,6 +927,7 @@ impl GracePeriod { online_cpus().contains(this_cpu), context.cpu == this_cpu, context.wf(), + context.scheduler == rcu_spec::rcu_scheduler(), old(self).tracked_closed_generations@.dom() == old(self).ghost_reports@.dom(), forall|i: int| 0 <= i < old(self).callback_summaries().len() ==> (#[trigger] old( @@ -839,8 +989,12 @@ impl GracePeriod { assert(self.callback_summaries() == old(self).callback_summaries()); assert forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) implies { &&& self.ghost_reports@[cpu].cpu == cpu + &&& self.ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler() &&& self.ghost_reports@[cpu].epoch == self@.epoch &&& self.ghost_reports@[cpu].matches_closed(self.tracked_closed_generations@[cpu]) + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).scheduler == rcu_spec::rcu_scheduler() &&& forall|i: int| 0 <= i < self.callback_summaries().len() ==> ( #[trigger] self.callback_summaries()[i]).retire_view.spec_le( @@ -853,8 +1007,14 @@ impl GracePeriod { ) } by { assert(self.tracked_closed_generations@.contains_key(cpu)); + assert forall|i: int| 0 <= i < self.callback_summaries().len() implies ( + #[trigger] self.callback_summaries()[i]).scheduler == rcu_spec::rcu_scheduler() by { + assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); + }; if cpu == this_cpu { assert(self.ghost_reports@[cpu] == report); + assert(report.scheduler == context.scheduler); + assert(report.scheduler == rcu_spec::rcu_scheduler()); assert(self.ghost_reports@[cpu].matches_closed( self.tracked_closed_generations@[cpu], )); @@ -873,7 +1033,9 @@ impl GracePeriod { } else { assert(self.ghost_reports@[cpu] == old_reports[cpu]); assert(old(self).ghost_reports@.contains_key(cpu)); + assert(old(self).ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler()); assert(self.ghost_reports@[cpu] == old(self).ghost_reports@[cpu]); + assert(self.ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler()); assert(self.tracked_closed_generations@[cpu] == old( self, ).tracked_closed_generations@[cpu]); @@ -917,6 +1079,9 @@ impl GracePeriod { &&& self.tracked_cpu_mask@.wf() &&& self.ghost_reports@.dom() == self.tracked_cpu_mask@.cpus() &&& self.tracked_closed_generations@.dom() == self.ghost_reports@.dom() + &&& forall|i: int| + 0 <= i < self.callback_summaries().len() ==> ( + #[trigger] self.callback_summaries()[i]).scheduler == rcu_spec::rcu_scheduler() &&& forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) ==> self.ghost_reports@[cpu].matches_closed( self.tracked_closed_generations@[cpu], @@ -924,6 +1089,7 @@ impl GracePeriod { &&& forall|cpu: CpuId| #[trigger] self.ghost_reports@.contains_key(cpu) ==> { &&& self.ghost_reports@[cpu].cpu == cpu + &&& self.ghost_reports@[cpu].scheduler == rcu_spec::rcu_scheduler() &&& self.ghost_reports@[cpu].epoch == self@.epoch &&& forall|i: int| 0 <= i < self.callback_summaries().len() ==> ( @@ -1119,6 +1285,7 @@ impl State { fn enqueue_after_grace_period(&mut self, callback: RcuCallback) -> (started_gp: bool) requires callback.wf(), + callback@.scheduler == rcu_spec::rcu_scheduler(), callback@.retire_epoch == old(self).next_callback_epoch(), callback@.retire_view.spec_le(old(self).lock_view()), ensures @@ -1194,6 +1361,7 @@ impl State { online_cpus().contains(this_cpu), context.cpu == this_cpu, context.wf(), + context.scheduler == rcu_spec::rcu_scheduler(), forall|i: int| 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( self, @@ -1222,11 +1390,13 @@ impl State { requires self.wf(), old(context).wf(), + old(context).scheduler() == rcu_spec::rcu_scheduler(), old(context).is_quiescent(), cpu == old(context).cpu(), self.tracked_retired_facts@.observed_by(old(context).irc11_view()), ensures res@.cpu == cpu, + res@.scheduler == rcu_spec::rcu_scheduler(), res@.view == old(context).irc11_view(), res@.wf(), self.tracked_retired_facts@.records().subset_of(res@.closed.known_retired()), @@ -1277,6 +1447,7 @@ impl State { online_cpus().contains(this_cpu), context.cpu == this_cpu, context.wf(), + context.scheduler == rcu_spec::rcu_scheduler(), forall|i: int| 0 <= i < old(self).current_gp.callback_summaries().len() ==> (#[trigger] old( self, @@ -1343,6 +1514,11 @@ impl State { proof { completed_cpu_mask = self.current_gp.tracked_cpu_mask@.cpus(); completed_reports = self.current_gp.ghost_reports@; + assert forall|cpu: CpuId| #[trigger] + completed_reports.contains_key(cpu) implies completed_reports[cpu].scheduler + == rcu_spec::rcu_scheduler() by { + assert(self.current_gp.ghost_reports@.contains_key(cpu)); + }; completed_closed_generations_view = self.current_gp.tracked_closed_generations@; assert(completed_cpu_mask == online_cpus()); assert(completed_closed_generations_view.dom() == completed_reports.dom()); @@ -1425,6 +1601,7 @@ impl State { assert forall|i: int, cpu: CpuId| 0 <= i < completed.callbacks().len() && #[trigger] completed.reports().contains_key(cpu) implies { + &&& completed.reports()[cpu].scheduler == completed.callbacks()[i].scheduler &&& (#[trigger] completed.callbacks()[i]).retire_view.spec_le( completed.reports()[cpu].view, ) @@ -1433,7 +1610,9 @@ impl State { ) } by { assert(completed.callbacks()[i] == initial_current_callbacks[i]); + assert(completed.callbacks()[i].scheduler == rcu_spec::rcu_scheduler()); assert(completed.reports() == completed_reports); + assert(completed.reports()[cpu].scheduler == rcu_spec::rcu_scheduler()); assert(completed.closed_generations()[cpu].known_retired() == completed_closed_generations_view[cpu].known_retired()); }; @@ -1496,6 +1675,10 @@ impl State { #[trigger] callback_summaries(self.next_callbacks)[i]).retire_view.spec_le( self.lock_view(), ) + &&& forall|i: int| + 0 <= i < callback_summaries(self.next_callbacks).len() ==> ( + #[trigger] callback_summaries(self.next_callbacks)[i]).scheduler + == rcu_spec::rcu_scheduler() &&& self.tracked_retired_facts@.observed_by(self.lock_view()) &&& forall|i: int| 0 <= i < self.current_gp.callback_summaries().len() @@ -1655,7 +1838,17 @@ impl RcuMonitor { Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), cert@.removal().observed_by(old(session).irc11_view()), + forall|permit: RcuReclaimPermit| + permit.wf() + && permit.callback().domain == cert@.domain() + && permit.callback().obj == cert@.obj() + && permit.callback().removal == cert@.removal() + && permit.callback().retire_observation_registry + == cert@.retire_observation_registry() + && permit.callback().scheduler == old(session).scheduler() + ==> raw.call_requires(permit), ensures final(session).wf(), final(session).task() == old(session).task(), @@ -1672,7 +1865,7 @@ impl RcuMonitor { )] pub(super) fn after_grace_period( &self, - raw: RawCallback, + raw: RawCallbackWithProof, cert: Tracked, ) { proof { @@ -1737,6 +1930,7 @@ impl RcuMonitor { Tracked(session): Tracked<&mut RunningTaskContext>, requires old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), old(session).is_quiescent(), ensures final(session).wf(), diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index be67db469..287a8c8a0 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -465,6 +465,7 @@ impl RunningTaskContext { !sched_view.cpu_rcu_participant_is_stored(cpu), sched_view.current.contains_key(cpu), sched_view.current[cpu] == Some(task_view.task()), + crate::specs::mm::cpu::online_cpus().contains(cpu), ensures res.scheduler() == task_view.scheduler(), res.task() == task_view.task(), @@ -581,6 +582,7 @@ impl RunningTaskContext { &&& self.rcu_binding().locals_key() == self.core_handle.expected_locals_key() &&& self.rcu_binding().single_local_id() == self.rcu_participant_id() &&& self.rcu_participant.cpu() == self.cpu() + &&& crate::specs::mm::cpu::online_cpus().contains(self.cpu()) &&& self.rcu_participant_view().spec_le(self.irc11_view()) } @@ -594,6 +596,15 @@ impl RunningTaskContext { { } + /// A running task is checked out on a CPU in the scheduler's online set. + pub proof fn lemma_cpu_online(tracked &self) + requires + self.wf(), + ensures + crate::specs::mm::cpu::online_cpus().contains(self.cpu()), + { + } + /// Relates this running context to the scheduler snapshot from which its /// task view was checked out. pub closed spec fn wf_scheduler(self, sched_view: SchedulerView) -> bool { @@ -740,6 +751,7 @@ impl RunningTaskContext { binding.cpu() == self.cpu(), binding.owner_id() == self.core_owner_id(), binding.locals_key() == seq![self.rcu_participant_id()], + binding.locals_key().len() == 1, binding.single_local_id() == self.rcu_participant_id(), { self.rcu_binding.tracked_duplicate() diff --git a/ostd/src/task/scheduler/mod.rs b/ostd/src/task/scheduler/mod.rs index 87b1aefee..f6e3085ab 100644 --- a/ostd/src/task/scheduler/mod.rs +++ b/ostd/src/task/scheduler/mod.rs @@ -1411,8 +1411,8 @@ impl SchedulerGhostState { /// later when the task registry is introduced. pub uninterp spec fn runnable_id(runnable: &RoArc) -> Loc; -pub open spec fn valid_cpu(_cpu: CpuId) -> bool { - true +pub open spec fn valid_cpu(cpu: CpuId) -> bool { + crate::specs::mm::cpu::online_cpus().contains(cpu) } pub open spec fn can_enqueue(view: SchedulerView, task: Loc, flags: EnqueueFlags) -> bool { diff --git a/verified_libs/vstd_extra/src/atomic_weak.rs b/verified_libs/vstd_extra/src/atomic_weak.rs index d33a199e5..cd625e167 100644 --- a/verified_libs/vstd_extra/src/atomic_weak.rs +++ b/verified_libs/vstd_extra/src/atomic_weak.rs @@ -1,2405 +1,12 @@ -//! Weak-memory atomic wrappers used by the verification layer. -//! -//! This module is a TCB boundary: executable atomic operations are connected to -//! Rust atomics with `external_body`, while proofs rely only on the ghost specs -//! below. Concrete wrappers currently cover Rust integer atomics, `AtomicBoolW`, -//! and `AtomicPtrW`, all using the same view/history model. -//! -//! We focus on the repaired C11/RC11-style memory model, where relaxed behavior -//! is modeled as reading from previously written messages in a location’s modi- -//! fication history, subject to coherence. In particular, relaxed reads may ob- -//! serve stale writes, but a thread’s view prevents it from going backwards, -//! and reads do not observe future writes that have not been added to the history. -//! -//! # References -//! -//! - [RCU Verification](https://dl.acm.org/doi/pdf/10.1145/3729246) -use core::sync::atomic::{ - AtomicBool, AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, - AtomicU32, AtomicUsize, Ordering, -}; - -#[cfg(target_has_atomic = "64")] -use core::sync::atomic::{AtomicI64, AtomicU64}; - -use vstd::assert_sets_equal; -use vstd::invariant::{AtomicInvariant, InvariantPredicate}; -use vstd::prelude::*; -use vstd::resource::Loc; -use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo}; -use vstd::seq::Seq; -use vstd::thread_view::Objective; - -verus! { - -// The "global" memory is defined within the invariant we need to preserve and, -// by the definition of Iris operations, invariant can be opened by a thread -// provided that the invariant holds and it can close afterwards provided that -// the invariant holds as well. -// -// Thanks to Verus' native support for the semantics, we only need to define -// what means for `atomic` and we can freely open the invariant and provide -// customized macros for doing ergonomic updates on both the physical resources -// and the ghost tokens like message histories, views, etc. -/// An `AtomicId` is just an abstract identifier (memory location) of one atomic object. -pub type AtomicId = Loc; - -/// Logical timestamp into one atomic object's message history. -/// Timestamp 0 is always the initial message installed by `new`. -pub type Timestamp = nat; - -/// A thread-local weak-memory view. -/// -/// `seen[id] = ts` means this thread has advanced past all messages for `id` -/// older than `ts`; future reads from that atomic must not go backwards. -/// -/// Typically, if another thread has published a message with timestamp `ts` for `id`, -/// and the reader reads the message via some atomic operations, then the reader's -/// thread view will advance to at least `ts` for `id`. -pub ghost struct WmView { - pub seen: IMap, -} - -impl WmView { - /// Creates an empty view. - pub open spec fn empty() -> Self { - WmView { seen: IMap::empty() } - } - - pub open spec fn seen_at(self, id: AtomicId) -> Timestamp { - if self.seen.contains_key(id) { - self.seen[id] - } else { - // Missing entries are equivalent to only seeing the initial write. - 0nat - } - } - - /// Monotonically advance the view for one atomic object. - /// - /// Just as the name indicates, `observe` means that the current thread has observed - /// a message written by another thread with a specific timestamp; because the atomic - /// operation never "goes back", the thread's view for that atomic must advance to at - /// least that timestamp. - /// - /// "During a read from `l`, a thread can observe any message `m` from `M(l)` where - /// `m.time >= V(l)`, and updates its view to incorporate `m.time`." - pub open spec fn observe(self, id: AtomicId, ts: Timestamp) -> Self { - WmView { - seen: self.seen.insert( - id, - if self.seen_at(id) <= ts { - ts - } else { - self.seen_at(id) - }, - ), - } - } - - /// Pointwise maximum of two views. - /// - /// This is the ghost effect of an acquire read: the reader imports the - /// release view carried by the message it read. - pub open spec fn join(self, other: Self) -> Self { - WmView { - seen: IMap::new( - |id: AtomicId| self.seen.contains_key(id) || other.seen.contains_key(id), - |id: AtomicId| - if self.seen_at(id) <= other.seen_at(id) { - other.seen_at(id) - } else { - self.seen_at(id) - }, - ), - } - } - - /// Partial ordering two threads' views. - pub open spec fn spec_le(self, other: Self) -> bool { - forall|id: AtomicId| #[trigger] self.seen_at(id) <= other.seen_at(id) - } - - pub proof fn lemma_join_left(self, other: Self) - ensures - self.spec_le(self.join(other)), - { - } - - pub proof fn lemma_join_right(self, other: Self) - ensures - other.spec_le(self.join(other)), - { - } - - /// Observing one location can only advance a thread view. - pub proof fn lemma_observe(self, id: AtomicId, ts: Timestamp) - ensures - self.spec_le(self.observe(id, ts)), - { - } - - /// An acquire read can only advance a thread view. - pub proof fn lemma_acquire(self, id: AtomicId, ts: Timestamp, published: Self) - ensures - self.spec_le(self.observe(id, ts).join(published)), - { - self.lemma_observe(id, ts); - self.observe(id, ts).lemma_join_left(published); - self.lemma_spec_le_transitive(self.observe(id, ts), self.observe(id, ts).join(published)); - } - - pub proof fn lemma_spec_le_transitive(self, middle: Self, upper: Self) - requires - self.spec_le(middle), - middle.spec_le(upper), - ensures - self.spec_le(upper), - { - } -} - -/// One message in an atomic object's modification history. -/// -/// `view` is the release view published with this value. Relaxed stores publish -/// only their own timestamp; release stores publish the writer's current view. -pub ghost struct Msg { - pub value: V, - pub view: WmView, -} - -pub type History = Seq>; - -/// User-supplied invariant predicate for a weak-memory atomic. -/// -/// This mirrors `vstd::atomic_ghost::AtomicInvariantPredicate`, except the -/// predicate is over the whole message history rather than one current value. -pub trait WeakAtomicInvariantPredicate { - spec fn atomic_inv(k: K, history: History, g: G) -> bool; -} - -/// Authoritative ghost state for one atomic object's history. -/// -/// This is intentionally a thin wrapper around vstd's map resource algebra: -/// [`GhostMapAuth`] owns the authoritative timestamp-to-message map, while `len` -/// records that the domain is the contiguous range `0..len`. -/// -/// The proof-facing atomic wrapper below stores this token inside an -/// `AtomicInvariant` next to the executable atomic. -pub tracked struct HistAuth { - auth: GhostMapAuth>, - // Private: code outside this TCB module must not forge the history length, - // which would desynchronize `len` from the authoritative map domain. - ghost len: nat, -} - -// The authoritative map and its logical length are global facts about one -// atomic location. They contain no thread-subjective view permission. -unsafe impl Objective for HistAuth { - -} - -proof fn lemma_timestamp_range_insert_last(hi: Timestamp) - ensures - Set::range(0nat, hi).insert(hi) == Set::range(0nat, hi + 1), -{ - broadcast use vstd::set_lib::range_set_properties; - - assert_sets_equal!(Set::range(0nat, hi).insert(hi), Set::range(0nat, hi + 1), ts: Timestamp => { - if Set::range(0nat, hi).insert(hi).contains(ts) { - if ts != hi { - assert(Set::range(0nat, hi).contains(ts)); - assert(ts < hi); - } - assert(ts < hi + 1); - assert(Set::range(0nat, hi + 1).contains(ts)); - } - - if Set::range(0nat, hi + 1).contains(ts) { - assert(ts < hi + 1); - if ts == hi { - assert(Set::range(0nat, hi).insert(hi).contains(ts)); - } else { - assert(ts < hi); - assert(Set::range(0nat, hi).contains(ts)); - assert(Set::range(0nat, hi).insert(hi).contains(ts)); - } - } - }); -} - -impl HistAuth { - pub closed spec fn id(self) -> AtomicId { - self.auth.id() - } - - pub closed spec fn map(self) -> Map> { - self.auth@ - } - - pub closed spec fn len(self) -> nat { - self.len - } - - pub open spec fn history(self) -> History - recommends - self.wf(), - { - Seq::new(self.len(), |i: int| self.map()[i as nat]) - } - - pub open spec fn wf(self) -> bool { - &&& self.len() > 0 - &&& self.map().dom() == Set::range(0nat, self.len()) - } - - pub open spec fn valid_ts(self, ts: Timestamp) -> bool { - ts < self.len() - } - - pub open spec fn msg_at(self, ts: Timestamp) -> Msg - recommends - self.valid_ts(ts), - { - self.map()[ts] - } - - /// RC11-style relaxed readability: a read may choose any message that is - /// not older than the thread's current view for this location. - pub open spec fn readable(self, view: WmView, ts: Timestamp) -> bool { - &&& self.valid_ts(ts) - &&& view.seen_at(self.id()) <= ts - } - - /// Append one message to the authoritative history and return a persistent - /// snapshot for the newly allocated timestamp. - pub proof fn append_msg(tracked &mut self, msg: Msg) -> (tracked snap: MsgSnap) - requires - old(self).wf(), - ensures - final(self).id() == old(self).id(), - final(self).history() == old(self).history().push(msg), - final(self).wf(), - snap.id() == final(self).id(), - snap.ts() == old(self).history().len(), - snap.msg() == msg, - snap.agrees_with(*final(self)), - { - let ghost ts = self.len(); - let ghost old_dom = self.map().dom(); - - let tracked pt = self.auth.insert(ts, msg); - self.len = self.len + 1; - - // Full-crate verification does not reliably rediscover this range/domain - // fact after the ghost-map insert, so keep the append step explicit. - lemma_timestamp_range_insert_last(ts); - assert(old_dom == Set::range(0nat, ts)); - assert(self.map().dom() == old_dom.insert(ts)); - assert(self.map().dom() == Set::range(0nat, ts + 1)); - assert(ts + 1 == self.len()); - assert(self.map().dom() == Set::range(0nat, self.len())); - - let tracked psnap = pt.persist(); - MsgSnap { snap: psnap } - } -} - -/// A stable proof handle for one message; a snapshot of the message. -/// -/// The underlying vstd token is persistent/duplicable, so a message snapshot -/// can be copied through proofs without granting permission to mutate history. -/// -/// Stores return snapshots so higher layers can connect a concrete write to -/// later ownership-transfer predicates without exposing the whole history. -pub tracked struct MsgSnap { - snap: GhostPersistentPointsTo>, -} - -impl MsgSnap { - pub closed spec fn id(self) -> AtomicId { - self.snap.id() - } - - /// Fetch the timestamp of this specific message. - pub closed spec fn ts(self) -> Timestamp { - self.snap.key() - } - - /// Fetch the ghost message value of this specific message. - pub closed spec fn msg(self) -> Msg { - self.snap.value() - } - - pub open spec fn agrees_with(self, auth: HistAuth) -> bool { - &&& self.id() == auth.id() - &&& auth.valid_ts(self.ts()) - &&& self.msg() == auth.msg_at(self.ts()) - } - - pub proof fn duplicate(tracked &self) -> (tracked snap: MsgSnap) - ensures - snap.id() == self.id(), - snap.ts() == self.ts(), - snap.msg() == self.msg(), - { - let tracked psnap = self.snap.duplicate(); - MsgSnap { snap: psnap } - } - - pub proof fn agree(tracked &self, tracked auth: &HistAuth) - requires - self.id() == auth.id(), - auth.wf(), - ensures - self.agrees_with(*auth), - { - self.snap.agree(&auth.auth); - assert(auth.map().contains_pair(self.ts(), self.msg())); - } -} - -} // verus! -/// Generate the proof-facing wrapper for one concrete weak-memory atomic type. -/// -/// The generated type keeps the executable TCB wrapper separate from the -/// invariant protocol. Adding `AtomicU32W` or `AtomicBoolW` later should require -/// a new executable wrapper plus one macro invocation, not another copy of the -/// invariant glue. -macro_rules! declare_weak_atomic_type { - ($weak_atomic:ident, $pred_adapter:ident, $raw_atomic:ident, $value_ty:ty) => { - verus! { - /// Predicate adapter stored inside `AtomicInvariant`. - /// - /// The invariant contains the authoritative history and user ghost - /// state. The constant pairs the user key `K` with the logical atomic id. - pub struct $pred_adapter { - p: Pred, - } - - impl InvariantPredicate<(K, AtomicId), (HistAuth<$value_ty>, G)> for $pred_adapter< - Pred, - > where Pred: WeakAtomicInvariantPredicate { - open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<$value_ty>, G)) -> bool { - let (k, id) = k_id; - let (hist, g) = hist_g; - &&& hist.id() == id - &&& hist.wf() - &&& Pred::atomic_inv(k, hist.history(), g) - } - } - - /// A weak-memory atomic with an `atomic_ghost`-style invariant. - /// - /// The executable atomic remains the TCB wrapper. This proof-facing - /// wrapper stores the authoritative history in an `AtomicInvariant` and - /// exposes `well_formed`/`type_inv` predicates tying that history to the - /// executable atomic id. As in `vstd::atomic_ghost`, outer data - /// structures put this predicate in their own - /// `#[verifier::type_invariant]`. - pub struct $weak_atomic { - #[doc(hidden)] - atomic: $raw_atomic, - #[doc(hidden)] - atomic_inv: Tracked< - AtomicInvariant<(K, AtomicId), (HistAuth<$value_ty>, G), $pred_adapter>, - >, - } - - impl $weak_atomic { - pub closed spec fn constant(&self) -> K { - self.atomic_inv@.constant().0 - } - - /// Logical modification-history identity of this atomic. - pub closed spec fn id(&self) -> AtomicId { - self.atomic_inv@.constant().1 - } - - pub closed spec fn well_formed(&self) -> bool { - self.id() == self.atomic.id() - } - - /// Borrows the executable atomic for a client-specific invariant - /// transition. - #[doc(hidden)] - pub fn raw_atomic(&self) -> (res: &$raw_atomic) - requires - self.well_formed(), - ensures - res.id() == self.id(), - { - &self.atomic - } - - /// Borrows the invariant for a client-specific atomic operation. - /// - /// The fields remain private so the type invariant cannot be - /// bypassed by construction or replacement. - #[doc(hidden)] - pub proof fn tracked_atomic_inv( - tracked &self, - ) -> (tracked res: &AtomicInvariant< - (K, AtomicId), - (HistAuth<$value_ty>, G), - $pred_adapter, - >) - requires - self.well_formed(), - ensures - res.constant() == (self.constant(), self.id()), - { - self.atomic_inv.borrow() - } - - #[verifier::type_invariant] - pub closed spec fn type_inv(&self) -> bool { - self.well_formed() - } - } - - impl $weak_atomic where - Pred: WeakAtomicInvariantPredicate, - { - #[inline(always)] - pub const fn new( - Ghost(k): Ghost, - init: $value_ty, - Tracked(g): Tracked, - ) -> (res: Self) - requires - Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), - ensures - res.well_formed(), - res.constant() == k, - { - let (atomic, Tracked(hist)) = $raw_atomic::new(init); - let tracked pair = (hist, g); - assert($pred_adapter::::inv((k, atomic.id()), pair)); - let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); - $weak_atomic { atomic, atomic_inv: Tracked(atomic_inv) } - } - - #[inline(always)] - pub fn load_relaxed( - &self, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - ensures - old(tv)@.spec_le(final(tv)@), - { - let result; - let ghost start_view = tv@; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); - proof { - start_view.lemma_observe(self.id(), result.1@); - pair = (hist, g); - } - }); - result - } - - #[inline(always)] - pub fn load_acquire( - &self, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } - } - } - }; -} - -declare_weak_atomic_type!(WeakAtomicU8, WeakAtomicPredU8, AtomicU8W, u8); -declare_weak_atomic_type!(WeakAtomicU16, WeakAtomicPredU16, AtomicU16W, u16); -declare_weak_atomic_type!(WeakAtomicU32, WeakAtomicPredU32, AtomicU32W, u32); -declare_weak_atomic_type!(WeakAtomicUsize, WeakAtomicPredUsize, AtomicUsizeW, usize); -declare_weak_atomic_type!(WeakAtomicBool, WeakAtomicPredBool, AtomicBoolW, bool); - -#[cfg(target_has_atomic = "64")] -declare_weak_atomic_type!(WeakAtomicU64, WeakAtomicPredU64, AtomicU64W, u64); - -declare_weak_atomic_type!(WeakAtomicI8, WeakAtomicPredI8, AtomicI8W, i8); -declare_weak_atomic_type!(WeakAtomicI16, WeakAtomicPredI16, AtomicI16W, i16); -declare_weak_atomic_type!(WeakAtomicI32, WeakAtomicPredI32, AtomicI32W, i32); -declare_weak_atomic_type!(WeakAtomicIsize, WeakAtomicPredIsize, AtomicIsizeW, isize); - -#[cfg(target_has_atomic = "64")] -declare_weak_atomic_type!(WeakAtomicI64, WeakAtomicPredI64, AtomicI64W, i64); - -verus! { - -/// Predicate adapter for weak-memory pointer atomics. -/// -/// The history stores raw pointer values. This tracks the atomic pointer value -/// itself; ownership of the pointee must be modeled by the user ghost state `G`. -pub struct WeakAtomicPredPtr { - t: T, - p: Pred, -} - -impl InvariantPredicate< - (K, AtomicId), - (HistAuth<*mut T>, G), -> for WeakAtomicPredPtr where Pred: WeakAtomicInvariantPredicate { - open spec fn inv(k_id: (K, AtomicId), hist_g: (HistAuth<*mut T>, G)) -> bool { - let (k, id) = k_id; - let (hist, g) = hist_g; - &&& hist.id() == id - &&& hist.wf() - &&& Pred::atomic_inv(k, hist.history(), g) - } -} - -/// Weak-memory atomic pointer with an `atomic_ghost`-style invariant. -/// -/// This is the pointer analogue of [`WeakAtomicUsize`]. It deliberately models -/// only the atomic pointer value and its release/acquire synchronization history; -/// any ownership or validity claim about the pointed-to allocation belongs in -/// the user-supplied ghost state `G` and invariant predicate. -#[verifier::accept_recursive_types(T)] -pub struct WeakAtomicPtr { - #[doc(hidden)] - atomic: AtomicPtrW, - #[doc(hidden)] - atomic_inv: Tracked< - AtomicInvariant<(K, AtomicId), (HistAuth<*mut T>, G), WeakAtomicPredPtr>, - >, -} - -impl WeakAtomicPtr { - pub closed spec fn constant(&self) -> K { - self.atomic_inv@.constant().0 - } - - /// Logical modification-history identity of this atomic pointer. - pub closed spec fn id(&self) -> AtomicId { - self.atomic_inv@.constant().1 - } - - pub closed spec fn well_formed(&self) -> bool { - self.id() == self.atomic.id() - } - - /// Borrows the executable pointer atomic for a client-specific invariant - /// transition. - #[doc(hidden)] - pub fn raw_atomic(&self) -> (res: &AtomicPtrW) - requires - self.well_formed(), - ensures - res.id() == self.id(), - { - &self.atomic - } - - /// Borrows the invariant for a client-specific atomic operation. - #[doc(hidden)] - pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: &AtomicInvariant< - (K, AtomicId), - (HistAuth<*mut T>, G), - WeakAtomicPredPtr, - >) - requires - self.well_formed(), - ensures - res.constant() == (self.constant(), self.id()), - { - self.atomic_inv.borrow() - } - - #[verifier::type_invariant] - pub closed spec fn type_inv(&self) -> bool { - self.well_formed() - } -} - -impl WeakAtomicPtr where - Pred: WeakAtomicInvariantPredicate, - { - #[inline(always)] - pub const fn new(Ghost(k): Ghost, init: *mut T, Tracked(g): Tracked) -> (res: Self) - requires - Pred::atomic_inv(k, seq![Msg { value: init, view: WmView::empty() }], g), - ensures - res.well_formed(), - res.constant() == k, - { - let (atomic, Tracked(hist)) = AtomicPtrW::::new(init); - let tracked pair = (hist, g); - assert(WeakAtomicPredPtr::::inv((k, atomic.id()), pair)); - let tracked atomic_inv = AtomicInvariant::new((k, atomic.id()), pair, 0); - WeakAtomicPtr { atomic, atomic_inv: Tracked(atomic_inv) } - } - - #[inline(always)] - pub fn load_relaxed(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( - *mut T, - Ghost, - )) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_relaxed(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } - - #[inline(always)] - pub fn load_acquire(&self, Tracked(tv): Tracked<&mut ThreadView>) -> (res: ( - *mut T, - Ghost, - )) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - result = self.atomic.load_acquire(Tracked(&hist), Tracked(tv)); - proof { - pair = (hist, g); - } - }); - result - } -} - -pub struct TrueWeakAtomicInv; - -impl WeakAtomicInvariantPredicate for TrueWeakAtomicInv { - open spec fn atomic_inv(k: K, history: History, g: G) -> bool { - true - } -} - -impl WeakAtomicPtr { - /// Release-store helper for users with the trivial atomic invariant. - /// - /// This keeps early weak-memory clients from depending on the macro while - /// we are still shaping the client-specific ghost state. - #[inline(always)] - pub fn store_release_simple(&self, value: *mut T, Tracked(tv): Tracked<&mut ThreadView>) { - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - let _snap = self.atomic.store_release(Tracked(&mut hist), Tracked(tv), value); - proof { - pair = (hist, g); - } - }); - } - - /// Strong AcqRel/Acquire CAS helper for users with the trivial invariant. - #[inline(always)] - pub fn compare_exchange_acqrel_acquire_simple( - &self, - current: *mut T, - new: *mut T, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (Result<*mut T, *mut T>, Ghost)) { - let result; - proof { - use_type_invariant(self); - } - vstd::invariant::open_atomic_invariant!(self.atomic_inv.borrow() => pair => { - let tracked (mut hist, g) = pair; - proof { - assert(hist.id() == self.atomic_inv@.constant().1); - assert(self.atomic_inv@.constant().1 == self.atomic.id()); - assert(hist.id() == self.atomic.id()); - } - let cas_result = self.atomic.compare_exchange_acqrel_acquire( - Tracked(&mut hist), - Tracked(tv), - current, - new, - ); - result = (cas_result.0, cas_result.1); - proof { - pair = (hist, g); - } - }); - result - } -} - -/// Similar to Verus' macro [`atomic_with_ghost!`] for atomics with ghost state, -/// but for weak-memory atomics with per-thread view tokens and message histories. -/// -/// The macro opens the atomic invariant, performs the specified operation, and -/// provides the previous history, new history, and operation snapshot to the user- -/// provided proof block. The user can then write proofs about the effects of the -/// operation on the history and thread view, using the snapshot to connect to the -/// authoritative history. -#[macro_export] -macro_rules! weak_atomic_with_ghost { - ( - $atomic:expr => compare_exchange_acqrel_acquire($current:expr, $new:expr, $tv:expr); - update $prev:ident -> $next:ident; - returning $ret:ident; - timestamp $ts:ident; - message $msg:ident; - snapshot $snap:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let result; - let atomic = &($atomic); - let current = $current; - let new = $new; - proof { - use_type_invariant(atomic); - } - let raw_atomic = atomic.raw_atomic(); - ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { - #[allow(unused_mut)] - let tracked (mut hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.id()); - assert(raw_atomic.id() == atomic.id()); - assert(hist.id() == raw_atomic.id()); - } - let ghost $prev = hist.history(); - let cas_result = raw_atomic.compare_exchange_acqrel_acquire( - Tracked(&mut hist), - $tv, - current, - new, - ); - result = (cas_result.0, cas_result.1); - let ghost $next = hist.history(); - let ghost $ret = cas_result.0; - let ghost $ts = cas_result.1@; - let ghost $msg = $prev[$ts as int]; - - proof { - let tracked $snap = cas_result.2.get(); - $b - } - - proof { - pair = (hist, $g); - } - }); - result - }} - }; - ( - $atomic:expr => load_acquire($tv:expr); - returning $ret:ident; - timestamp $ts:ident; - message $msg:ident; - history $history:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let result; - let atomic = &($atomic); - proof { - use_type_invariant(atomic); - } - let raw_atomic = atomic.raw_atomic(); - ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { - #[allow(unused_mut)] - let tracked (hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.id()); - assert(raw_atomic.id() == atomic.id()); - assert(hist.id() == raw_atomic.id()); - } - let ghost $history = hist.history(); - result = raw_atomic.load_acquire(Tracked(&hist), $tv); - let ghost $ret = result.0; - let ghost $ts = result.1@; - let ghost $msg = hist.msg_at($ts); - - proof { $b } - - proof { - pair = (hist, $g); - } - }); - result - }} - }; - ( - $atomic:expr => load_relaxed($tv:expr); - returning $ret:ident; - timestamp $ts:ident; - message $msg:ident; - history $history:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let result; - let atomic = &($atomic); - proof { - use_type_invariant(atomic); - } - let raw_atomic = atomic.raw_atomic(); - ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { - #[allow(unused_mut)] - let tracked (hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.id()); - assert(raw_atomic.id() == atomic.id()); - assert(hist.id() == raw_atomic.id()); - } - let ghost $history = hist.history(); - result = raw_atomic.load_relaxed(Tracked(&hist), $tv); - let ghost $ret = result.0; - let ghost $ts = result.1@; - let ghost $msg = hist.msg_at($ts); - - proof { $b } - - proof { - pair = (hist, $g); - } - }); - result - }} - }; - ( - $atomic:expr => store_release($value:expr, $tv:expr); - update $prev:ident -> $next:ident; - snapshot $snap:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let atomic = &($atomic); - let value = $value; - proof { - use_type_invariant(atomic); - } - let raw_atomic = atomic.raw_atomic(); - ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { - #[allow(unused_mut)] - let tracked (mut hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.id()); - assert(raw_atomic.id() == atomic.id()); - assert(hist.id() == raw_atomic.id()); - } - let ghost $prev = hist.history(); - let snap_tracked = raw_atomic.store_release(Tracked(&mut hist), $tv, value); - let ghost $next = hist.history(); - - proof { - let tracked $snap = snap_tracked.get(); - $b - } - - proof { - pair = (hist, $g); - } - }); - }} - }; - ( - $atomic:expr => store_relaxed($value:expr, $tv:expr); - update $prev:ident -> $next:ident; - snapshot $snap:ident; - ghost $g:ident => $b:block - ) => { - ::vstd::prelude::verus_exec_expr! {{ - let atomic = &($atomic); - let value = $value; - proof { - use_type_invariant(atomic); - } - let raw_atomic = atomic.raw_atomic(); - ::vstd::invariant::open_atomic_invariant!(atomic.tracked_atomic_inv() => pair => { - #[allow(unused_mut)] - let tracked (mut hist, mut $g) = pair; - proof { - assert(hist.id() == atomic.id()); - assert(raw_atomic.id() == atomic.id()); - assert(hist.id() == raw_atomic.id()); - } - let ghost $prev = hist.history(); - let snap_tracked = raw_atomic.store_relaxed(Tracked(&mut hist), $tv, value); - let ghost $next = hist.history(); - - proof { - let tracked $snap = snap_tracked.get(); - $b - } - - proof { - pair = (hist, $g); - } - }); - }} - }; -} - -pub use weak_atomic_with_ghost; - -/// Explicit per-thread view token. -/// -/// Passing this token through atomic operations makes the weak-memory effects -/// visible in specs instead of hiding them in global or thread-local state. -/// -/// # Soundness -/// -/// The wrapped view is private and can only evolve through the TCB atomic -/// operations in this module. Those operations maintain the invariant that a -/// view never claims a timestamp at or beyond the length of that location's -/// history: loads observe an existing message, stores observe the message they -/// just appended, and acquire joins only import message views that were built -/// from existing timestamps. This keeps the `readable`-based postconditions of -/// loads satisfiable. Do not add raw mutators (e.g. an unconditional -/// `observe`/`join` proof fn): a forged view claiming an unwritten timestamp -/// would make the next load's postcondition vacuously false. -pub tracked struct ThreadView { - ghost view: WmView, -} - -impl View for ThreadView { - type V = WmView; - - closed spec fn view(&self) -> WmView { - self.view - } -} - -impl ThreadView { - /// Creates a fresh token holding the empty view. - /// - /// The empty view is the weakest token: it lower-bounds every location at - /// timestamp 0, so minting one is always sound — the holder merely - /// forfeits all ordering knowledge. Note that minting a fresh view - /// mid-thread over-approximates real executions (it forgets per-location - /// coherence the thread has already observed) and publishes nothing useful - /// through release stores, so executable code should thread one token per - /// logical operation or critical section, and eventually one per task. - /// - /// This generic constructor is public because schedulers live outside - /// `vstd_extra`. Minting a fresh empty view is sound but loses ordering - /// knowledge. Production OSTD code therefore calls it only when registering - /// a task or CPU, then moves that same linear token through schedule-in and - /// schedule-out. - pub proof fn new() -> (tracked res: Self) - ensures - res@ == WmView::empty(), - { - ThreadView { view: WmView::empty() } - } - - /// Imports observations from another genuine thread/CPU view. - /// - /// Unlike a raw ghost mutator, this operation cannot introduce an - /// unwritten timestamp: both operands are tracked `ThreadView` values that - /// originated from the weak-memory TCB. Scheduler context switches use it - /// to transfer observations between a CPU view and a task view. - pub proof fn tracked_join(tracked &mut self, tracked other: &Self) - ensures - final(self)@ == old(self)@.join(other@), - { - self.view = self.view.join(other.view); - } -} - -#[repr(transparent)] -#[verifier::external_body] -/// TCB wrapper around Rust's `AtomicUsize`. -/// -/// The executable field is the real atomic object. The proof layer sees only -/// the specs below, plus the uninterpreted logical identity `id`. -pub struct AtomicUsizeW { - value: AtomicUsize, -} - -impl AtomicUsizeW { - /// Logical identity of this atomic object. - /// - /// `id` has no runtime representation; it indexes ghost histories and - /// thread views. The `new` spec ties the fresh history to this identity. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: usize) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = AtomicUsizeW { value: AtomicUsize::new(init) }; - (atomic, Tracked::assume_new()) - } - - /// Relaxed load: choose a readable message and advance only this location's - /// timestamp in the caller's thread view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (usize, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - /// Acquire load: same readable message choice as relaxed, plus import the - /// release view carried by the selected message. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (usize, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) - } - - /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` - /// failure ordering. - /// - /// On success, RMW atomicity forces the read to be the latest message in - /// the modification history, and the new release message is appended - /// immediately after it. On failure, the operation is only an acquire - /// load: it may read *any* readable message whose value differs from - /// `current`, not necessarily the latest one. A strong CAS merely never - /// fails after reading a value equal to `current`. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - current: usize, - new: usize, - ) -> (res: (Result, Ghost, Tracked>>)) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& v == current - &&& read_msg.value == current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& v == read_msg.value - &&& read_msg.value != current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind - { - let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); - (result, Ghost::assume_new(), Tracked::assume_new()) - } - - /// Relaxed store: append a new message whose published view contains only - /// this store's own timestamp. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: usize, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - /// Release store: append a new message carrying the writer's current view, - /// then advance the writer's view for this location. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: usize, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } -} - -} // verus! -/// Generate a TCB executable wrapper around one Rust integer atomic type. -/// -/// All integer atomics share the same weak-memory history shape: load chooses a -/// readable message, stores append a message, and CAS either reads the latest -/// message and appends its write right after it (success), or acts as an -/// acquire read of any readable message with a different value (failure). -macro_rules! declare_integer_atomic_wrapper { - ($wrapper:ident, $rust_atomic:ident, $value_ty:ty) => { - verus! { - #[repr(transparent)] - #[verifier::external_body] - /// TCB wrapper around a Rust integer atomic. - pub struct $wrapper { - value: $rust_atomic, - } - - impl $wrapper { - /// Logical identity of this atomic object. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: $value_ty) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = $wrapper { value: $rust_atomic::new(init) }; - (atomic, Tracked::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: ($value_ty, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - current: $value_ty, - new: $value_ty, - ) -> (res: ( - Result<$value_ty, $value_ty>, - Ghost, - Tracked>>, - )) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& v == current - &&& read_msg.value == current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& v == read_msg.value - &&& read_msg.value != current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind - { - let result = self.value.compare_exchange( - current, - new, - Ordering::AcqRel, - Ordering::Acquire, - ); - (result, Ghost::assume_new(), Tracked::assume_new()) - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: $value_ty, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth<$value_ty>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: $value_ty, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } - } - } - }; -} - -declare_integer_atomic_wrapper!(AtomicU8W, AtomicU8, u8); - -declare_integer_atomic_wrapper!(AtomicU16W, AtomicU16, u16); - -declare_integer_atomic_wrapper!(AtomicU32W, AtomicU32, u32); - -declare_integer_atomic_wrapper!(AtomicIsizeW, AtomicIsize, isize); - -declare_integer_atomic_wrapper!(AtomicI8W, AtomicI8, i8); - -declare_integer_atomic_wrapper!(AtomicI16W, AtomicI16, i16); - -declare_integer_atomic_wrapper!(AtomicI32W, AtomicI32, i32); - -#[cfg(target_has_atomic = "64")] -declare_integer_atomic_wrapper!(AtomicU64W, AtomicU64, u64); - -#[cfg(target_has_atomic = "64")] -declare_integer_atomic_wrapper!(AtomicI64W, AtomicI64, i64); - -verus! { - -#[repr(transparent)] -#[verifier::external_body] -/// TCB wrapper around Rust's `AtomicBool`. -/// -/// Bool atomics share the load/store/CAS weak-memory protocol with integer -/// atomics, but they are not numeric atomics: this wrapper intentionally exposes -/// no arithmetic or bitwise fetch operations. -pub struct AtomicBoolW { - value: AtomicBool, -} - -impl AtomicBoolW { - /// Logical identity of this atomic object. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: bool) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = AtomicBoolW { value: AtomicBool::new(init) }; - (atomic, Tracked::assume_new()) - } - - /// Relaxed load: choose a readable bool message and advance only this - /// location's timestamp in the caller's thread view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (bool, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - /// Acquire load: same bool choice as relaxed, plus import the release view - /// carried by the selected message. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (bool, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& res.0 == auth.msg_at(ts).value - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) - } - - /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` - /// failure ordering. - /// - /// On success it reads the latest message and appends `new` immediately - /// after it; on failure it acts as an acquire load that may read any - /// readable message with a different value, importing that message's view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - current: bool, - new: bool, - ) -> (res: (Result, Ghost, Tracked>>)) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& v == current - &&& read_msg.value == current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& v == read_msg.value - &&& read_msg.value != current - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind - { - let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); - (result, Ghost::assume_new(), Tracked::assume_new()) - } - - /// Relaxed store: append a bool-valued message whose published view contains - /// only this store's own timestamp. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: bool, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - /// Release store: append a bool-valued message carrying the writer's current - /// view, then advance the writer's view for this location. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth>, - Tracked(tv): Tracked<&mut ThreadView>, - value: bool, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } -} - -#[repr(transparent)] -#[verifier::accept_recursive_types(T)] -#[verifier::external_body] -/// TCB wrapper around Rust's `AtomicPtr`. -/// -/// This wrapper tracks the pointer value in the weak-memory history, but it -/// does not claim ownership of, or permission to dereference, the pointee. A -/// higher-level invariant must connect pointer values to `PointsTo`, refcount, -/// hazard-pointer, RCU, or other ownership ghost state when dereference safety -/// matters. -pub struct AtomicPtrW { - value: AtomicPtr, -} - -impl AtomicPtrW { - /// Logical identity of this atomic pointer object. - pub uninterp spec fn id(&self) -> AtomicId; - - #[inline(always)] - #[verifier::external_body] - pub const fn new(init: *mut T) -> (res: (Self, Tracked>)) - ensures - res.1@.id() == res.0.id(), - res.1@.history() == seq![Msg { value: init, view: WmView::empty() }], - res.1@.wf(), - { - let atomic = AtomicPtrW { value: AtomicPtr::new(init) }; - (atomic, Tracked::assume_new()) - } - - /// Relaxed load: choose a readable pointer message and advance only this - /// location's timestamp in the caller's thread view. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_relaxed( - &self, - Tracked(auth): Tracked<&HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (*mut T, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& equal(res.0, auth.msg_at(ts).value) - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Relaxed); - (value, Ghost::assume_new()) - } - - /// Acquire load: same pointer choice as relaxed, plus import the release - /// view carried by the selected message. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn load_acquire( - &self, - Tracked(auth): Tracked<&HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - ) -> (res: (*mut T, Ghost)) - requires - auth.id() == self.id(), - auth.wf(), - ensures - ({ - let ts = res.1@; - &&& auth.readable(old(tv)@, ts) - &&& equal(res.0, auth.msg_at(ts).value) - &&& final(tv)@ == old(tv)@.observe(self.id(), ts).join(auth.msg_at(ts).view) - }), - opens_invariants none - no_unwind - { - let value = self.value.load(Ordering::Acquire); - (value, Ghost::assume_new()) - } - - /// Relaxed store: append a pointer-valued message whose published view - /// contains only this store's own timestamp. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_relaxed( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: *mut T, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: WmView::empty().observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Relaxed); - Tracked::assume_new() - } - - /// Release store: append a pointer-valued message carrying the writer's - /// current view, then advance the writer's view for this location. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn store_release( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: *mut T, - ) -> (snap: Tracked>) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let ts = old(auth).history().len(); - let msg = Msg { value, view: old(tv)@.observe(self.id(), ts) }; - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), ts) - &&& snap@.id() == self.id() - &&& snap@.ts() == ts - &&& snap@.msg() == msg - &&& snap@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - self.value.store(value, Ordering::Release); - Tracked::assume_new() - } - - /// Release swap: return the latest pointer and append a new release - /// message in the same atomic read-modify-write step. - /// - /// `Ordering::Release` does not acquire the old message's view. The caller - /// observes the new modification-order timestamp and publishes its existing - /// thread view, exactly as for a release store. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn swap_release( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - value: *mut T, - ) -> (res: (*mut T, Tracked>)) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let write_ts = old(auth).history().len(); - let old_msg = old(auth).history()[(write_ts - 1) as int]; - let write_msg = Msg { value, view: old(tv)@.observe(self.id(), write_ts) }; - &&& write_ts >= 1 - &&& equal(res.0, old_msg.value) - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == old(tv)@.observe(self.id(), write_ts) - &&& res.1@.id() == self.id() - &&& res.1@.ts() == write_ts - &&& res.1@.msg() == write_msg - &&& res.1@.agrees_with(*final(auth)) - }), - opens_invariants none - no_unwind - { - let old = self.value.swap(value, Ordering::Release); - (old, Tracked::assume_new()) - } -} - -impl AtomicPtrW { - /// Strong compare-exchange with `AcqRel` success ordering and `Acquire` - /// failure ordering. - /// - /// Pointer CAS compares runtime pointer identity, which Verus models as - /// address equality for sized pointers. The returned pointer and written - /// message still carry the full pointer value, including provenance. - /// - /// On success the read is the latest message and the write is appended - /// immediately after it; on failure the operation is an acquire load that - /// may read any readable message whose address differs from `current`. - #[inline(always)] - #[verifier::external_body] - #[verifier::atomic] - pub fn compare_exchange_acqrel_acquire( - &self, - Tracked(auth): Tracked<&mut HistAuth<*mut T>>, - Tracked(tv): Tracked<&mut ThreadView>, - current: *mut T, - new: *mut T, - ) -> (res: (Result<*mut T, *mut T>, Ghost, Tracked>>)) - requires - old(auth).id() == self.id(), - old(auth).wf(), - ensures - ({ - let read_ts = res.1@; - let read_msg = old(auth).msg_at(read_ts); - let after_read = old(tv)@.observe(self.id(), read_ts).join(read_msg.view); - &&& old(auth).readable(old(tv)@, read_ts) - &&& match res.0 { - Ok(v) => { - let write_ts = old(auth).history().len(); - let write_msg = Msg { - value: new, - view: after_read.observe(self.id(), write_ts), - }; - &&& read_ts + 1 == old(auth).history().len() - &&& current.addr() == read_msg.value.addr() - &&& equal(v, read_msg.value) - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history().push(write_msg) - &&& final(auth).wf() - &&& final(tv)@ == after_read.observe(self.id(), write_ts) - &&& res.2@ is Some - &&& res.2@->Some_0.id() == self.id() - &&& res.2@->Some_0.ts() == write_ts - &&& res.2@->Some_0.msg() == write_msg - &&& res.2@->Some_0.agrees_with(*final(auth)) - }, - Err(v) => { - &&& current.addr() != read_msg.value.addr() - &&& equal(v, read_msg.value) - &&& final(auth).id() == old(auth).id() - &&& final(auth).history() == old(auth).history() - &&& final(auth).wf() - &&& final(tv)@ == after_read - &&& res.2@ is None - }, - } - }), - opens_invariants none - no_unwind - { - let result = self.value.compare_exchange(current, new, Ordering::AcqRel, Ordering::Acquire); - (result, Ghost::assume_new(), Tracked::assume_new()) - } -} - -#[cfg(verus_keep_ghost)] -fn smoke_test_weak_atomic_with_ghost() { - let atomic = WeakAtomicUsize::<(), (), TrueWeakAtomicInv>::new(Ghost(()), 0, Tracked(())); - let tracked mut tv = ThreadView::new(); - weak_atomic_with_ghost! { - atomic => store_release(1, Tracked(&mut tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(snap.msg().value == 1); - } - } - weak_atomic_with_ghost! { - atomic => store_relaxed(2, Tracked(&mut tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(snap.msg().value == 2); - } - } - let _ = - weak_atomic_with_ghost! { - atomic => load_acquire(Tracked(&mut tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(ret == msg.value); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - atomic => load_relaxed(Tracked(&mut tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(ret == msg.value); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - atomic => compare_exchange_acqrel_acquire(2, 3, Tracked(&mut tv)); - update prev -> next; - returning ret; - timestamp ts; - message msg; - snapshot snap; - ghost g => { - assert(ts < prev.len()); - match ret { - Result::Ok(v) => { - assert(ts + 1 == prev.len()); - assert(v == 2); - assert(msg.value == 2); - match snap { - Option::Some(s) => { - assert(next == prev.push(s.msg())); - assert(s.msg().value == 3); - }, - Option::None => { - assert(false); - }, - } - }, - Result::Err(v) => { - assert(v == msg.value); - assert(msg.value != 2); - match snap { - Option::Some(_) => { - assert(false); - }, - Option::None => {}, - } - assert(next == prev); - }, - } - } - }; - - let bool_atomic = WeakAtomicBool::<(), (), TrueWeakAtomicInv>::new( - Ghost(()), - false, - Tracked(()), - ); - let tracked mut bool_tv = ThreadView::new(); - weak_atomic_with_ghost! { - bool_atomic => store_release(true, Tracked(&mut bool_tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(snap.msg().value == true); - } - } - let _ = - weak_atomic_with_ghost! { - bool_atomic => load_acquire(Tracked(&mut bool_tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(ret == msg.value); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - bool_atomic => compare_exchange_acqrel_acquire(true, false, Tracked(&mut bool_tv)); - update prev -> next; - returning ret; - timestamp ts; - message msg; - snapshot snap; - ghost g => { - assert(ts < prev.len()); - match ret { - Result::Ok(v) => { - assert(ts + 1 == prev.len()); - assert(v == true); - assert(msg.value == true); - match snap { - Option::Some(s) => { - assert(next == prev.push(s.msg())); - assert(s.msg().value == false); - }, - Option::None => { - assert(false); - }, - } - }, - Result::Err(v) => { - assert(v == msg.value); - assert(msg.value != true); - match snap { - Option::Some(_) => { - assert(false); - }, - Option::None => {}, - } - assert(next == prev); - }, - } - } - }; - - let null = core::ptr::null_mut::(); - let ptr_atomic = WeakAtomicPtr::::new( - Ghost(()), - null, - Tracked(()), - ); - let tracked mut ptr_tv = ThreadView::new(); - weak_atomic_with_ghost! { - ptr_atomic => store_release(null, Tracked(&mut ptr_tv)); - update prev -> next; - snapshot snap; - ghost g => { - assert(next == prev.push(snap.msg())); - assert(snap.ts() == prev.len()); - assert(equal(snap.msg().value, null)); - } - } - let _ = - weak_atomic_with_ghost! { - ptr_atomic => load_acquire(Tracked(&mut ptr_tv)); - returning ret; - timestamp ts; - message msg; - history history; - ghost g => { - assert(equal(ret, msg.value)); - assert(ts < history.len()); - } - }; - let _ = - weak_atomic_with_ghost! { - ptr_atomic => compare_exchange_acqrel_acquire(null, null, Tracked(&mut ptr_tv)); - update prev -> next; - returning ret; - timestamp ts; - message msg; - snapshot snap; - ghost g => { - assert(ts < prev.len()); - match ret { - Result::Ok(v) => { - assert(ts + 1 == prev.len()); - assert(equal(v, msg.value)); - assert(msg.value.addr() == null.addr()); - match snap { - Option::Some(s) => { - assert(next == prev.push(s.msg())); - assert(equal(s.msg().value, null)); - }, - Option::None => { - assert(false); - }, - } - }, - Result::Err(v) => { - assert(equal(v, msg.value)); - assert(msg.value.addr() != null.addr()); - match snap { - Option::Some(_) => { - assert(false); - }, - Option::None => {}, - } - assert(next == prev); - }, - } - } - }; -} - -#[cfg(verus_keep_ghost)] -pub struct MessagePassingDataInv; - -#[cfg(verus_keep_ghost)] -impl WeakAtomicInvariantPredicate<(), usize, ()> for MessagePassingDataInv { - open spec fn atomic_inv(k: (), history: History, g: ()) -> bool { - &&& history.len() >= 1 - &&& history[0].value == 0 - &&& forall|i: int| 1 <= i < history.len() ==> #[trigger] history[i].value == 1 - } -} - -#[cfg(verus_keep_ghost)] -pub struct MessagePassingFlagInv; - -#[cfg(verus_keep_ghost)] -impl WeakAtomicInvariantPredicate for MessagePassingFlagInv { - open spec fn atomic_inv(data_id: AtomicId, history: History, g: ()) -> bool { - &&& history.len() >= 1 - &&& history[0].value == 0 - &&& forall|i: int| - 1 <= i < history.len() ==> { - &&& #[trigger] history[i].value == 1 - &&& history[i].view.seen_at(data_id) >= 1 - } - } -} - -#[cfg(verus_keep_ghost)] -proof fn preserve_message_passing_data_inv_on_push( - prev: History, - next: History, - msg: Msg, -) - requires - MessagePassingDataInv::atomic_inv((), prev, ()), - next == prev.push(msg), - msg.value == 1, - ensures - MessagePassingDataInv::atomic_inv((), next, ()), -{ - assert(next.len() >= 1); - assert(next[0].value == 0); - assert forall|i: int| 1 <= i < next.len() implies #[trigger] next[i].value == 1 by { - if i == prev.len() { - assert(next[i] == msg); - } else { - assert(i < prev.len()); - } - }; -} - -#[cfg(verus_keep_ghost)] -proof fn preserve_message_passing_flag_inv_on_push( - data_id: AtomicId, - prev: History, - next: History, - msg: Msg, -) - requires - MessagePassingFlagInv::atomic_inv(data_id, prev, ()), - next == prev.push(msg), - msg.value == 1, - msg.view.seen_at(data_id) >= 1, - ensures - MessagePassingFlagInv::atomic_inv(data_id, next, ()), -{ - assert(next.len() >= 1); - assert(next[0].value == 0); - assert forall|i: int| 1 <= i < next.len() implies { - &&& #[trigger] next[i].value == 1 - &&& next[i].view.seen_at(data_id) >= 1 - } by { - if i == prev.len() { - assert(next[i] == msg); - } else { - assert(i < prev.len()); - } - }; -} - -#[cfg(verus_keep_ghost)] -proof fn prove_message_passing_data_read( - data_id: AtomicId, - history: History, - ret: usize, - ts: Timestamp, - msg: Msg, - tv: WmView, -) - requires - MessagePassingDataInv::atomic_inv((), history, ()), - ts < history.len(), - ret == msg.value, - msg == history[ts as int], - tv.seen_at(data_id) >= 1, - tv.seen_at(data_id) <= ts, - ensures - ret == 1, -{ - assert(ts >= 1); - assert(history[ts as int].value == 1); -} - -// #[cfg(verus_keep_ghost)] -// fn message_passing_release_acquire_threads_can_prove() { -// let data = std::sync::Arc::new( -// WeakAtomicUsize::<(), (), MessagePassingDataInv>::new(Ghost(()), 0, Tracked(())), -// ); -// let ghost data_id = data.atomic.id(); -// let flag = std::sync::Arc::new( -// WeakAtomicUsize::::new(Ghost(data_id), 0, Tracked(())), -// ); -// let data_writer = data.clone(); -// let flag_writer = flag.clone(); -// let data_reader = data.clone(); -// let flag_reader = flag.clone(); -// let writer = vstd::thread::spawn( -// move || -// { -// let tracked mut writer_tv = ThreadView::new(); -// weak_atomic_with_ghost! { -// *data_writer => store_release(1, Tracked(&mut writer_tv)); -// update prev -> next; -// snapshot snap; -// ghost g => { -// preserve_message_passing_data_inv_on_push(prev, next, snap.msg()); -// assert(writer_tv.view.seen_at(data_id) >= 1); -// } -// } -// let ghost before_flag = writer_tv.view; -// assert(before_flag.seen_at(data_id) >= 1); -// weak_atomic_with_ghost! { -// *flag_writer => store_release(1, Tracked(&mut writer_tv)); -// update prev -> next; -// snapshot snap; -// ghost g => { -// assert(snap.msg().view == before_flag.observe(flag_writer.atomic.id(), snap.ts())); -// assert(snap.msg().view.seen_at(data_id) >= 1); -// preserve_message_passing_flag_inv_on_push(data_id, prev, next, snap.msg()); -// } -// } -// }, -// ); -// let reader = vstd::thread::spawn( -// move || -// { -// let tracked mut reader_tv = ThreadView::new(); -// let flag_result = -// weak_atomic_with_ghost! { -// *flag_reader => load_acquire(Tracked(&mut reader_tv)); -// returning ret; -// timestamp ts; -// message msg; -// history history; -// ghost g => { -// if ret == 1 { -// assert(msg.value == 1); -// if ts == 0 { -// assert(history[0].value == 0); -// assert(false); -// } -// assert(ts >= 1); -// assert(history[ts as int].value == 1); -// assert(history[ts as int].view.seen_at(data_id) >= 1); -// assert(msg == history[ts as int]); -// assert(msg.view.seen_at(data_id) >= 1); -// assert(reader_tv.view.seen_at(data_id) >= 1); -// } -// } -// }; -// if flag_result.0 == 1 { -// assert(reader_tv.view.seen_at(data_id) >= 1); -// let data_result = -// weak_atomic_with_ghost! { -// *data_reader => load_relaxed(Tracked(&mut reader_tv)); -// returning ret; -// timestamp ts; -// message msg; -// history history; -// ghost g => { -// prove_message_passing_data_read(data_id, history, ret, ts, msg, reader_tv.view); -// } -// }; -// assert(data_result.0 == 1); -// } -// }, -// ); -// let _ = writer.join(); -// let _ = reader.join(); -// } -} // verus! +//! Compatibility exports for the native IRC11 weak-memory model. +//! +//! The original version of this module implemented a second weak-memory +//! history, thread-view, and atomic-wrapper semantics. Verus now provides +//! those primitives in `vstd::atomic_weak`, and [`crate::atomic_irc11`] adds +//! only the adapters that are still missing upstream, such as weak +//! `AtomicPtr` support. +//! +//! New code should import [`crate::atomic_irc11`] directly. This module remains +//! as a source-compatible path and deliberately contains no independent +//! weak-memory semantics. +pub use crate::atomic_irc11::*; diff --git a/verified_libs/vstd_extra/src/raw_callback.rs b/verified_libs/vstd_extra/src/raw_callback.rs index 034659e91..d6b1d1901 100644 --- a/verified_libs/vstd_extra/src/raw_callback.rs +++ b/verified_libs/vstd_extra/src/raw_callback.rs @@ -5,6 +5,7 @@ //! [`RawCallbackContext`], while the executable runner erasure is kept behind //! trusted `external_body` wrappers. use alloc::boxed::Box; +use core::marker::PhantomData; use vstd::prelude::*; @@ -31,6 +32,28 @@ pub trait RawCallbackContext: Send + 'static { fn run(self); } +/// A one-shot callback that transports a tracked proof value to its runner. +/// +/// This is the proof-carrying counterpart of [`RawCallback`]. The executable +/// representation remains a thin data pointer plus a monomorphized runner, +/// while `S` records the tracked value that must be supplied at invocation. +#[must_use] +pub struct RawCallbackWithProof { + data: *mut (), + run: usize, + _proof: PhantomData, +} + +/// Captured context for a [`RawCallbackWithProof`]. +pub trait RawCallbackContextWithProof: Send + 'static { + spec fn call_requires(&self, proof: S) -> bool; + + fn run(self, proof: Tracked) + requires + self.call_requires(proof@), + ; +} + struct RawDropContext { value: T, } @@ -42,6 +65,14 @@ unsafe impl Send for RawCallback { } +// SAFETY: `RawCallbackWithProof::new` only accepts `C: Send + 'static` +// payloads, and `call_once` consumes the payload through the matching +// monomorphized runner. `S` is proof-only and has no runtime representation. +#[verifier::external] +unsafe impl Send for RawCallbackWithProof { + +} + impl RawCallbackContext for RawDropContext { #[inline] #[verifier::external_body] @@ -82,10 +113,52 @@ impl RawCallback { } } +impl RawCallbackWithProof { + /// Proof obligation retained across executable callback type erasure. + pub uninterp spec fn call_requires(&self, proof: S) -> bool; + + /// Builds a proof-carrying callback from an explicit captured context. + #[inline] + #[verifier::external_body] + pub fn new>(context: C) -> (res: Self) + ensures + forall|proof: S| res.call_requires(proof) == context.call_requires(proof), + { + let payload = Box::new(RawCallbackPayloadWithProof { context, _proof: PhantomData }); + Self { + data: Box::into_raw(payload).cast::<()>(), + run: raw_callback_with_proof_runner::(), + _proof: PhantomData, + } + } + + /// Runs the callback exactly once and transfers `proof` to its context. + /// + /// # Safety + /// + /// The caller must ensure this callback has not already been run and will + /// not be run again. + #[inline] + #[verifier::external_body] + pub unsafe fn call_once(self, proof: Tracked) + requires + self.call_requires(proof@), + { + unsafe { + call_raw_callback_with_proof(self.data, self.run, proof); + } + } +} + struct RawCallbackPayload { context: C, } +struct RawCallbackPayloadWithProof, S> { + context: C, + _proof: PhantomData, +} + #[inline] #[verifier::external_body] fn raw_callback_runner() -> usize { @@ -109,4 +182,30 @@ unsafe fn run_raw_callback(data: *mut ()) { context.run(); } +#[inline] +#[verifier::external_body] +fn raw_callback_with_proof_runner, S>() -> usize { + run_raw_callback_with_proof:: as *const () as usize +} + +#[inline] +#[verifier::external_body] +unsafe fn call_raw_callback_with_proof(data: *mut (), run: usize, proof: Tracked) { + let run: unsafe fn (*mut (), Tracked) = unsafe { core::mem::transmute(run) }; + unsafe { + run(data, proof); + } +} + +#[inline] +#[verifier::external_body] +unsafe fn run_raw_callback_with_proof, S>( + data: *mut (), + proof: Tracked, +) { + let payload = unsafe { Box::from_raw(data.cast::>()) }; + let RawCallbackPayloadWithProof { context, _proof } = *payload; + context.run(proof); +} + } // verus! diff --git a/verified_libs/vstd_extra/src/rcu_read_pool.rs b/verified_libs/vstd_extra/src/rcu_read_pool.rs index bd47f52c3..a82882622 100644 --- a/verified_libs/vstd_extra/src/rcu_read_pool.rs +++ b/verified_libs/vstd_extra/src/rcu_read_pool.rs @@ -411,6 +411,20 @@ impl RcuReadPoolRegistry { self.pools.contains_key(key) } + /// Relates keyed lookup to membership in the registry's key set. + pub proof fn lemma_contains_iff_key(tracked &self, key: K) + ensures + self.contains(key) <==> self.keys().contains(key), + { + } + + /// Relates registry membership to the complete key set for all keys. + pub proof fn lemma_all_contains_iff_keys(tracked &self) + ensures + forall|key: K| #[trigger] self.keys().contains(key) ==> self.contains(key), + { + } + pub closed spec fn pool(self, key: K) -> RcuReadPool recommends self.contains(key), @@ -520,6 +534,20 @@ impl RcuTrackedReadPoolRegistry { self.pools.contains_key(key) } + /// Relates keyed lookup to membership in the registry's key set. + pub proof fn lemma_contains_iff_key(tracked &self, key: K) + ensures + self.contains(key) <==> self.keys().contains(key), + { + } + + /// Relates registry membership to the complete key set for all keys. + pub proof fn lemma_all_contains_iff_keys(tracked &self) + ensures + forall|key: K| #[trigger] self.keys().contains(key) ==> self.contains(key), + { + } + pub closed spec fn pool(self, key: K) -> RcuReadPool recommends self.contains(key), @@ -547,6 +575,53 @@ impl RcuTrackedReadPoolRegistry { self.active[lease_id] } + /// Borrows the client witness associated with one active lease. + /// + /// The witness remains owned by the registry until the matching lease is + /// returned. Reclamation proofs use this borrow to show that an allegedly + /// active lease is incompatible with a completed grace period. + pub proof fn tracked_borrow_active_witness(tracked &self, lease_id: nat) -> (tracked witness: + &W) + requires + self.active_ids().contains(lease_id), + ensures + *witness == self.active_record(lease_id).witness(), + { + let tracked record = self.active.tracked_borrow(lease_id); + &record.witness + } + + /// Mutably borrows an active witness while preserving the registry. + /// + /// Resource-algebra validation may require a mutable receiver even when + /// its postcondition leaves the witness unchanged. + pub proof fn tracked_borrow_active_witness_mut( + tracked &mut self, + lease_id: nat, + ) -> (tracked witness: &mut W) + requires + old(self).active_ids().contains(lease_id), + ensures + *witness == old(self).active_record(lease_id).witness(), + final(self).keys() == old(self).keys(), + final(self).active_ids() == old(self).active_ids(), + final(self).next_lease() == old(self).next_lease(), + final(self).active_record(lease_id).key() == old(self).active_record(lease_id).key(), + final(self).active_record(lease_id).pool_id() == old(self).active_record( + lease_id, + ).pool_id(), + final(self).active_record(lease_id).fraction() == old(self).active_record( + lease_id, + ).fraction(), + final(self).active_record(lease_id).witness() == *final(witness), + forall|other: nat| + other != lease_id && old(self).active_ids().contains(other) + ==> final(self).active_record(other) == old(self).active_record(other), + { + let tracked record = self.active.tracked_borrow_mut(lease_id); + &mut record.witness + } + pub open spec fn has_active(self, key: K) -> bool { exists|lease_id: nat| self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == key @@ -579,6 +654,9 @@ impl RcuTrackedReadPoolRegistry { final(self).keys() == old(self).keys().insert(key), final(self).active_ids() == old(self).active_ids(), final(self).next_lease() == old(self).next_lease(), + forall|lease_id: nat| + old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) + == old(self).active_record(lease_id), final(self).contains(key), final(self).pool(key).resource() == resource, final(self).pool(key).fraction() == 1real, @@ -645,6 +723,8 @@ impl RcuTrackedReadPoolRegistry { ensures final(self).wf(), final(self).keys() == old(self).keys(), + forall|candidate: K| #[trigger] + final(self).contains(candidate) == old(self).contains(candidate), final(self).next_lease() == old(self).next_lease() + 1, lease.lease_id() == old(self).next_lease(), lease.key() == key, @@ -744,6 +824,8 @@ impl RcuTrackedReadPoolRegistry { ensures final(self).wf(), final(self).keys() == old(self).keys(), + forall|candidate: K| #[trigger] + final(self).contains(candidate) == old(self).contains(candidate), final(self).next_lease() == old(self).next_lease(), final(self).active_ids() == old(self).active_ids().remove(lease.lease_id()), witness == old(self).active_record(lease.lease_id()).witness(), @@ -824,7 +906,11 @@ impl RcuTrackedReadPoolRegistry { final(self).wf(), final(self).keys() == old(self).keys().remove(key), final(self).active_ids() == old(self).active_ids(), + final(self).active_records() == old(self).active_records(), final(self).next_lease() == old(self).next_lease(), + forall|lease_id: nat| + old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) + == old(self).active_record(lease_id), !final(self).contains(key), resource == old(self).pool(key).resource(), forall|other: K| From 176bcd93f500361cb9c82ba54aa7542fd688833d Mon Sep 17 00:00:00 2001 From: Hiroki Date: Wed, 5 Aug 2026 04:09:23 -0400 Subject: [PATCH 40/47] fix merge --- ostd/src/sync/rwlock.rs | 2 +- ostd/src/sync/rwmutex.rs | 2 +- ostd/src/task/preempt/guard.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ostd/src/sync/rwlock.rs b/ostd/src/sync/rwlock.rs index 789e0f4d6..970a9bd96 100644 --- a/ostd/src/sync/rwlock.rs +++ b/ostd/src/sync/rwlock.rs @@ -4,7 +4,7 @@ use vstd::cell::{self, CellId, pcell::*}; use vstd::prelude::*; use vstd::resource::Loc; use vstd::thread_view::Objective; -use vstd_extra::resource::ghost_resource::{count::*, csum::*, excl::*, tokens::*}; +use vstd_extra::resource::ghost_resource::{count_auth::*, count_ghost::*, csum::*, excl::*}; use vstd_extra::sum::*; use vstd_extra::{prelude::*, resource}; diff --git a/ostd/src/sync/rwmutex.rs b/ostd/src/sync/rwmutex.rs index d6284456c..d0c96953d 100644 --- a/ostd/src/sync/rwmutex.rs +++ b/ostd/src/sync/rwmutex.rs @@ -4,7 +4,7 @@ use vstd::cell::{self, CellId, pcell::*}; use vstd::prelude::*; use vstd::resource::Loc; use vstd::thread_view::Objective; -use vstd_extra::resource::ghost_resource::{count::*, csum::*, excl::*, tokens::*}; +use vstd_extra::resource::ghost_resource::{count_auth::*, count_ghost::*, csum::*, excl::*}; use vstd_extra::sum::*; use core::{ diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index 287a8c8a0..a2178ec76 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -2,7 +2,7 @@ use vstd::thread_view::{ThreadView as Irc11ThreadView, ViewSeen}; use vstd::{prelude::*, resource::Loc}; use vstd_extra::atomic_irc11::ThreadViewOrder; -use vstd_extra::resource::ghost_resource::{count::CountGhost, tokens::CountGhostResource}; +use vstd_extra::resource::ghost_resource::count_ghost::{CountGhost, CountGhostResource}; use crate::{ specs::sync::{ @@ -153,7 +153,7 @@ impl PreemptThreadViewSession { PREEMPT_SESSION_FRACTIONS, >::alloc(state); assert(tokens.is_full()); - tokens.validate_full(); + tokens.validate(); assert(tokens.frac() == PREEMPT_SESSION_FRACTIONS); let tracked res = PreemptThreadViewSession { task_view, tokens }; assert(res.available_fractions() == PREEMPT_SESSION_FRACTIONS); @@ -373,7 +373,7 @@ impl PreemptThreadViewSession { quiescent_generation: generation + 1, }; self.tokens.update(state); - self.tokens.validate_full(); + self.tokens.validate(); assert(self.tokens.frac() == PREEMPT_SESSION_FRACTIONS); assert(self.tokens@.task == self.task_view.task()); generation From 18391de3cb0c69bf04e86fc722d3d27e1fc7a33e Mon Sep 17 00:00:00 2001 From: Hiroki Date: Fri, 7 Aug 2026 05:36:06 -0400 Subject: [PATCH 41/47] Verify linked-list RCU traversal reclamation --- ostd/specs/sync/rcu.rs | 2061 +++++++++++++++++- ostd/specs/sync/rcu_cpu.rs | 221 +- ostd/specs/sync/weak_memory.rs | 1922 ++++++++++++++++ ostd/src/sync/rcu/mod.rs | 480 +++- verified_libs/vstd_extra/src/atomic_irc11.rs | 22 + 5 files changed, 4625 insertions(+), 81 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index dabc6ddff..1f1afb068 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -1821,6 +1821,15 @@ impl RcuLinkView { marker: self.marker, } } + + /// Observing one source never moves any source's traversal position + /// backwards. + pub proof fn lemma_observe_monotonic(self, obj: nat, n: LinkIndex, other: nat) + ensures + self.seen_at(other) <= self.observe(obj, n).seen_at(other), + self.seen_at(obj) <= n ==> self.observe(obj, n).seen_at(obj) == n, + { + } } /// Paper-style `SeenRemoved(D, LV)`. @@ -3430,6 +3439,92 @@ impl RcuReadGuardToken { { self.base.tracked_protect(info); } + + /// Records a coherent observation of the next link event from an already + /// protected source. The returned source witness is refreshed to carry the + /// guard's advanced link-view snapshot. + pub proof fn tracked_observe_link( + tracked &mut self, + tracked from: RcuProtectedPtr, + n: LinkIndex, + ) -> (tracked res: RcuProtectedPtr) + requires + old(self).wf(), + from.protected_by(*old(self)), + old(self).seen_at(from.obj()) <= n, + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected(), + final(self).seen_removed().removed == old(self).seen_removed().removed, + final(self).link_view() == old(self).link_view().observe(from.obj(), n), + final(self).seen_at(from.obj()) == n, + res.domain() == from.domain(), + res.obj() == from.obj(), + res.ptr() == from.ptr(), + res.protected_by(*final(self)), + { + let ghost old_expired = self.expired(); + let ghost old_removed = self.seen_removed.removed; + let ghost seen_removed = RcuSeenRemoved { + removed: old_removed, + link_view: self.seen_removed.link_view.observe(from.obj(), n), + }; + self.seen_removed = seen_removed; + assert(self.expired() == old_expired); + assert(self.seen_removed.removed == old_removed); + assert(self.expired().subset_of(self.seen_removed.removed)); + assert(self.wf()); + RcuProtectedPtr { domain: from.domain(), obj: from.obj(), ptr: from.ptr(), seen_removed } + } + + /// In-place form of [`tracked_observe_link`](Self::tracked_observe_link). + /// This is convenient for atomic loads, which must advance the guard and + /// its source protection witness to the same traversal snapshot. + pub proof fn tracked_observe_link_in_place( + tracked &mut self, + tracked from: &mut RcuProtectedPtr, + n: LinkIndex, + ) + requires + old(self).wf(), + old(from).protected_by(*old(self)), + old(self).seen_at(old(from).obj()) <= n, + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).tid() == old(self).tid(), + final(self).reader_registry() == old(self).reader_registry(), + final(self).reader() == old(self).reader(), + final(self).root() == old(self).root(), + final(self).start_view() == old(self).start_view(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).expired() == old(self).expired(), + final(self).protected() == old(self).protected(), + final(self).seen_removed().removed == old(self).seen_removed().removed, + final(self).link_view() == old(self).link_view().observe(old(from).obj(), n), + final(self).seen_at(old(from).obj()) == n, + final(from).domain() == old(from).domain(), + final(from).obj() == old(from).obj(), + final(from).ptr() == old(from).ptr(), + final(from).protected_by(*final(self)), + { + let ghost seen_removed = RcuSeenRemoved { + removed: self.seen_removed.removed, + link_view: self.seen_removed.link_view.observe(from.obj(), n), + }; + self.seen_removed = seen_removed; + from.seen_removed = seen_removed; + assert(self.wf()); + assert(from.protected_by(*self)); + } } /// A pointer protected by a live read-side guard. @@ -3462,6 +3557,24 @@ impl RcuProtectedPtr { self.seen_removed } + /// Duplicates the persistent fact that this pointer is protected by its + /// recorded guard snapshot. + /// + /// The witness contains only ghost identities and observations; it owns + /// no linear physical resource. Traversal can therefore retain one copy + /// while another is recorded in a physical read lease. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res == *self, + { + RcuProtectedPtr { + domain: self.domain, + obj: self.obj, + ptr: self.ptr, + seen_removed: self.seen_removed, + } + } + pub open spec fn protected_by(self, guard: RcuReadGuardToken) -> bool { &&& self.domain() == guard.domain() &&& self.seen_removed() == guard.seen_removed() @@ -3611,6 +3724,12 @@ pub proof fn protect_link( to_protected.protected_by(*final(guard)), final(guard).wf(), final(guard).domain() == old(guard).domain(), + final(guard).tid() == old(guard).tid(), + final(guard).reader_registry() == old(guard).reader_registry(), + final(guard).reader() == old(guard).reader(), + final(guard).root() == old(guard).root(), + final(guard).start_view() == old(guard).start_view(), + final(guard).retire_observation_registry() == old(guard).retire_observation_registry(), final(guard).expired() == old(guard).expired(), final(guard).seen_removed() == old(guard).seen_removed(), S::node_inv(to, to_info.obj(), g), @@ -3640,92 +3759,1920 @@ pub struct LinkedListNode; /// Paper-style ghost state for a linked list. /// /// `successors[p]` is the successor history for `p`, corresponding to -/// `RcuPointsTo(p, s)`. +/// `RcuPointsTo(p, s)`. A non-null event records both the pointer and its AId; +/// retaining the AId prevents an old history event from being reinterpreted as +/// a different allocation after address reuse. /// /// `incoming_all[p]` is the set of all incoming edges that have ever pointed to /// `p`, corresponding to the authoritative incoming set in `RcuPointedBy(p, B)`. /// -/// `current_incoming[p]` is the current incoming set `B`. It is not required for -/// the simple one-step traversal proof below, but keeping it in the ghost state -/// makes the example match the paper's predicate shape. pub ghost struct LinkedListGhost { pub root: *mut LinkedListNode, pub root_obj: nat, - pub objects: Map<*mut LinkedListNode, nat>, - pub successors: Map<*mut LinkedListNode, Seq>>, + /// Historical allocation registry, keyed by AId rather than address. + /// Distinct reclaimed and newly registered objects may therefore retain + /// the same pointer without collapsing their identities. + pub objects: Map, + /// Per-allocation link histories. An address is not a valid history key: + /// after reuse, the old and new allocations must have disjoint histories. + pub successors: Map>>, pub incoming_all: Map>, - pub current_incoming: Map>, } -pub struct LinkedListTraversalSpec; +impl LinkedListGhost { + /// Current `RcuPointedBy` set, derived from the latest event of every + /// source. Keeping it derived avoids a second mutable representation of + /// the same link relation. + pub open spec fn current_incoming(self, to_obj: nat) -> Set { + if self.incoming_all.contains_key(to_obj) { + self.incoming_all[to_obj].filter( + |edge: LinkEdge| + self.objects.contains_key(edge.0) && self.successors.contains_key(edge.0) + && self.successors[edge.0].len() > 0 && edge.1 + == self.successors[edge.0].len() - 1 + && self.successors[edge.0].last() is Some + && self.successors[edge.0].last()->Some_0.1 == to_obj, + ) + } else { + Set::empty() + } + } -impl RcuTraversalSafety for LinkedListTraversalSpec { - type Node = LinkedListNode; + /// Every recorded history event names a registered allocation and appears + /// in that allocation's authoritative incoming-edge history. + pub open spec fn wf(self) -> bool { + &&& self.objects.contains_pair(self.root_obj, self.root) + &&& self.successors.dom() == self.objects.dom() + &&& self.incoming_all.dom() == self.objects.dom() + &&& forall|from_obj: nat, n: LinkIndex| + #![trigger self.objects.contains_key(from_obj), self.successors[from_obj][n as int]] + self.objects.contains_key(from_obj) && n < self.successors[from_obj].len() + && self.successors[from_obj][n as int] is Some ==> { + let event = self.successors[from_obj][n as int]->Some_0; + &&& self.objects.contains_pair(event.1, event.0) + &&& self.incoming_all[event.1].contains((from_obj, n)) + } + } - type Ghost = LinkedListGhost; + /// A link view cannot claim an event newer than the source's history. + pub open spec fn bounds(self, view: RcuLinkView) -> bool { + forall|from_obj: nat| #[trigger] + self.objects.contains_key(from_obj) && view.seen.contains_key(from_obj) ==> { + &&& self.successors[from_obj].len() > 0 + &&& view.seen_at(from_obj) < self.successors[from_obj].len() + } + } +} - open spec fn root_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { - &&& p == g.root - &&& obj == g.root_obj - &&& g.objects.contains_pair(p, obj) - &&& g.successors.contains_key(p) - &&& g.incoming_all.contains_key(obj) +/// Linear writer authority for the linked-list traversal history. +/// +/// Clients can inspect [`LinkedListGhost`] snapshots, but only this tracked +/// authority can append link events and turn a node's unique base permission +/// into [`RcuRetirePerm`]. This is the proof-only analogue of owning all +/// `RcuPointsTo`/`RcuPointedBy` resources for one list. +pub tracked struct LinkedListTraversalAuth { + ghost domain: Loc, + ghost state: LinkedListGhost, + ghost removed: Set, + infos: Map>, + retire_perms: Map>, +} + +impl LinkedListTraversalAuth { + pub closed spec fn domain(self) -> Loc { + self.domain } - open spec fn node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { - &&& g.objects.contains_pair(p, obj) - &&& g.successors.contains_key(p) - &&& g.incoming_all.contains_key(obj) + pub closed spec fn state(self) -> LinkedListGhost { + self.state } - open spec fn link_inv( - from: *mut LinkedListNode, - from_obj: nat, - n: LinkIndex, - to: *mut LinkedListNode, - to_obj: nat, - g: LinkedListGhost, - ) -> bool { - &&& g.objects.contains_pair(from, from_obj) - &&& g.objects.contains_pair(to, to_obj) - &&& g.successors.contains_key(from) - &&& n < g.successors[from].len() - &&& g.successors[from][n as int] == Some(to) - &&& g.successors.contains_key(to) - &&& g.incoming_all.contains_key(to_obj) - &&& g.incoming_all[to_obj].contains((from_obj, n)) + /// Objects whose removal has already been certified by this authority. + pub closed spec fn removed(self) -> Set { + self.removed } - open spec fn seen_removed_sound( - seen_removed: RcuSeenRemoved, - g: LinkedListGhost, - ) -> bool { - forall|to_obj: nat| #[trigger] - seen_removed.removed.contains(to_obj) ==> { - &&& g.incoming_all.contains_key(to_obj) - &&& forall|edge: LinkEdge| #[trigger] - g.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge) + pub closed spec fn has_retire_perm(self, obj: nat) -> bool { + self.retire_perms.contains_key(obj) + } + + pub closed spec fn has_info(self, obj: nat) -> bool { + self.infos.contains_key(obj) + } + + pub closed spec fn info(self, obj: nat) -> RcuBlockInfo + recommends + self.has_info(obj), + { + self.infos[obj] + } + + pub closed spec fn wf(self) -> bool { + &&& self.state().wf() + &&& self.infos.dom() == self.state().incoming_all.dom() + &&& forall|obj: nat| #[trigger] + self.infos.contains_key(obj) ==> { + let info = self.infos[obj]; + &&& info.wf() + &&& info.domain() == self.domain() + &&& info.obj() == obj + &&& self.state().objects.contains_pair(obj, info.ptr()) + } + &&& forall|obj: nat| #[trigger] + self.retire_perms.contains_key(obj) ==> { + let perm = self.retire_perms[obj]; + &&& perm.wf() + &&& perm.domain() == self.domain() + &&& self.state().objects.contains_pair(obj, perm.ptr()) + &&& perm.obj() == obj } } - proof fn root_is_node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) { + /// Starts an authoritative list with one registered root allocation. + pub proof fn tracked_new( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + ) -> (tracked res: Self) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + ensures + res.wf(), + res.domain() == root_info.domain(), + res.state().root == root_info.ptr(), + res.state().root_obj == root_info.obj(), + res.state().objects == Map::empty().insert(root_info.obj(), root_info.ptr()), + res.state().successors == Map::empty().insert(root_info.obj(), Seq::empty()), + res.state().incoming_all == Map::empty().insert(root_info.obj(), Set::empty()), + res.removed() == Set::empty(), + res.has_info(root_info.obj()), + res.info(root_info.obj()).ptr() == root_info.ptr(), + res.has_retire_perm(root_info.obj()), + forall|obj: nat| #[trigger] res.has_retire_perm(obj) <==> obj == root_info.obj(), + { + let ghost state = LinkedListGhost { + root: root_info.ptr(), + root_obj: root_info.obj(), + objects: Map::empty().insert(root_info.obj(), root_info.ptr()), + successors: Map::empty().insert(root_info.obj(), Seq::empty()), + incoming_all: Map::empty().insert(root_info.obj(), Set::empty()), + }; + let tracked mut infos = Map::tracked_empty(); + let tracked saved_root_info = root_info.tracked_duplicate(); + infos.tracked_insert(root_info.obj(), saved_root_info); + let tracked mut retire_perms = Map::tracked_empty(); + retire_perms.tracked_insert(root_info.obj(), root_retire); + LinkedListTraversalAuth { + domain: root_info.domain(), + state, + removed: Set::empty(), + infos, + retire_perms, + } } - proof fn link_preserves_protection( + /// Adds a separately registered allocation to this list's traversal + /// authority. Registration alone does not publish an incoming link. + pub proof fn tracked_register_node( + tracked &mut self, + tracked info: &RcuBlockInfo, + tracked retire: RcuBaseRetirePerm, + ) + requires + old(self).wf(), + info.wf(), + retire.wf(), + info.domain() == old(self).domain(), + retire.domain() == old(self).domain(), + retire.obj() == info.obj(), + retire.ptr() == info.ptr(), + !old(self).state().objects.contains_key(info.obj()), + !old(self).state().incoming_all.contains_key(info.obj()), + !old(self).has_retire_perm(info.obj()), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects.insert(info.obj(), info.ptr()), + final(self).state().successors == old(self).state().successors.insert( + info.obj(), + Seq::empty(), + ), + final(self).state().incoming_all == old(self).state().incoming_all.insert( + info.obj(), + Set::empty(), + ), + final(self).removed() == old(self).removed(), + final(self).has_info(info.obj()), + final(self).info(info.obj()).ptr() == info.ptr(), + final(self).has_retire_perm(info.obj()), + { + let ghost old_state = self.state; + self.state = LinkedListGhost { + root: self.state.root, + root_obj: self.state.root_obj, + objects: self.state.objects.insert(info.obj(), info.ptr()), + successors: self.state.successors.insert(info.obj(), Seq::empty()), + incoming_all: self.state.incoming_all.insert(info.obj(), Set::empty()), + }; + let tracked saved_info = info.tracked_duplicate(); + self.infos.tracked_insert(info.obj(), saved_info); + self.retire_perms.tracked_insert(info.obj(), retire); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert(self.state.incoming_all.dom() == self.state.objects.dom()); + assert forall|from_obj: nat, n: LinkIndex| + #![trigger self.state.objects.contains_key(from_obj), self.state.successors[from_obj][n as int]] + self.state.objects.contains_key(from_obj) && n < self.state.successors[from_obj].len() + && self.state.successors[from_obj][n as int] is Some implies { + let event = self.state.successors[from_obj][n as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((from_obj, n)) + } by { + assert(from_obj != info.obj()); + assert(old_state.objects.contains_key(from_obj)); + assert(old_state.successors[from_obj] == self.state.successors[from_obj]); + }; + assert(self.state.wf()); + assert(self.infos.dom() == self.state.incoming_all.dom()); + assert forall|obj: nat| #[trigger] self.infos.contains_key(obj) implies { + let saved = self.infos[obj]; + &&& saved.wf() + &&& saved.domain() == self.domain() + &&& saved.obj() == obj + &&& self.state.objects.contains_pair(obj, saved.ptr()) + } by { + if obj != info.obj() { + assert(old(self).infos.contains_key(obj)); + } + }; + assert forall|obj: nat| #[trigger] self.retire_perms.contains_key(obj) implies { + let perm = self.retire_perms[obj]; + &&& perm.wf() + &&& perm.domain() == self.domain() + &&& self.state.objects.contains_pair(obj, perm.ptr()) + &&& perm.obj() == obj + } by { + if obj != info.obj() { + assert(old(self).retire_perms.contains_key(obj)); + } + }; + } + + /// Copies the persistent allocation identity used by an atomic link + /// message without exposing the authority's internal registry. + pub proof fn tracked_info_for(tracked &self, obj: nat) -> (tracked res: RcuBlockInfo< + LinkedListNode, + >) + requires + self.wf(), + self.has_info(obj), + ensures + res.wf(), + res.domain() == self.domain(), + res.obj() == obj, + res.ptr() == self.info(obj).ptr(), + self.state().objects.contains_pair(obj, res.ptr()), + { + let tracked info = self.infos.tracked_borrow(obj); + info.tracked_duplicate() + } + + /// Opens the registry-domain consequence needed by traversal adapters. + pub proof fn lemma_has_info_for_object(tracked &self, obj: nat) + requires + self.wf(), + self.state().incoming_all.contains_key(obj), + ensures + self.has_info(obj), + { + assert(self.infos.dom() == self.state().incoming_all.dom()); + } + + /// Installs the initial null event for a newly created atomic link. + /// Subsequent changes to this source must use the publish/unlink rules. + pub proof fn tracked_initialize_null( + tracked &mut self, + from: *mut LinkedListNode, + from_obj: nat, + ) -> (n: LinkIndex) + requires + old(self).wf(), + old(self).state().objects.contains_pair(from_obj, from), + old(self).state().successors[from_obj].len() == 0, + !old(self).removed().contains(from_obj), + ensures + n == 0, + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects, + final(self).state().successors == old(self).state().successors.insert( + from_obj, + old(self).state().successors[from_obj].push(None), + ), + final(self).state().incoming_all == old(self).state().incoming_all, + final(self).removed() == old(self).removed(), + forall|obj: nat| #[trigger] + final(self).has_retire_perm(obj) == old(self).has_retire_perm(obj), + { + let ghost old_state = self.state; + self.state = LinkedListGhost { + root: old_state.root, + root_obj: old_state.root_obj, + objects: old_state.objects, + successors: old_state.successors.insert( + from_obj, + old_state.successors[from_obj].push(None), + ), + incoming_all: old_state.incoming_all, + }; + assert(old_state.successors.contains_key(from_obj)); + assert(self.state.objects.contains_pair(self.state.root_obj, self.state.root)); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert forall|source: *mut LinkedListNode, source_obj: nat, i: LinkIndex| + #![trigger self.state.objects.contains_pair(source_obj, source), self.state.successors[source_obj][i as int]] + self.state.objects.contains_pair(source_obj, source) + && self.state.successors.contains_key(source_obj) && i + < self.state.successors[source_obj].len() + && self.state.successors[source_obj][i as int] is Some implies { + let event = self.state.successors[source_obj][i as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((source_obj, i)) + } by { + if source_obj == from_obj { + assert(i == 0); + assert(self.state.successors[source_obj][i as int] is None); + assert(false); + } + assert(old_state.objects.contains_pair(source_obj, source)); + assert(old_state.successors[source_obj] == self.state.successors[source_obj]); + }; + assert(self.state.wf()); + assert(self.infos.dom() == self.state.incoming_all.dom()); + 0 + } + + /// Publishes (or replaces with) a non-null successor and returns the new + /// source-history index. + pub proof fn tracked_publish_link( + tracked &mut self, from: *mut LinkedListNode, from_obj: nat, - n: LinkIndex, to: *mut LinkedListNode, to_obj: nat, - seen_removed: RcuSeenRemoved, - g: LinkedListGhost, - ) { - if seen_removed.removed.contains(to_obj) { - assert(g.incoming_all[to_obj].contains((from_obj, n))); - assert(seen_removed.dead_edge((from_obj, n))); - assert(false); - } + ) -> (n: LinkIndex) + requires + old(self).wf(), + old(self).state().objects.contains_pair(from_obj, from), + old(self).state().objects.contains_pair(to_obj, to), + !old(self).removed().contains(from_obj), + !old(self).removed().contains(to_obj), + ensures + n == old(self).state().successors[from_obj].len(), + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects, + final(self).state().successors == old(self).state().successors.insert( + from_obj, + old(self).state().successors[from_obj].push(Some((to, to_obj))), + ), + final(self).state().incoming_all == old(self).state().incoming_all.insert( + to_obj, + old(self).state().incoming_all[to_obj].insert((from_obj, n)), + ), + final(self).removed() == old(self).removed(), + forall|obj: nat| #[trigger] + final(self).has_retire_perm(obj) == old(self).has_retire_perm(obj), + LinkedListTraversalSpec::link_inv(from, from_obj, n, to, to_obj, final(self).state()), + { + let ghost old_state = self.state; + let n = old_state.successors[from_obj].len(); + self.state = LinkedListGhost { + root: old_state.root, + root_obj: old_state.root_obj, + objects: old_state.objects, + successors: old_state.successors.insert( + from_obj, + old_state.successors[from_obj].push(Some((to, to_obj))), + ), + incoming_all: old_state.incoming_all.insert( + to_obj, + old_state.incoming_all[to_obj].insert((from_obj, n)), + ), + }; + assert(old_state.successors.contains_key(from_obj)); + assert(old_state.incoming_all.contains_key(to_obj)); + assert(self.state.objects.contains_pair(self.state.root_obj, self.state.root)); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert(self.state.incoming_all.dom() == self.state.objects.dom()); + assert forall|source: *mut LinkedListNode, source_obj: nat, i: LinkIndex| + #![trigger self.state.objects.contains_pair(source_obj, source), self.state.successors[source_obj][i as int]] + self.state.objects.contains_pair(source_obj, source) + && self.state.successors.contains_key(source_obj) && i + < self.state.successors[source_obj].len() + && self.state.successors[source_obj][i as int] is Some implies { + let event = self.state.successors[source_obj][i as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((source_obj, i)) + } by { + if source_obj == from_obj && i == n { + assert(self.state.successors[source_obj][i as int] == Some((to, to_obj))); + } else { + assert(i < old_state.successors[source_obj].len()); + assert(self.state.successors[source_obj][i as int] + == old_state.successors[source_obj][i as int]); + let event = old_state.successors[source_obj][i as int]->Some_0; + assert(old_state.incoming_all[event.1].contains((source_obj, i))); + assert(self.state.incoming_all[event.1].contains((source_obj, i))); + } + }; + assert(self.state.wf()); + assert(self.infos.dom() == self.state.incoming_all.dom()); + assert forall|obj: nat| #[trigger] self.infos.contains_key(obj) implies { + let info = self.infos[obj]; + &&& info.wf() + &&& info.domain() == self.domain() + &&& info.obj() == obj + &&& self.state.objects.contains_pair(obj, info.ptr()) + } by { + let ghost saved = self.infos[obj]; + assert(old_state.objects.contains_pair(obj, saved.ptr())); + }; + n + } + + /// Appends a null event after the expected current successor. The old + /// incoming edge remains in `incoming_all`, but is no longer current. + pub proof fn tracked_unlink( + tracked &mut self, + from: *mut LinkedListNode, + from_obj: nat, + to: *mut LinkedListNode, + to_obj: nat, + ) -> (n: LinkIndex) + requires + old(self).wf(), + old(self).state().objects.contains_pair(from_obj, from), + old(self).state().objects.contains_pair(to_obj, to), + !old(self).removed().contains(from_obj), + old(self).state().successors[from_obj].len() > 0, + old(self).state().successors[from_obj].last() == Some((to, to_obj)), + ensures + n == old(self).state().successors[from_obj].len(), + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state().root == old(self).state().root, + final(self).state().root_obj == old(self).state().root_obj, + final(self).state().objects == old(self).state().objects, + final(self).state().successors == old(self).state().successors.insert( + from_obj, + old(self).state().successors[from_obj].push(None), + ), + final(self).state().incoming_all == old(self).state().incoming_all, + final(self).removed() == old(self).removed(), + forall|obj: nat| #[trigger] + final(self).has_retire_perm(obj) == old(self).has_retire_perm(obj), + !final(self).state().current_incoming(to_obj).contains((from_obj, (n - 1) as nat)), + { + let ghost old_state = self.state; + let n = old_state.successors[from_obj].len(); + self.state = LinkedListGhost { + root: old_state.root, + root_obj: old_state.root_obj, + objects: old_state.objects, + successors: old_state.successors.insert( + from_obj, + old_state.successors[from_obj].push(None), + ), + incoming_all: old_state.incoming_all, + }; + assert(old_state.successors.contains_key(from_obj)); + assert(self.state.objects.contains_pair(self.state.root_obj, self.state.root)); + assert(self.state.successors.dom() == self.state.objects.dom()); + assert forall|source: *mut LinkedListNode, source_obj: nat, i: LinkIndex| + #![trigger self.state.objects.contains_pair(source_obj, source), self.state.successors[source_obj][i as int]] + self.state.objects.contains_pair(source_obj, source) + && self.state.successors.contains_key(source_obj) && i + < self.state.successors[source_obj].len() + && self.state.successors[source_obj][i as int] is Some implies { + let event = self.state.successors[source_obj][i as int]->Some_0; + &&& self.state.objects.contains_pair(event.1, event.0) + &&& self.state.incoming_all[event.1].contains((source_obj, i)) + } by { + if source_obj == from_obj && i == n { + assert(self.state.successors[source_obj][i as int] is None); + assert(false); + } + assert(i < old_state.successors[source_obj].len()); + assert(self.state.successors[source_obj][i as int] + == old_state.successors[source_obj][i as int]); + }; + assert(self.state.wf()); + n } + + /// Applies the paper's traversal retire rule. The prior observation must + /// already cover every historical incoming edge; this authority is the + /// only public producer of the resulting high-level retire permission. + pub proof fn tracked_retire_node( + tracked &mut self, + obj: nat, + prior: RcuSeenRemoved, + ) -> (tracked res: RcuRetirePerm) + requires + old(self).wf(), + old(self).has_retire_perm(obj), + old(self).has_info(obj), + obj != old(self).state().root_obj, + old(self).state().incoming_all[obj].len() > 0, + prior.removed == old(self).removed(), + old(self).state().bounds(prior.link_view), + LinkedListTraversalSpec::seen_removed_sound(prior, old(self).state()), + forall|edge: LinkEdge| #[trigger] + old(self).state().incoming_all[obj].contains(edge) ==> prior.dead_edge(edge), + ensures + final(self).wf(), + final(self).domain() == old(self).domain(), + final(self).state() == old(self).state(), + final(self).removed() == old(self).removed().insert(obj), + !final(self).has_retire_perm(obj), + final(self).has_info(obj), + final(self).info(obj) == old(self).info(obj), + res.wf(), + res.ready_to_retire(), + res.domain() == old(self).domain(), + res.obj() == obj, + old(self).state().objects.contains_pair(obj, res.ptr()), + res.seen_removed().removed == prior.removed.insert(obj), + res.seen_removed().link_view == prior.link_view, + LinkedListTraversalSpec::seen_removed_sound(res.seen_removed(), final(self).state()), + { + assert(self.state().objects.contains_pair(obj, self.retire_perms[obj].ptr())); + let tracked base = self.retire_perms.tracked_remove(obj); + let ghost seen_removed = RcuSeenRemoved { + removed: prior.removed.insert(obj), + link_view: prior.link_view, + }; + self.removed = self.removed.insert(obj); + assert forall|to_obj: nat| #[trigger] seen_removed.removed.contains(to_obj) implies { + &&& self.state.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + self.state.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge) + } by { + if to_obj == obj { + assert(self.state.incoming_all.contains_key(obj)); + } else { + assert(prior.removed.contains(to_obj)); + } + }; + RcuRetirePerm { base, seen_removed } + } +} + +/// Native IRC11 timestamp metadata for one linked-list atomic link. +/// +/// Native histories use abstract timestamps, whereas the traversal proof uses +/// dense per-source event indices. This linear ghost state records their +/// explicit correspondence; no equality between the two namespaces is +/// assumed. +pub tracked struct LinkedListLinkObservation { + fact: GhostPersistentPointsTo, + ghost loc: Irc11AtomicId, + ghost view: Irc11ThreadView, +} + +impl LinkedListLinkObservation { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + vstd_extra::atomic_irc11::timestamp_in_view(self.loc(), self.view()) == Some( + self.timestamp(), + ) + } + + /// Persistent registry that certifies this timestamp/index pair. + pub closed spec fn registry(self) -> Loc { + self.fact.id() + } + + /// Native IRC11 timestamp observed by the load. + pub closed spec fn timestamp(self) -> nat { + self.fact.key() + } + + /// Dense traversal-history index corresponding to [`Self::timestamp`]. + pub closed spec fn index(self) -> LinkIndex { + self.fact.value() + } + + /// Native atomic location whose timestamp was observed. + pub closed spec fn loc(self) -> Irc11AtomicId { + self.loc + } + + /// Subjective view immediately after the load that minted this token. + pub closed spec fn view(self) -> Irc11ThreadView { + self.view + } + + /// Exposes the native view projection retained by this persistent + /// observation without revealing its private representation. + pub proof fn lemma_view_timestamp(tracked &self) + ensures + vstd_extra::atomic_irc11::timestamp_in_view(self.loc(), self.view()) == Some( + self.timestamp(), + ), + { + use_type_invariant(self); + } + + /// Duplicates the persistent timestamp/index observation. + pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) + ensures + res.registry() == self.registry(), + res.timestamp() == self.timestamp(), + res.index() == self.index(), + res.loc() == self.loc(), + res.view() == self.view(), + { + use_type_invariant(self); + LinkedListLinkObservation { fact: self.fact.duplicate(), loc: self.loc, view: self.view } + } +} + +pub tracked struct LinkedListAtomicLinkGhost { + ghost source: *mut LinkedListNode, + ghost source_obj: nat, + ghost timestamp_to_index: Map, + timestamp_registry: GhostMapAuth, + timestamp_facts: Map>, + ghost current_timestamp: nat, +} + +impl LinkedListAtomicLinkGhost { + pub closed spec fn source(self) -> *mut LinkedListNode { + self.source + } + + pub closed spec fn source_obj(self) -> nat { + self.source_obj + } + + pub closed spec fn timestamps(self) -> Map { + self.timestamp_to_index + } + + /// Append-only registry used to issue persistent load observations. + pub closed spec fn timestamp_registry(self) -> Loc { + self.timestamp_registry.id() + } + + pub closed spec fn certified_timestamps(self) -> Map { + self.timestamp_registry@ + } + + pub closed spec fn observation_facts(self) -> Map< + nat, + GhostPersistentPointsTo, + > { + self.timestamp_facts + } + + pub closed spec fn current_timestamp(self) -> nat { + self.current_timestamp + } + + pub open spec fn index_at(self, timestamp: nat) -> LinkIndex + recommends + self.timestamps().contains_key(timestamp), + { + self.timestamps()[timestamp] + } + + /// Agreement among the native atomic history, dense traversal history, + /// and persistent allocation identities retained by the list authority. + pub open spec fn wf( + self, + history: Irc11History<*mut LinkedListNode>, + auth: LinkedListTraversalAuth, + ) -> bool { + &&& auth.wf() + &&& auth.state().objects.contains_pair(self.source_obj(), self.source()) + &&& !auth.removed().contains(self.source_obj()) + &&& history.is_max_timestamp(self.current_timestamp()) + &&& self.timestamps().dom() == history.dom() + &&& self.certified_timestamps() == self.timestamps() + &&& self.observation_facts().dom() == self.timestamps().dom() + &&& forall|timestamp: nat| #[trigger] + self.observation_facts().contains_key(timestamp) ==> { + let fact = self.observation_facts()[timestamp]; + &&& fact.id() == self.timestamp_registry() + &&& fact.key() == timestamp + &&& fact.value() == self.timestamps()[timestamp] + } + &&& auth.state().successors[self.source_obj()].len() > 0 + &&& self.index_at(self.current_timestamp()) + 1 + == auth.state().successors[self.source_obj()].len() + &&& forall|timestamp: nat| + history.contains_timestamp(timestamp) ==> { + let n = #[trigger] self.timestamps()[timestamp]; + &&& n < auth.state().successors[self.source_obj()].len() + &&& match auth.state().successors[self.source_obj()][n as int] { + None => history.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& history.value(timestamp).addr() != 0 + &&& equal(ptr, history.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } + &&& forall|earlier: nat, later: nat| + #![trigger self.timestamps()[earlier], self.timestamps()[later]] + history.contains_timestamp(earlier) && history.contains_timestamp(later) ==> (earlier + < later <==> self.index_at(earlier) < self.index_at(later)) + } + + /// Creates the timestamp mapping for an atomic link initialized to null. + pub proof fn tracked_initial_null( + history: Irc11History<*mut LinkedListNode>, + timestamp: nat, + message_view: Irc11ThreadView, + tracked auth: &LinkedListTraversalAuth, + source: *mut LinkedListNode, + source_obj: nat, + ) -> (tracked res: Self) + requires + auth.wf(), + auth.state().objects.contains_pair(source_obj, source), + !auth.removed().contains(source_obj), + auth.state().successors[source_obj] == Seq::empty().push(None), + history.is_singleton(timestamp, (core::ptr::null_mut(), message_view)), + ensures + res.wf(history, *auth), + res.source() == source, + res.source_obj() == source_obj, + res.timestamps() == Map::empty().insert(timestamp, 0), + res.current_timestamp() == timestamp, + { + assert(history.is_max_timestamp(timestamp)); + assert(history.dom() == Set::empty().insert(timestamp)) by { + assert forall|ts: nat| + history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { + if history.dom().contains(ts) { + assert(history.contains_timestamp(ts)); + assert(ts == timestamp); + } + }; + }; + let tracked (mut timestamp_registry, _timestamp_entries) = GhostMapAuth::new(Map::empty()); + let tracked initial_fact = timestamp_registry.insert(timestamp, 0).persist(); + let tracked mut timestamp_facts = Map::tracked_empty(); + timestamp_facts.tracked_insert(timestamp, initial_fact); + let tracked res = LinkedListAtomicLinkGhost { + source, + source_obj, + timestamp_to_index: Map::empty().insert(timestamp, 0), + timestamp_registry, + timestamp_facts, + current_timestamp: timestamp, + }; + assert forall|ts: nat| history.contains_timestamp(ts) implies { + let n = #[trigger] res.timestamps()[ts]; + &&& n < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][n as int] { + None => history.value(ts).addr() == 0, + Some((ptr, obj)) => { + &&& history.value(ts).addr() != 0 + &&& equal(ptr, history.value(ts)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + assert(ts == timestamp); + }; + assert forall|earlier: nat, later: nat| + #![trigger res.timestamps()[earlier], res.timestamps()[later]] + history.contains_timestamp(earlier) && history.contains_timestamp(later) implies ( + earlier < later <==> res.index_at(earlier) < res.index_at(later)) by { + assert(earlier == timestamp); + assert(later == timestamp); + }; + res + } + + /// Issues persistent evidence for one native timestamp's dense index. + pub proof fn tracked_observation_at( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &LinkedListTraversalAuth, + timestamp: nat, + loc: Irc11AtomicId, + view: Irc11ThreadView, + ) -> (tracked res: LinkedListLinkObservation) + requires + self.wf(history, *auth), + history.contains_timestamp(timestamp), + vstd_extra::atomic_irc11::timestamp_in_view(loc, view) == Some(timestamp), + ensures + res.registry() == self.timestamp_registry(), + res.timestamp() == timestamp, + res.index() == self.index_at(timestamp), + res.loc() == loc, + res.view() == view, + { + let tracked fact = self.timestamp_facts.tracked_borrow(timestamp).duplicate(); + LinkedListLinkObservation { fact, loc, view } + } + + /// Agrees a prior persistent observation with the current append-only + /// timestamp map. This remains valid after later CAS updates. + pub proof fn lemma_observation_agrees( + tracked &self, + tracked observation: &LinkedListLinkObservation, + ) + requires + observation.registry() == self.timestamp_registry(), + self.certified_timestamps() == self.timestamps(), + ensures + self.timestamps().contains_pair(observation.timestamp(), observation.index()), + { + observation.fact.agree(&self.timestamp_registry); + } + + /// Records a successful native CAS that publishes a non-null successor. + /// + /// `load_timestamp` and `store_timestamp` are supplied by the native + /// [`UpdateData`](vstd::atomic_weak::UpdateData). The traversal index is + /// allocated independently by [`LinkedListTraversalAuth`]. + pub proof fn tracked_cas_publish( + tracked &mut self, + tracked auth: &mut LinkedListTraversalAuth, + prev: Irc11History<*mut LinkedListNode>, + next: Irc11History<*mut LinkedListNode>, + load_timestamp: nat, + store_timestamp: nat, + to: *mut LinkedListNode, + to_obj: nat, + message_view: Irc11ThreadView, + ) -> (n: LinkIndex) + requires + old(self).wf(prev, *old(auth)), + prev.is_max_timestamp(load_timestamp), + store_timestamp == load_timestamp + 1, + next == prev.insert(store_timestamp, to, message_view), + to.addr() != 0, + old(auth).state().objects.contains_pair(to_obj, to), + !old(auth).removed().contains(to_obj), + ensures + n == old(auth).state().successors[old(self).source_obj()].len(), + final(self).wf(next, *final(auth)), + final(self).source() == old(self).source(), + final(self).source_obj() == old(self).source_obj(), + final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).timestamps() == old(self).timestamps().insert(store_timestamp, n), + final(self).current_timestamp() == store_timestamp, + final(auth).domain() == old(auth).domain(), + final(auth).state().root == old(auth).state().root, + final(auth).state().root_obj == old(auth).state().root_obj, + final(auth).state().objects == old(auth).state().objects, + final(auth).state().successors == old(auth).state().successors.insert( + old(self).source_obj(), + old(auth).state().successors[old(self).source_obj()].push(Some((to, to_obj))), + ), + final(auth).state().incoming_all == old(auth).state().incoming_all.insert( + to_obj, + old(auth).state().incoming_all[to_obj].insert((old(self).source_obj(), n)), + ), + final(auth).removed() == old(auth).removed(), + forall|obj: nat| #[trigger] + final(auth).has_retire_perm(obj) == old(auth).has_retire_perm(obj), + LinkedListTraversalSpec::link_inv( + final(self).source(), + final(self).source_obj(), + n, + to, + to_obj, + final(auth).state(), + ), + { + let ghost source = self.source; + let ghost source_obj = self.source_obj; + let ghost old_timestamps = self.timestamp_to_index; + let ghost old_state = auth.state(); + let ghost old_current = self.current_timestamp; + + assert(prev.contains_timestamp(old_current)); + assert(prev.contains_timestamp(load_timestamp)); + assert(load_timestamp <= old_current); + assert(old_current <= load_timestamp); + assert(load_timestamp == old_current); + assert(!prev.contains_timestamp(store_timestamp)); + + let n = auth.tracked_publish_link(source, source_obj, to, to_obj); + self.timestamp_to_index = old_timestamps.insert(store_timestamp, n); + let tracked fact = self.timestamp_registry.insert(store_timestamp, n).persist(); + self.timestamp_facts.tracked_insert(store_timestamp, fact); + self.current_timestamp = store_timestamp; + + assert(next.is_max_timestamp(store_timestamp)) by { + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies timestamp + <= store_timestamp by { + if timestamp != store_timestamp { + assert(prev.contains_timestamp(timestamp)); + assert(timestamp <= load_timestamp); + } + }; + }; + assert(self.timestamps().dom() == next.dom()); + assert(self.index_at(store_timestamp) == n); + assert(n + 1 == auth.state().successors[source_obj].len()); + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies { + let index = #[trigger] self.timestamps()[timestamp]; + &&& index < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][index as int] { + None => next.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& next.value(timestamp).addr() != 0 + &&& equal(ptr, next.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + if timestamp == store_timestamp { + assert(auth.state().successors[source_obj][n as int] == Some((to, to_obj))); + assert(auth.has_info(to_obj)); + assert(auth.info(to_obj).ptr() == to); + } else { + assert(prev.contains_timestamp(timestamp)); + let ghost index = old_timestamps[timestamp]; + assert(index < old_state.successors[source_obj].len()); + assert(auth.state().successors[source_obj][index as int] + == old_state.successors[source_obj][index as int]); + assert(next.value(timestamp) == prev.value(timestamp)); + match old_state.successors[source_obj][index as int] { + None => {}, + Some((ptr, obj)) => { + assert(old(auth).has_info(obj)); + assert(auth.has_info(obj)); + assert(auth.info(obj).ptr() == ptr); + }, + } + } + }; + assert forall|earlier: nat, later: nat| + #![trigger self.timestamps()[earlier], self.timestamps()[later]] + next.contains_timestamp(earlier) && next.contains_timestamp(later) implies (earlier + < later <==> self.index_at(earlier) < self.index_at(later)) by { + if earlier == store_timestamp { + if later != store_timestamp { + assert(prev.contains_timestamp(later)); + assert(later <= load_timestamp); + assert(old_timestamps[later] < old_state.successors[source_obj].len()); + assert(self.index_at(later) < n); + } + } else if later == store_timestamp { + assert(prev.contains_timestamp(earlier)); + assert(earlier <= load_timestamp); + assert(old_timestamps[earlier] < old_state.successors[source_obj].len()); + assert(self.index_at(earlier) < n); + } else { + assert(prev.contains_timestamp(earlier)); + assert(prev.contains_timestamp(later)); + } + }; + n + } + + /// Records a successful native CAS that replaces the current successor + /// with null. The detached edge remains in the append-only traversal + /// history and can subsequently be discharged by a reader observation. + pub proof fn tracked_cas_unlink( + tracked &mut self, + tracked auth: &mut LinkedListTraversalAuth, + prev: Irc11History<*mut LinkedListNode>, + next: Irc11History<*mut LinkedListNode>, + load_timestamp: nat, + store_timestamp: nat, + to: *mut LinkedListNode, + to_obj: nat, + message_view: Irc11ThreadView, + ) -> (n: LinkIndex) + requires + old(self).wf(prev, *old(auth)), + prev.is_max_timestamp(load_timestamp), + store_timestamp == load_timestamp + 1, + next == prev.insert(store_timestamp, core::ptr::null_mut(), message_view), + old(auth).state().successors[old(self).source_obj()].last() == Some((to, to_obj)), + ensures + n == old(auth).state().successors[old(self).source_obj()].len(), + final(self).wf(next, *final(auth)), + final(self).source() == old(self).source(), + final(self).source_obj() == old(self).source_obj(), + final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).timestamps() == old(self).timestamps().insert(store_timestamp, n), + final(self).current_timestamp() == store_timestamp, + final(auth).domain() == old(auth).domain(), + final(auth).state().root == old(auth).state().root, + final(auth).state().root_obj == old(auth).state().root_obj, + final(auth).state().objects == old(auth).state().objects, + final(auth).state().successors == old(auth).state().successors.insert( + old(self).source_obj(), + old(auth).state().successors[old(self).source_obj()].push(None), + ), + final(auth).state().incoming_all == old(auth).state().incoming_all, + final(auth).removed() == old(auth).removed(), + forall|obj: nat| #[trigger] + final(auth).has_retire_perm(obj) == old(auth).has_retire_perm(obj), + final(auth).state().successors[final(self).source_obj()][n as int] is None, + !final(auth).state().current_incoming(to_obj).contains( + (final(self).source_obj(), (n - 1) as nat), + ), + { + let ghost source = self.source; + let ghost source_obj = self.source_obj; + let ghost old_timestamps = self.timestamp_to_index; + let ghost old_state = auth.state(); + let ghost old_current = self.current_timestamp; + + assert(prev.contains_timestamp(old_current)); + assert(prev.contains_timestamp(load_timestamp)); + assert(load_timestamp <= old_current); + assert(old_current <= load_timestamp); + assert(load_timestamp == old_current); + assert(!prev.contains_timestamp(store_timestamp)); + + let n = auth.tracked_unlink(source, source_obj, to, to_obj); + self.timestamp_to_index = old_timestamps.insert(store_timestamp, n); + let tracked fact = self.timestamp_registry.insert(store_timestamp, n).persist(); + self.timestamp_facts.tracked_insert(store_timestamp, fact); + self.current_timestamp = store_timestamp; + + assert(next.is_max_timestamp(store_timestamp)) by { + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies timestamp + <= store_timestamp by { + if timestamp != store_timestamp { + assert(prev.contains_timestamp(timestamp)); + assert(timestamp <= load_timestamp); + } + }; + }; + assert(self.timestamps().dom() == next.dom()); + assert(self.index_at(store_timestamp) == n); + assert(n + 1 == auth.state().successors[source_obj].len()); + assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies { + let index = #[trigger] self.timestamps()[timestamp]; + &&& index < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][index as int] { + None => next.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& next.value(timestamp).addr() != 0 + &&& equal(ptr, next.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + if timestamp == store_timestamp { + assert(auth.state().successors[source_obj][n as int] is None); + } else { + assert(prev.contains_timestamp(timestamp)); + let ghost index = old_timestamps[timestamp]; + assert(index < old_state.successors[source_obj].len()); + assert(auth.state().successors[source_obj][index as int] + == old_state.successors[source_obj][index as int]); + assert(next.value(timestamp) == prev.value(timestamp)); + match old_state.successors[source_obj][index as int] { + None => {}, + Some((ptr, obj)) => { + assert(old(auth).has_info(obj)); + assert(auth.has_info(obj)); + assert(auth.info(obj).ptr() == ptr); + }, + } + } + }; + assert forall|earlier: nat, later: nat| + #![trigger self.timestamps()[earlier], self.timestamps()[later]] + next.contains_timestamp(earlier) && next.contains_timestamp(later) implies (earlier + < later <==> self.index_at(earlier) < self.index_at(later)) by { + if earlier == store_timestamp { + if later != store_timestamp { + assert(prev.contains_timestamp(later)); + assert(later <= load_timestamp); + assert(old_timestamps[later] < old_state.successors[source_obj].len()); + assert(self.index_at(later) < n); + } + } else if later == store_timestamp { + assert(prev.contains_timestamp(earlier)); + assert(earlier <= load_timestamp); + assert(old_timestamps[earlier] < old_state.successors[source_obj].len()); + assert(self.index_at(earlier) < n); + } else { + assert(prev.contains_timestamp(earlier)); + assert(prev.contains_timestamp(later)); + } + }; + n + } + + /// Resolves an observed native atomic message to its traversal event and + /// persistent child identity. + pub proof fn tracked_info_at( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &LinkedListTraversalAuth, + timestamp: nat, + ) -> (tracked res: Option>) + requires + self.wf(history, *auth), + history.contains_timestamp(timestamp), + ensures + self.index_at(timestamp) < auth.state().successors[self.source_obj()].len(), + match res { + None => { + &&& history.value(timestamp).addr() == 0 + &&& auth.state().successors[self.source_obj()][self.index_at( + timestamp, + ) as int] is None + }, + Some(info) => { + &&& history.value(timestamp).addr() != 0 + &&& info.wf() + &&& info.domain() == auth.domain() + &&& equal(info.ptr(), history.value(timestamp)) + &&& LinkedListTraversalSpec::link_inv( + self.source(), + self.source_obj(), + self.index_at(timestamp), + info.ptr(), + info.obj(), + auth.state(), + ) + }, + }, + { + let ghost n = self.index_at(timestamp); + match auth.state().successors[self.source_obj()][n as int] { + None => None, + Some((ptr, obj)) => { + let tracked info = auth.tracked_info_for(obj); + assert(equal(info.ptr(), history.value(timestamp))); + Some(info) + }, + } + } + + /// Connects a native atomic load to the paper's guarded traversal rule. + /// + /// The source witness is refreshed in place with the observed link index. + /// A non-null message additionally installs the loaded allocation in the + /// guard's protection map and returns its protected witness. + pub proof fn tracked_load_and_protect( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &LinkedListTraversalAuth, + tracked guard: &mut RcuReadGuardToken, + tracked from: &mut RcuProtectedPtr, + timestamp: nat, + ) -> (tracked res: Option>) + requires + self.wf(history, *auth), + history.contains_timestamp(timestamp), + old(guard).wf(), + old(guard).domain() == auth.domain(), + old(from).protected_by(*old(guard)), + old(from).ptr() == self.source(), + old(from).obj() == self.source_obj(), + old(guard).seen_at(old(from).obj()) <= self.index_at(timestamp), + LinkedListTraversalSpec::seen_removed_sound(old(guard).seen_removed(), auth.state()), + ensures + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).tid() == old(guard).tid(), + final(guard).reader_registry() == old(guard).reader_registry(), + final(guard).reader() == old(guard).reader(), + final(guard).root() == old(guard).root(), + final(guard).start_view() == old(guard).start_view(), + final(guard).retire_observation_registry() == old(guard).retire_observation_registry(), + final(guard).expired() == old(guard).expired(), + final(guard).seen_removed().removed == old(guard).seen_removed().removed, + final(guard).seen_at(self.source_obj()) == self.index_at(timestamp), + LinkedListTraversalSpec::seen_removed_sound(final(guard).seen_removed(), auth.state()), + final(from).ptr() == self.source(), + final(from).obj() == self.source_obj(), + final(from).domain() == final(guard).domain(), + final(from).seen_removed() == final(guard).seen_removed(), + !final(from).seen_removed().removed.contains(final(from).obj()), + res is None ==> final(guard).protected() == old(guard).protected(), + (res is Some) == (history.value(timestamp).addr() != 0), + match res { + None => history.value(timestamp).addr() == 0, + Some(child) => { + &&& equal(child.ptr(), history.value(timestamp)) + &&& child.domain() == auth.domain() + &&& child.protected_by(*final(guard)) + &&& LinkedListTraversalSpec::node_inv(child.ptr(), child.obj(), auth.state()) + }, + }, + { + let ghost n = self.index_at(timestamp); + let tracked info = self.tracked_info_at(history, auth, timestamp); + let ghost old_seen_removed = guard.seen_removed(); + let ghost old_domain = guard.domain(); + let ghost old_tid = guard.tid(); + let ghost old_reader_registry = guard.reader_registry(); + let ghost old_reader = guard.reader(); + let ghost old_root = guard.root(); + let ghost old_start_view = guard.start_view(); + let ghost old_retire_observation_registry = guard.retire_observation_registry(); + let ghost old_expired = guard.expired(); + let ghost old_protected = guard.protected(); + assert(old_domain == old(guard).domain()); + assert(old_tid == old(guard).tid()); + assert(old_reader_registry == old(guard).reader_registry()); + assert(old_reader == old(guard).reader()); + assert(old_root == old(guard).root()); + assert(old_start_view == old(guard).start_view()); + assert(old_retire_observation_registry == old(guard).retire_observation_registry()); + assert(old_expired == old(guard).expired()); + assert(old_protected == old(guard).protected()); + guard.tracked_observe_link_in_place(from, n); + linked_list_observe_preserves_seen_removed_sound( + old_seen_removed, + auth.state(), + self.source_obj(), + n, + ); + assert(from.protected_by(*guard)); + assert(guard.domain() == old_domain); + assert(guard.tid() == old_tid); + assert(guard.reader_registry() == old_reader_registry); + assert(guard.reader() == old_reader); + assert(guard.root() == old_root); + assert(guard.start_view() == old_start_view); + assert(guard.retire_observation_registry() == old_retire_observation_registry); + assert(guard.expired() == old_expired); + assert(guard.protected() == old_protected); + let tracked res; + match info { + None => { + assert(guard.domain() == old(guard).domain()); + assert(guard.tid() == old(guard).tid()); + assert(guard.reader_registry() == old(guard).reader_registry()); + assert(guard.reader() == old(guard).reader()); + assert(guard.root() == old(guard).root()); + assert(guard.start_view() == old(guard).start_view()); + assert(guard.retire_observation_registry() == old( + guard, + ).retire_observation_registry()); + assert(guard.expired() == old(guard).expired()); + assert(guard.protected() == old(guard).protected()); + res = None; + }, + Some(info) => { + assert(LinkedListTraversalSpec::node_inv(from.ptr(), from.obj(), auth.state())); + let tracked child = protect_link::( + guard, + from, + &info, + n, + info.ptr(), + auth.state(), + ); + res = Some(child); + }, + } + assert(guard.domain() == old(guard).domain()); + assert(guard.tid() == old(guard).tid()); + assert(guard.reader_registry() == old(guard).reader_registry()); + assert(guard.reader() == old(guard).reader()); + assert(guard.root() == old(guard).root()); + assert(guard.start_view() == old(guard).start_view()); + assert(guard.retire_observation_registry() == old(guard).retire_observation_registry()); + assert(guard.expired() == old(guard).expired()); + res + } +} + +pub struct LinkedListTraversalSpec; + +impl RcuTraversalSafety for LinkedListTraversalSpec { + type Node = LinkedListNode; + + type Ghost = LinkedListGhost; + + open spec fn root_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { + &&& p == g.root + &&& obj == g.root_obj + &&& g.objects.contains_pair(obj, p) + &&& g.successors.contains_key(obj) + &&& g.incoming_all.contains_key(obj) + } + + open spec fn node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) -> bool { + &&& g.objects.contains_pair(obj, p) + &&& g.successors.contains_key(obj) + &&& g.incoming_all.contains_key(obj) + } + + open spec fn link_inv( + from: *mut LinkedListNode, + from_obj: nat, + n: LinkIndex, + to: *mut LinkedListNode, + to_obj: nat, + g: LinkedListGhost, + ) -> bool { + &&& g.objects.contains_pair(from_obj, from) + &&& g.objects.contains_pair(to_obj, to) + &&& g.successors.contains_key(from_obj) + &&& n < g.successors[from_obj].len() + &&& g.successors[from_obj][n as int] == Some((to, to_obj)) + &&& g.successors.contains_key(to_obj) + &&& g.incoming_all.contains_key(to_obj) + &&& g.incoming_all[to_obj].contains((from_obj, n)) + } + + open spec fn seen_removed_sound( + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + ) -> bool { + forall|to_obj: nat| #[trigger] + seen_removed.removed.contains(to_obj) ==> { + &&& g.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge) + } + } + + proof fn root_is_node_inv(p: *mut LinkedListNode, obj: nat, g: LinkedListGhost) { + } + + proof fn link_preserves_protection( + from: *mut LinkedListNode, + from_obj: nat, + n: LinkIndex, + to: *mut LinkedListNode, + to_obj: nat, + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + ) { + if seen_removed.removed.contains(to_obj) { + assert(g.incoming_all[to_obj].contains((from_obj, n))); + assert(seen_removed.dead_edge((from_obj, n))); + assert(false); + } + } +} + +/// Advancing one source observation preserves every previously established +/// dead-edge fact in a linked-list `SeenRemoved` snapshot. +pub proof fn linked_list_observe_preserves_seen_removed_sound( + seen_removed: RcuSeenRemoved, + g: LinkedListGhost, + source_obj: nat, + n: LinkIndex, +) + requires + LinkedListTraversalSpec::seen_removed_sound(seen_removed, g), + ensures + LinkedListTraversalSpec::seen_removed_sound( + RcuSeenRemoved { + removed: seen_removed.removed, + link_view: seen_removed.link_view.observe(source_obj, n), + }, + g, + ), +{ + let ghost observed = RcuSeenRemoved { + removed: seen_removed.removed, + link_view: seen_removed.link_view.observe(source_obj, n), + }; + assert forall|to_obj: nat| #[trigger] observed.removed.contains(to_obj) implies { + &&& g.incoming_all.contains_key(to_obj) + &&& forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> observed.dead_edge(edge) + } by { + assert(seen_removed.removed.contains(to_obj)); + assert(g.incoming_all.contains_key(to_obj)); + assert forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) implies observed.dead_edge(edge) by { + assert(seen_removed.dead_edge(edge)); + if !seen_removed.removed.contains(edge.0) { + seen_removed.link_view.lemma_observe_monotonic(source_obj, n, edge.0); + assert(observed.seen_at(edge.0) >= seen_removed.seen_at(edge.0)); + } + }; + }; +} + +/// A latest link from a predecessor not already in `D` cannot be dead in a +/// bounded link view. Consequently the traversal retire rule cannot be +/// applied while that incoming edge remains live. +pub proof fn linked_list_live_edge_blocks_retire( + g: LinkedListGhost, + seen_removed: RcuSeenRemoved, + from: *mut LinkedListNode, + from_obj: nat, + n: LinkIndex, + to: *mut LinkedListNode, + to_obj: nat, +) + requires + g.wf(), + g.bounds(seen_removed.link_view), + LinkedListTraversalSpec::link_inv(from, from_obj, n, to, to_obj, g), + g.successors[from_obj].len() == n + 1, + !seen_removed.removed.contains(from_obj), + ensures + g.current_incoming(to_obj).contains((from_obj, n)), + !seen_removed.dead_edge((from_obj, n)), + !(forall|edge: LinkEdge| #[trigger] + g.incoming_all[to_obj].contains(edge) ==> seen_removed.dead_edge(edge)), +{ + if seen_removed.link_view.seen.contains_key(from_obj) { + assert(seen_removed.seen_at(from_obj) < g.successors[from_obj].len()); + } else { + assert(seen_removed.seen_at(from_obj) == 0); + } + assert(seen_removed.seen_at(from_obj) <= n); + assert(g.successors[from_obj].last() == Some((to, to_obj))); + assert(g.current_incoming(to_obj).contains((from_obj, n))) by { + assert(g.objects.contains_key(from_obj)); + assert(g.successors.contains_key(from_obj)); + } +} + +/// End-to-end ghost example for an internal list node. Publishing creates the +/// historical incoming edge at index 0; unlinking appends a newer null event; +/// observing index 1 then lets the authority consume the child's unique base +/// permission and produce the traversal-level retire permission. +pub proof fn linked_list_unlink_enables_retire( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked child_info: &RcuBlockInfo, + tracked child_retire: RcuBaseRetirePerm, +) -> (tracked retired: RcuRetirePerm) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + child_info.wf(), + child_retire.wf(), + child_info.domain() == root_info.domain(), + child_retire.domain() == root_info.domain(), + child_retire.obj() == child_info.obj(), + child_retire.ptr() == child_info.ptr(), + child_info.ptr() != root_info.ptr(), + child_info.obj() != root_info.obj(), + ensures + retired.wf(), + retired.ready_to_retire(), + retired.domain() == root_info.domain(), + retired.obj() == child_info.obj(), + LinkedListTraversalSpec::seen_removed_sound( + retired.seen_removed(), + LinkedListGhost { + root: root_info.ptr(), + root_obj: root_info.obj(), + objects: Map::empty().insert(root_info.obj(), root_info.ptr()).insert( + child_info.obj(), + child_info.ptr(), + ), + successors: Map::empty().insert(root_info.obj(), Seq::empty()).insert( + child_info.obj(), + Seq::empty(), + ).insert( + root_info.obj(), + Seq::empty().push(Some((child_info.ptr(), child_info.obj()))).push(None), + ), + incoming_all: Map::empty().insert(root_info.obj(), Set::empty()).insert( + child_info.obj(), + Set::empty().insert((root_info.obj(), 0)), + ), + }, + ), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + auth.tracked_register_node(child_info, child_retire); + let n = auth.tracked_publish_link( + root_info.ptr(), + root_info.obj(), + child_info.ptr(), + child_info.obj(), + ); + assert(n == 0); + let observed = auth.tracked_unlink( + root_info.ptr(), + root_info.obj(), + child_info.ptr(), + child_info.obj(), + ); + assert(observed == 1); + let ghost prior = RcuSeenRemoved { + removed: Set::empty(), + link_view: RcuLinkView::empty().observe(root_info.obj(), observed), + }; + assert(auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut LinkedListNode, from_obj: nat| #[trigger] + auth.state().objects.contains_pair(from_obj, from) && prior.link_view.seen.contains_key( + from_obj, + ) implies { + &&& auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) < auth.state().successors[from_obj].len() + } by { + assert(from == root_info.ptr()); + assert(from_obj == root_info.obj()); + }; + } + assert(LinkedListTraversalSpec::seen_removed_sound(prior, auth.state())); + assert forall|edge: LinkEdge| #[trigger] + auth.state().incoming_all[child_info.obj()].contains(edge) implies prior.dead_edge( + edge, + ) by { + assert(edge == (root_info.obj(), 0)); + assert(prior.seen_at(root_info.obj()) == observed); + }; + auth.tracked_retire_node(child_info.obj(), prior) +} + +/// Regression proof for the fully dynamic traversal authority. +/// +/// The old allocation is reachable from two distinct predecessors. Both +/// incoming edges must be unlinked and observed before retirement. A fresh +/// AId is then registered at exactly the same address, published, unlinked, +/// and published again. The old history entries continue naming `old_obj`, +/// while both publications of the reused allocation name `reused_obj`. +pub proof fn linked_list_multiple_predecessors_republish_reused_address( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked left_info: &RcuBlockInfo, + tracked left_retire: RcuBaseRetirePerm, + tracked right_info: &RcuBlockInfo, + tracked right_retire: RcuBaseRetirePerm, + tracked old_info: &RcuBlockInfo, + tracked old_retire: RcuBaseRetirePerm, + tracked reused_info: &RcuBlockInfo, + tracked reused_retire: RcuBaseRetirePerm, +) -> (tracked res: (LinkedListTraversalAuth, RcuRetirePerm)) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + left_info.wf(), + left_retire.wf(), + left_info.domain() == root_info.domain(), + left_retire.domain() == root_info.domain(), + left_retire.obj() == left_info.obj(), + left_retire.ptr() == left_info.ptr(), + right_info.wf(), + right_retire.wf(), + right_info.domain() == root_info.domain(), + right_retire.domain() == root_info.domain(), + right_retire.obj() == right_info.obj(), + right_retire.ptr() == right_info.ptr(), + old_info.wf(), + old_retire.wf(), + old_info.domain() == root_info.domain(), + old_retire.domain() == root_info.domain(), + old_retire.obj() == old_info.obj(), + old_retire.ptr() == old_info.ptr(), + reused_info.wf(), + reused_retire.wf(), + reused_info.domain() == root_info.domain(), + reused_retire.domain() == root_info.domain(), + reused_retire.obj() == reused_info.obj(), + reused_retire.ptr() == reused_info.ptr(), + old_info.ptr() == reused_info.ptr(), + root_info.obj() != left_info.obj(), + root_info.obj() != right_info.obj(), + root_info.obj() != old_info.obj(), + root_info.obj() != reused_info.obj(), + left_info.obj() != right_info.obj(), + left_info.obj() != old_info.obj(), + left_info.obj() != reused_info.obj(), + right_info.obj() != old_info.obj(), + right_info.obj() != reused_info.obj(), + old_info.obj() != reused_info.obj(), + ensures + res.0.wf(), + res.0.removed().contains(old_info.obj()), + res.0.has_retire_perm(reused_info.obj()), + res.0.state().objects.contains_pair(old_info.obj(), old_info.ptr()), + res.0.state().objects.contains_pair(reused_info.obj(), old_info.ptr()), + res.0.state().successors[left_info.obj()][0] == Some((old_info.ptr(), old_info.obj())), + res.0.state().successors[left_info.obj()][2] == Some( + (reused_info.ptr(), reused_info.obj()), + ), + res.0.state().successors[left_info.obj()][4] == Some( + (reused_info.ptr(), reused_info.obj()), + ), + res.0.state().incoming_all[old_info.obj()].contains((left_info.obj(), 0)), + res.0.state().incoming_all[old_info.obj()].contains((right_info.obj(), 0)), + res.0.state().incoming_all[reused_info.obj()].contains((left_info.obj(), 2)), + res.0.state().incoming_all[reused_info.obj()].contains((left_info.obj(), 4)), + res.0.state().current_incoming(reused_info.obj()).contains((left_info.obj(), 4)), + res.1.wf(), + res.1.ready_to_retire(), + res.1.obj() == old_info.obj(), + res.1.ptr() == old_info.ptr(), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + auth.tracked_register_node(left_info, left_retire); + auth.tracked_register_node(right_info, right_retire); + auth.tracked_register_node(old_info, old_retire); + + let root_left = auth.tracked_publish_link( + root_info.ptr(), + root_info.obj(), + left_info.ptr(), + left_info.obj(), + ); + let left_old = auth.tracked_publish_link( + left_info.ptr(), + left_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + let root_right = auth.tracked_publish_link( + root_info.ptr(), + root_info.obj(), + right_info.ptr(), + right_info.obj(), + ); + let right_old = auth.tracked_publish_link( + right_info.ptr(), + right_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + assert(root_left == 0); + assert(left_old == 0); + assert(root_right == 1); + assert(right_old == 0); + assert(auth.state().current_incoming(old_info.obj()).contains((left_info.obj(), left_old))); + assert(auth.state().current_incoming(old_info.obj()).contains((right_info.obj(), right_old))); + + let left_unlink = auth.tracked_unlink( + left_info.ptr(), + left_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + let right_unlink = auth.tracked_unlink( + right_info.ptr(), + right_info.obj(), + old_info.ptr(), + old_info.obj(), + ); + assert(left_unlink == 1); + assert(right_unlink == 1); + let ghost prior = RcuSeenRemoved { + removed: Set::empty(), + link_view: RcuLinkView::empty().observe(left_info.obj(), left_unlink).observe( + right_info.obj(), + right_unlink, + ), + }; + assert(auth.state().bounds(prior.link_view)) by { + assert forall|from_obj: nat| #[trigger] + auth.state().objects.contains_key(from_obj) && prior.link_view.seen.contains_key( + from_obj, + ) implies { + &&& auth.state().successors[from_obj].len() > 0 + &&& prior.seen_at(from_obj) < auth.state().successors[from_obj].len() + } by { + if from_obj == left_info.obj() { + assert(prior.seen_at(from_obj) == left_unlink); + } else { + assert(from_obj == right_info.obj()); + assert(prior.seen_at(from_obj) == right_unlink); + } + }; + } + assert(LinkedListTraversalSpec::seen_removed_sound(prior, auth.state())); + assert forall|edge: LinkEdge| #[trigger] + auth.state().incoming_all[old_info.obj()].contains(edge) implies prior.dead_edge(edge) by { + if edge.0 == left_info.obj() { + assert(edge == (left_info.obj(), left_old)); + assert(prior.seen_at(left_info.obj()) == left_unlink); + } else { + assert(edge == (right_info.obj(), right_old)); + assert(prior.seen_at(right_info.obj()) == right_unlink); + } + }; + let tracked retired = auth.tracked_retire_node(old_info.obj(), prior); + + auth.tracked_register_node(reused_info, reused_retire); + let first_republication = auth.tracked_publish_link( + left_info.ptr(), + left_info.obj(), + reused_info.ptr(), + reused_info.obj(), + ); + assert(first_republication == 2); + let reused_unlink = auth.tracked_unlink( + left_info.ptr(), + left_info.obj(), + reused_info.ptr(), + reused_info.obj(), + ); + assert(reused_unlink == 3); + let second_republication = auth.tracked_publish_link( + left_info.ptr(), + left_info.obj(), + reused_info.ptr(), + reused_info.obj(), + ); + assert(second_republication == 4); + assert(auth.state().objects[old_info.obj()] == old_info.ptr()); + assert(auth.state().objects[reused_info.obj()] == reused_info.ptr()); + assert(old_info.ptr() == reused_info.ptr()); + (auth, retired) +} + +/// End-to-end writer example connecting successful native IRC11 CAS updates +/// to the traversal retire rule. +/// +/// Native timestamps are kept abstract. The proof relies only on the CAS +/// contract's successor timestamps and on `LinkedListAtomicLinkGhost`'s +/// explicit timestamp-to-index correspondence. +pub proof fn linked_list_native_cas_unlink_enables_retire( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked child_info: &RcuBlockInfo, + tracked child_retire: RcuBaseRetirePerm, + initial_history: Irc11History<*mut LinkedListNode>, + published_history: Irc11History<*mut LinkedListNode>, + unlinked_history: Irc11History<*mut LinkedListNode>, + initial_timestamp: nat, + published_timestamp: nat, + unlinked_timestamp: nat, + initial_view: Irc11ThreadView, + published_view: Irc11ThreadView, + unlinked_view: Irc11ThreadView, +) -> (tracked retired: RcuRetirePerm) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + child_info.wf(), + child_retire.wf(), + child_info.domain() == root_info.domain(), + child_retire.domain() == root_info.domain(), + child_retire.obj() == child_info.obj(), + child_retire.ptr() == child_info.ptr(), + child_info.ptr().addr() != 0, + child_info.ptr() != root_info.ptr(), + child_info.obj() != root_info.obj(), + initial_history.is_singleton(initial_timestamp, (core::ptr::null_mut(), initial_view)), + published_timestamp == initial_timestamp + 1, + published_history == initial_history.insert( + published_timestamp, + child_info.ptr(), + published_view, + ), + unlinked_timestamp == published_timestamp + 1, + unlinked_history == published_history.insert( + unlinked_timestamp, + core::ptr::null_mut(), + unlinked_view, + ), + ensures + retired.wf(), + retired.ready_to_retire(), + retired.domain() == root_info.domain(), + retired.obj() == child_info.obj(), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + auth.tracked_register_node(child_info, child_retire); + let initial_index = auth.tracked_initialize_null(root_info.ptr(), root_info.obj()); + assert(initial_index == 0); + let tracked mut link = LinkedListAtomicLinkGhost::tracked_initial_null( + initial_history, + initial_timestamp, + initial_view, + &auth, + root_info.ptr(), + root_info.obj(), + ); + + let published_index = link.tracked_cas_publish( + &mut auth, + initial_history, + published_history, + initial_timestamp, + published_timestamp, + child_info.ptr(), + child_info.obj(), + published_view, + ); + assert(published_index == 1); + let unlinked_index = link.tracked_cas_unlink( + &mut auth, + published_history, + unlinked_history, + published_timestamp, + unlinked_timestamp, + child_info.ptr(), + child_info.obj(), + unlinked_view, + ); + assert(unlinked_index == 2); + + let ghost prior = RcuSeenRemoved { + removed: Set::empty(), + link_view: RcuLinkView::empty().observe(root_info.obj(), unlinked_index), + }; + assert(auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut LinkedListNode, from_obj: nat| #[trigger] + auth.state().objects.contains_pair(from_obj, from) && prior.link_view.seen.contains_key( + from_obj, + ) implies { + &&& auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) < auth.state().successors[from_obj].len() + } by { + assert(from == root_info.ptr()); + assert(from_obj == root_info.obj()); + }; + } + assert(LinkedListTraversalSpec::seen_removed_sound(prior, auth.state())); + assert forall|edge: LinkEdge| #[trigger] + auth.state().incoming_all[child_info.obj()].contains(edge) implies prior.dead_edge( + edge, + ) by { + assert(edge == (root_info.obj(), published_index)); + assert(prior.seen_at(root_info.obj()) == unlinked_index); + }; + auth.tracked_retire_node(child_info.obj(), prior) +} + +/// Uses an authoritative history snapshot to discharge the structural +/// premises of [`protect_link`]. The remaining `seen_removed_sound` premise is +/// the reader-side observation carried by the live guard. +pub proof fn linked_list_authorized_protect_next( + tracked auth: &LinkedListTraversalAuth, + tracked guard: &mut RcuReadGuardToken, + tracked root_info: &RcuBlockInfo, + tracked next_info: &RcuBlockInfo, + n: LinkIndex, +) -> (tracked next_protected: RcuProtectedPtr) + requires + auth.wf(), + old(guard).can_protect(*root_info), + old(guard).can_base_protect(*next_info), + root_info.domain() == auth.domain(), + next_info.domain() == auth.domain(), + root_info.ptr() == auth.state().root, + root_info.obj() == auth.state().root_obj, + auth.state().objects.contains_pair(next_info.obj(), next_info.ptr()), + n < auth.state().successors[root_info.obj()].len(), + auth.state().successors[root_info.obj()][n as int] == Some( + (next_info.ptr(), next_info.obj()), + ), + LinkedListTraversalSpec::seen_removed_sound(old(guard).seen_removed(), auth.state()), + old(guard).seen_at(root_info.obj()) <= n, + ensures + next_protected.ptr() == next_info.ptr(), + next_protected.obj() == next_info.obj(), + next_protected.domain() == auth.domain(), + next_protected.protected_by(*final(guard)), + final(guard).wf(), + LinkedListTraversalSpec::node_inv(next_info.ptr(), next_info.obj(), auth.state()), +{ + assert(LinkedListTraversalSpec::root_inv(root_info.ptr(), root_info.obj(), auth.state())); + assert(LinkedListTraversalSpec::link_inv( + root_info.ptr(), + root_info.obj(), + n, + next_info.ptr(), + next_info.obj(), + auth.state(), + )); + linked_list_protect_next_example( + guard, + root_info, + next_info, + root_info.ptr(), + n, + next_info.ptr(), + auth.state(), + ) } /// Example: after protecting the root, following a non-stale successor-history diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs index 02360faaa..9c9845a70 100644 --- a/ostd/specs/sync/rcu_cpu.rs +++ b/ostd/specs/sync/rcu_cpu.rs @@ -1299,6 +1299,21 @@ impl CpuRcuReadGuardToken { ensures res.0 == self.paper_guard(), res.1 == self.reader_fragment(), + res.2 == self.binding(), + res.0.domain() == self.domain(), + res.0.reader_registry() == self.reader_registry(), + res.0.reader() == self.reader_context(), + res.0.root() == self.root(), + res.0.start_view() == self.start_view(), + res.0.retire_observation_registry() == self.retire_observation_registry(), + res.0.expired() == self.expired(), + res.0.seen_removed() == self.seen_removed(), + res.0.protected() == self.protected(), + res.1.participant_id() == self.participant_id(), + res.1.cpu() == self.cpu(), + res.1.generation() == self.generation(), + res.1.participant_view() == self.participant_view(), + res.1.known_retired() == self.known_retired(), res.2.registry() == self.scheduler(), res.2.cpu() == self.cpu(), res.2.locals_key().len() == 1, @@ -1306,6 +1321,7 @@ impl CpuRcuReadGuardToken { res.0.wf(), res.1.wf(), res.0.reader().cpu == res.1.cpu(), + res.0.reader().generation == res.1.generation(), res.1.participant_view().spec_le(res.0.start_view()), forall|record: RcuRetiredRecord| #[trigger] res.1.known_retired().contains(record) && record.domain == res.0.domain() @@ -2064,6 +2080,18 @@ impl RcuRootPermissionState { self.reclaim_state@ } + /// Every allocation retained by the append-only identity registry has a + /// corresponding reclaim-state cell, including after that cell changes + /// from `Some(ptr)` to `None` at reclamation. + pub proof fn lemma_allocation_has_reclaim_state(tracked &self, obj: nat) + requires + self.wf(), + self.allocations().contains(obj), + ensures + self.reclaim_states().dom().contains(obj), + { + } + pub closed spec fn unretired_claims(self) -> Map>> { self.unretired_claims } @@ -2391,27 +2419,31 @@ impl RcuRootPermissionState { RcuReclaimClaim { points_to } } - /// Splits a physical read lease for the object selected by a guarded load. + /// Splits a physical read lease for an allocation already protected by a + /// traversal-level guard. /// - /// The CPU reader fragment is split at the same time and its matching half - /// is retained in the active registry record. - pub proof fn tracked_split_loaded( + /// Unlike [`Self::tracked_split_loaded`], this transition borrows the + /// explicit [`RcuProtectedPtr`] minted by a traversal step. This lets an + /// internal-link load use the same AId-indexed physical pool as a direct + /// root load without reconstructing protection from the guard map. A + /// persistent copy is retained in the active registry record while the + /// caller keeps the original for further traversal. The CPU reader + /// fragment is split at the same time. + pub proof fn tracked_split_protected( tracked &mut self, tracked guard: CpuRcuReadGuardToken, - tracked info: &RcuBlockInfo, + tracked protected: &RcuProtectedPtr, ) -> (tracked res: (CpuRcuReadGuardToken, RcuRootReadLease)) requires old(self).wf(), - old(self).contains(info.obj()), + old(self).contains(protected.obj()), guard.wf(), guard.scheduler() == old(self).scheduler(), - info.wf(), - info.domain() == old(self).domain(), + protected.domain() == old(self).domain(), + protected.protected_by(guard.paper_guard()), guard.domain() == old(self).domain(), guard.root() == old(self).root(), guard.retire_observation_registry() == old(self).retire_observation_registry(), - guard.protects(info.addr(), info.obj()), - !guard.seen_removed().removed.contains(info.obj()), online_cpus().contains(guard.cpu()), ensures final(self).wf(), @@ -2451,8 +2483,8 @@ impl RcuRootPermissionState { res.0.seen_removed() == guard.seen_removed(), res.0.protected() == guard.protected(), res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, - res.1.key() == info.obj(), - res.1.resource() == old(self).ownership(info.obj()), + res.1.key() == protected.obj(), + res.1.resource() == old(self).ownership(protected.obj()), res.1.active_registry() == old(self).active_lease_registry(), res.1.participant_id() == guard.participant_id(), res.1.reader_fraction() == res.0.reader_fragment().fraction(), @@ -2460,26 +2492,25 @@ impl RcuRootPermissionState { res.1.root() == guard.root(), res.1.reader_context() == guard.reader_context(), res.1.start_view() == guard.start_view(), - res.1.protected_addr() == info.addr(), + res.1.protected_addr() == protected.ptr().addr(), final(self).active_ids() == old(self).active_ids().insert(res.1.lease_id()), final(self).active_record(res.1.lease_id()).witness().paper_guard() == guard.paper_guard(), - final(self).active_record(res.1.lease_id()).witness().protected().obj() == info.obj(), + final(self).active_record(res.1.lease_id()).witness().protected() == *protected, { - let tracked (guard, witness) = CpuRcuReadLeaseWitness::tracked_from_loaded_guard( + let tracked saved_protected = protected.tracked_duplicate(); + let tracked (guard, witness) = CpuRcuReadLeaseWitness::tracked_from_guard( guard, - info, + saved_protected, ); - let tracked lease = self.registry.split_lease(info.obj(), witness); + let tracked lease = self.registry.split_lease(protected.obj(), witness); let ghost binding = RcuActiveLeaseBinding::from_record( self.active_record(lease.lease_id()), ); reveal(RcuActiveLeaseBinding::from_record); assert(self.active_record(lease.lease_id()).witness() == witness); - assert(self.active_record(lease.lease_id()).witness().protected().ptr() == info.ptr()); - info.lemma_wf_facts(); - assert(info.addr() == info.ptr().addr()); - assert(binding.protected_addr == info.addr()); + assert(self.active_record(lease.lease_id()).witness().protected() == *protected); + assert(binding.protected_addr == protected.ptr().addr()); let tracked active = self.active_leases.insert(lease.lease_id(), binding); assert forall|lease_id: nat| #[trigger] self.active_ids().contains(lease_id) implies { let record = self.active_record(lease_id); @@ -2494,7 +2525,7 @@ impl RcuRootPermissionState { } by { if lease_id == lease.lease_id() { assert(self.active_record(lease_id).witness() == witness); - assert(self.active_record(lease_id).key() == info.obj()); + assert(self.active_record(lease_id).key() == protected.obj()); } else { assert(old(self).active_ids().contains(lease_id)); assert(self.active_record(lease_id) == old(self).active_record(lease_id)); @@ -2504,6 +2535,89 @@ impl RcuRootPermissionState { (guard, lease) } + /// Splits a physical read lease for the object selected by a direct-root + /// guarded load. + /// + /// The direct-root adapter materializes the same traversal protection + /// witness and delegates to [`Self::tracked_split_protected`]. + pub proof fn tracked_split_loaded( + tracked &mut self, + tracked guard: CpuRcuReadGuardToken, + tracked info: &RcuBlockInfo, + ) -> (tracked res: (CpuRcuReadGuardToken, RcuRootReadLease)) + requires + old(self).wf(), + old(self).contains(info.obj()), + guard.wf(), + guard.scheduler() == old(self).scheduler(), + info.wf(), + info.domain() == old(self).domain(), + guard.domain() == old(self).domain(), + guard.root() == old(self).root(), + guard.retire_observation_registry() == old(self).retire_observation_registry(), + guard.protects(info.addr(), info.obj()), + !guard.seen_removed().removed.contains(info.obj()), + online_cpus().contains(guard.cpu()), + ensures + final(self).wf(), + final(self).scheduler() == old(self).scheduler(), + final(self).domain() == old(self).domain(), + final(self).root() == old(self).root(), + final(self).retire_observation_registry() == old(self).retire_observation_registry(), + final(self).reclaim_registry() == old(self).reclaim_registry(), + final(self).active_lease_registry() == old(self).active_lease_registry(), + final(self).keys() == old(self).keys(), + final(self).allocations() == old(self).allocations(), + final(self).reclaim_states() == old(self).reclaim_states(), + final(self).reclaimed() == old(self).reclaimed(), + final(self).unretired_claims() == old(self).unretired_claims(), + forall|candidate: nat| #[trigger] + final(self).has_unretired_claim(candidate) == old(self).has_unretired_claim( + candidate, + ), + forall|obj: nat| #[trigger] + old(self).contains(obj) ==> final(self).ownership(obj) == old(self).ownership(obj), + res.0.wf(), + res.0.paper_guard() == guard.paper_guard(), + res.0.binding() == guard.binding(), + res.0.participant_id() == guard.participant_id(), + res.0.scheduler() == guard.scheduler(), + res.0.cpu() == guard.cpu(), + res.0.generation() == guard.generation(), + res.0.participant_view() == guard.participant_view(), + res.0.known_retired() == guard.known_retired(), + res.0.domain() == guard.domain(), + res.0.root() == guard.root(), + res.0.reader_registry() == guard.reader_registry(), + res.0.retire_observation_registry() == guard.retire_observation_registry(), + res.0.reader_context() == guard.reader_context(), + res.0.start_view() == guard.start_view(), + res.0.expired() == guard.expired(), + res.0.seen_removed() == guard.seen_removed(), + res.0.protected() == guard.protected(), + res.0.reader_fragment().fraction() == guard.reader_fragment().fraction() / 2real, + res.1.key() == info.obj(), + res.1.resource() == old(self).ownership(info.obj()), + res.1.active_registry() == old(self).active_lease_registry(), + res.1.participant_id() == guard.participant_id(), + res.1.reader_fraction() == res.0.reader_fragment().fraction(), + res.1.domain() == guard.domain(), + res.1.root() == guard.root(), + res.1.reader_context() == guard.reader_context(), + res.1.start_view() == guard.start_view(), + res.1.protected_addr() == info.addr(), + final(self).active_ids() == old(self).active_ids().insert(res.1.lease_id()), + final(self).active_record(res.1.lease_id()).witness().paper_guard() + == guard.paper_guard(), + final(self).active_record(res.1.lease_id()).witness().protected().obj() == info.obj(), + { + let tracked protected = RcuProtectedPtr::tracked_from_guard(&guard.paper_guard, info); + let tracked res = self.tracked_split_protected(guard, &protected); + info.lemma_wf_facts(); + assert(res.1.protected_addr() == info.ptr().addr()); + res + } + /// Returns one physical lease and rejoins its CPU fragment with the /// executable guard that originally issued it. pub proof fn tracked_return_loaded( @@ -2621,6 +2735,69 @@ impl RcuRootPermissionState { guard } + /// A completed grace period rules out every still-active lease for the + /// retired allocation. + /// + /// Any lease coexisting with the persistent closed-generation report must + /// be a later reader. Such a reader already knows the retirement record, + /// so its paper guard marks the allocation expired. That contradicts the + /// same lease's protection witness. + pub proof fn lemma_completed_excludes_active( + tracked &mut self, + tracked completed: &RcuReclaimedWitness, + obj: nat, + ) + requires + old(self).wf(), + completed.wf(), + completed.scheduler() == old(self).scheduler(), + completed.record().domain == old(self).domain(), + completed.record().obj == obj, + completed.record().retire_observation_registry == old( + self, + ).retire_observation_registry(), + completed.record().removal.root == old(self).root(), + ensures + *final(self) == *old(self), + !final(self).has_active(obj), + { + if self.has_active(obj) { + assert(exists|lease_id: nat| + self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == obj); + let ghost lease_id = choose|lease_id: nat| + self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == obj; + let ghost record = self.active_record(lease_id); + assert(record.key() == obj); + assert(record.witness().wf()); + assert(record.witness().binding().registry() == self.scheduler()); + assert(record.witness().protected().obj() == obj); + assert(record.witness().protected().domain() == self.domain()); + assert(record.witness().paper_guard().domain() == self.domain()); + assert(record.witness().paper_guard().root() == self.root()); + assert(record.witness().paper_guard().retire_observation_registry() + == self.retire_observation_registry()); + let tracked witness = self.registry.tracked_borrow_active_witness_mut(lease_id); + assert(*witness == record.witness()); + let tracked closed = completed.tracked_closed_generation(witness.reader().cpu()); + witness.lemma_same_participant_as_closed(closed); + assert(closed.participant_id() == witness.reader().participant_id()); + closed.lemma_later_lease_witness_ref(witness); + assert(closed.known_retired().contains(completed.record())); + assert(completed.record().domain == witness.paper_guard().domain()); + assert(completed.record().retire_observation_registry + == witness.paper_guard().retire_observation_registry()); + assert(completed.record().removal.root == witness.paper_guard().root()); + assert(witness.reader().known_retired().contains(completed.record())); + assert(witness.paper_guard().expired().contains(obj)); + assert(witness.protected().protected_by(witness.paper_guard())); + assert(!witness.paper_guard().seen_removed().removed.contains(obj)); + assert(witness.paper_guard().expired().subset_of( + witness.paper_guard().seen_removed().removed, + )); + assert(false); + } + } + /// Recovers one allocation after completion has ruled out every active /// lease for its identity. pub proof fn tracked_reclaim( diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index d333b0da6..03369de9a 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -6,12 +6,14 @@ //! [`vstd_extra::atomic_irc11`]. use core::{marker::PhantomData, sync::atomic::Ordering}; +use super::rcu::RcuTraversalSafety; use super::{rcu as rcu_spec, rcu_cpu as rcu_cpu_spec}; use crate::specs::mm::cpu::online_cpus; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::modes::tracked_static_ref; use vstd::prelude::*; use vstd::resource::Loc; +use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; use vstd::thread_view::Objective; use vstd_extra::atomic_irc11::{ AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, PAtomicWeakPtr, @@ -275,6 +277,1926 @@ impl RcuRetiredRootObject { } } +/// Writer resources obtained after the traversal proof has certified that the +/// fixed child has no live incoming edge. +/// +/// This is the handoff between the linked-list layer and base RCU: `retire` +/// authorizes the paper's `rcu-retire` transition, while `claim` is the unique +/// right to recover the physical ownership after a completed grace period. +pub tracked struct LinkedListDetachedChild { + object: rcu_spec::RcuObjectId, + retire: rcu_spec::RcuRetirePerm, + claim: rcu_cpu_spec::RcuReclaimClaim, + ghost removal: rcu_spec::RcuRemovalObservation, +} + +impl LinkedListDetachedChild { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + &&& self.object().wf() + &&& self.retire().wf() + &&& self.retire().ready_to_retire() + &&& self.object().domain() == self.retire().domain() + &&& self.object().obj() == self.retire().obj() + &&& self.object().ptr() == self.retire().ptr() + &&& self.claim().obj() == self.object().obj() + &&& self.claim().is_pending() + &&& equal(self.claim().ptr(), self.object().ptr()) + } + + pub closed spec fn object(self) -> rcu_spec::RcuObjectId { + self.object + } + + pub closed spec fn retire(self) -> rcu_spec::RcuRetirePerm { + self.retire + } + + pub closed spec fn claim(self) -> rcu_cpu_spec::RcuReclaimClaim { + self.claim + } + + pub closed spec fn removal(self) -> rcu_spec::RcuRemovalObservation { + self.removal + } + + /// Performs the base-RCU retirement transition after traversal retirement. + pub proof fn tracked_retire( + tracked self, + tracked domain: &mut rcu_spec::RcuDomainAuth, + ) -> (tracked res: LinkedListRetiredChild) + requires + old(domain).wf(), + self.retire().belongs_to(*old(domain)), + self.object().domain() == old(domain).id(), + ensures + final(domain).wf(), + final(domain).id() == old(domain).id(), + res.object() == self.object(), + res.claim() == self.claim(), + res.retired().domain() == self.object().domain(), + res.retired().obj() == self.object().obj(), + res.retired().ptr() == self.object().ptr(), + res.retired().removal() == self.removal(), + { + use_type_invariant(&self); + let ghost object_value = self.object(); + let ghost claim_value = self.claim(); + assert(self.object().domain() == old(domain).id()); + let tracked LinkedListDetachedChild { object, retire, claim, removal } = self; + let tracked retired = domain.tracked_retire(retire, removal); + assert(object == object_value); + assert(claim == claim_value); + assert(object.wf()); + assert(retired.wf()); + assert(retired.domain() == old(domain).id()); + assert(object.domain() == retired.domain()); + assert(object.obj() == retired.obj()); + assert(object.ptr() == retired.ptr()); + assert(claim.obj() == object.obj()); + assert(claim.is_pending()); + assert(equal(claim.ptr(), object.ptr())); + LinkedListRetiredChild { object, retired, claim } + } +} + +/// A retired linked-list child paired with the claim that will eventually +/// recover its physical ownership. +pub tracked struct LinkedListRetiredChild { + object: rcu_spec::RcuObjectId, + retired: rcu_spec::RcuRetired, + claim: rcu_cpu_spec::RcuReclaimClaim, +} + +impl LinkedListRetiredChild { + #[verifier::type_invariant] + pub closed spec fn type_inv(self) -> bool { + &&& self.object().wf() + &&& self.retired().wf() + &&& self.object().domain() == self.retired().domain() + &&& self.object().obj() == self.retired().obj() + &&& self.object().ptr() == self.retired().ptr() + &&& self.claim().obj() == self.object().obj() + &&& self.claim().is_pending() + &&& equal(self.claim().ptr(), self.object().ptr()) + } + + pub closed spec fn object(self) -> rcu_spec::RcuObjectId { + self.object + } + + pub closed spec fn retired(self) -> rcu_spec::RcuRetired { + self.retired + } + + pub closed spec fn claim(self) -> rcu_cpu_spec::RcuReclaimClaim { + self.claim + } + + /// Compresses the typed traversal retirement into the type-erased safety + /// certificate consumed by the existing callback monitor. + pub proof fn tracked_certify_callback(tracked self) -> (tracked res: ( + rcu_spec::RcuObjectId, + rcu_spec::RcuCallbackSafety, + rcu_cpu_spec::RcuReclaimClaim, + )) + ensures + res.0 == self.object(), + res.2 == self.claim(), + res.1.domain() == res.0.domain(), + res.1.obj() == res.0.obj(), + res.1.removal() == self.retired().removal(), + res.1.retire_observation_registry() == self.retired().retire_observation_registry(), + rcu_spec::callback_safety_from_traversal(res.1, res.0), + { + use_type_invariant(&self); + let tracked LinkedListRetiredChild { object, retired, claim } = self; + let tracked cert = rcu_spec::certify_callback_from_retired(&object, retired); + (object, cert, claim) + } +} + +/// Immutable identities carried by the native atomic invariant for the +/// two-node linked-list traversal example. +pub ghost struct LinkedListAtomicKey { + pub scheduler: Loc, + pub domain: Loc, + pub root: Loc, + pub retire_observation_registry: Loc, + pub reclaim_registry: Loc, + pub active_lease_registry: Loc, + pub lifecycle: Loc, + pub timestamp_registry: Loc, + pub source: *mut rcu_spec::LinkedListNode, + pub source_obj: nat, + pub child: *mut rcu_spec::LinkedListNode, + pub child_obj: nat, +} + +/// Writer-visible lifecycle of the fixed child managed by one concrete link. +/// +/// The two halves of a [`GhostVar`] keep this phase synchronized between the +/// executable wrapper and its atomic invariant. In particular, a writer that +/// has moved the child to `Retired` can no longer call the publication rule, +/// while stale readers may continue returning leases until `Reclaimed`. +pub ghost enum LinkedListChildPhase { + /// The child is registered but has never been published by this link. + Unpublished, + /// The child is the latest value of the link. + Linked { index: rcu_spec::LinkIndex, timestamp: Timestamp }, + /// The child was removed by the recorded native atomic message. + Unlinked { index: rcu_spec::LinkIndex, removal: rcu_spec::RcuRemovalObservation }, + /// Traversal retirement and the physical reclaim claim left the invariant. + Retired { index: rcu_spec::LinkIndex, removal: rcu_spec::RcuRemovalObservation }, + /// Grace-period completion recovered the physical ownership resource. + Reclaimed { index: rcu_spec::LinkIndex, removal: rcu_spec::RcuRemovalObservation }, +} + +impl LinkedListChildPhase { + pub open spec fn is_reclaimed(self) -> bool { + self is Reclaimed + } + + pub open spec fn is_unpublished(self) -> bool { + self is Unpublished + } + + pub open spec fn is_linked(self) -> bool { + self is Linked + } + + pub open spec fn is_unlinked(self) -> bool { + self is Unlinked + } + + pub open spec fn is_retired(self) -> bool { + self is Retired + } +} + +/// Complete state protected by one native linked-list link invariant. +pub tracked struct LinkedListAtomicState { + pub(crate) points_to: AtomicPointsTo<*mut rcu_spec::LinkedListNode>, + pub(crate) link: rcu_spec::LinkedListAtomicLinkGhost, + pub(crate) auth: rcu_spec::LinkedListTraversalAuth, + pub(crate) permissions: rcu_cpu_spec::RcuRootPermissionState, + pub(crate) lifecycle: GhostVarAuth, +} + +unsafe impl Objective for LinkedListAtomicState { + +} + +impl LinkedListAtomicState { + pub closed spec fn points_to(self) -> AtomicPointsTo<*mut rcu_spec::LinkedListNode> { + self.points_to + } + + pub closed spec fn link(self) -> rcu_spec::LinkedListAtomicLinkGhost { + self.link + } + + pub closed spec fn auth(self) -> rcu_spec::LinkedListTraversalAuth { + self.auth + } + + pub closed spec fn permissions(self) -> rcu_cpu_spec::RcuRootPermissionState< + rcu_spec::LinkedListNode, + O, + > { + self.permissions + } + + pub closed spec fn lifecycle(self) -> GhostVarAuth { + self.lifecycle + } +} + +/// Native IRC11 invariant for a link whose only non-null value is one +/// pre-registered child. +/// +/// Restricting the first executable-style wrapper to two nodes keeps the +/// atomic protocol closed while the general node-registration and physical +/// permission pools are still being designed. +pub struct LinkedListAtomicInv { + _marker: PhantomData, +} + +impl InvariantPredicate< + (LinkedListAtomicKey, Irc11AtomicId), + LinkedListAtomicState, +> for LinkedListAtomicInv where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + open spec fn inv( + key_loc: (LinkedListAtomicKey, Irc11AtomicId), + state: LinkedListAtomicState, + ) -> bool { + let (key, loc) = key_loc; + let permissions = state.permissions(); + &&& state.points_to().loc() == loc + &&& key.source.addr() != 0 + &&& key.child.addr() != 0 + &&& key.source.addr() != key.child.addr() + &&& key.source_obj != key.child_obj + &&& state.auth().wf() + &&& state.auth().domain() == key.domain + &&& state.auth().state().root == key.source + &&& state.auth().state().root_obj == key.source_obj + &&& state.auth().state().objects == Map::empty().insert(key.source_obj, key.source).insert( + key.child_obj, + key.child, + ) + &&& state.auth().state().incoming_all.contains_key(key.child_obj) + &&& state.auth().state().successors.contains_key(key.source_obj) + &&& permissions.wf() + &&& permissions.scheduler() == key.scheduler + &&& permissions.domain() == key.domain + &&& permissions.root() == key.root + &&& permissions.retire_observation_registry() == key.retire_observation_registry + &&& permissions.reclaim_registry() == key.reclaim_registry + &&& permissions.active_lease_registry() == key.active_lease_registry + &&& permissions.allocations() == Set::empty().insert(key.child_obj) + &&& state.lifecycle().id() == key.lifecycle + &&& state.link().source() == key.source + &&& state.link().source_obj() == key.source_obj + &&& state.link().timestamp_registry() == key.timestamp_registry + &&& state.link().wf(state.points_to().hist(), state.auth()) + &&& forall|n: rcu_spec::LinkIndex| + n < state.auth().state().successors[key.source_obj].len() + && state.auth().state().successors[key.source_obj][n as int] is Some + ==> #[trigger] state.auth().state().successors[key.source_obj][n as int] == Some( + (key.child, key.child_obj), + ) + &&& match state.lifecycle()@ { + LinkedListChildPhase::Unpublished => { + &&& state.auth().removed() == Set::::empty() + &&& state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& state.auth().state().successors[key.source_obj].len() == 1 + &&& state.auth().state().successors[key.source_obj].last() is None + &&& state.auth().state().incoming_all[key.child_obj] == Set::< + rcu_spec::LinkEdge, + >::empty() + }, + LinkedListChildPhase::Linked { index, timestamp } => { + &&& state.auth().removed() == Set::::empty() + &&& state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& state.link().current_timestamp() == timestamp + &&& state.link().index_at(timestamp) == index + &&& index + 1 == state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj].last() == Some( + (key.child, key.child_obj), + ) + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, index), + ) + }, + LinkedListChildPhase::Unlinked { index, removal } => { + &&& state.auth().removed() == Set::::empty() + &&& state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& removal.root == key.root + &&& state.link().current_timestamp() == removal.timestamp + &&& state.link().index_at(removal.timestamp) == index + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& index + 1 == state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj].last() is None + &&& index > 0 + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, (index - 1) as nat), + ) + }, + LinkedListChildPhase::Retired { index, removal } => { + &&& state.auth().removed() == Set::::empty().insert(key.child_obj) + &&& !state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::empty().insert(key.child_obj) + &&& permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] == Some(key.child) + &&& !permissions.has_unretired_claim(key.child_obj) + &&& OwnPred::owns(key.child, permissions.ownership(key.child_obj)) + &&& removal.root == key.root + &&& state.link().current_timestamp() == removal.timestamp + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& state.auth().state().successors[key.source_obj].last() is None + &&& index > 0 + &&& state.link().index_at(removal.timestamp) == index + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, (index - 1) as nat), + ) + }, + LinkedListChildPhase::Reclaimed { index, removal } => { + &&& state.auth().removed() == Set::::empty().insert(key.child_obj) + &&& !state.auth().has_retire_perm(key.child_obj) + &&& permissions.keys() == Set::::empty() + &&& !permissions.contains(key.child_obj) + &&& permissions.reclaim_states()[key.child_obj] is None + &&& !permissions.has_unretired_claim(key.child_obj) + &&& permissions.reclaimed().contains_key(key.child_obj) + &&& permissions.reclaimed()[key.child_obj].record().removal == removal + &&& removal.root == key.root + &&& state.link().current_timestamp() == removal.timestamp + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& state.auth().state().successors[key.source_obj].last() is None + &&& index > 0 + &&& state.link().index_at(removal.timestamp) == index + &&& state.auth().state().incoming_all[key.child_obj] == Set::empty().insert( + (key.source_obj, (index - 1) as nat), + ) + }, + } + } +} + +pub type LinkedListAtomicInvariant = AtomicInvariant< + (LinkedListAtomicKey, Irc11AtomicId), + LinkedListAtomicState, + LinkedListAtomicInv, +>; + +/// Executable-style native weak atomic used to close the linked-list +/// traversal invariant before generalizing it to arbitrary data structures. +pub struct LinkedListWeakAtomicLink { + atomic: PAtomicWeakPtr, + child: *mut rcu_spec::LinkedListNode, + tracked_atomic_inv: Tracked<&'static LinkedListAtomicInvariant>, + tracked_child_phase: Tracked>, +} + +impl LinkedListWeakAtomicLink { + pub closed spec fn constant(&self) -> LinkedListAtomicKey { + self.tracked_atomic_inv@.constant().0 + } + + pub closed spec fn native_loc(&self) -> Irc11AtomicId { + self.atomic.loc() + } + + /// Namespace of the atomic invariant owned by this wrapper. + pub closed spec fn invariant_namespace(&self) -> int { + self.tracked_atomic_inv@.namespace() + } + + pub closed spec fn child_ptr(&self) -> *mut rcu_spec::LinkedListNode { + self.child + } + + /// Runtime child pointer retained by this one-link wrapper. + #[inline(always)] + pub(crate) fn child_raw(&self) -> (res: *mut rcu_spec::LinkedListNode) + ensures + equal(res, self.child_ptr()), + { + self.child + } + + pub closed spec fn child_phase(&self) -> LinkedListChildPhase { + self.tracked_child_phase@.view() + } + + pub closed spec fn well_formed(&self) -> bool { + &&& self.tracked_atomic_inv@.constant().1 == self.native_loc() + &&& self.child_ptr() == self.constant().child + &&& self.child_ptr().addr() != 0 + &&& self.tracked_child_phase@.id() == self.constant().lifecycle + } + + /// Exposes the structural consequences of the closed wrapper invariant to + /// executable adapters without exposing its proof-resource representation. + pub proof fn lemma_well_formed_facts(&self) + requires + self.well_formed(), + ensures + self.child_ptr() == self.constant().child, + self.child_ptr().addr() != 0, + { + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } +} + +impl LinkedListWeakAtomicLink where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { + /// Creates a null link with a pre-registered source and child. + pub const fn new( + Ghost(scheduler): Ghost, + Ghost(root): Ghost, + Ghost(retire_observation_registry): Ghost, + child: *mut rcu_spec::LinkedListNode, + Tracked(source_info): Tracked<&rcu_spec::RcuBlockInfo>, + Tracked(source_retire): Tracked>, + Tracked(child_info): Tracked<&rcu_spec::RcuBlockInfo>, + Tracked(child_retire): Tracked>, + Tracked(child_ownership): Tracked, + ) -> (res: Self) + requires + source_info.wf(), + source_retire.wf(), + source_retire.domain() == source_info.domain(), + source_retire.obj() == source_info.obj(), + source_retire.ptr() == source_info.ptr(), + child_info.wf(), + child_retire.wf(), + child_info.domain() == source_info.domain(), + child_retire.domain() == source_info.domain(), + child_retire.obj() == child_info.obj(), + child_retire.ptr() == child_info.ptr(), + child == child_info.ptr(), + source_info.addr() != 0, + child_info.addr() != 0, + source_info.addr() != child_info.addr(), + source_info.ptr() != child_info.ptr(), + source_info.obj() != child_info.obj(), + OwnPred::owns(child, child_ownership), + ensures + res.well_formed(), + res.constant().scheduler == scheduler, + res.constant().domain == source_info.domain(), + res.constant().root == root, + res.constant().retire_observation_registry == retire_observation_registry, + res.constant().source == source_info.ptr(), + res.constant().source_obj == source_info.obj(), + res.constant().child == child_info.ptr(), + res.constant().child_obj == child_info.obj(), + res.child_ptr() == child, + res.child_phase() is Unpublished, + { + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + PAtomicWeakPtr::new(core::ptr::null_mut()); + let tracked mut auth = rcu_spec::LinkedListTraversalAuth::tracked_new( + source_info, + source_retire, + ); + proof { + assert(auth.has_retire_perm(source_info.obj())); + assert(!auth.has_retire_perm(child_info.obj())); + auth.tracked_register_node(child_info, child_retire); + } + let ghost initial_index = auth.tracked_initialize_null( + source_info.ptr(), + source_info.obj(), + ); + let tracked link = rcu_spec::LinkedListAtomicLinkGhost::tracked_initial_null( + points_to.hist(), + timestamp, + initial_view@, + &auth, + source_info.ptr(), + source_info.obj(), + ); + proof_decl! { + let tracked mut permissions: + rcu_cpu_spec::RcuRootPermissionState; + let ghost reclaim_registry: Loc; + let ghost active_lease_registry: Loc; + let ghost child_ownership_value: O; + } + proof { + permissions = + rcu_cpu_spec::RcuRootPermissionState::empty( + scheduler, + source_info.domain(), + root, + retire_observation_registry, + ); + reclaim_registry = permissions.reclaim_registry(); + active_lease_registry = permissions.active_lease_registry(); + child_ownership_value = child_ownership; + assert(permissions.allocations() == Set::::empty()); + permissions.tracked_insert(child_info, child_ownership); + } + let tracked (lifecycle, lifecycle_peer) = GhostVarAuth::new( + LinkedListChildPhase::Unpublished, + ); + let tracked state = LinkedListAtomicState { points_to, link, auth, permissions, lifecycle }; + let ghost key = LinkedListAtomicKey { + scheduler, + domain: source_info.domain(), + root, + retire_observation_registry, + reclaim_registry, + active_lease_registry, + lifecycle: state.lifecycle().id(), + timestamp_registry: state.link().timestamp_registry(), + source: source_info.ptr(), + source_obj: source_info.obj(), + child: child_info.ptr(), + child_obj: child_info.obj(), + }; + proof { + source_info.lemma_wf_facts(); + child_info.lemma_wf_facts(); + assert(initial_index == 0); + assert(state.points_to().loc() == atomic.loc()); + assert(key.source.addr() == source_info.addr()); + assert(key.child.addr() == child_info.addr()); + assert(state.auth().wf()); + assert(state.auth().domain() == key.domain); + assert(state.auth().removed() == Set::::empty()); + assert(state.auth().state().root == key.source); + assert(state.auth().state().root_obj == key.source_obj); + assert(state.auth().state().objects == Map::empty().insert( + key.source_obj, + key.source, + ).insert(key.child_obj, key.child)); + assert(state.permissions().wf()); + assert(state.permissions().scheduler() == key.scheduler); + assert(state.permissions().domain() == key.domain); + assert(state.permissions().root() == key.root); + assert(state.permissions().retire_observation_registry() + == key.retire_observation_registry); + assert(state.permissions().reclaim_registry() == key.reclaim_registry); + assert(state.permissions().active_lease_registry() == key.active_lease_registry); + assert(state.permissions().allocations() == Set::empty().insert(key.child_obj)); + assert(state.permissions().keys() == Set::empty().insert(key.child_obj)); + assert(state.permissions().contains(key.child_obj)); + assert(state.permissions().reclaim_states()[key.child_obj] == Some(key.child)); + assert(state.permissions().has_unretired_claim(key.child_obj)); + assert(state.permissions().ownership(key.child_obj) == child_ownership_value); + assert(OwnPred::owns(key.child, state.permissions().ownership(key.child_obj))); + assert(state.lifecycle()@ is Unpublished); + assert(state.link().source() == key.source); + assert(state.link().source_obj() == key.source_obj); + assert(state.link().wf(state.points_to().hist(), state.auth())); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth().state().successors[key.source_obj].len() + && state.auth().state().successors[key.source_obj][n as int] is Some implies #[trigger] state.auth().state().successors[key.source_obj][n as int] + == Some((key.child, key.child_obj)) by { + assert(n == 0); + assert(state.auth().state().successors[key.source_obj][n as int] is None); + assert(false); + }; + assert(LinkedListAtomicInv::::inv((key, atomic.loc()), state)); + } + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); + let tracked atomic_inv = tracked_static_ref(atomic_inv); + LinkedListWeakAtomicLink { + atomic, + child, + tracked_atomic_inv: Tracked(atomic_inv), + tracked_child_phase: Tracked(lifecycle_peer), + } + } + + fn raw_atomic(&self) -> (res: &PAtomicWeakPtr) + requires + self.well_formed(), + ensures + res.loc() == self.native_loc(), + opens_invariants none + no_unwind + { + &self.atomic + } + + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: + &'static LinkedListAtomicInvariant) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.native_loc()), + { + self.tracked_atomic_inv.get() + } + + /// Acquire-loads the native link and applies the paper's guarded traversal + /// rule in the same atomic-invariant opening. + /// + /// `previous` is absent for the first observation and otherwise carries + /// the persistent timestamp/index fact returned by the preceding load. + /// Its native view lower bound prevents a later load from moving the + /// guard's dense traversal position backwards. + #[inline(always)] + pub fn load_acquire_and_protect( + &self, + Tracked(guard): Tracked<&mut rcu_spec::RcuReadGuardToken>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + !self.child_phase().is_reclaimed(), + old(guard).wf(), + old(guard).domain() == self.constant().domain, + old(guard).seen_removed().removed == Set::::empty(), + match previous { + None => old(guard).seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& old(guard).seen_at(self.constant().source_obj) == observation.index() + }, + }, + old(from).protected_by(*old(guard)), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).seen_removed().removed == old(guard).seen_removed().removed, + final(guard).seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.4@.registry() == self.constant().timestamp_registry, + res.4@.loc() == self.native_loc(), + res.4@.timestamp() == res.1@, + res.4@.index() == res.2@, + res.4@.view() == final(tv)@, + (res.3@ is Some) == (res.0.addr() != 0), + match res.3@ { + None => res.0.addr() == 0, + Some(child) => { + &&& equal(child.ptr(), res.0) + &&& child.ptr() == self.constant().child + &&& child.obj() == self.constant().child_obj + &&& child.domain() == self.constant().domain + &&& child.protected_by(*final(guard)) + }, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(&*self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.lifecycle@ == self.child_phase()); + match state.lifecycle@ { + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + _ => {}, + } + assert(state.permissions.contains(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(state.points_to.loc() == self.native_loc()); + match previous { + None => { + assert(guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + use_type_invariant(observation); + observation.lemma_view_timestamp(); + assert(vstd_extra::atomic_irc11::timestamp_in_view( + observation.loc(), + observation.view(), + ) == Some(observation.timestamp())); + assert(observation.loc() == self.native_loc()); + state.link.lemma_observation_agrees(observation); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(state.points_to.hist().contains_timestamp( + observation.timestamp(), + )); + vstd_extra::atomic_irc11::axiom_get_timestamp_is_location_projection( + &state.points_to, + observation.view(), + ); + assert(vstd_extra::atomic_irc11::timestamp_in_view( + self.native_loc(), + observation.view(), + ) == Some(observation.timestamp())); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + assert(state.points_to.hist().contains_timestamp(timestamp)); + match previous { + None => { + assert(guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.timestamp() <= timestamp); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(observation.index() <= index); + assert(guard.seen_at(from.obj()) == observation.index()); + }, + } + assert(guard.seen_at(from.obj()) <= index); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + old(guard).seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + old(guard).seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> old(guard).seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + guard, + from, + timestamp, + ); + let tracked observation; + } + proof { + vstd_extra::atomic_irc11::axiom_get_timestamp_is_location_projection( + &state.points_to, + tv@, + ); + assert(vstd_extra::atomic_irc11::timestamp_in_view( + self.native_loc(), + tv@, + ) == Some(timestamp)); + observation = state.link.tracked_observation_at( + state.points_to.hist(), + &state.auth, + timestamp, + self.native_loc(), + tv@, + ); + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + match &protected { + None => { + assert(loaded.0.addr() == 0); + }, + Some(child) => { + assert(state.auth.state().successors[self.constant().source_obj][index as int] + is Some); + assert(state.auth.state().successors[self.constant().source_obj][index as int] + == Some((self.constant().child, self.constant().child_obj))); + assert(child.ptr() == self.constant().child); + assert(child.obj() == self.constant().child_obj); + }, + } + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(protected), + Tracked(observation), + ); + }); + result + } + + /// Acquire-loads an internal link, applies the paper traversal rule, and + /// splits a physical read lease for the observed child in one atomic + /// invariant opening. + /// + /// A non-null result retains the child protection witness for subsequent + /// traversal while the lease supplies the physical ownership resource + /// used to derive a concrete `RefPermission`. A null result leaves the + /// CPU reader fraction unchanged. + #[inline(always)] + pub fn load_acquire_and_lease_cpu( + &self, + Tracked(guard): Tracked>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>, + Tracked>>, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + !self.child_phase().is_reclaimed(), + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + online_cpus().contains(guard.cpu()), + guard.seen_removed().removed == Set::::empty(), + match previous { + None => guard.paper_guard().seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& guard.paper_guard().seen_at(self.constant().source_obj) + == observation.index() + }, + }, + old(from).protected_by(guard.paper_guard()), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + res.3@.wf(), + res.3@.binding() == guard.binding(), + res.3@.participant_id() == guard.participant_id(), + res.3@.cpu() == guard.cpu(), + res.3@.generation() == guard.generation(), + res.3@.participant_view() == guard.participant_view(), + res.3@.known_retired() == guard.known_retired(), + res.3@.scheduler() == guard.scheduler(), + res.3@.domain() == guard.domain(), + res.3@.root() == guard.root(), + res.3@.reader_registry() == guard.reader_registry(), + res.3@.retire_observation_registry() == guard.retire_observation_registry(), + res.3@.reader_context() == guard.reader_context(), + res.3@.start_view() == guard.start_view(), + res.3@.expired() == guard.expired(), + res.3@.seen_removed().removed == guard.seen_removed().removed, + res.3@.paper_guard().seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.6@.registry() == self.constant().timestamp_registry, + res.6@.loc() == self.native_loc(), + res.6@.timestamp() == res.1@, + res.6@.index() == res.2@, + res.6@.view() == final(tv)@, + (res.4@ is Some) == (res.0.addr() != 0), + (res.5@ is Some) == (res.0.addr() != 0), + match (res.4@, res.5@) { + (None, None) => { + &&& res.0.addr() == 0 + &&& res.3@.reader_fragment() == guard.reader_fragment() + }, + (Some(child), Some(lease)) => { + &&& equal(child.ptr(), res.0) + &&& child.ptr() == self.constant().child + &&& child.obj() == self.constant().child_obj + &&& child.domain() == self.constant().domain + &&& child.protected_by(res.3@.paper_guard()) + &&& res.3@.reader_fragment().fraction() == guard.reader_fragment().fraction() + / 2real + &&& lease.key() == child.obj() + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == res.3@.participant_id() + &&& lease.reader_fraction() == res.3@.reader_fragment().fraction() + &&& lease.domain() == res.3@.domain() + &&& lease.root() == res.3@.root() + &&& lease.reader_context() == res.3@.reader_context() + &&& lease.start_view() == res.3@.start_view() + &&& lease.protected_addr() == child.ptr().addr() + &&& OwnPred::owns(child.ptr(), lease.resource()) + }, + _ => false, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(self); + } + proof_decl! { + let tracked (mut paper_guard, cpu_reader, binding) = guard.tracked_into_parts(); + } + proof { + assert(paper_guard == guard.paper_guard()); + assert(cpu_reader == guard.reader_fragment()); + assert(binding == guard.binding()); + assert(paper_guard.domain() == self.constant().domain); + assert(paper_guard.seen_removed().removed == Set::::empty()); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost permissions_before = state.permissions; + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.lifecycle@ == self.child_phase()); + match state.lifecycle@ { + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + _ => {}, + } + assert(state.permissions.contains(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(state.points_to.loc() == self.native_loc()); + match previous { + None => { + assert(paper_guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + use_type_invariant(observation); + observation.lemma_view_timestamp(); + assert(vstd_extra::atomic_irc11::timestamp_in_view( + observation.loc(), + observation.view(), + ) == Some(observation.timestamp())); + assert(observation.loc() == self.native_loc()); + state.link.lemma_observation_agrees(observation); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(state.points_to.hist().contains_timestamp( + observation.timestamp(), + )); + vstd_extra::atomic_irc11::axiom_get_timestamp_is_location_projection( + &state.points_to, + observation.view(), + ); + assert(vstd_extra::atomic_irc11::timestamp_in_view( + self.native_loc(), + observation.view(), + ) == Some(observation.timestamp())); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + assert(state.points_to.hist().contains_timestamp(timestamp)); + match previous { + None => { + assert(paper_guard.seen_at(from.obj()) == 0); + }, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.timestamp() <= timestamp); + assert(state.link.index_at(observation.timestamp()) + == observation.index()); + assert(observation.index() <= index); + assert(paper_guard.seen_at(from.obj()) == observation.index()); + }, + } + assert(paper_guard.seen_at(from.obj()) <= index); + assert(paper_guard.domain() == state.auth.domain()); + assert(state.auth.state().successors.contains_key(self.constant().source_obj)); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + paper_guard.seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + paper_guard.seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> paper_guard.seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + &mut paper_guard, + from, + timestamp, + ); + let tracked cpu_guard = rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new( + paper_guard, + cpu_reader, + binding, + ); + let tracked final_guard; + let tracked lease; + let tracked observation; + } + proof { + vstd_extra::atomic_irc11::axiom_get_timestamp_is_location_projection( + &state.points_to, + tv@, + ); + assert(vstd_extra::atomic_irc11::timestamp_in_view( + self.native_loc(), + tv@, + ) == Some(timestamp)); + observation = state.link.tracked_observation_at( + state.points_to.hist(), + &state.auth, + timestamp, + self.native_loc(), + tv@, + ); + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + match &protected { + None => { + assert(loaded.0.addr() == 0); + final_guard = cpu_guard; + lease = None; + }, + Some(child) => { + assert(state.auth.state().successors[ + self.constant().source_obj + ][index as int] is Some); + assert(state.auth.state().successors[ + self.constant().source_obj + ][index as int] == Some(( + self.constant().child, + self.constant().child_obj, + ))); + assert(child.ptr() == self.constant().child); + assert(child.obj() == self.constant().child_obj); + assert(state.permissions.contains(child.obj())); + let ghost child_ownership = state.permissions.ownership(child.obj()); + assert(OwnPred::owns(child.ptr(), child_ownership)); + let tracked split = state.permissions.tracked_split_protected( + cpu_guard, + child, + ); + final_guard = split.0; + lease = Some(split.1); + assert(split.1.resource() == child_ownership); + assert(OwnPred::owns(child.ptr(), split.1.resource())); + assert(state.permissions.ownership(child.obj()) == child_ownership); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + }, + } + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() + == permissions_before.reclaim_states()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + state.permissions.lemma_contains_iff_key(self.constant().child_obj); + assert(state.permissions.contains(self.constant().child_obj)); + assert(state.permissions.reclaim_states()[self.constant().child_obj] + == Some(self.constant().child)); + match state.lifecycle@ { + LinkedListChildPhase::Unpublished + | LinkedListChildPhase::Linked { index: _, timestamp: _ } + | LinkedListChildPhase::Unlinked { index: _, removal: _ } => { + assert(state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Retired { index: _, removal: _ } => { + assert(!state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + } + assert(OwnPred::owns( + self.constant().child, + permissions_before.ownership(self.constant().child_obj), + )); + assert(state.permissions.ownership(self.constant().child_obj) + == permissions_before.ownership(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(final_guard), + Tracked(protected), + Tracked(lease), + Tracked(observation), + ); + }); + result + } + + /// Returns a child traversal lease and rejoins its saved CPU reader + /// fraction with the live guard. + #[verifier::atomic] + pub fn return_child_lease_cpu( + &self, + Tracked(lease): Tracked>>, + Tracked(guard): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked>) + requires + self.well_formed(), + !self.child_phase().is_reclaimed(), + match lease { + None => true, + Some(lease) => { + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + }, + }, + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + old(tv)@.spec_le(final(tv)@), + res@.wf(), + res@.paper_guard() == guard.paper_guard(), + res@.binding() == guard.binding(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + res@.participant_view() == guard.participant_view(), + res@.known_retired() == guard.known_retired(), + res@.domain() == guard.domain(), + res@.root() == guard.root(), + res@.reader_registry() == guard.reader_registry(), + res@.retire_observation_registry() == guard.retire_observation_registry(), + res@.reader_context() == guard.reader_context(), + res@.start_view() == guard.start_view(), + res@.expired() == guard.expired(), + res@.seen_removed() == guard.seen_removed(), + res@.protected() == guard.protected(), + res@.reader_fragment().fraction() == match lease { + None => guard.reader_fragment().fraction(), + Some(_) => guard.reader_fragment().fraction() * 2real, + }, + no_unwind + { + let raw_atomic = &self.atomic; + proof_decl! { + let tracked final_guard; + } + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + assert(state.lifecycle@ == self.child_phase()); + match state.lifecycle@ { + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + _ => {}, + } + match lease { + None => { + final_guard = guard; + }, + Some(lease) => { + final_guard = state.permissions.tracked_return_loaded(lease, guard); + }, + } + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() + == permissions_before.reclaim_states()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + state.permissions.lemma_contains_iff_key(self.constant().child_obj); + assert(state.permissions.contains(self.constant().child_obj)); + assert(state.permissions.reclaim_states()[self.constant().child_obj] + == Some(self.constant().child)); + match state.lifecycle@ { + LinkedListChildPhase::Unpublished + | LinkedListChildPhase::Linked { index: _, timestamp: _ } + | LinkedListChildPhase::Unlinked { index: _, removal: _ } => { + assert(state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Retired { index: _, removal: _ } => { + assert(!state.permissions.has_unretired_claim(self.constant().child_obj)); + }, + LinkedListChildPhase::Reclaimed { index: _, removal: _ } => assert(false), + } + assert(OwnPred::owns( + self.constant().child, + permissions_before.ownership(self.constant().child_obj), + )); + assert(state.permissions.ownership(self.constant().child_obj) + == permissions_before.ownership(self.constant().child_obj)); + assert(OwnPred::owns( + self.constant().child, + state.permissions.ownership(self.constant().child_obj), + )); + assert(LinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + Tracked(final_guard) + } + + /// Publishes the pre-registered child with a native AcqRel/Acquire CAS. + /// The successful branch appends the matching traversal event before the + /// atomic invariant is closed. + #[inline(always)] + pub fn compare_exchange_publish_child(&mut self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).child_phase().is_unpublished(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> res.0->Ok_0.addr() == 0, + res.0 is Ok ==> final(self).child_phase() is Linked, + res.0 is Ok ==> final(self).child_phase()->Linked_index == res.1@->Some_0, + res.0 is Err ==> final(self).child_phase() is Unpublished, + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let child = self.child; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + assert(state.lifecycle@ is Unpublished); + assert(state.points_to.loc() == native_loc); + } + let ghost prev = state.points_to.hist(); + let ghost old_successors = state.auth.state().successors[key.source_obj]; + let cas = raw_atomic.compare_exchange( + null, + child, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost published_index: Option; + } + proof { + match cas.0 { + Result::Ok(_) => { + assert(next == prev.insert( + update.load_timestamp + 1, + child, + update.store_message_view, + )); + let index = state.link.tracked_cas_publish( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + child, + key.child_obj, + update.store_message_view, + ); + published_index = Some(index); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Linked { + index, + timestamp: update.load_timestamp + 1, + }, + ); + assert(state.auth.state().incoming_all[key.child_obj].contains(( + key.source_obj, + index, + ))); + assert(state.auth.state().incoming_all[key.child_obj].len() > 0); + assert(state.auth.removed() == Set::::empty()); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth.state().successors[key.source_obj].len() + && state.auth.state().successors[key.source_obj][n as int] is Some implies + #[trigger] state.auth.state().successors[key.source_obj][n as int] == Some(( + key.child, + key.child_obj, + )) by { + if n == index { + } else { + assert(n < old_successors.len()); + assert(state.auth.state().successors[key.source_obj][n as int] == old_successors[n as int]); + } + }; + }, + Result::Err(_) => { + published_index = None; + }, + } + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + } + result = (cas.0, Ghost(published_index)); + }); + result + } + + /// Unlinks the pre-registered child with a native AcqRel/Acquire CAS. + /// A successful address comparison is resolved through the invariant to + /// the child's persistent object identity before the null traversal event + /// is appended. + #[inline(always)] + pub fn compare_exchange_unlink_child(&mut self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).child_phase().is_linked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> res.0->Ok_0.addr() == final(self).child_ptr().addr(), + res.0 is Ok ==> final(self).child_phase() is Unlinked, + res.0 is Ok ==> final(self).child_phase()->Unlinked_index == res.1@->Some_0, + res.0 is Ok ==> final(self).child_phase()->Unlinked_removal.root + == final(self).constant().root, + res.0 is Ok ==> final(self).child_phase()->Unlinked_removal.observed_by(final(tv)@), + res.0 is Err ==> final(self).child_phase() is Linked, + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let child = self.child; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + assert(state.lifecycle@ is Linked); + assert(state.points_to.loc() == native_loc); + } + let ghost prev = state.points_to.hist(); + let ghost old_successors = state.auth.state().successors[key.source_obj]; + let cas = raw_atomic.compare_exchange( + child, + null, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost unlinked_index: Option; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + let ghost current_index = state.link.index_at(current_timestamp); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp <= current_timestamp); + assert(current_timestamp <= update.load_timestamp); + assert(current_timestamp == update.load_timestamp); + assert(state.lifecycle@->Linked_timestamp == current_timestamp); + assert(state.lifecycle@->Linked_index == current_index); + assert(equal(prev.value(update.load_timestamp), value)); + assert(value.addr() == child.addr()); + assert(prev.value(current_timestamp).addr() == child.addr()); + assert(current_index + 1 == old_successors.len()); + assert(old_successors[current_index as int] is Some) by { + if old_successors[current_index as int] is None { + assert(prev.value(current_timestamp).addr() == 0); + assert(false); + } + }; + assert(old_successors[current_index as int] == Some(( + key.child, + key.child_obj, + ))); + assert(old_successors.last() == Some(( + key.child, + key.child_obj, + ))); + assert(state.auth.state().objects.contains_pair( + key.source_obj, + key.source, + )); + assert(state.auth.state().objects.contains_pair( + key.child_obj, + key.child, + )); + assert(state.auth.state().incoming_all.contains_key(key.child_obj)); + assert(current_index < old_successors.len()); + assert(state.auth.state().successors[key.source_obj][current_index as int] + == Some((key.child, key.child_obj))); + assert(state.auth.state().incoming_all[key.child_obj].len() > 0); + assert(next == prev.insert( + update.load_timestamp + 1, + null, + update.store_message_view, + )); + let index = state.link.tracked_cas_unlink( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + key.child, + key.child_obj, + update.store_message_view, + ); + unlinked_index = Some(index); + let ghost removal = rcu_spec::RcuRemovalObservation { + root: key.root, + timestamp: update.load_timestamp + 1, + message_view: update.store_message_view, + }; + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Unlinked { index, removal }, + ); + assert(state.auth.removed() == Set::::empty()); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth.state().successors[key.source_obj].len() + && state.auth.state().successors[key.source_obj][n as int] is Some implies + #[trigger] state.auth.state().successors[key.source_obj][n as int] == Some(( + key.child, + key.child_obj, + )) by { + assert(n < old_successors.len()); + assert(state.auth.state().successors[key.source_obj][n as int] == old_successors[n as int]); + }; + assert(state.lifecycle@ is Unlinked); + assert(state.link.current_timestamp() == removal.timestamp); + assert(state.link.index_at(removal.timestamp) == index); + assert(state.points_to.hist().thread_view(removal.timestamp) + == removal.message_view); + assert(index + 1 + == state.auth.state().successors[key.source_obj].len()); + assert(state.auth.state().successors[key.source_obj].last() is None); + assert(state.auth.state().incoming_all[key.child_obj].len() > 0); + }, + Result::Err(_) => { + unlinked_index = None; + }, + } + assert(LinkedListAtomicInv::::inv( + (key, native_loc), + state, + )); + } + result = (cas.0, Ghost(unlinked_index)); + }); + result + } + + /// Converts a successfully unlinked child into the writer resources + /// required by base RCU and by eventual physical reclamation. + /// + /// The traversal observation records the unlink event itself. Since the + /// only historical incoming edge is the immediately preceding publish, + /// this proves that every incoming edge is dead and consumes the child's + /// unique traversal retire permission. + #[verifier::atomic] + pub fn retire_unlinked_child(&mut self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: Tracked< + LinkedListDetachedChild, + >) + requires + old(self).well_formed(), + old(self).child_phase().is_unlinked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).child_phase() is Retired, + final(self).child_phase()->Retired_index == old(self).child_phase()->Unlinked_index, + final(self).child_phase()->Retired_removal == old(self).child_phase()->Unlinked_removal, + final(self).child_phase()->Retired_removal.root == final(self).constant().root, + res@.object().wf(), + res@.object().domain() == final(self).constant().domain, + res@.object().obj() == final(self).constant().child_obj, + res@.object().ptr() == final(self).constant().child, + res@.retire().wf(), + res@.retire().ready_to_retire(), + res@.retire().domain() == final(self).constant().domain, + res@.retire().obj() == final(self).constant().child_obj, + res@.retire().ptr() == final(self).constant().child, + res@.claim().registry() == final(self).constant().reclaim_registry, + res@.claim().obj() == final(self).constant().child_obj, + res@.claim().is_pending(), + res@.claim().ptr() == final(self).constant().child, + res@.removal() == final(self).child_phase()->Retired_removal, + res@.removal().root == final(self).constant().root, + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked detached; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.lifecycle@ is Unlinked); + let ghost index = state.lifecycle@->Unlinked_index; + let ghost removal = state.lifecycle@->Unlinked_removal; + let ghost prior = rcu_spec::RcuSeenRemoved { + removed: Set::empty(), + link_view: rcu_spec::RcuLinkView::empty().observe(key.source_obj, index), + }; + assert(state.auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut rcu_spec::LinkedListNode, from_obj: nat| #[trigger] + state.auth.state().objects.contains_pair(from_obj, from) + && prior.link_view.seen.contains_key(from_obj) implies { + &&& state.auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) + < state.auth.state().successors[from_obj].len() + } by { + assert(from_obj == key.source_obj); + assert(from == key.source); + assert(prior.link_view.seen_at(from_obj) == index); + assert(index + 1 == state.auth.state().successors[from_obj].len()); + }; + } + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + prior, + state.auth.state(), + )); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[key.child_obj].contains(edge) implies + prior.dead_edge(edge) by { + assert(edge == (key.source_obj, (index - 1) as nat)); + assert(prior.seen_at(key.source_obj) == index); + }; + state.auth.lemma_has_info_for_object(key.child_obj); + assert(state.auth.has_info(key.child_obj)); + assert(state.permissions.contains(key.child_obj)); + let ghost auth_before = state.auth; + let ghost permissions_before = state.permissions; + let ghost child_ownership = state.permissions.ownership(key.child_obj); + let tracked object = state.auth.tracked_info_for(key.child_obj); + let tracked retire = state.auth.tracked_retire_node(key.child_obj, prior); + assert(auth_before.state().objects.contains_pair(key.child_obj, retire.ptr())); + assert(retire.ptr() == key.child); + let tracked claim = state.permissions.tracked_retire(key.child_obj); + assert(claim.ptr() == key.child); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Retired { index, removal }, + ); + assert(state.auth.removed() == Set::::empty().insert(key.child_obj)); + assert(!state.auth.has_retire_perm(key.child_obj)); + assert(state.permissions.keys() == Set::empty().insert(key.child_obj)); + state.permissions.lemma_contains_iff_key(key.child_obj); + assert(state.permissions.contains(key.child_obj)); + assert(state.permissions.allocations().contains(key.child_obj)); + assert(state.permissions.reclaim_states()[key.child_obj] == Some(key.child)); + assert(!state.permissions.has_unretired_claim(key.child_obj)); + assert(state.permissions.ownership(key.child_obj) == child_ownership); + assert(OwnPred::owns( + key.child, + state.permissions.ownership(key.child_obj), + )); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(object.ptr() == key.child); + assert(retire.ptr() == key.child); + assert(claim.ptr() == key.child); + detached = LinkedListDetachedChild { object, retire, claim, removal }; + } + }); + Tracked(detached) + } + + /// Reclaims the retired child's physical ownership after the existing RCU + /// monitor has produced a completed grace-period witness. + #[verifier::atomic] + pub fn reclaim_retired_child( + &mut self, + Tracked(claim): Tracked>, + Tracked(completed): Tracked, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked) + requires + old(self).well_formed(), + old(self).child_phase().is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == old(self).constant().child_obj, + claim.is_pending(), + claim.ptr() == old(self).constant().child, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == old(self).constant().child_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).child_phase()->Retired_removal, + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).child_phase() is Reclaimed, + final(self).child_phase()->Reclaimed_index == old(self).child_phase()->Retired_index, + final(self).child_phase()->Reclaimed_removal == old( + self, + ).child_phase()->Retired_removal, + OwnPred::owns(final(self).constant().child, res@), + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked ownership; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.lifecycle@ is Retired); + let ghost index = state.lifecycle@->Retired_index; + let ghost removal = state.lifecycle@->Retired_removal; + assert(completed.record().removal == removal); + assert(completed.record().removal.root == key.root); + state.permissions.lemma_completed_excludes_active( + &completed, + key.child_obj, + ); + assert(!state.permissions.has_active(key.child_obj)); + ownership = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Reclaimed { index, removal }, + ); + assert(state.permissions.keys() == Set::::empty()); + state.permissions.lemma_contains_iff_key(key.child_obj); + assert(!state.permissions.contains(key.child_obj)); + assert(state.permissions.allocations().contains(key.child_obj)); + assert(state.permissions.reclaim_states()[key.child_obj] is None); + assert(!state.permissions.has_unretired_claim(key.child_obj)); + assert(state.permissions.reclaimed().contains_key(key.child_obj)); + assert(state.permissions.reclaimed()[key.child_obj].record().removal == removal); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(OwnPred::owns(key.child, ownership)); + } + }); + Tracked(ownership) + } + + /// Proof-mode form used by a type-erased callback after its executable + /// monitor has supplied an open-invariant credit and a completed grace + /// period. No additional atomic access is needed at runtime: the callback + /// owns this link and only consumes proof resources before deallocating the + /// recovered smart pointer. + pub proof fn tracked_reclaim_retired_child( + tracked &mut self, + tracked claim: rcu_cpu_spec::RcuReclaimClaim, + tracked completed: rcu_cpu_spec::RcuReclaimedWitness, + tracked credit: vstd::invariant::OpenInvariantCredit, + ) -> (tracked ownership: O) + requires + old(self).well_formed(), + old(self).child_phase().is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == old(self).constant().child_obj, + claim.is_pending(), + claim.ptr() == old(self).constant().child, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == old(self).constant().child_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).child_phase()->Retired_removal, + ensures + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).child_phase() is Reclaimed, + final(self).child_phase()->Reclaimed_index == old(self).child_phase()->Retired_index, + final(self).child_phase()->Reclaimed_removal == old( + self, + ).child_phase()->Retired_removal, + OwnPred::owns(final(self).constant().child, ownership), + opens_invariants [ self.invariant_namespace() ] + { + use_type_invariant(&*self); + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked mut recovered; + vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + state.lifecycle.agree(self.tracked_child_phase.borrow()); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.lifecycle@ is Retired); + let ghost index = state.lifecycle@->Retired_index; + let ghost removal = state.lifecycle@->Retired_removal; + assert(completed.record().removal == removal); + assert(completed.record().removal.root == key.root); + state.permissions.lemma_completed_excludes_active( + &completed, + key.child_obj, + ); + assert(!state.permissions.has_active(key.child_obj)); + recovered = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_child_phase.borrow_mut(), + LinkedListChildPhase::Reclaimed { index, removal }, + ); + assert(state.permissions.keys() == Set::::empty()); + state.permissions.lemma_contains_iff_key(key.child_obj); + assert(!state.permissions.contains(key.child_obj)); + assert(state.permissions.allocations().contains(key.child_obj)); + state.permissions.lemma_allocation_has_reclaim_state(key.child_obj); + assert(state.permissions.reclaim_states()[key.child_obj] is None); + assert(!state.permissions.has_unretired_claim(key.child_obj)); + assert(state.permissions.reclaimed().contains_key(key.child_obj)); + assert(state.permissions.reclaimed()[key.child_obj].record().removal == removal); + assert(LinkedListAtomicInv::::inv((key, native_loc), state)); + assert(OwnPred::owns(key.child, recovered)); + }); + recovered + } +} + /// OSTD's RCU-specific specialization of the generic weak pointer atomic. /// /// This is an RCU client of Verus' native IRC11 protocol. The only local TCB diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 4423faaee..9ec45efe5 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -134,6 +134,7 @@ //! because Verus cannot yet attach this invariant-opening transition to Rust's //! implicit `Drop::drop(&mut self)`. Runtime destruction still restores the //! executable preemption counter through `DisabledPreemptGuard`. +use alloc::boxed::Box; use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; use vstd::invariant::InvariantPredicate; @@ -148,8 +149,8 @@ use crate::{ sync::{ rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, weak_memory::{ - RcuRetiredRootObject, RcuRootAtomicInv, RcuRootAtomicInvariant, RcuRootAtomicState, - RcuWeakAtomicPtr, + LinkedListRetiredChild, LinkedListWeakAtomicLink, RcuRetiredRootObject, + RcuRootAtomicInv, RcuRootAtomicInvariant, RcuRootAtomicState, RcuWeakAtomicPtr, }, }, task::InAtomicMode, @@ -198,6 +199,231 @@ impl rcu_spec::RcuRootOwnershipPredicate< } } +/// Concrete linked-list atomic whose pointee ownership is a real smart-pointer +/// permission understood by [`NonNullPtrRef`]. +type RcuLinkedListAtomicLink

= LinkedListWeakAtomicLink< +

::Permission, + RcuPointerOwnership

, +>; + +/// One loaded internal child together with its physical RCU read lease. +/// +/// This is the linked-list counterpart of [`RcuReadGuardInner`]. In +/// particular, [`Self::get`] derives `P::RefPermission` from the lease and +/// invokes the same verified `raw_as_ref` boundary as a direct-root read. +struct LinkedListChildReadGuard<'a, P> where P: NonNullPtr { + obj_ptr: *mut rcu_spec::LinkedListNode, + link: &'a RcuLinkedListAtomicLink

, + proof_active: bool, + tracked_guard: Tracked>>, + tracked_child: Tracked>>, + tracked_lease: Tracked>>, + tracked_observation: Tracked>, +} + +impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr { + #[verifier::type_invariant] + closed spec fn type_inv(&self) -> bool { + &&& self.link.well_formed() + &&& self.proof_active ==> { + &&& self.tracked_guard@ is Some + &&& self.tracked_observation@ is Some + &&& self.tracked_guard@->Some_0.wf() + &&& self.tracked_guard@->Some_0.scheduler() == self.link.constant().scheduler + &&& self.tracked_guard@->Some_0.domain() == self.link.constant().domain + &&& self.tracked_guard@->Some_0.root() == self.link.constant().root + &&& self.tracked_guard@->Some_0.retire_observation_registry() + == self.link.constant().retire_observation_registry + &&& self.tracked_observation@->Some_0.registry() + == self.link.constant().timestamp_registry + &&& self.tracked_observation@->Some_0.loc() == self.link.native_loc() + &&& self.tracked_guard@->Some_0.paper_guard().seen_at(self.link.constant().source_obj) + == self.tracked_observation@->Some_0.index() + &&& (self.tracked_child@ is Some) == (self.obj_ptr.addr() != 0) + &&& (self.tracked_lease@ is Some) == (self.obj_ptr.addr() != 0) + &&& match (self.tracked_child@, self.tracked_lease@) { + (None, None) => self.obj_ptr.addr() == 0, + (Some(child), Some(lease)) => { + &&& equal(child.ptr(), self.obj_ptr) + &&& child.ptr() == self.link.constant().child + &&& child.obj() == self.link.constant().child_obj + &&& child.protected_by(self.tracked_guard@->Some_0.paper_guard()) + &&& lease.key() == child.obj() + &&& lease.active_registry() == self.link.constant().active_lease_registry + &&& lease.participant_id() == self.tracked_guard@->Some_0.participant_id() + &&& lease.reader_fraction() + == self.tracked_guard@->Some_0.reader_fragment().fraction() + &&& lease.domain() == self.tracked_guard@->Some_0.domain() + &&& lease.root() == self.tracked_guard@->Some_0.root() + &&& lease.reader_context() == self.tracked_guard@->Some_0.reader_context() + &&& lease.start_view() == self.tracked_guard@->Some_0.start_view() + &&& lease.protected_addr() == child.ptr().addr() + &&& RcuPointerOwnership::

::owns(child.ptr(), lease.resource()) + }, + _ => false, + } + } + } + + /// Loads an internal child and retains both its traversal witness and its + /// physical lease. `previous` may be supplied to repeat a load from the + /// same source without resetting the dense traversal view. + fn load( + link: &'a RcuLinkedListAtomicLink

, + Tracked(guard): Tracked>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Self) + requires + link.well_formed(), + !link.child_phase().is_reclaimed(), + guard.wf(), + guard.scheduler() == link.constant().scheduler, + guard.domain() == link.constant().domain, + guard.root() == link.constant().root, + guard.retire_observation_registry() == link.constant().retire_observation_registry, + online_cpus().contains(guard.cpu()), + guard.seen_removed().removed == Set::::empty(), + match previous { + None => guard.paper_guard().seen_at(link.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == link.constant().timestamp_registry + &&& observation.loc() == link.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& guard.paper_guard().seen_at(link.constant().source_obj) + == observation.index() + }, + }, + old(from).protected_by(guard.paper_guard()), + old(from).ptr() == link.constant().source, + old(from).obj() == link.constant().source_obj, + ensures + res.type_inv(), + res.proof_active, + old(tv)@.spec_le(final(tv)@), + final(from).ptr() == link.constant().source, + final(from).obj() == link.constant().source_obj, + { + let ( + obj_ptr, + _timestamp, + _index, + Tracked(guard), + Tracked(child), + Tracked(lease), + Tracked(observation), + ) = link.load_acquire_and_lease_cpu( + Tracked(guard), + Tracked(from), + Tracked(previous), + Tracked(tv), + ); + let res = Self { + obj_ptr, + link, + proof_active: true, + tracked_guard: Tracked(Some(guard)), + tracked_child: Tracked(child), + tracked_lease: Tracked(lease), + tracked_observation: Tracked(Some(observation)), + }; + proof { + assert(res.type_inv()); + } + res + } + + /// Obtains the smart pointer's real shared-reference representation from + /// the internal child's physical lease. + fn get<'b>(&'b self) -> Option<

>::Ref> where P: NonNullPtrRef<'b> + requires + self.proof_active, + { + proof { + use_type_invariant(self); + reveal(LinkedListChildReadGuard::type_inv); + if self.obj_ptr.addr() != 0 { + assert(self.tracked_lease@ is Some); + assert(RcuPointerOwnership::

::owns( + self.obj_ptr, + self.tracked_lease@->Some_0.resource(), + )); + assert(P::ptr_perm_match(self.obj_ptr, self.tracked_lease@->Some_0.resource())); + assert(self.tracked_lease@->Some_0.resource().inv()); + } + } + NonNull::new(self.obj_ptr).map( + |ptr| + requires + self.tracked_lease@ is Some, + P::ptr_perm_match(ptr.view_ptr_mut(), self.tracked_lease@->Some_0.resource()), + { + proof_decl! { + let tracked lease = self.tracked_lease.tracked_borrow(); + let tracked ref_perm = P::borrow_perm_as_ref_perm(lease.borrow()); + } + unsafe { P::raw_as_ref(ptr, Tracked(ref_perm)) } + }, + ) + } + + /// Returns the physical lease, the traversal witness, and the updated CPU + /// guard so the caller may continue traversing. + fn finish(self, Tracked(tv): Tracked<&mut ViewSeen>) -> (res: Tracked< + ( + rcu_cpu_spec::CpuRcuReadGuardToken, + Option>, + rcu_spec::LinkedListLinkObservation, + ), + >) + requires + self.type_inv(), + self.proof_active, + !self.link.child_phase().is_reclaimed(), + ensures + old(tv)@.spec_le(final(tv)@), + res@.0.wf(), + res@.0.paper_guard().seen_at(self.link.constant().source_obj) == res@.2.index(), + res@.2.registry() == self.link.constant().timestamp_registry, + res@.2.loc() == self.link.native_loc(), + (res@.1 is Some) == (self.obj_ptr.addr() != 0), + { + let mut this = self; + proof { + use_type_invariant(&this); + reveal(LinkedListChildReadGuard::type_inv); + } + this.proof_active = false; + proof_decl! { + let tracked guard = this.tracked_guard.borrow_mut().tracked_take(); + let tracked mut child = None; + vstd::modes::tracked_swap(this.tracked_child.borrow_mut(), &mut child); + let tracked mut lease = None; + vstd::modes::tracked_swap(this.tracked_lease.borrow_mut(), &mut lease); + let tracked observation = this.tracked_observation.borrow_mut().tracked_take(); + } + let Tracked(guard) = this.link.return_child_lease_cpu( + Tracked(lease), + Tracked(guard), + Tracked(tv), + ); + Tracked((guard, child, observation)) + } +} + +impl<'a> LinkedListChildReadGuard<'a, Box> { + /// Concrete acceptance path: turn the boxed child's leased + /// `RefPermission` into an actual Rust shared reference and dereference + /// the node allocation. + fn deref_box_child<'b>(&'b self) -> Option<&'b rcu_spec::LinkedListNode> + requires + self.proof_active, + { + self.get().map(|child| child.deref_target()) + } +} + /// The weak-memory atomic slot used by RCU. /// /// `bool` is the constant key: `true` means the public cell may contain null @@ -263,6 +489,125 @@ struct RcuDropCallbackContext { >, } +/// Type-erased callback payload for an internal linked-list child. +/// +/// The callback owns the entire link wrapper after unlink/retire. This gives +/// it exclusive access to the link's phase token and physical permission pool +/// when the monitor eventually supplies a reclaim permit. +struct LinkedListDropCallbackContext

where + P: NonNullPtr + Send, + { + pointer: NonNull, + link: RcuLinkedListAtomicLink

, + tracked_object: Tracked>, + tracked_claim: Tracked>, + ghost_removal: Ghost, + ghost_retire_observation_registry: Ghost, + ghost_scheduler: Ghost, +} + +// SAFETY: the context owns the detached `P` allocation and the unique link +// wrapper that protects its proof-only permission pool. No borrowed runtime +// state crosses into the monitor queue. +#[verifier::external] +unsafe impl

Send for LinkedListDropCallbackContext

where + P: NonNullPtr + Send, + { + +} + +impl

LinkedListDropCallbackContext

where + P: NonNullPtr + Send, + { + pub closed spec fn permit_matches(&self, permit: monitor::RcuReclaimPermit) -> bool { + &&& permit.wf() + &&& permit.callback().domain == self.tracked_object@.domain() + &&& permit.callback().obj == self.tracked_object@.obj() + &&& permit.callback().removal == self.ghost_removal@ + &&& permit.callback().retire_observation_registry == self.ghost_retire_observation_registry@ + &&& permit.callback().scheduler == self.ghost_scheduler@ + } + + #[verifier::type_invariant] + closed spec fn type_inv(self) -> bool { + &&& self.link.well_formed() + &&& (self.link.child_phase().is_retired() || self.link.child_phase().is_reclaimed()) + &&& self.tracked_object@.wf() + &&& equal(self.tracked_object@.ptr(), self.pointer.view_ptr_mut()) + &&& self.tracked_object@.domain() == self.link.constant().domain + &&& self.tracked_object@.obj() == self.link.constant().child_obj + &&& equal(self.tracked_object@.ptr(), self.link.constant().child) + &&& self.ghost_scheduler@ == self.link.constant().scheduler + &&& match self.link.child_phase() { + crate::specs::sync::weak_memory::LinkedListChildPhase::Retired { index: _, removal } + | crate::specs::sync::weak_memory::LinkedListChildPhase::Reclaimed { + index: _, + removal, + } => self.ghost_removal@ == removal, + _ => false, + } + &&& self.ghost_removal@.root == self.link.constant().root + &&& self.ghost_retire_observation_registry@ + == self.link.constant().retire_observation_registry + &&& self.link.child_phase().is_retired() + &&& self.tracked_claim@.registry() == self.link.constant().reclaim_registry + &&& self.tracked_claim@.obj() == self.tracked_object@.obj() + &&& self.tracked_claim@.is_pending() + &&& equal(self.tracked_claim@.ptr(), self.pointer.view_ptr_mut()) + } +} + +impl

RawCallbackContextWithProof for LinkedListDropCallbackContext< + P, +> where P: NonNullPtr + Send { + open spec fn call_requires(&self, permit: monitor::RcuReclaimPermit) -> bool { + self.permit_matches(permit) + } + + fn run(self, Tracked(permit): Tracked) { + let Tracked(credit) = vstd::invariant::create_open_invariant_credit(); + proof_decl! { + let tracked permission; + let tracked completed; + } + proof { + use_type_invariant(&self); + use_type_invariant(&permit); + reveal(LinkedListDropCallbackContext::type_inv); + permit.lemma_authorizes_callback(); + let ghost callback = permit.callback(); + assert(self.permit_matches(permit)); + assert(permit.authorizes(callback)); + assert(self.link.child_phase().is_retired()); + completed = permit.tracked_into_reclaimed_witness(callback); + assert(completed.wf()); + assert(completed.scheduler() == self.link.constant().scheduler); + assert(completed.record() == callback.retired_record()); + assert(completed.record().domain == self.link.constant().domain); + assert(completed.record().obj == self.link.constant().child_obj); + assert(completed.record().retire_observation_registry + == self.link.constant().retire_observation_registry); + assert(completed.record().removal == self.link.child_phase()->Retired_removal); + } + let LinkedListDropCallbackContext { + pointer, + mut link, + tracked_object: _, + tracked_claim, + ghost_removal: _, + ghost_retire_observation_registry: _, + ghost_scheduler: _, + } = self; + proof { + permission = link.tracked_reclaim_retired_child(tracked_claim.get(), completed, credit); + assert(RcuPointerOwnership::

::owns(pointer.as_ptr(), permission)); + assert(P::ptr_perm_match(pointer.as_ptr(), permission)); + assert(permission.inv()); + } + let _pointer = unsafe { P::from_raw(pointer, Tracked(permission)) }; + } +} + // SAFETY: the callback consumes the same owning pointer type `P` that was // accepted by the RCU cell. The tracked permission has no runtime payload. #[verifier::external] @@ -591,6 +936,137 @@ fn callback_from_detached( (RawCallbackWithProof::new(context), Tracked(cert)) } +/// Erases a retired internal child into the real monitor callback pipeline. +/// The link moves into the callback context, so the successful callback is the +/// only code that can change its phase to `Reclaimed` and recover `P`'s full +/// physical permission. +fn callback_from_linked_list_child

( + link: RcuLinkedListAtomicLink

, + Tracked(retired): Tracked, +) -> (res: ( + RawCallbackWithProof, + Tracked, +)) where P: NonNullPtr + Send + requires + link.well_formed(), + link.child_phase().is_retired(), + retired.object().wf(), + retired.object().domain() == link.constant().domain, + retired.object().obj() == link.constant().child_obj, + equal(retired.object().ptr(), link.constant().child), + retired.claim().registry() == link.constant().reclaim_registry, + retired.claim().obj() == link.constant().child_obj, + retired.claim().is_pending(), + equal(retired.claim().ptr(), link.constant().child), + retired.retired().removal() == link.child_phase()->Retired_removal, + link.child_phase()->Retired_removal.root == link.constant().root, + retired.retired().retire_observation_registry() + == link.constant().retire_observation_registry, + ensures + res.1@.domain() == link.constant().domain, + res.1@.obj() == link.constant().child_obj, + res.1@.removal() == link.child_phase()->Retired_removal, + res.1@.retire_observation_registry() == link.constant().retire_observation_registry, + forall|permit: monitor::RcuReclaimPermit| + permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj + == res.1@.obj() && permit.callback().removal == res.1@.removal() + && permit.callback().retire_observation_registry + == res.1@.retire_observation_registry() && permit.callback().scheduler + == link.constant().scheduler ==> res.0.call_requires(permit), +{ + proof_decl! { + let tracked (object, cert, claim) = retired.tracked_certify_callback(); + } + let child_ptr = link.child_raw(); + proof { + link.lemma_well_formed_facts(); + assert(link.well_formed()); + assert(link.child_ptr().addr() != 0); + assert(equal(child_ptr, link.child_ptr())); + assert(child_ptr.addr() != 0); + } + let pointer = unsafe { NonNull::new_unchecked(child_ptr) }; + proof { + assert(equal(pointer.view_ptr_mut(), child_ptr)); + assert(link.child_ptr().addr() != 0); + assert(object.wf()); + assert(link.child_phase() is Retired); + assert(equal(object.ptr(), pointer.view_ptr_mut())); + assert(equal(object.ptr(), link.constant().child)); + assert(object.domain() == link.constant().domain); + assert(object.obj() == link.constant().child_obj); + assert(claim.registry() == link.constant().reclaim_registry); + assert(claim.obj() == object.obj()); + assert(claim.is_pending()); + assert(equal(claim.ptr(), pointer.view_ptr_mut())); + assert(cert.removal() == link.child_phase()->Retired_removal); + assert(cert.removal().root == link.constant().root); + assert(cert.retire_observation_registry() == link.constant().retire_observation_registry); + } + let ghost scheduler = link.constant().scheduler; + let context = LinkedListDropCallbackContext::

{ + pointer, + link, + tracked_object: Tracked(object), + tracked_claim: Tracked(claim), + ghost_removal: Ghost(cert.removal()), + ghost_retire_observation_registry: Ghost(cert.retire_observation_registry()), + ghost_scheduler: Ghost(scheduler), + }; + proof { + use_type_invariant(&context); + } + (RawCallbackWithProof::new(context), Tracked(cert)) +} + +/// Schedules one retired internal child on the existing `call_rcu` monitor +/// path. This is intentionally kept private to the linked-list acceptance +/// case until a production data-structure adapter chooses its public API. +fn after_grace_period_linked_list_child

( + link: RcuLinkedListAtomicLink

, + Tracked(retired): Tracked, + Tracked(session): Tracked<&mut RunningTaskContext>, +) where P: NonNullPtr + Send + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + link.well_formed(), + link.constant().scheduler == old(session).scheduler(), + link.child_phase().is_retired(), + retired.object().wf(), + retired.object().domain() == link.constant().domain, + retired.object().obj() == link.constant().child_obj, + equal(retired.object().ptr(), link.constant().child), + retired.claim().registry() == link.constant().reclaim_registry, + retired.claim().obj() == link.constant().child_obj, + retired.claim().is_pending(), + equal(retired.claim().ptr(), link.constant().child), + retired.retired().removal() == link.child_phase()->Retired_removal, + link.child_phase()->Retired_removal.root == link.constant().root, + retired.retired().removal().observed_by(old(session).irc11_view()), + retired.retired().retire_observation_registry() + == link.constant().retire_observation_registry, + ensures + final(session).wf(), + final(session).task() == old(session).task(), + final(session).scheduler() == old(session).scheduler(), + final(session).cpu() == old(session).cpu(), + final(session).session_id() == old(session).session_id(), + final(session).quiescent_generation() == old(session).quiescent_generation(), + final(session).available_fractions() == old(session).available_fractions(), + final(session).preempt_depth() == old(session).preempt_depth(), + final(session).rcu_participant_id() == old(session).rcu_participant_id(), + final(session).rcu_generation() == old(session).rcu_generation(), + final(session).rcu_participant_view() == old(session).rcu_participant_view(), + final(session).rcu_fraction() == old(session).rcu_fraction(), +{ + let (callback, cert) = callback_from_linked_list_child::

(link, Tracked(retired)); + if let Some(monitor) = RCU_MONITOR.get() { + #[verus_spec(with Tracked(session))] + monitor.after_grace_period(callback, cert); + } +} + impl RcuInner

{ closed spec fn is_nullable(self) -> bool { self.ghost_nullable@ diff --git a/verified_libs/vstd_extra/src/atomic_irc11.rs b/verified_libs/vstd_extra/src/atomic_irc11.rs index 582c640c3..7feb3bb99 100644 --- a/verified_libs/vstd_extra/src/atomic_irc11.rs +++ b/verified_libs/vstd_extra/src/atomic_irc11.rs @@ -32,6 +32,28 @@ verus! { /// Logical timestamp used by one native atomic history. pub type Timestamp = nat; +/// Canonical per-location projection of a native subjective thread view. +/// +/// Upstream exposes this projection only through +/// [`AtomicPointsTo::get_timestamp`], whose receiver also contains the +/// location's changing history. Naming the projection independently lets a +/// client retain a load observation while the authoritative history grows. +pub uninterp spec fn timestamp_in_view(loc: AtomicId, view: ThreadView) -> Option; + +/// `AtomicPointsTo::get_timestamp` depends only on the atomic location and the +/// supplied native thread view, not on the current version of its append-only +/// history. +/// +/// This is a thin exposure of the native IRC11 view projection; it does not +/// introduce a second memory model or any new ordering relation. +pub axiom fn axiom_get_timestamp_is_location_projection( + tracked points_to: &AtomicPointsTo, + view: ThreadView, +) + ensures + points_to.get_timestamp(view) == timestamp_in_view(points_to.loc(), view), +; + /// Compatibility vocabulary for ordering native subjective views. /// /// `old.spec_le(new)` is only notation for the native relation From 9249f65d61fe3acef56ffe737f7937df953da197 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Sun, 9 Aug 2026 22:45:00 -0400 Subject: [PATCH 42/47] Remove RCU view projection axiom --- ostd/specs/sync/rcu.rs | 194 +++++++++++++++---- ostd/specs/sync/weak_memory.rs | 75 +++---- ostd/src/sync/rcu/mod.rs | 8 +- verified_libs/vstd_extra/src/atomic_irc11.rs | 22 --- 4 files changed, 192 insertions(+), 107 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 1f1afb068..3ddb20cd3 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -21,11 +21,15 @@ //! for an internal node with arbitrary incoming links; such nodes require the //! tracked `RcuPointedBy` transition from the traversal layer. //! -//! Two paper-level connections remain deliberately incomplete. -//! `Guard-seen-retired` still needs a persistent start-snapshot resource, not -//! only the guard's pure `expired` set, and monitor completion still needs the -//! per-CPU closed-generation resources. Until both are connected, callback -//! completion is not an end-to-end reclamation-safety theorem. +//! The paper-level reclamation chain is connected through persistent reader +//! start snapshots, per-CPU closed-generation resources, physical read leases, +//! and type-erased callback reclaim permits. The linked-list acceptance path +//! additionally connects native internal-link loads and unlink CAS events to +//! AId-keyed traversal authority. Remaining limitations are integration +//! boundaries rather than missing logical steps: the linked-list adapter is +//! still private and acceptance-specific, `read_with()` retains a trusted +//! shared-reference bridge, and implicit Rust `Drop` cannot yet carry the +//! verified consuming transition. use core::marker::PhantomData; use crate::specs::mm::cpu::CpuId; @@ -4344,16 +4348,13 @@ impl LinkedListTraversalAuth { /// assumed. pub tracked struct LinkedListLinkObservation { fact: GhostPersistentPointsTo, - ghost loc: Irc11AtomicId, - ghost view: Irc11ThreadView, + native_fact: GhostPersistentPointsTo, } impl LinkedListLinkObservation { #[verifier::type_invariant] pub closed spec fn type_inv(self) -> bool { - vstd_extra::atomic_irc11::timestamp_in_view(self.loc(), self.view()) == Some( - self.timestamp(), - ) + self.native_fact.value().2 == self.timestamp() } /// Persistent registry that certifies this timestamp/index pair. @@ -4361,6 +4362,16 @@ impl LinkedListLinkObservation { self.fact.id() } + /// Persistent registry that certifies the native view observation. + pub closed spec fn native_registry(self) -> Loc { + self.native_fact.id() + } + + /// Fresh identifier allocated for this native load observation. + pub closed spec fn native_observation_id(self) -> nat { + self.native_fact.key() + } + /// Native IRC11 timestamp observed by the load. pub closed spec fn timestamp(self) -> nat { self.fact.key() @@ -4373,36 +4384,29 @@ impl LinkedListLinkObservation { /// Native atomic location whose timestamp was observed. pub closed spec fn loc(self) -> Irc11AtomicId { - self.loc + self.native_fact.value().0 } /// Subjective view immediately after the load that minted this token. pub closed spec fn view(self) -> Irc11ThreadView { - self.view - } - - /// Exposes the native view projection retained by this persistent - /// observation without revealing its private representation. - pub proof fn lemma_view_timestamp(tracked &self) - ensures - vstd_extra::atomic_irc11::timestamp_in_view(self.loc(), self.view()) == Some( - self.timestamp(), - ), - { - use_type_invariant(self); + self.native_fact.value().1 } /// Duplicates the persistent timestamp/index observation. pub proof fn tracked_duplicate(tracked &self) -> (tracked res: Self) ensures res.registry() == self.registry(), + res.native_registry() == self.native_registry(), res.timestamp() == self.timestamp(), res.index() == self.index(), res.loc() == self.loc(), res.view() == self.view(), { use_type_invariant(self); - LinkedListLinkObservation { fact: self.fact.duplicate(), loc: self.loc, view: self.view } + LinkedListLinkObservation { + fact: self.fact.duplicate(), + native_fact: self.native_fact.duplicate(), + } } } @@ -4412,6 +4416,8 @@ pub tracked struct LinkedListAtomicLinkGhost { ghost timestamp_to_index: Map, timestamp_registry: GhostMapAuth, timestamp_facts: Map>, + native_observation_registry: GhostMapAuth, + ghost next_observation: nat, ghost current_timestamp: nat, } @@ -4444,6 +4450,22 @@ impl LinkedListAtomicLinkGhost { self.timestamp_facts } + /// Append-only registry of native `(location, view, timestamp)` load facts. + pub closed spec fn native_observation_registry(self) -> Loc { + self.native_observation_registry.id() + } + + pub closed spec fn native_observations(self) -> Map< + nat, + (Irc11AtomicId, Irc11ThreadView, nat), + > { + self.native_observation_registry@ + } + + pub closed spec fn next_observation(self) -> nat { + self.next_observation + } + pub closed spec fn current_timestamp(self) -> nat { self.current_timestamp } @@ -4476,6 +4498,9 @@ impl LinkedListAtomicLinkGhost { &&& fact.key() == timestamp &&& fact.value() == self.timestamps()[timestamp] } + &&& forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) ==> observation_id + < self.next_observation() &&& auth.state().successors[self.source_obj()].len() > 0 &&& self.index_at(self.current_timestamp()) + 1 == auth.state().successors[self.source_obj()].len() @@ -4499,6 +4524,20 @@ impl LinkedListAtomicLinkGhost { < later <==> self.index_at(earlier) < self.index_at(later)) } + /// Every issued native observation remains valid for the current + /// append-only atomic points-to resource. + pub open spec fn native_observations_wf( + self, + points_to: AtomicPointsTo<*mut LinkedListNode>, + ) -> bool { + forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) ==> { + let observation = self.native_observations()[observation_id]; + &&& observation.0 == points_to.loc() + &&& points_to.get_timestamp(observation.1) == Some(observation.2) + } + } + /// Creates the timestamp mapping for an atomic link initialized to null. pub proof fn tracked_initial_null( history: Irc11History<*mut LinkedListNode>, @@ -4519,6 +4558,8 @@ impl LinkedListAtomicLinkGhost { res.source() == source, res.source_obj() == source_obj, res.timestamps() == Map::empty().insert(timestamp, 0), + res.native_observations() == Map::empty(), + res.next_observation() == 0, res.current_timestamp() == timestamp, { assert(history.is_max_timestamp(timestamp)); @@ -4535,12 +4576,17 @@ impl LinkedListAtomicLinkGhost { let tracked initial_fact = timestamp_registry.insert(timestamp, 0).persist(); let tracked mut timestamp_facts = Map::tracked_empty(); timestamp_facts.tracked_insert(timestamp, initial_fact); + let tracked (native_observation_registry, _native_observations) = GhostMapAuth::new( + Map::empty(), + ); let tracked res = LinkedListAtomicLinkGhost { source, source_obj, timestamp_to_index: Map::empty().insert(timestamp, 0), timestamp_registry, timestamp_facts, + native_observation_registry, + next_observation: 0, current_timestamp: timestamp, }; assert forall|ts: nat| history.contains_timestamp(ts) implies { @@ -4570,26 +4616,68 @@ impl LinkedListAtomicLinkGhost { /// Issues persistent evidence for one native timestamp's dense index. pub proof fn tracked_observation_at( - tracked &self, - history: Irc11History<*mut LinkedListNode>, + tracked &mut self, + tracked points_to: &AtomicPointsTo<*mut LinkedListNode>, tracked auth: &LinkedListTraversalAuth, timestamp: nat, - loc: Irc11AtomicId, view: Irc11ThreadView, ) -> (tracked res: LinkedListLinkObservation) requires - self.wf(history, *auth), - history.contains_timestamp(timestamp), - vstd_extra::atomic_irc11::timestamp_in_view(loc, view) == Some(timestamp), + old(self).wf(points_to.hist(), *auth), + old(self).native_observations_wf(*points_to), + points_to.hist().contains_timestamp(timestamp), + points_to.get_timestamp(view) == Some(timestamp), ensures - res.registry() == self.timestamp_registry(), + final(self).wf(points_to.hist(), *auth), + final(self).native_observations_wf(*points_to), + final(self).source() == old(self).source(), + final(self).source_obj() == old(self).source_obj(), + final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).native_observation_registry() == old(self).native_observation_registry(), + final(self).timestamps() == old(self).timestamps(), + final(self).current_timestamp() == old(self).current_timestamp(), + res.registry() == final(self).timestamp_registry(), + res.native_registry() == final(self).native_observation_registry(), res.timestamp() == timestamp, - res.index() == self.index_at(timestamp), - res.loc() == loc, + res.index() == final(self).index_at(timestamp), + res.loc() == points_to.loc(), res.view() == view, { + let ghost old_native_observations = self.native_observations(); + let ghost observation_id = self.next_observation; + assert(!old_native_observations.contains_key(observation_id)) by { + if old_native_observations.contains_key(observation_id) { + assert(observation_id < self.next_observation); + assert(false); + } + }; + let tracked native_fact = self.native_observation_registry.insert( + observation_id, + (points_to.loc(), view, timestamp), + ).persist(); + self.next_observation = observation_id + 1; let tracked fact = self.timestamp_facts.tracked_borrow(timestamp).duplicate(); - LinkedListLinkObservation { fact, loc, view } + let tracked res = LinkedListLinkObservation { fact, native_fact }; + assert forall|id: nat| #[trigger] self.native_observations().contains_key(id) implies id + < self.next_observation by { + if id == observation_id { + } else { + assert(old_native_observations.contains_key(id)); + assert(old_native_observations[id] == self.native_observations()[id]); + } + }; + assert forall|id: nat| #[trigger] self.native_observations().contains_key(id) implies { + let observation = self.native_observations()[id]; + &&& observation.0 == points_to.loc() + &&& points_to.get_timestamp(observation.1) == Some(observation.2) + } by { + if id == observation_id { + } else { + assert(old_native_observations.contains_key(id)); + assert(old_native_observations[id] == self.native_observations()[id]); + } + }; + res } /// Agrees a prior persistent observation with the current append-only @@ -4607,6 +4695,24 @@ impl LinkedListAtomicLinkGhost { observation.fact.agree(&self.timestamp_registry); } + /// Agrees a prior persistent native observation with the current + /// append-only observation registry. + pub proof fn lemma_native_observation_agrees( + tracked &self, + tracked observation: &LinkedListLinkObservation, + ) + requires + observation.native_registry() == self.native_observation_registry(), + ensures + self.native_observations().contains_pair( + observation.native_observation_id(), + (observation.loc(), observation.view(), observation.timestamp()), + ), + { + use_type_invariant(observation); + observation.native_fact.agree(&self.native_observation_registry); + } + /// Records a successful native CAS that publishes a non-null successor. /// /// `load_timestamp` and `store_timestamp` are supplied by the native @@ -4637,6 +4743,9 @@ impl LinkedListAtomicLinkGhost { final(self).source() == old(self).source(), final(self).source_obj() == old(self).source_obj(), final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).native_observation_registry() == old(self).native_observation_registry(), + final(self).native_observations() == old(self).native_observations(), + final(self).next_observation() == old(self).next_observation(), final(self).timestamps() == old(self).timestamps().insert(store_timestamp, n), final(self).current_timestamp() == store_timestamp, final(auth).domain() == old(auth).domain(), @@ -4666,6 +4775,8 @@ impl LinkedListAtomicLinkGhost { let ghost source = self.source; let ghost source_obj = self.source_obj; let ghost old_timestamps = self.timestamp_to_index; + let ghost old_native_observations = self.native_observations(); + let ghost old_next_observation = self.next_observation(); let ghost old_state = auth.state(); let ghost old_current = self.current_timestamp; @@ -4692,6 +4803,11 @@ impl LinkedListAtomicLinkGhost { }; }; assert(self.timestamps().dom() == next.dom()); + assert(self.native_observations() == old_native_observations); + assert(self.next_observation() == old_next_observation); + assert forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) implies observation_id + < self.next_observation() by {}; assert(self.index_at(store_timestamp) == n); assert(n + 1 == auth.state().successors[source_obj].len()); assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies { @@ -4778,6 +4894,9 @@ impl LinkedListAtomicLinkGhost { final(self).source() == old(self).source(), final(self).source_obj() == old(self).source_obj(), final(self).timestamp_registry() == old(self).timestamp_registry(), + final(self).native_observation_registry() == old(self).native_observation_registry(), + final(self).native_observations() == old(self).native_observations(), + final(self).next_observation() == old(self).next_observation(), final(self).timestamps() == old(self).timestamps().insert(store_timestamp, n), final(self).current_timestamp() == store_timestamp, final(auth).domain() == old(auth).domain(), @@ -4800,6 +4919,8 @@ impl LinkedListAtomicLinkGhost { let ghost source = self.source; let ghost source_obj = self.source_obj; let ghost old_timestamps = self.timestamp_to_index; + let ghost old_native_observations = self.native_observations(); + let ghost old_next_observation = self.next_observation(); let ghost old_state = auth.state(); let ghost old_current = self.current_timestamp; @@ -4826,6 +4947,11 @@ impl LinkedListAtomicLinkGhost { }; }; assert(self.timestamps().dom() == next.dom()); + assert(self.native_observations() == old_native_observations); + assert(self.next_observation() == old_next_observation); + assert forall|observation_id: nat| #[trigger] + self.native_observations().contains_key(observation_id) implies observation_id + < self.next_observation() by {}; assert(self.index_at(store_timestamp) == n); assert(n + 1 == auth.state().successors[source_obj].len()); assert forall|timestamp: nat| next.contains_timestamp(timestamp) implies { diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 03369de9a..e4592434f 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -427,6 +427,7 @@ pub ghost struct LinkedListAtomicKey { pub active_lease_registry: Loc, pub lifecycle: Loc, pub timestamp_registry: Loc, + pub native_observation_registry: Loc, pub source: *mut rcu_spec::LinkedListNode, pub source_obj: nat, pub child: *mut rcu_spec::LinkedListNode, @@ -515,9 +516,9 @@ impl LinkedListAtomicState { /// Native IRC11 invariant for a link whose only non-null value is one /// pre-registered child. /// -/// Restricting the first executable-style wrapper to two nodes keeps the -/// atomic protocol closed while the general node-registration and physical -/// permission pools are still being designed. +/// Restricting the first executable-style wrapper to two nodes keeps this +/// acceptance protocol closed while a production data-structure adapter and +/// its public node-registration API are still being selected. pub struct LinkedListAtomicInv { _marker: PhantomData, } @@ -561,7 +562,9 @@ impl InvariantPredicate< &&& state.link().source() == key.source &&& state.link().source_obj() == key.source_obj &&& state.link().timestamp_registry() == key.timestamp_registry + &&& state.link().native_observation_registry() == key.native_observation_registry &&& state.link().wf(state.points_to().hist(), state.auth()) + &&& state.link().native_observations_wf(state.points_to()) &&& forall|n: rcu_spec::LinkIndex| n < state.auth().state().successors[key.source_obj].len() && state.auth().state().successors[key.source_obj][n as int] is Some @@ -835,6 +838,7 @@ impl LinkedListWeakAtomicLink LinkedListWeakAtomicLink LinkedListWeakAtomicLink old(guard).seen_at(self.constant().source_obj) == 0, Some(observation) => { &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry &&& observation.loc() == self.native_loc() &&& old(tv)@.contains(observation.view()) &&& old(guard).seen_at(self.constant().source_obj) == observation.index() @@ -964,6 +978,7 @@ impl LinkedListWeakAtomicLink LinkedListWeakAtomicLink { use_type_invariant(observation); - observation.lemma_view_timestamp(); - assert(vstd_extra::atomic_irc11::timestamp_in_view( - observation.loc(), - observation.view(), - ) == Some(observation.timestamp())); assert(observation.loc() == self.native_loc()); state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); assert(state.link.index_at(observation.timestamp()) == observation.index()); assert(state.points_to.hist().contains_timestamp( observation.timestamp(), )); - vstd_extra::atomic_irc11::axiom_get_timestamp_is_location_projection( - &state.points_to, - observation.view(), - ); - assert(vstd_extra::atomic_irc11::timestamp_in_view( - self.native_loc(), - observation.view(), - ) == Some(observation.timestamp())); assert(state.points_to.get_timestamp(observation.view()) == Some(observation.timestamp())); state.points_to.get_timestamp_monotonic( @@ -1090,19 +1093,10 @@ impl LinkedListWeakAtomicLink LinkedListWeakAtomicLink guard.paper_guard().seen_at(self.constant().source_obj) == 0, Some(observation) => { &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry &&& observation.loc() == self.native_loc() &&& old(tv)@.contains(observation.view()) &&& guard.paper_guard().seen_at(self.constant().source_obj) @@ -1204,6 +1199,7 @@ impl LinkedListWeakAtomicLink LinkedListWeakAtomicLink { use_type_invariant(observation); - observation.lemma_view_timestamp(); - assert(vstd_extra::atomic_irc11::timestamp_in_view( - observation.loc(), - observation.view(), - ) == Some(observation.timestamp())); assert(observation.loc() == self.native_loc()); state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); assert(state.link.index_at(observation.timestamp()) == observation.index()); assert(state.points_to.hist().contains_timestamp( observation.timestamp(), )); - vstd_extra::atomic_irc11::axiom_get_timestamp_is_location_projection( - &state.points_to, - observation.view(), - ); - assert(vstd_extra::atomic_irc11::timestamp_in_view( - self.native_loc(), - observation.view(), - ) == Some(observation.timestamp())); assert(state.points_to.get_timestamp(observation.view()) == Some(observation.timestamp())); state.points_to.get_timestamp_monotonic( @@ -1367,19 +1351,10 @@ impl LinkedListWeakAtomicLink LinkedListWeakAtomicLinkRetired_removal, OwnPred::owns(final(self).constant().child, ownership), - opens_invariants [ self.invariant_namespace() ] + opens_invariants [self.invariant_namespace()] { use_type_invariant(&*self); let ghost key = self.constant(); diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 9ec45efe5..94389b291 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -236,6 +236,8 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtrSome_0.registry() == self.link.constant().timestamp_registry + &&& self.tracked_observation@->Some_0.native_registry() + == self.link.constant().native_observation_registry &&& self.tracked_observation@->Some_0.loc() == self.link.native_loc() &&& self.tracked_guard@->Some_0.paper_guard().seen_at(self.link.constant().source_obj) == self.tracked_observation@->Some_0.index() @@ -267,7 +269,9 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr, Tracked(guard): Tracked>, @@ -289,6 +293,7 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr guard.paper_guard().seen_at(link.constant().source_obj) == 0, Some(observation) => { &&& observation.registry() == link.constant().timestamp_registry + &&& observation.native_registry() == link.constant().native_observation_registry &&& observation.loc() == link.native_loc() &&& old(tv)@.contains(observation.view()) &&& guard.paper_guard().seen_at(link.constant().source_obj) @@ -386,6 +391,7 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr Option; - -/// `AtomicPointsTo::get_timestamp` depends only on the atomic location and the -/// supplied native thread view, not on the current version of its append-only -/// history. -/// -/// This is a thin exposure of the native IRC11 view projection; it does not -/// introduce a second memory model or any new ordering relation. -pub axiom fn axiom_get_timestamp_is_location_projection( - tracked points_to: &AtomicPointsTo, - view: ThreadView, -) - ensures - points_to.get_timestamp(view) == timestamp_in_view(points_to.loc(), view), -; - /// Compatibility vocabulary for ordering native subjective views. /// /// `old.spec_le(new)` is only notation for the native relation From efaa6dd21162fc304e1df2ecdba7baa6e34357e9 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 10 Aug 2026 00:10:42 -0400 Subject: [PATCH 43/47] Generalize native RCU link targets --- ostd/specs/sync/rcu.rs | 291 +++++++++++++ ostd/specs/sync/weak_memory.rs | 727 +++++++++++++++++++++++++++++++++ 2 files changed, 1018 insertions(+) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 3ddb20cd3..2542775fe 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -3892,6 +3892,39 @@ impl LinkedListTraversalAuth { } } + /// Opens the registry-domain equalities hidden by [`Self::wf`]. + /// Native-link adapters use these facts to move between an AId's object, + /// successor, incoming-edge, and persistent-info entries. + pub proof fn lemma_registry_domains(tracked &self) + requires + self.wf(), + ensures + self.state().wf(), + self.state().successors.dom() == self.state().objects.dom(), + self.state().incoming_all.dom() == self.state().objects.dom(), + forall|obj: nat| #[trigger] + self.has_info(obj) <==> self.state().objects.contains_key(obj), + { + } + + /// A fresh AId cannot already own any list-local registration resource. + pub proof fn lemma_unregistered_has_no_resources(tracked &self, obj: nat) + requires + self.wf(), + !self.state().objects.contains_key(obj), + ensures + !self.state().successors.contains_key(obj), + !self.state().incoming_all.contains_key(obj), + !self.has_info(obj), + !self.has_retire_perm(obj), + { + self.lemma_registry_domains(); + if self.has_retire_perm(obj) { + assert(self.retire_perms.contains_key(obj)); + assert(self.state().objects.contains_pair(obj, self.retire_perms[obj].ptr())); + } + } + /// Starts an authoritative list with one registered root allocation. pub proof fn tracked_new( tracked root_info: &RcuBlockInfo, @@ -4614,6 +4647,83 @@ impl LinkedListAtomicLinkGhost { res } + /// Registers a fresh target while preserving the native link/history + /// correspondence. Registration adds an empty successor history and an + /// empty incoming-edge set for the new AId, so every pre-existing native + /// timestamp continues to resolve to the same traversal event. + pub proof fn tracked_register_target( + tracked &self, + history: Irc11History<*mut LinkedListNode>, + tracked auth: &mut LinkedListTraversalAuth, + tracked info: &RcuBlockInfo, + tracked retire: RcuBaseRetirePerm, + ) + requires + self.wf(history, *old(auth)), + info.wf(), + retire.wf(), + info.domain() == old(auth).domain(), + retire.domain() == old(auth).domain(), + retire.obj() == info.obj(), + retire.ptr() == info.ptr(), + !old(auth).state().objects.contains_key(info.obj()), + !old(auth).state().incoming_all.contains_key(info.obj()), + !old(auth).has_retire_perm(info.obj()), + ensures + self.wf(history, *final(auth)), + final(auth).wf(), + final(auth).domain() == old(auth).domain(), + final(auth).state().root == old(auth).state().root, + final(auth).state().root_obj == old(auth).state().root_obj, + final(auth).state().objects == old(auth).state().objects.insert(info.obj(), info.ptr()), + final(auth).state().successors == old(auth).state().successors.insert( + info.obj(), + Seq::empty(), + ), + final(auth).state().incoming_all == old(auth).state().incoming_all.insert( + info.obj(), + Set::empty(), + ), + final(auth).removed() == old(auth).removed(), + final(auth).has_info(info.obj()), + final(auth).info(info.obj()).ptr() == info.ptr(), + final(auth).has_retire_perm(info.obj()), + { + let ghost old_state = auth.state(); + let ghost source_obj = self.source_obj(); + assert(old_state.objects.contains_key(source_obj)); + assert(info.obj() != source_obj); + auth.tracked_register_node(info, retire); + auth.lemma_registry_domains(); + assert(auth.state().successors[source_obj] == old_state.successors[source_obj]); + assert forall|timestamp: nat| history.contains_timestamp(timestamp) implies { + let n = #[trigger] self.timestamps()[timestamp]; + &&& n < auth.state().successors[source_obj].len() + &&& match auth.state().successors[source_obj][n as int] { + None => history.value(timestamp).addr() == 0, + Some((ptr, obj)) => { + &&& history.value(timestamp).addr() != 0 + &&& equal(ptr, history.value(timestamp)) + &&& auth.has_info(obj) + &&& auth.info(obj).ptr() == ptr + }, + } + } by { + assert(auth.state().successors[source_obj][self.timestamps()[timestamp] as int] + == old_state.successors[source_obj][self.timestamps()[timestamp] as int]); + match old_state.successors[source_obj][self.timestamps()[timestamp] as int] { + None => {}, + Some((ptr, obj)) => { + assert(old_state.objects.contains_pair(obj, ptr)); + assert(auth.state().objects.contains_pair(obj, ptr)); + assert(auth.has_info(obj)); + assert(auth.info(obj).ptr() == ptr); + }, + } + }; + assert(self.wf(history, *auth)); + } + /// Issues persistent evidence for one native timestamp's dense index. pub proof fn tracked_observation_at( tracked &mut self, @@ -5748,6 +5858,187 @@ pub proof fn linked_list_native_cas_unlink_enables_retire( auth.tracked_retire_node(child_info.obj(), prior) } +/// Regression proof for one native link publishing multiple registered AIds. +/// +/// `first` and `second` deliberately name distinct allocation identities at +/// the same address. The native CAS values are therefore indistinguishable by +/// address, while the append-only traversal history still records the exact +/// AId selected by each publication. +pub proof fn linked_list_native_single_link_multiple_registered_aids( + tracked root_info: &RcuBlockInfo, + tracked root_retire: RcuBaseRetirePerm, + tracked first_info: &RcuBlockInfo, + tracked first_retire: RcuBaseRetirePerm, + tracked second_info: &RcuBlockInfo, + tracked second_retire: RcuBaseRetirePerm, + initial_history: Irc11History<*mut LinkedListNode>, + initial_timestamp: nat, + initial_view: Irc11ThreadView, + first_publish_view: Irc11ThreadView, + first_unlink_view: Irc11ThreadView, + second_publish_view: Irc11ThreadView, + second_unlink_view: Irc11ThreadView, + first_republish_view: Irc11ThreadView, +) -> (tracked res: (LinkedListTraversalAuth, LinkedListAtomicLinkGhost)) + requires + root_info.wf(), + root_retire.wf(), + root_retire.domain() == root_info.domain(), + root_retire.obj() == root_info.obj(), + root_retire.ptr() == root_info.ptr(), + first_info.wf(), + first_retire.wf(), + first_info.domain() == root_info.domain(), + first_retire.domain() == root_info.domain(), + first_retire.obj() == first_info.obj(), + first_retire.ptr() == first_info.ptr(), + second_info.wf(), + second_retire.wf(), + second_info.domain() == root_info.domain(), + second_retire.domain() == root_info.domain(), + second_retire.obj() == second_info.obj(), + second_retire.ptr() == second_info.ptr(), + first_info.obj() != root_info.obj(), + second_info.obj() != root_info.obj(), + first_info.obj() != second_info.obj(), + first_info.ptr() == second_info.ptr(), + initial_history.is_singleton(initial_timestamp, (core::ptr::null_mut(), initial_view)), + ensures + res.0.wf(), + res.0.removed() == Set::::empty(), + res.0.state().objects.contains_pair(first_info.obj(), first_info.ptr()), + res.0.state().objects.contains_pair(second_info.obj(), first_info.ptr()), + res.0.state().successors[root_info.obj()][0] is None, + res.0.state().successors[root_info.obj()][1] == Some((first_info.ptr(), first_info.obj())), + res.0.state().successors[root_info.obj()][2] is None, + res.0.state().successors[root_info.obj()][3] == Some( + (second_info.ptr(), second_info.obj()), + ), + res.0.state().successors[root_info.obj()][4] is None, + res.0.state().successors[root_info.obj()][5] == Some((first_info.ptr(), first_info.obj())), + res.0.state().incoming_all[first_info.obj()].contains((root_info.obj(), 1)), + res.0.state().incoming_all[first_info.obj()].contains((root_info.obj(), 5)), + res.0.state().incoming_all[second_info.obj()].contains((root_info.obj(), 3)), + res.1.current_timestamp() == initial_timestamp + 5, + res.1.wf( + initial_history.insert( + initial_timestamp + 1, + first_info.ptr(), + first_publish_view, + ).insert(initial_timestamp + 2, core::ptr::null_mut(), first_unlink_view).insert( + initial_timestamp + 3, + second_info.ptr(), + second_publish_view, + ).insert(initial_timestamp + 4, core::ptr::null_mut(), second_unlink_view).insert( + initial_timestamp + 5, + first_info.ptr(), + first_republish_view, + ), + res.0, + ), +{ + let tracked mut auth = LinkedListTraversalAuth::tracked_new(root_info, root_retire); + let initial_index = auth.tracked_initialize_null(root_info.ptr(), root_info.obj()); + assert(initial_index == 0); + let tracked mut link = LinkedListAtomicLinkGhost::tracked_initial_null( + initial_history, + initial_timestamp, + initial_view, + &auth, + root_info.ptr(), + root_info.obj(), + ); + link.tracked_register_target(initial_history, &mut auth, first_info, first_retire); + link.tracked_register_target(initial_history, &mut auth, second_info, second_retire); + + let ghost first_published = initial_history.insert( + initial_timestamp + 1, + first_info.ptr(), + first_publish_view, + ); + let first_index = link.tracked_cas_publish( + &mut auth, + initial_history, + first_published, + initial_timestamp, + initial_timestamp + 1, + first_info.ptr(), + first_info.obj(), + first_publish_view, + ); + assert(first_index == 1); + + let ghost first_unlinked = first_published.insert( + initial_timestamp + 2, + core::ptr::null_mut(), + first_unlink_view, + ); + let first_unlink_index = link.tracked_cas_unlink( + &mut auth, + first_published, + first_unlinked, + initial_timestamp + 1, + initial_timestamp + 2, + first_info.ptr(), + first_info.obj(), + first_unlink_view, + ); + assert(first_unlink_index == 2); + + let ghost second_published = first_unlinked.insert( + initial_timestamp + 3, + second_info.ptr(), + second_publish_view, + ); + let second_index = link.tracked_cas_publish( + &mut auth, + first_unlinked, + second_published, + initial_timestamp + 2, + initial_timestamp + 3, + second_info.ptr(), + second_info.obj(), + second_publish_view, + ); + assert(second_index == 3); + + let ghost second_unlinked = second_published.insert( + initial_timestamp + 4, + core::ptr::null_mut(), + second_unlink_view, + ); + let second_unlink_index = link.tracked_cas_unlink( + &mut auth, + second_published, + second_unlinked, + initial_timestamp + 3, + initial_timestamp + 4, + second_info.ptr(), + second_info.obj(), + second_unlink_view, + ); + assert(second_unlink_index == 4); + + let ghost first_republished = second_unlinked.insert( + initial_timestamp + 5, + first_info.ptr(), + first_republish_view, + ); + let first_republish_index = link.tracked_cas_publish( + &mut auth, + second_unlinked, + first_republished, + initial_timestamp + 4, + initial_timestamp + 5, + first_info.ptr(), + first_info.obj(), + first_republish_view, + ); + assert(first_republish_index == 5); + assert(second_info.ptr() == first_info.ptr()); + (auth, link) +} + /// Uses an authoritative history snapshot to discharge the structural /// premises of [`protect_link`]. The remaining `seen_removed_sound` premise is /// the reader-side observation carried by the live guard. diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index e4592434f..04e26a786 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -416,6 +416,733 @@ impl LinkedListRetiredChild { } } +/// Immutable identities carried by a native linked-list link whose target is +/// selected from an extensible allocation-ID registry. +pub ghost struct RegisteredLinkedListAtomicKey { + pub domain: Loc, + pub root: Loc, + pub registry: Loc, + pub current: Loc, + pub timestamp_registry: Loc, + pub native_observation_registry: Loc, + pub source: *mut rcu_spec::LinkedListNode, + pub source_obj: nat, +} + +/// Complete proof state for one native link with arbitrary registered targets. +/// +/// `registry` is append-only in the current API. `current` records the AId of +/// the latest non-null successor independently of its address, which is what +/// makes a successful address-based CAS safe when a reclaimed address is later +/// reused by a fresh allocation identity. +pub tracked struct RegisteredLinkedListAtomicState { + pub(crate) points_to: AtomicPointsTo<*mut rcu_spec::LinkedListNode>, + pub(crate) link: rcu_spec::LinkedListAtomicLinkGhost, + pub(crate) auth: rcu_spec::LinkedListTraversalAuth, + pub(crate) registry: GhostVarAuth>, + pub(crate) current: GhostVarAuth>, +} + +unsafe impl Objective for RegisteredLinkedListAtomicState { + +} + +impl RegisteredLinkedListAtomicState { + pub closed spec fn points_to(self) -> AtomicPointsTo<*mut rcu_spec::LinkedListNode> { + self.points_to + } + + pub closed spec fn link(self) -> rcu_spec::LinkedListAtomicLinkGhost { + self.link + } + + pub closed spec fn auth(self) -> rcu_spec::LinkedListTraversalAuth { + self.auth + } + + pub closed spec fn registry(self) -> GhostVarAuth> { + self.registry + } + + pub closed spec fn current(self) -> GhostVarAuth> { + self.current + } +} + +/// Native IRC11 invariant for one link ranging over any registered AId. +pub struct RegisteredLinkedListAtomicInv; + +impl InvariantPredicate< + (RegisteredLinkedListAtomicKey, Irc11AtomicId), + RegisteredLinkedListAtomicState, +> for RegisteredLinkedListAtomicInv { + open spec fn inv( + key_loc: (RegisteredLinkedListAtomicKey, Irc11AtomicId), + state: RegisteredLinkedListAtomicState, + ) -> bool { + let (key, loc) = key_loc; + &&& state.points_to().loc() == loc + &&& key.source.addr() != 0 + &&& state.auth().wf() + &&& state.auth().domain() == key.domain + &&& state.auth().state().root == key.source + &&& state.auth().state().root_obj == key.source_obj + &&& state.auth().removed() == Set::::empty() + &&& state.registry().id() == key.registry + &&& state.registry()@ == state.auth().state().objects + &&& state.current().id() == key.current + &&& state.link().source() == key.source + &&& state.link().source_obj() == key.source_obj + &&& state.link().timestamp_registry() == key.timestamp_registry + &&& state.link().native_observation_registry() == key.native_observation_registry + &&& state.link().wf(state.points_to().hist(), state.auth()) + &&& state.link().native_observations_wf(state.points_to()) + &&& match state.current()@ { + None => state.auth().state().successors[key.source_obj].last() is None, + Some(obj) => { + &&& obj != key.source_obj + &&& state.registry()@.contains_key(obj) + &&& state.auth().state().successors[key.source_obj].last() == Some( + (state.registry()@[obj], obj), + ) + }, + } + } +} + +pub type RegisteredLinkedListAtomicInvariant = AtomicInvariant< + (RegisteredLinkedListAtomicKey, Irc11AtomicId), + RegisteredLinkedListAtomicState, + RegisteredLinkedListAtomicInv, +>; + +/// A real weak atomic link whose non-null values are selected by AId from an +/// extensible registry rather than fixed by the wrapper's constructor. +pub struct RegisteredLinkedListWeakAtomicLink { + atomic: PAtomicWeakPtr, + tracked_atomic_inv: Tracked<&'static RegisteredLinkedListAtomicInvariant>, + tracked_registry: Tracked>>, + tracked_current: Tracked>>, +} + +impl RegisteredLinkedListWeakAtomicLink { + pub closed spec fn constant(&self) -> RegisteredLinkedListAtomicKey { + self.tracked_atomic_inv@.constant().0 + } + + pub closed spec fn native_loc(&self) -> Irc11AtomicId { + self.atomic.loc() + } + + pub closed spec fn invariant_namespace(&self) -> int { + self.tracked_atomic_inv@.namespace() + } + + pub closed spec fn registered_targets(&self) -> Map { + self.tracked_registry@.view() + } + + pub closed spec fn current_target(&self) -> Option { + self.tracked_current@.view() + } + + pub closed spec fn well_formed(&self) -> bool { + &&& self.tracked_atomic_inv@.constant().1 == self.native_loc() + &&& self.tracked_registry@.id() == self.constant().registry + &&& self.tracked_current@.id() == self.constant().current + &&& self.registered_targets().contains_pair( + self.constant().source_obj, + self.constant().source, + ) + } + + #[verifier::type_invariant] + pub closed spec fn type_inv(&self) -> bool { + self.well_formed() + } + + /// Creates a null native link. Additional target AIds can be registered + /// after construction and then selected by the publication CAS. + pub const fn new( + Ghost(root): Ghost, + Tracked(source_info): Tracked<&rcu_spec::RcuBlockInfo>, + Tracked(source_retire): Tracked>, + ) -> (res: Self) + requires + source_info.wf(), + source_retire.wf(), + source_retire.domain() == source_info.domain(), + source_retire.obj() == source_info.obj(), + source_retire.ptr() == source_info.ptr(), + ensures + res.well_formed(), + res.constant().domain == source_info.domain(), + res.constant().root == root, + res.constant().source == source_info.ptr(), + res.constant().source_obj == source_info.obj(), + res.registered_targets() == Map::empty().insert(source_info.obj(), source_info.ptr()), + res.current_target() is None, + { + let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = + PAtomicWeakPtr::new(core::ptr::null_mut()); + let tracked mut auth = rcu_spec::LinkedListTraversalAuth::tracked_new( + source_info, + source_retire, + ); + let ghost initial_index = auth.tracked_initialize_null( + source_info.ptr(), + source_info.obj(), + ); + let tracked link = rcu_spec::LinkedListAtomicLinkGhost::tracked_initial_null( + points_to.hist(), + timestamp, + initial_view@, + &auth, + source_info.ptr(), + source_info.obj(), + ); + let tracked (registry, registry_peer) = GhostVarAuth::new( + Map::empty().insert(source_info.obj(), source_info.ptr()), + ); + let tracked (current, current_peer) = GhostVarAuth::new(None); + let tracked state = RegisteredLinkedListAtomicState { + points_to, + link, + auth, + registry, + current, + }; + let ghost key = RegisteredLinkedListAtomicKey { + domain: source_info.domain(), + root, + registry: state.registry().id(), + current: state.current().id(), + timestamp_registry: state.link().timestamp_registry(), + native_observation_registry: state.link().native_observation_registry(), + source: source_info.ptr(), + source_obj: source_info.obj(), + }; + proof { + source_info.lemma_wf_facts(); + assert(initial_index == 0); + assert(state.points_to().loc() == atomic.loc()); + assert(state.auth().state().objects == Map::empty().insert(key.source_obj, key.source)); + assert(state.auth().removed() == Set::::empty()); + assert(state.registry()@ == state.auth().state().objects); + assert(state.link().native_observations() == Map::empty()); + assert(state.link().native_observations_wf(state.points_to())) by { + assert forall|observation_id: nat| #[trigger] + state.link().native_observations().contains_key(observation_id) implies { + let observation = state.link().native_observations()[observation_id]; + &&& observation.0 == state.points_to().loc() + &&& state.points_to().get_timestamp(observation.1) == Some(observation.2) + } by {}; + }; + assert(RegisteredLinkedListAtomicInv::inv((key, atomic.loc()), state)); + } + let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); + let tracked atomic_inv = tracked_static_ref(atomic_inv); + RegisteredLinkedListWeakAtomicLink { + atomic, + tracked_atomic_inv: Tracked(atomic_inv), + tracked_registry: Tracked(registry_peer), + tracked_current: Tracked(current_peer), + } + } + + fn raw_atomic(&self) -> (res: &PAtomicWeakPtr) + requires + self.well_formed(), + ensures + res.loc() == self.native_loc(), + opens_invariants none + no_unwind + { + &self.atomic + } + + pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: + &'static RegisteredLinkedListAtomicInvariant) + requires + self.well_formed(), + ensures + res.constant() == (self.constant(), self.native_loc()), + { + self.tracked_atomic_inv.get() + } + + /// Adds a fresh allocation identity to this link's target registry under + /// the caller's writer-side invariant-opening authority. + pub proof fn tracked_register_target( + tracked &mut self, + target: *mut rcu_spec::LinkedListNode, + tracked info: &rcu_spec::RcuBlockInfo, + tracked retire: rcu_spec::RcuBaseRetirePerm, + tracked credit: vstd::invariant::OpenInvariantCredit, + ) + requires + old(self).well_formed(), + info.wf(), + retire.wf(), + info.domain() == old(self).constant().domain, + retire.domain() == old(self).constant().domain, + retire.obj() == info.obj(), + retire.ptr() == info.ptr(), + equal(target, info.ptr()), + info.obj() != old(self).constant().source_obj, + !old(self).registered_targets().contains_key(info.obj()), + ensures + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets().insert( + info.obj(), + target, + ), + final(self).current_target() == old(self).current_target(), + opens_invariants [self.invariant_namespace()] + { + use_type_invariant(&*self); + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + let ghost old_auth = state.auth; + let ghost old_objects = state.auth.state().objects; + let ghost old_registry = state.registry@; + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + assert(old_registry == self.registered_targets()); + assert(!old_objects.contains_key(info.obj())); + state.auth.lemma_registry_domains(); + assert(state.auth.state().successors.contains_key(key.source_obj)); + state.auth.lemma_unregistered_has_no_resources(info.obj()); + state.link.tracked_register_target( + state.points_to.hist(), + &mut state.auth, + info, + retire, + ); + state.registry.update( + self.tracked_registry.borrow_mut(), + old_registry.insert(info.obj(), target), + ); + assert(state.registry@ == state.auth.state().objects); + state.auth.lemma_registry_domains(); + match state.current@ { + None => { + assert(state.auth.state().successors[key.source_obj] + == old_auth.state().successors[key.source_obj]); + }, + Some(obj) => { + assert(obj != info.obj()); + assert(old_registry.contains_key(obj)); + assert(state.registry@.contains_key(obj)); + assert(state.registry@[obj] == old_registry[obj]); + assert(state.auth.state().successors[key.source_obj] + == old_auth.state().successors[key.source_obj]); + }, + } + assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + }); + } + + /// Acquire-loads the native link and resolves a non-null message to the + /// exact registered AId selected by that historical message. + #[inline(always)] + pub fn load_acquire_and_protect( + &self, + Tracked(guard): Tracked<&mut rcu_spec::RcuReadGuardToken>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>>, + Tracked, + )) + requires + self.well_formed(), + old(guard).wf(), + old(guard).domain() == self.constant().domain, + old(guard).seen_removed().removed == Set::::empty(), + match previous { + None => old(guard).seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& old(guard).seen_at(self.constant().source_obj) == observation.index() + }, + }, + old(from).protected_by(*old(guard)), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, + ensures + old(tv)@.spec_le(final(tv)@), + final(guard).wf(), + final(guard).domain() == old(guard).domain(), + final(guard).seen_removed().removed == old(guard).seen_removed().removed, + final(guard).seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.4@.registry() == self.constant().timestamp_registry, + res.4@.native_registry() == self.constant().native_observation_registry, + res.4@.loc() == self.native_loc(), + res.4@.timestamp() == res.1@, + res.4@.index() == res.2@, + res.4@.view() == final(tv)@, + (res.3@ is Some) == (res.0.addr() != 0), + match res.3@ { + None => res.0.addr() == 0, + Some(child) => { + &&& equal(child.ptr(), res.0) + &&& child.domain() == self.constant().domain + &&& child.protected_by(*final(guard)) + &&& self.registered_targets().contains_pair(child.obj(), child.ptr()) + }, + }, + no_unwind + { + let result; + let ghost view_before = tv@; + proof { + use_type_invariant(&*self); + } + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + assert(RegisteredLinkedListAtomicInv::inv( + (self.constant(), self.native_loc()), + state, + )); + match previous { + None => {}, + Some(observation) => { + use_type_invariant(observation); + state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } + } + let loaded = raw_atomic.load( + Ordering::Acquire, + Tracked(tv), + Tracked(&state.points_to), + ); + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); + proof { + match previous { + None => {}, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.index() <= index); + }, + } + assert(guard.seen_at(from.obj()) <= index); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + old(guard).seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + old(guard).seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> old(guard).seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + guard, + from, + timestamp, + ); + let tracked observation = state.link.tracked_observation_at( + &state.points_to, + &state.auth, + timestamp, + tv@, + ); + } + proof { + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + match &protected { + None => {}, + Some(child) => { + assert(state.auth.state().objects.contains_pair( + child.obj(), + child.ptr(), + )); + assert(state.registry@.contains_pair(child.obj(), child.ptr())); + assert(self.registered_targets().contains_pair( + child.obj(), + child.ptr(), + )); + }, + } + assert(RegisteredLinkedListAtomicInv::inv( + (self.constant(), self.native_loc()), + state, + )); + } + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(protected), + Tracked(observation), + ); + }); + result + } + + /// Publishes any registered target from a null link. + #[inline(always)] + pub fn compare_exchange_publish( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).current_target() is None, + target_obj != old(self).constant().source_obj, + old(self).registered_targets().contains_pair(target_obj, target), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> final(self).current_target() == Some(target_obj), + res.0 is Err ==> final(self).current_target() is None, + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + assert(state.current@ is None); + assert(state.auth.state().objects.contains_pair(target_obj, target)); + state.auth.lemma_registry_domains(); + assert(state.auth.state().incoming_all.contains_key(target_obj)); + state.auth.lemma_has_info_for_object(target_obj); + let tracked target_info = state.auth.tracked_info_for(target_obj); + assert(target_info.ptr() == target); + target_info.lemma_wf_facts(); + assert(target.addr() != 0); + assert(state.auth.state().successors.contains_key(key.source_obj)); + } + let ghost prev = state.points_to.hist(); + let cas = raw_atomic.compare_exchange( + null, + target, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost published_index: Option; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp == current_timestamp); + assert(equal(prev.value(current_timestamp), value)); + assert(value.addr() == 0); + assert(state.auth.state().successors[key.source_obj].last() is None); + assert(next == prev.insert( + update.load_timestamp + 1, + target, + update.store_message_view, + )); + let index = state.link.tracked_cas_publish( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + target, + target_obj, + update.store_message_view, + ); + state.current.update( + self.tracked_current.borrow_mut(), + Some(target_obj), + ); + published_index = Some(index); + }, + Result::Err(_) => { + published_index = None; + }, + } + assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + } + result = (cas.0, Ghost(published_index)); + }); + result + } + + /// Unlinks the currently selected AId. The AId precondition, rather than + /// pointer-address equality, identifies which logical allocation loses its + /// incoming edge. + #[inline(always)] + pub fn compare_exchange_unlink( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).current_target() == Some(target_obj), + old(self).registered_targets().contains_pair(target_obj, target), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> final(self).current_target() is None, + res.0 is Ok ==> res.1@->Some_0.1.root == final(self).constant().root, + res.0 is Ok ==> res.1@->Some_0.1.observed_by(final(tv)@), + res.0 is Err ==> final(self).current_target() == Some(target_obj), + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + assert(state.current@ == Some(target_obj)); + assert(state.auth.state().successors[key.source_obj].last() + == Some((target, target_obj))); + } + let ghost prev = state.points_to.hist(); + let cas = raw_atomic.compare_exchange( + target, + null, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost unlinked: Option<( + rcu_spec::LinkIndex, + rcu_spec::RcuRemovalObservation, + )>; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp == current_timestamp); + assert(equal(prev.value(current_timestamp), value)); + assert(value.addr() == target.addr()); + assert(next == prev.insert( + update.load_timestamp + 1, + null, + update.store_message_view, + )); + let index = state.link.tracked_cas_unlink( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + target, + target_obj, + update.store_message_view, + ); + let ghost removal = rcu_spec::RcuRemovalObservation { + root: key.root, + timestamp: update.load_timestamp + 1, + message_view: update.store_message_view, + }; + state.current.update(self.tracked_current.borrow_mut(), None); + unlinked = Some((index, removal)); + }, + Result::Err(_) => { + unlinked = None; + }, + } + assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + } + result = (cas.0, Ghost(unlinked)); + }); + result + } +} + /// Immutable identities carried by the native atomic invariant for the /// two-node linked-list traversal example. pub ghost struct LinkedListAtomicKey { From d30cd2f111268dffa1f07a2f77536b491ef3f6e0 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 10 Aug 2026 04:30:48 -0400 Subject: [PATCH 44/47] Generalize registered RCU reclamation --- ostd/specs/sync/rcu.rs | 33 +- ostd/specs/sync/rcu_cpu.rs | 24 +- ostd/specs/sync/weak_memory.rs | 1711 +++++++++++++++-- ostd/src/sync/rcu/mod.rs | 136 +- ostd/src/task/preempt/guard.rs | 14 +- verified_libs/vstd_extra/src/rcu_read_pool.rs | 6 +- 6 files changed, 1725 insertions(+), 199 deletions(-) diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index 2542775fe..d4f69e4c8 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -36,8 +36,8 @@ use crate::specs::mm::cpu::CpuId; use vstd::invariant::InvariantPredicate; use vstd::prelude::*; -use vstd::resource::Loc; use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; +use vstd::resource::Loc; use vstd::thread_view::Objective; use vstd_extra::atomic_irc11::{ AtomicHistory as Irc11History, AtomicId as Irc11AtomicId, AtomicPointsTo, @@ -4007,6 +4007,10 @@ impl LinkedListTraversalAuth { final(self).has_info(info.obj()), final(self).info(info.obj()).ptr() == info.ptr(), final(self).has_retire_perm(info.obj()), + forall|obj: nat| + obj != info.obj() ==> #[trigger] final(self).has_retire_perm(obj) == old( + self, + ).has_retire_perm(obj), { let ghost old_state = self.state; self.state = LinkedListGhost { @@ -4035,6 +4039,8 @@ impl LinkedListTraversalAuth { }; assert(self.state.wf()); assert(self.infos.dom() == self.state.incoming_all.dom()); + assert forall|obj: nat| obj != info.obj() implies #[trigger] self.has_retire_perm(obj) + == old(self).has_retire_perm(obj) by {}; assert forall|obj: nat| #[trigger] self.infos.contains_key(obj) implies { let saved = self.infos[obj]; &&& saved.wf() @@ -4340,8 +4346,15 @@ impl LinkedListTraversalAuth { final(self).state() == old(self).state(), final(self).removed() == old(self).removed().insert(obj), !final(self).has_retire_perm(obj), + forall|other: nat| + other != obj ==> #[trigger] final(self).has_retire_perm(other) == old( + self, + ).has_retire_perm(other), final(self).has_info(obj), final(self).info(obj) == old(self).info(obj), + forall|other: nat| #[trigger] final(self).has_info(other) == old(self).has_info(other), + forall|other: nat| #[trigger] + old(self).has_info(other) ==> final(self).info(other) == old(self).info(other), res.wf(), res.ready_to_retire(), res.domain() == old(self).domain(), @@ -4353,6 +4366,12 @@ impl LinkedListTraversalAuth { { assert(self.state().objects.contains_pair(obj, self.retire_perms[obj].ptr())); let tracked base = self.retire_perms.tracked_remove(obj); + assert forall|other: nat| other != obj implies #[trigger] self.has_retire_perm(other) + == old(self).has_retire_perm(other) by {}; + assert forall|other: nat| #[trigger] + self.has_info(other) == old(self).has_info(other) by {}; + assert forall|other: nat| #[trigger] old(self).has_info(other) implies self.info(other) + == old(self).info(other) by {}; let ghost seen_removed = RcuSeenRemoved { removed: prior.removed.insert(obj), link_view: prior.link_view, @@ -4688,6 +4707,10 @@ impl LinkedListAtomicLinkGhost { final(auth).has_info(info.obj()), final(auth).info(info.obj()).ptr() == info.ptr(), final(auth).has_retire_perm(info.obj()), + forall|obj: nat| + obj != info.obj() ==> #[trigger] final(auth).has_retire_perm(obj) == old( + auth, + ).has_retire_perm(obj), { let ghost old_state = auth.state(); let ghost source_obj = self.source_obj(); @@ -5217,6 +5240,14 @@ impl LinkedListAtomicLinkGhost { &&& child.domain() == auth.domain() &&& child.protected_by(*final(guard)) &&& LinkedListTraversalSpec::node_inv(child.ptr(), child.obj(), auth.state()) + &&& LinkedListTraversalSpec::link_inv( + self.source(), + self.source_obj(), + self.index_at(timestamp), + child.ptr(), + child.obj(), + auth.state(), + ) }, }, { diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs index 9c9845a70..08a9906ad 100644 --- a/ostd/specs/sync/rcu_cpu.rs +++ b/ostd/specs/sync/rcu_cpu.rs @@ -46,20 +46,20 @@ //! counter and is not an authority for this persistent CPU generation. Reader //! contexts obtain their CPU generation from [`CpuRcuReaderFragment`]. use crate::specs::{ - mm::cpu::{CpuId, online_cpus}, + mm::cpu::{online_cpus, CpuId}, task::cpu_core::{CpuCoreLocalState, CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreRegistration}, }; use vstd::{ modes::tracked_swap, prelude::*, resource::{ - Loc, agree::AgreementRA, algebra::{Resource, ResourceAlgebra}, frac::FractionRA, map::{GhostMapAuth, GhostPointsTo}, product::ProductRA, relations::frame_preserving_update_opt, + Loc, }, }; @@ -2009,6 +2009,16 @@ impl RcuRootPermissionState { self.registry.lemma_contains_iff_key(obj); } + /// Opens pool membership for every allocation identity at once. + pub proof fn lemma_all_contains_iff_keys(tracked &self) + ensures + forall|obj: nat| #[trigger] self.contains(obj) <==> self.keys().contains(obj), + { + reveal(RcuRootPermissionState::contains); + reveal(RcuRootPermissionState::keys); + self.registry.lemma_all_contains_iff_keys(); + } + /// Opens the allocation-state facts associated with a live permission pool. pub proof fn lemma_live_reclaim_state(tracked &self, obj: nat) requires @@ -2092,6 +2102,16 @@ impl RcuRootPermissionState { { } + /// Opens every append-only allocation cell in one quantified fact. + pub proof fn lemma_all_allocations_have_reclaim_states(tracked &self) + requires + self.wf(), + ensures + forall|obj: nat| #[trigger] + self.allocations().contains(obj) ==> self.reclaim_states().dom().contains(obj), + { + } + pub closed spec fn unretired_claims(self) -> Map>> { self.unretired_claims } diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index 04e26a786..a153470e8 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -12,8 +12,8 @@ use crate::specs::mm::cpu::online_cpus; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::modes::tracked_static_ref; use vstd::prelude::*; -use vstd::resource::Loc; use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; +use vstd::resource::Loc; use vstd::thread_view::Objective; use vstd_extra::atomic_irc11::{ AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, PAtomicWeakPtr, @@ -419,10 +419,15 @@ impl LinkedListRetiredChild { /// Immutable identities carried by a native linked-list link whose target is /// selected from an extensible allocation-ID registry. pub ghost struct RegisteredLinkedListAtomicKey { + pub scheduler: Loc, pub domain: Loc, pub root: Loc, + pub retire_observation_registry: Loc, + pub reclaim_registry: Loc, + pub active_lease_registry: Loc, pub registry: Loc, pub current: Loc, + pub lifecycle: Loc, pub timestamp_registry: Loc, pub native_observation_registry: Loc, pub source: *mut rcu_spec::LinkedListNode, @@ -435,19 +440,21 @@ pub ghost struct RegisteredLinkedListAtomicKey { /// the latest non-null successor independently of its address, which is what /// makes a successful address-based CAS safe when a reclaimed address is later /// reused by a fresh allocation identity. -pub tracked struct RegisteredLinkedListAtomicState { +pub tracked struct RegisteredLinkedListAtomicState { pub(crate) points_to: AtomicPointsTo<*mut rcu_spec::LinkedListNode>, pub(crate) link: rcu_spec::LinkedListAtomicLinkGhost, pub(crate) auth: rcu_spec::LinkedListTraversalAuth, pub(crate) registry: GhostVarAuth>, pub(crate) current: GhostVarAuth>, + pub(crate) permissions: rcu_cpu_spec::RcuRootPermissionState, + pub(crate) lifecycle: GhostVarAuth>, } -unsafe impl Objective for RegisteredLinkedListAtomicState { +unsafe impl Objective for RegisteredLinkedListAtomicState { } -impl RegisteredLinkedListAtomicState { +impl RegisteredLinkedListAtomicState { pub closed spec fn points_to(self) -> AtomicPointsTo<*mut rcu_spec::LinkedListNode> { self.points_to } @@ -467,36 +474,156 @@ impl RegisteredLinkedListAtomicState { pub closed spec fn current(self) -> GhostVarAuth> { self.current } + + pub closed spec fn permissions(self) -> rcu_cpu_spec::RcuRootPermissionState< + rcu_spec::LinkedListNode, + O, + > { + self.permissions + } + + pub closed spec fn lifecycle(self) -> GhostVarAuth> { + self.lifecycle + } +} + +/// Per-allocation part of the registered-link invariant. Keeping this fact +/// separate lets operations preserve unaffected AIds without unfolding the +/// complete native atomic protocol. +pub open spec fn registered_linked_list_target_inv( + key: RegisteredLinkedListAtomicKey, + state: RegisteredLinkedListAtomicState, + obj: nat, +) -> bool where OwnPred: rcu_spec::RcuRootOwnershipPredicate { + let phase = state.lifecycle()@[obj]; + &&& obj != key.source_obj + &&& state.registry()@.contains_key(obj) + &&& (state.auth().removed().contains(obj) <==> phase.is_retired() || phase.is_reclaimed()) + &&& (state.auth().has_retire_perm(obj) <==> !phase.is_retired() && !phase.is_reclaimed()) + &&& (state.permissions().contains(obj) <==> !phase.is_reclaimed()) + &&& state.permissions().reclaim_states()[obj] == if phase.is_reclaimed() { + None + } else { + Some(state.registry()@[obj]) + } + &&& (state.permissions().has_unretired_claim(obj) <==> !phase.is_retired() + && !phase.is_reclaimed()) + &&& (!phase.is_reclaimed() ==> OwnPred::owns( + state.registry()@[obj], + state.permissions().ownership(obj), + )) + &&& (phase.is_linked() <==> state.current()@ == Some(obj)) + &&& match phase { + LinkedListChildPhase::Unpublished => { + state.auth().state().incoming_all[obj] == Set::::empty() + }, + LinkedListChildPhase::Linked { index, timestamp } => { + &&& state.link().current_timestamp() == timestamp + &&& state.link().index_at(timestamp) == index + &&& index + 1 == state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj].last() == Some( + (state.registry()@[obj], obj), + ) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth().state().incoming_all[obj].contains(edge) ==> { + &&& edge.0 == key.source_obj + &&& edge.1 <= index + } + }, + LinkedListChildPhase::Unlinked { index, removal } + | LinkedListChildPhase::Retired { index, removal } => { + &&& removal.root == key.root + &&& state.points_to().hist().contains_timestamp(removal.timestamp) + &&& state.link().index_at(removal.timestamp) == index + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& index < state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj][index as int] is None + &&& index > 0 + &&& state.auth().state().successors[key.source_obj][(index - 1) as int] == Some( + (state.registry()@[obj], obj), + ) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth().state().incoming_all[obj].contains(edge) ==> { + &&& edge.0 == key.source_obj + &&& edge.1 < index + } + }, + LinkedListChildPhase::Reclaimed { index, removal } => { + &&& state.permissions().reclaimed().contains_key(obj) + &&& state.permissions().reclaimed()[obj].record().removal == removal + &&& removal.root == key.root + &&& state.points_to().hist().contains_timestamp(removal.timestamp) + &&& state.link().index_at(removal.timestamp) == index + &&& state.points_to().hist().thread_view(removal.timestamp) == removal.message_view + &&& index < state.auth().state().successors[key.source_obj].len() + &&& state.auth().state().successors[key.source_obj][index as int] is None + &&& index > 0 + &&& state.auth().state().successors[key.source_obj][(index - 1) as int] == Some( + (state.registry()@[obj], obj), + ) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth().state().incoming_all[obj].contains(edge) ==> { + &&& edge.0 == key.source_obj + &&& edge.1 < index + } + }, + } } /// Native IRC11 invariant for one link ranging over any registered AId. -pub struct RegisteredLinkedListAtomicInv; +pub struct RegisteredLinkedListAtomicInv { + _marker: PhantomData, +} -impl InvariantPredicate< +impl InvariantPredicate< (RegisteredLinkedListAtomicKey, Irc11AtomicId), - RegisteredLinkedListAtomicState, -> for RegisteredLinkedListAtomicInv { + RegisteredLinkedListAtomicState, +> for RegisteredLinkedListAtomicInv where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { open spec fn inv( key_loc: (RegisteredLinkedListAtomicKey, Irc11AtomicId), - state: RegisteredLinkedListAtomicState, + state: RegisteredLinkedListAtomicState, ) -> bool { let (key, loc) = key_loc; + let target_ids = state.registry()@.dom().remove(key.source_obj); &&& state.points_to().loc() == loc &&& key.source.addr() != 0 &&& state.auth().wf() &&& state.auth().domain() == key.domain &&& state.auth().state().root == key.source &&& state.auth().state().root_obj == key.source_obj - &&& state.auth().removed() == Set::::empty() + &&& state.auth().removed().subset_of(target_ids) &&& state.registry().id() == key.registry &&& state.registry()@ == state.auth().state().objects &&& state.current().id() == key.current + &&& state.lifecycle().id() == key.lifecycle + &&& state.lifecycle()@.dom() == target_ids + &&& state.permissions().wf() + &&& state.permissions().scheduler() == key.scheduler + &&& state.permissions().domain() == key.domain + &&& state.permissions().root() == key.root + &&& state.permissions().retire_observation_registry() == key.retire_observation_registry + &&& state.permissions().reclaim_registry() == key.reclaim_registry + &&& state.permissions().active_lease_registry() == key.active_lease_registry + &&& state.permissions().allocations() == target_ids + &&& forall|obj: nat| #[trigger] + target_ids.contains(obj) ==> registered_linked_list_target_inv::( + key, + state, + obj, + ) &&& state.link().source() == key.source &&& state.link().source_obj() == key.source_obj &&& state.link().timestamp_registry() == key.timestamp_registry &&& state.link().native_observation_registry() == key.native_observation_registry &&& state.link().wf(state.points_to().hist(), state.auth()) &&& state.link().native_observations_wf(state.points_to()) + &&& forall|n: rcu_spec::LinkIndex| + n < state.auth().state().successors[key.source_obj].len() + && state.auth().state().successors[key.source_obj][n as int] is Some + ==> #[trigger] state.auth().state().successors[key.source_obj][n as int]->Some_0.1 + != key.source_obj &&& match state.current()@ { None => state.auth().state().successors[key.source_obj].last() is None, Some(obj) => { @@ -510,22 +637,23 @@ impl InvariantPredicate< } } -pub type RegisteredLinkedListAtomicInvariant = AtomicInvariant< +pub type RegisteredLinkedListAtomicInvariant = AtomicInvariant< (RegisteredLinkedListAtomicKey, Irc11AtomicId), - RegisteredLinkedListAtomicState, - RegisteredLinkedListAtomicInv, + RegisteredLinkedListAtomicState, + RegisteredLinkedListAtomicInv, >; /// A real weak atomic link whose non-null values are selected by AId from an /// extensible registry rather than fixed by the wrapper's constructor. -pub struct RegisteredLinkedListWeakAtomicLink { +pub struct RegisteredLinkedListWeakAtomicLink { atomic: PAtomicWeakPtr, - tracked_atomic_inv: Tracked<&'static RegisteredLinkedListAtomicInvariant>, + tracked_atomic_inv: Tracked<&'static RegisteredLinkedListAtomicInvariant>, tracked_registry: Tracked>>, tracked_current: Tracked>>, + tracked_lifecycle: Tracked>>, } -impl RegisteredLinkedListWeakAtomicLink { +impl RegisteredLinkedListWeakAtomicLink { pub closed spec fn constant(&self) -> RegisteredLinkedListAtomicKey { self.tracked_atomic_inv@.constant().0 } @@ -546,10 +674,30 @@ impl RegisteredLinkedListWeakAtomicLink { self.tracked_current@.view() } + pub closed spec fn target_lifecycles(&self) -> Map { + self.tracked_lifecycle@.view() + } + + pub closed spec fn target_phase(&self, obj: nat) -> LinkedListChildPhase + recommends + self.target_lifecycles().contains_key(obj), + { + self.target_lifecycles()[obj] + } + + pub open spec fn no_reclaimed_targets(&self) -> bool { + forall|obj: nat| #[trigger] + self.registered_targets().dom().remove(self.constant().source_obj).contains(obj) + ==> self.target_lifecycles().contains_key(obj) && !self.target_phase( + obj, + ).is_reclaimed() + } + pub closed spec fn well_formed(&self) -> bool { &&& self.tracked_atomic_inv@.constant().1 == self.native_loc() &&& self.tracked_registry@.id() == self.constant().registry &&& self.tracked_current@.id() == self.constant().current + &&& self.tracked_lifecycle@.id() == self.constant().lifecycle &&& self.registered_targets().contains_pair( self.constant().source_obj, self.constant().source, @@ -560,11 +708,17 @@ impl RegisteredLinkedListWeakAtomicLink { pub closed spec fn type_inv(&self) -> bool { self.well_formed() } +} +impl RegisteredLinkedListWeakAtomicLink where + OwnPred: rcu_spec::RcuRootOwnershipPredicate, + { /// Creates a null native link. Additional target AIds can be registered /// after construction and then selected by the publication CAS. pub const fn new( + Ghost(scheduler): Ghost, Ghost(root): Ghost, + Ghost(retire_observation_registry): Ghost, Tracked(source_info): Tracked<&rcu_spec::RcuBlockInfo>, Tracked(source_retire): Tracked>, ) -> (res: Self) @@ -576,12 +730,15 @@ impl RegisteredLinkedListWeakAtomicLink { source_retire.ptr() == source_info.ptr(), ensures res.well_formed(), + res.constant().scheduler == scheduler, res.constant().domain == source_info.domain(), res.constant().root == root, + res.constant().retire_observation_registry == retire_observation_registry, res.constant().source == source_info.ptr(), res.constant().source_obj == source_info.obj(), res.registered_targets() == Map::empty().insert(source_info.obj(), source_info.ptr()), res.current_target() is None, + res.target_lifecycles() == Map::::empty(), { let (atomic, Tracked(points_to), Tracked(initial_view), Ghost(timestamp)) = PAtomicWeakPtr::new(core::ptr::null_mut()); @@ -605,18 +762,32 @@ impl RegisteredLinkedListWeakAtomicLink { Map::empty().insert(source_info.obj(), source_info.ptr()), ); let tracked (current, current_peer) = GhostVarAuth::new(None); + let tracked permissions = rcu_cpu_spec::RcuRootPermissionState::empty( + scheduler, + source_info.domain(), + root, + retire_observation_registry, + ); + let tracked (lifecycle, lifecycle_peer) = GhostVarAuth::new(Map::empty()); let tracked state = RegisteredLinkedListAtomicState { points_to, link, auth, registry, current, + permissions, + lifecycle, }; let ghost key = RegisteredLinkedListAtomicKey { + scheduler, domain: source_info.domain(), root, + retire_observation_registry, + reclaim_registry: state.permissions().reclaim_registry(), + active_lease_registry: state.permissions().active_lease_registry(), registry: state.registry().id(), current: state.current().id(), + lifecycle: state.lifecycle().id(), timestamp_registry: state.link().timestamp_registry(), native_observation_registry: state.link().native_observation_registry(), source: source_info.ptr(), @@ -629,6 +800,11 @@ impl RegisteredLinkedListWeakAtomicLink { assert(state.auth().state().objects == Map::empty().insert(key.source_obj, key.source)); assert(state.auth().removed() == Set::::empty()); assert(state.registry()@ == state.auth().state().objects); + assert(state.registry()@.dom().remove(key.source_obj) == Set::::empty()); + assert(state.lifecycle()@ == Map::::empty()); + assert(state.permissions().allocations() == Set::::empty()); + assert(state.permissions().keys() == Set::::empty()); + assert(state.permissions().unretired_claims().dom() == Set::::empty()); assert(state.link().native_observations() == Map::empty()); assert(state.link().native_observations_wf(state.points_to())) by { assert forall|observation_id: nat| #[trigger] @@ -638,7 +814,7 @@ impl RegisteredLinkedListWeakAtomicLink { &&& state.points_to().get_timestamp(observation.1) == Some(observation.2) } by {}; }; - assert(RegisteredLinkedListAtomicInv::inv((key, atomic.loc()), state)); + assert(RegisteredLinkedListAtomicInv::::inv((key, atomic.loc()), state)); } let tracked atomic_inv = AtomicInvariant::new((key, atomic.loc()), state, 0); let tracked atomic_inv = tracked_static_ref(atomic_inv); @@ -647,6 +823,7 @@ impl RegisteredLinkedListWeakAtomicLink { tracked_atomic_inv: Tracked(atomic_inv), tracked_registry: Tracked(registry_peer), tracked_current: Tracked(current_peer), + tracked_lifecycle: Tracked(lifecycle_peer), } } @@ -662,7 +839,7 @@ impl RegisteredLinkedListWeakAtomicLink { } pub proof fn tracked_atomic_inv(tracked &self) -> (tracked res: - &'static RegisteredLinkedListAtomicInvariant) + &'static RegisteredLinkedListAtomicInvariant) requires self.well_formed(), ensures @@ -678,6 +855,7 @@ impl RegisteredLinkedListWeakAtomicLink { target: *mut rcu_spec::LinkedListNode, tracked info: &rcu_spec::RcuBlockInfo, tracked retire: rcu_spec::RcuBaseRetirePerm, + tracked ownership: O, tracked credit: vstd::invariant::OpenInvariantCredit, ) requires @@ -691,6 +869,7 @@ impl RegisteredLinkedListWeakAtomicLink { equal(target, info.ptr()), info.obj() != old(self).constant().source_obj, !old(self).registered_targets().contains_key(info.obj()), + OwnPred::owns(target, ownership), ensures final(self).well_formed(), final(self).constant() == old(self).constant(), @@ -700,6 +879,10 @@ impl RegisteredLinkedListWeakAtomicLink { target, ), final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + info.obj(), + LinkedListChildPhase::Unpublished, + ), opens_invariants [self.invariant_namespace()] { use_type_invariant(&*self); @@ -707,14 +890,22 @@ impl RegisteredLinkedListWeakAtomicLink { let ghost native_loc = self.native_loc(); let tracked atomic_inv = self.tracked_atomic_inv.get(); vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + let ghost state_before = state; let ghost old_auth = state.auth; let ghost old_objects = state.auth.state().objects; let ghost old_registry = state.registry@; + let ghost old_lifecycle = state.lifecycle@; + let ghost old_permissions = state.permissions; state.registry.agree(self.tracked_registry.borrow()); state.current.agree(self.tracked_current.borrow()); - assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); assert(old_registry == self.registered_targets()); assert(!old_objects.contains_key(info.obj())); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_live_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); state.auth.lemma_registry_domains(); assert(state.auth.state().successors.contains_key(key.source_obj)); state.auth.lemma_unregistered_has_no_resources(info.obj()); @@ -728,6 +919,11 @@ impl RegisteredLinkedListWeakAtomicLink { self.tracked_registry.borrow_mut(), old_registry.insert(info.obj(), target), ); + state.permissions.tracked_insert(info, ownership); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert(info.obj(), LinkedListChildPhase::Unpublished), + ); assert(state.registry@ == state.auth.state().objects); state.auth.lemma_registry_domains(); match state.current@ { @@ -744,7 +940,76 @@ impl RegisteredLinkedListWeakAtomicLink { == old_auth.state().successors[key.source_obj]); }, } - assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + assert(state.registry@.dom().remove(key.source_obj) + == old_registry.dom().remove(key.source_obj).insert(info.obj())); + assert(state.permissions.allocations() + == old_permissions.allocations().insert(info.obj())); + assert(state.permissions.keys() == old_permissions.keys().insert(info.obj())); + assert(state.permissions.unretired_claims().dom() + == old_permissions.unretired_claims().dom().insert(info.obj())); + assert(state.lifecycle@.dom() == old_lifecycle.dom().insert(info.obj())); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + state.permissions.lemma_all_live_reclaim_states(); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(obj != key.source_obj); + assert(state.registry@.contains_key(obj)); + assert(state.permissions.allocations().contains(obj)); + if obj == info.obj() { + assert(state.registry@[obj] == target); + assert(state.permissions.reclaim_states()[obj] == Some(target)); + assert(state.permissions.ownership(obj) == ownership); + assert(state.permissions.unretired_claims().dom().contains(obj)); + assert(state.permissions.has_unretired_claim(obj)); + assert(state.lifecycle@[obj] is Unpublished); + assert(state.current@ != Some(obj)); + assert(state.auth.state().incoming_all[obj] + == Set::::empty()); + assert(!state.auth.removed().contains(obj)); + assert(state.auth.has_retire_perm(obj)); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + assert(old_registry.dom().remove(key.source_obj).contains(obj)); + assert(old_permissions.allocations().contains(obj)); + assert(state.permissions.allocations().contains(obj)); + assert(state.permissions.reclaim_states().dom().contains(obj)); + assert(state.registry@[obj] == old_registry[obj]); + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.current == state_before.current); + assert(state.auth.removed() == state_before.auth.removed()); + assert(state.auth.has_retire_perm(obj) + == state_before.auth.has_retire_perm(obj)); + assert(state.auth.state().successors[key.source_obj] + == state_before.auth.state().successors[key.source_obj]); + assert(state.auth.state().incoming_all[obj] + == state_before.auth.state().incoming_all[obj]); + assert(state.permissions.reclaim_states()[obj] + == old_permissions.reclaim_states()[obj]); + assert(state.permissions.has_unretired_claim(obj) + == old_permissions.has_unretired_claim(obj)); + assert(state.permissions.reclaimed() == old_permissions.reclaimed()); + if !state.lifecycle@[obj].is_reclaimed() { + assert(old_permissions.contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == old_permissions.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); }); } @@ -818,7 +1083,7 @@ impl RegisteredLinkedListWeakAtomicLink { proof { state.registry.agree(self.tracked_registry.borrow()); state.current.agree(self.tracked_current.borrow()); - assert(RegisteredLinkedListAtomicInv::inv( + assert(RegisteredLinkedListAtomicInv::::inv( (self.constant(), self.native_loc()), state, )); @@ -900,7 +1165,7 @@ impl RegisteredLinkedListWeakAtomicLink { )); }, } - assert(RegisteredLinkedListAtomicInv::inv( + assert(RegisteredLinkedListAtomicInv::::inv( (self.constant(), self.native_loc()), state, )); @@ -916,136 +1181,459 @@ impl RegisteredLinkedListWeakAtomicLink { result } - /// Publishes any registered target from a null link. + /// Acquire-loads any registered target and splits that target's AId-keyed + /// physical permission pool in the same native atomic invariant opening. #[inline(always)] - pub fn compare_exchange_publish( - &mut self, - target: *mut rcu_spec::LinkedListNode, - Ghost(target_obj): Ghost, + pub fn load_acquire_and_lease_cpu( + &self, + Tracked(guard): Tracked>, + Tracked(from): Tracked<&mut rcu_spec::RcuProtectedPtr>, + Tracked(previous): Tracked>, Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( - Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, - Ghost>, + *mut rcu_spec::LinkedListNode, + Ghost, + Ghost, + Tracked>, + Tracked>>, + Tracked>>, + Tracked, )) requires - old(self).well_formed(), - old(self).current_target() is None, - target_obj != old(self).constant().source_obj, - old(self).registered_targets().contains_pair(target_obj, target), + self.well_formed(), + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + online_cpus().contains(guard.cpu()), + guard.seen_removed().removed == Set::::empty(), + match previous { + None => guard.paper_guard().seen_at(self.constant().source_obj) == 0, + Some(observation) => { + &&& observation.registry() == self.constant().timestamp_registry + &&& observation.native_registry() == self.constant().native_observation_registry + &&& observation.loc() == self.native_loc() + &&& old(tv)@.contains(observation.view()) + &&& guard.paper_guard().seen_at(self.constant().source_obj) + == observation.index() + }, + }, + old(from).protected_by(guard.paper_guard()), + old(from).ptr() == self.constant().source, + old(from).obj() == self.constant().source_obj, ensures old(tv)@.spec_le(final(tv)@), - final(self).well_formed(), - final(self).constant() == old(self).constant(), - final(self).native_loc() == old(self).native_loc(), - final(self).registered_targets() == old(self).registered_targets(), - (res.0 is Ok) == (res.1@ is Some), - res.0 is Ok ==> final(self).current_target() == Some(target_obj), - res.0 is Err ==> final(self).current_target() is None, + res.3@.wf(), + res.3@.binding() == guard.binding(), + res.3@.participant_id() == guard.participant_id(), + res.3@.cpu() == guard.cpu(), + res.3@.generation() == guard.generation(), + res.3@.participant_view() == guard.participant_view(), + res.3@.known_retired() == guard.known_retired(), + res.3@.scheduler() == guard.scheduler(), + res.3@.domain() == guard.domain(), + res.3@.root() == guard.root(), + res.3@.reader_registry() == guard.reader_registry(), + res.3@.retire_observation_registry() == guard.retire_observation_registry(), + res.3@.reader_context() == guard.reader_context(), + res.3@.start_view() == guard.start_view(), + res.3@.expired() == guard.expired(), + res.3@.seen_removed().removed == guard.seen_removed().removed, + res.3@.paper_guard().seen_at(self.constant().source_obj) == res.2@, + final(from).ptr() == self.constant().source, + final(from).obj() == self.constant().source_obj, + res.6@.registry() == self.constant().timestamp_registry, + res.6@.native_registry() == self.constant().native_observation_registry, + res.6@.loc() == self.native_loc(), + res.6@.timestamp() == res.1@, + res.6@.index() == res.2@, + res.6@.view() == final(tv)@, + (res.4@ is Some) == (res.0.addr() != 0), + res.5@ is Some ==> res.4@ is Some, + match (res.4@, res.5@) { + (None, None) => { + &&& res.0.addr() == 0 + &&& res.3@.reader_fragment() == guard.reader_fragment() + }, + (Some(child), Some(lease)) => { + &&& equal(child.ptr(), res.0) + &&& child.domain() == self.constant().domain + &&& child.protected_by(res.3@.paper_guard()) + &&& child.obj() != self.constant().source_obj + &&& self.registered_targets().contains_pair(child.obj(), child.ptr()) + &&& self.target_lifecycles().contains_key(child.obj()) + &&& !self.target_phase(child.obj()).is_reclaimed() + &&& res.3@.reader_fragment().fraction() == guard.reader_fragment().fraction() + / 2real + &&& lease.key() == child.obj() + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == res.3@.participant_id() + &&& lease.reader_fraction() == res.3@.reader_fragment().fraction() + &&& lease.domain() == res.3@.domain() + &&& lease.root() == res.3@.root() + &&& lease.reader_context() == res.3@.reader_context() + &&& lease.start_view() == res.3@.start_view() + &&& lease.protected_addr() == child.ptr().addr() + &&& OwnPred::owns(child.ptr(), lease.resource()) + }, + (Some(child), None) => { + &&& equal(child.ptr(), res.0) + &&& child.domain() == self.constant().domain + &&& child.protected_by(res.3@.paper_guard()) + &&& child.obj() != self.constant().source_obj + &&& self.registered_targets().contains_pair(child.obj(), child.ptr()) + &&& self.target_lifecycles().contains_key(child.obj()) + &&& self.target_phase(child.obj()).is_reclaimed() + &&& res.3@.reader_fragment() == guard.reader_fragment() + }, + (None, Some(_)) => false, + }, no_unwind { let result; + let ghost view_before = tv@; proof { - use_type_invariant(&*self); + use_type_invariant(self); } - let raw_atomic = &self.atomic; - let null = core::ptr::null_mut(); proof_decl! { - let ghost key = self.constant(); - let ghost native_loc = self.native_loc(); - let tracked atomic_inv = self.tracked_atomic_inv.get(); - let tracked release_view = ReleaseViewSeen::new(); + let tracked (mut paper_guard, cpu_reader, binding) = guard.tracked_into_parts(); } - vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let raw_atomic = self.raw_atomic(); + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost state_before = state; + let ghost permissions_before = state.permissions; proof { state.registry.agree(self.tracked_registry.borrow()); state.current.agree(self.tracked_current.borrow()); - assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); - assert(state.current@ is None); - assert(state.auth.state().objects.contains_pair(target_obj, target)); - state.auth.lemma_registry_domains(); - assert(state.auth.state().incoming_all.contains_key(target_obj)); - state.auth.lemma_has_info_for_object(target_obj); - let tracked target_info = state.auth.tracked_info_for(target_obj); - assert(target_info.ptr() == target); - target_info.lemma_wf_facts(); - assert(target.addr() != 0); - assert(state.auth.state().successors.contains_key(key.source_obj)); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + match previous { + None => {}, + Some(observation) => { + use_type_invariant(observation); + state.link.lemma_observation_agrees(observation); + state.link.lemma_native_observation_agrees(observation); + assert(state.points_to.get_timestamp(observation.view()) + == Some(observation.timestamp())); + state.points_to.get_timestamp_monotonic( + view_before, + observation.view(), + ); + }, + } } - let ghost prev = state.points_to.hist(); - let cas = raw_atomic.compare_exchange( - null, - target, - Ordering::AcqRel, + let loaded = raw_atomic.load( Ordering::Acquire, Tracked(tv), - Tracked(release_view), - Tracked(&mut state.points_to), + Tracked(&state.points_to), ); - let ghost update = cas.2@; - let ghost next = state.points_to.hist(); - proof_decl! { - let ghost published_index: Option; - } + let ghost timestamp = loaded.2@.timestamp; + let ghost index = state.link.index_at(timestamp); proof { - match cas.0 { - Result::Ok(value) => { - let ghost current_timestamp = state.link.current_timestamp(); - assert(prev.is_max_timestamp(update.load_timestamp)); - assert(prev.contains_timestamp(current_timestamp)); - assert(prev.contains_timestamp(update.load_timestamp)); - assert(update.load_timestamp == current_timestamp); - assert(equal(prev.value(current_timestamp), value)); - assert(value.addr() == 0); - assert(state.auth.state().successors[key.source_obj].last() is None); - assert(next == prev.insert( - update.load_timestamp + 1, - target, - update.store_message_view, - )); - let index = state.link.tracked_cas_publish( - &mut state.auth, - prev, - next, - update.load_timestamp, - update.load_timestamp + 1, - target, - target_obj, - update.store_message_view, - ); - state.current.update( - self.tracked_current.borrow_mut(), - Some(target_obj), - ); - published_index = Some(index); + match previous { + None => {}, + Some(observation) => { + assert(state.points_to.get_timestamp(view_before).is_some()); + assert(observation.timestamp() + <= state.points_to.get_timestamp(view_before).unwrap()); + assert(state.points_to.get_timestamp(view_before).unwrap() <= timestamp); + assert(observation.index() <= index); }, - Result::Err(_) => { - published_index = None; + } + assert(paper_guard.seen_at(from.obj()) <= index); + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + paper_guard.seen_removed(), + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + paper_guard.seen_removed().removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> paper_guard.seen_removed().dead_edge(edge) + } by {}; + }; + } + proof_decl! { + let tracked protected = state.link.tracked_load_and_protect( + state.points_to.hist(), + &state.auth, + &mut paper_guard, + from, + timestamp, + ); + let tracked cpu_guard = rcu_cpu_spec::CpuRcuReadGuardToken::tracked_new( + paper_guard, + cpu_reader, + binding, + ); + let tracked final_guard; + let tracked lease; + let tracked observation; + } + proof { + observation = state.link.tracked_observation_at( + &state.points_to, + &state.auth, + timestamp, + tv@, + ); + assert(equal(state.points_to.hist().value(timestamp), loaded.0)); + state.permissions.lemma_all_contains_iff_keys(); + match &protected { + None => { + final_guard = cpu_guard; + lease = None; + }, + Some(child) => { + assert(rcu_spec::LinkedListTraversalSpec::link_inv( + self.constant().source, + self.constant().source_obj, + index, + child.ptr(), + child.obj(), + state.auth.state(), + )); + assert(state.auth.state().successors[ + self.constant().source_obj + ][index as int] == Some((child.ptr(), child.obj()))); + assert(state.registry@.contains_pair(child.obj(), child.ptr())); + assert(child.obj() != self.constant().source_obj); + assert(state.registry@.dom().remove( + self.constant().source_obj, + ).contains(child.obj())); + assert(registered_linked_list_target_inv::( + self.constant(), + state, + child.obj(), + )); + if state.lifecycle@[child.obj()].is_reclaimed() { + final_guard = cpu_guard; + lease = None; + } else { + assert(state.permissions.contains(child.obj())); + let ghost ownership = state.permissions.ownership(child.obj()); + assert(OwnPred::owns(child.ptr(), ownership)); + let tracked split = state.permissions.tracked_split_protected( + cpu_guard, + child, + ); + final_guard = split.0; + lease = Some(split.1); + assert(split.1.resource() == ownership); + assert(OwnPred::owns(child.ptr(), split.1.resource())); + } }, } - assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() == permissions_before.reclaim_states()); + assert(state.permissions.reclaimed() == permissions_before.reclaimed()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + assert(state.points_to == state_before.points_to); + assert(state.link.source() == state_before.link.source()); + assert(state.link.source_obj() == state_before.link.source_obj()); + assert(state.link.timestamps() == state_before.link.timestamps()); + assert(state.link.current_timestamp() + == state_before.link.current_timestamp()); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle == state_before.lifecycle); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(self.constant().source_obj).contains(obj) implies + registered_linked_list_target_inv::( + self.constant(), + state, + obj, + ) by { + assert(registered_linked_list_target_inv::( + self.constant(), + state_before, + obj, + )); + if !state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.contains(obj)); + assert(permissions_before.keys().contains(obj)); + assert(state.permissions.keys().contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + }; + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); } - result = (cas.0, Ghost(published_index)); + result = ( + loaded.0, + Ghost(timestamp), + Ghost(index), + Tracked(final_guard), + Tracked(protected), + Tracked(lease), + Tracked(observation), + ); }); result } - /// Unlinks the currently selected AId. The AId precondition, rather than - /// pointer-address equality, identifies which logical allocation loses its - /// incoming edge. + /// Returns an AId-keyed traversal lease and rejoins its saved CPU reader + /// fraction with the live guard. + #[verifier::atomic] + pub fn return_registered_lease_cpu( + &self, + Tracked(lease): Tracked>>, + Tracked(guard): Tracked>, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked>) + requires + self.well_formed(), + match lease { + None => true, + Some(lease) => { + &&& lease.active_registry() == self.constant().active_lease_registry + &&& lease.participant_id() == guard.participant_id() + &&& lease.reader_fraction() == guard.reader_fragment().fraction() + &&& lease.domain() == guard.domain() + &&& lease.root() == guard.root() + &&& lease.reader_context() == guard.reader_context() + &&& lease.start_view() == guard.start_view() + &&& guard.protects(lease.protected_addr(), lease.key()) + }, + }, + guard.wf(), + guard.scheduler() == self.constant().scheduler, + guard.domain() == self.constant().domain, + guard.root() == self.constant().root, + guard.retire_observation_registry() == self.constant().retire_observation_registry, + ensures + old(tv)@.spec_le(final(tv)@), + res@.wf(), + res@.paper_guard() == guard.paper_guard(), + res@.binding() == guard.binding(), + res@.participant_id() == guard.participant_id(), + res@.cpu() == guard.cpu(), + res@.generation() == guard.generation(), + res@.participant_view() == guard.participant_view(), + res@.known_retired() == guard.known_retired(), + res@.domain() == guard.domain(), + res@.root() == guard.root(), + res@.reader_registry() == guard.reader_registry(), + res@.retire_observation_registry() == guard.retire_observation_registry(), + res@.reader_context() == guard.reader_context(), + res@.start_view() == guard.start_view(), + res@.expired() == guard.expired(), + res@.seen_removed() == guard.seen_removed(), + res@.protected() == guard.protected(), + res@.reader_fragment().fraction() == match lease { + None => guard.reader_fragment().fraction(), + Some(_) => guard.reader_fragment().fraction() * 2real, + }, + no_unwind + { + let raw_atomic = &self.atomic; + proof_decl! { + let tracked final_guard; + } + vstd::invariant::open_atomic_invariant!(self.tracked_atomic_inv() => state => { + let ghost state_before = state; + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + state.permissions.lemma_all_contains_iff_keys(); + match lease { + None => { + final_guard = guard; + }, + Some(lease) => { + final_guard = state.permissions.tracked_return_loaded(lease, guard); + }, + } + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.keys() == permissions_before.keys()); + assert(state.permissions.reclaim_states() == permissions_before.reclaim_states()); + assert(state.permissions.reclaimed() == permissions_before.reclaimed()); + assert(state.permissions.unretired_claims() + == permissions_before.unretired_claims()); + assert(state.points_to == state_before.points_to); + assert(state.link.source() == state_before.link.source()); + assert(state.link.source_obj() == state_before.link.source_obj()); + assert(state.link.timestamps() == state_before.link.timestamps()); + assert(state.link.current_timestamp() + == state_before.link.current_timestamp()); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle == state_before.lifecycle); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_live_reclaim_states(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(self.constant().source_obj).contains(obj) implies + registered_linked_list_target_inv::( + self.constant(), + state, + obj, + ) by { + assert(registered_linked_list_target_inv::( + self.constant(), + state_before, + obj, + )); + if !state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.contains(obj)); + assert(permissions_before.keys().contains(obj)); + assert(state.permissions.keys().contains(obj)); + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + }; + assert(RegisteredLinkedListAtomicInv::::inv( + (self.constant(), self.native_loc()), + state, + )); + } + }); + Tracked(final_guard) + } + + /// Publishes any registered target from a null link. #[inline(always)] - pub fn compare_exchange_unlink( + pub fn compare_exchange_publish( &mut self, target: *mut rcu_spec::LinkedListNode, Ghost(target_obj): Ghost, Tracked(tv): Tracked<&mut ViewSeen>, ) -> (res: ( Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, - Ghost>, + Ghost>, )) requires old(self).well_formed(), - old(self).current_target() == Some(target_obj), + old(self).current_target() is None, + target_obj != old(self).constant().source_obj, old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_unpublished() || old(self).target_phase( + target_obj, + ).is_unlinked(), ensures old(tv)@.spec_le(final(tv)@), final(self).well_formed(), @@ -1053,10 +1641,18 @@ impl RegisteredLinkedListWeakAtomicLink { final(self).native_loc() == old(self).native_loc(), final(self).registered_targets() == old(self).registered_targets(), (res.0 is Ok) == (res.1@ is Some), - res.0 is Ok ==> final(self).current_target() is None, - res.0 is Ok ==> res.1@->Some_0.1.root == final(self).constant().root, - res.0 is Ok ==> res.1@->Some_0.1.observed_by(final(tv)@), - res.0 is Err ==> final(self).current_target() == Some(target_obj), + res.0 is Ok ==> final(self).current_target() == Some(target_obj), + res.0 is Ok ==> final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Linked { + index: res.1@->Some_0, + timestamp: final(self).target_phase(target_obj)->Linked_timestamp, + }, + ), + res.0 is Ok ==> final(self).target_phase(target_obj).is_linked(), + res.0 is Ok ==> final(self).target_phase(target_obj)->Linked_index == res.1@->Some_0, + res.0 is Err ==> final(self).current_target() is None, + res.0 is Err ==> final(self).target_lifecycles() == old(self).target_lifecycles(), no_unwind { let result; @@ -1072,18 +1668,28 @@ impl RegisteredLinkedListWeakAtomicLink { let tracked release_view = ReleaseViewSeen::new(); } vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; proof { state.registry.agree(self.tracked_registry.borrow()); state.current.agree(self.tracked_current.borrow()); - assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); - assert(state.current@ == Some(target_obj)); - assert(state.auth.state().successors[key.source_obj].last() - == Some((target, target_obj))); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.current@ is None); + assert(state.auth.state().objects.contains_pair(target_obj, target)); + state.auth.lemma_registry_domains(); + assert(state.auth.state().incoming_all.contains_key(target_obj)); + state.auth.lemma_has_info_for_object(target_obj); + let tracked target_info = state.auth.tracked_info_for(target_obj); + assert(target_info.ptr() == target); + target_info.lemma_wf_facts(); + assert(target.addr() != 0); + assert(state.auth.state().successors.contains_key(key.source_obj)); } let ghost prev = state.points_to.hist(); let cas = raw_atomic.compare_exchange( - target, null, + target, Ordering::AcqRel, Ordering::Acquire, Tracked(tv), @@ -1093,10 +1699,7 @@ impl RegisteredLinkedListWeakAtomicLink { let ghost update = cas.2@; let ghost next = state.points_to.hist(); proof_decl! { - let ghost unlinked: Option<( - rcu_spec::LinkIndex, - rcu_spec::RcuRemovalObservation, - )>; + let ghost published_index: Option; } proof { match cas.0 { @@ -1107,13 +1710,14 @@ impl RegisteredLinkedListWeakAtomicLink { assert(prev.contains_timestamp(update.load_timestamp)); assert(update.load_timestamp == current_timestamp); assert(equal(prev.value(current_timestamp), value)); - assert(value.addr() == target.addr()); + assert(value.addr() == 0); + assert(state.auth.state().successors[key.source_obj].last() is None); assert(next == prev.insert( update.load_timestamp + 1, - null, + target, update.store_message_view, )); - let index = state.link.tracked_cas_unlink( + let index = state.link.tracked_cas_publish( &mut state.auth, prev, next, @@ -1123,24 +1727,853 @@ impl RegisteredLinkedListWeakAtomicLink { target_obj, update.store_message_view, ); - let ghost removal = rcu_spec::RcuRemovalObservation { - root: key.root, - timestamp: update.load_timestamp + 1, - message_view: update.store_message_view, - }; - state.current.update(self.tracked_current.borrow_mut(), None); - unlinked = Some((index, removal)); + state.current.update( + self.tracked_current.borrow_mut(), + Some(target_obj), + ); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Linked { + index, + timestamp: update.load_timestamp + 1, + }, + ), + ); + published_index = Some(index); }, Result::Err(_) => { - unlinked = None; + published_index = None; }, } - assert(RegisteredLinkedListAtomicInv::inv((key, native_loc), state)); - } - result = (cas.0, Ghost(unlinked)); - }); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + match cas.0 { + Result::Ok(_) => { + if obj == target_obj { + assert(state.lifecycle@[obj] is Linked); + assert(state.current@ == Some(obj)); + assert(state.auth.state().incoming_all[obj] + == state_before.auth.state().incoming_all[obj].insert(( + key.source_obj, + published_index->Some_0, + ))); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) implies { + &&& edge.0 == key.source_obj + &&& edge.1 <= published_index->Some_0 + } by { + if edge != (key.source_obj, published_index->Some_0) { + assert(state_before.auth.state().incoming_all[obj].contains( + edge, + )); + match old_lifecycle[obj] { + LinkedListChildPhase::Unpublished => assert(false), + LinkedListChildPhase::Unlinked { + index: old_index, + removal: _, + } => { + assert(edge.1 < old_index); + assert(old_index + < state_before.auth.state().successors[ + key.source_obj + ].len()); + }, + _ => assert(false), + } + } + }; + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(!old_lifecycle[obj].is_linked()); + assert(state.current@ != Some(obj)); + } + }, + Result::Err(_) => { + assert(state.lifecycle@ == old_lifecycle); + assert(state.current@ is None); + }, + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + state.auth.lemma_registry_domains(); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + } + result = (cas.0, Ghost(published_index)); + }); result } + + /// Unlinks the currently selected AId. The AId precondition, rather than + /// pointer-address equality, identifies which logical allocation loses its + /// incoming edge. + #[inline(always)] + pub fn compare_exchange_unlink( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: ( + Result<*mut rcu_spec::LinkedListNode, *mut rcu_spec::LinkedListNode>, + Ghost>, + )) + requires + old(self).well_formed(), + old(self).current_target() == Some(target_obj), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_linked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + (res.0 is Ok) == (res.1@ is Some), + res.0 is Ok ==> final(self).current_target() is None, + res.0 is Ok ==> res.1@->Some_0.1.root == final(self).constant().root, + res.0 is Ok ==> res.1@->Some_0.1.observed_by(final(tv)@), + res.0 is Ok ==> final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Unlinked { + index: res.1@->Some_0.0, + removal: res.1@->Some_0.1, + }, + ), + res.0 is Ok ==> final(self).target_phase(target_obj).is_unlinked(), + res.0 is Err ==> final(self).current_target() == Some(target_obj), + res.0 is Err ==> final(self).target_lifecycles() == old(self).target_lifecycles(), + no_unwind + { + let result; + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + let null = core::ptr::null_mut(); + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked release_view = ReleaseViewSeen::new(); + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(state.current@ == Some(target_obj)); + state.auth.lemma_registry_domains(); + assert(state.auth.state().successors.contains_key(key.source_obj)); + assert(state.auth.state().successors[key.source_obj].last() + == Some((target, target_obj))); + } + let ghost prev = state.points_to.hist(); + let cas = raw_atomic.compare_exchange( + target, + null, + Ordering::AcqRel, + Ordering::Acquire, + Tracked(tv), + Tracked(release_view), + Tracked(&mut state.points_to), + ); + let ghost update = cas.2@; + let ghost next = state.points_to.hist(); + proof_decl! { + let ghost unlinked: Option<( + rcu_spec::LinkIndex, + rcu_spec::RcuRemovalObservation, + )>; + } + proof { + match cas.0 { + Result::Ok(value) => { + let ghost current_timestamp = state.link.current_timestamp(); + assert(prev.is_max_timestamp(update.load_timestamp)); + assert(prev.contains_timestamp(current_timestamp)); + assert(prev.contains_timestamp(update.load_timestamp)); + assert(update.load_timestamp == current_timestamp); + assert(equal(prev.value(current_timestamp), value)); + assert(value.addr() == target.addr()); + assert(next == prev.insert( + update.load_timestamp + 1, + null, + update.store_message_view, + )); + let index = state.link.tracked_cas_unlink( + &mut state.auth, + prev, + next, + update.load_timestamp, + update.load_timestamp + 1, + target, + target_obj, + update.store_message_view, + ); + let ghost removal = rcu_spec::RcuRemovalObservation { + root: key.root, + timestamp: update.load_timestamp + 1, + message_view: update.store_message_view, + }; + state.current.update(self.tracked_current.borrow_mut(), None); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Unlinked { index, removal }, + ), + ); + unlinked = Some((index, removal)); + }, + Result::Err(_) => { + unlinked = None; + }, + } + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + match cas.0 { + Result::Ok(_) => { + if obj == target_obj { + assert(state.lifecycle@[obj] is Unlinked); + assert(state.auth.state().incoming_all[obj] + == state_before.auth.state().incoming_all[obj]); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) implies { + &&& edge.0 == key.source_obj + &&& edge.1 < unlinked->Some_0.0 + } by { + assert(edge.1 + <= old_lifecycle[obj]->Linked_index); + assert(old_lifecycle[obj]->Linked_index + 1 + == unlinked->Some_0.0); + }; + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(obj != target_obj); + } + assert(!state.lifecycle@[obj].is_linked()); + assert(state.current@ is None); + }, + Result::Err(_) => { + assert(state.lifecycle@ == old_lifecycle); + assert(state.current@ == Some(target_obj)); + }, + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + state.auth.lemma_registry_domains(); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + } + result = (cas.0, Ghost(unlinked)); + }); + result + } + + /// Converts one unlinked registered AId into the traversal-retire and + /// physical-reclaim resources needed by the existing RCU monitor. + #[verifier::atomic] + pub fn retire_unlinked_target( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked) + requires + old(self).well_formed(), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_unlinked(), + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Retired { + index: old(self).target_phase(target_obj)->Unlinked_index, + removal: old(self).target_phase(target_obj)->Unlinked_removal, + }, + ), + final(self).target_phase(target_obj).is_retired(), + res@.object().wf(), + res@.object().domain() == final(self).constant().domain, + res@.object().obj() == target_obj, + res@.object().ptr() == target, + res@.retire().wf(), + res@.retire().ready_to_retire(), + res@.retire().domain() == final(self).constant().domain, + res@.retire().obj() == target_obj, + res@.retire().ptr() == target, + res@.claim().registry() == final(self).constant().reclaim_registry, + res@.claim().obj() == target_obj, + res@.claim().is_pending(), + res@.claim().ptr() == target, + res@.removal() == old(self).target_phase(target_obj)->Unlinked_removal, + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked detached; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(registered_linked_list_target_inv::( + key, + state, + target_obj, + )); + state.auth.lemma_registry_domains(); + state.permissions.lemma_all_unretired_domains(); + assert(state.auth.state().successors.contains_key(key.source_obj)); + assert(state.auth.state().incoming_all.contains_key(target_obj)); + assert(state.lifecycle@[target_obj] is Unlinked); + let ghost index = state.lifecycle@[target_obj]->Unlinked_index; + let ghost removal = state.lifecycle@[target_obj]->Unlinked_removal; + let ghost latest = (state.auth.state().successors[key.source_obj].len() - 1) + as nat; + let ghost prior = rcu_spec::RcuSeenRemoved { + removed: state.auth.removed(), + link_view: rcu_spec::RcuLinkView::empty().observe(key.source_obj, latest), + }; + assert(state.auth.state().successors[key.source_obj].len() > 0); + assert(latest + 1 == state.auth.state().successors[key.source_obj].len()); + assert(state.auth.state().bounds(prior.link_view)) by { + assert forall|from: *mut rcu_spec::LinkedListNode, from_obj: nat| #[trigger] + state.auth.state().objects.contains_pair(from_obj, from) + && prior.link_view.seen.contains_key(from_obj) implies { + &&& state.auth.state().successors[from_obj].len() > 0 + &&& prior.link_view.seen_at(from_obj) + < state.auth.state().successors[from_obj].len() + } by { + assert(from_obj == key.source_obj); + assert(state.auth.state().successors.contains_key(from_obj)); + assert(prior.link_view.seen_at(from_obj) == latest); + }; + } + assert(rcu_spec::LinkedListTraversalSpec::seen_removed_sound( + prior, + state.auth.state(), + )) by { + assert forall|obj: nat| #[trigger] + prior.removed.contains(obj) implies { + &&& state.auth.state().incoming_all.contains_key(obj) + &&& forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) + ==> prior.dead_edge(edge) + } by { + assert(state.auth.removed().contains(obj)); + assert(state.registry@.dom().remove(key.source_obj).contains(obj)); + assert(registered_linked_list_target_inv::(key, state, obj)); + assert(state.registry@.contains_key(obj)); + assert(state.auth.state().objects.contains_key(obj)); + assert(state.auth.state().incoming_all.contains_key(obj)); + assert(state.lifecycle@[obj].is_retired() + || state.lifecycle@[obj].is_reclaimed()); + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[obj].contains(edge) implies + prior.dead_edge(edge) by { + match state.lifecycle@[obj] { + LinkedListChildPhase::Retired { + index: removed_index, + removal: _, + } + | LinkedListChildPhase::Reclaimed { + index: removed_index, + removal: _, + } => { + assert(edge.0 == key.source_obj); + assert(edge.1 < removed_index); + assert(removed_index + < state.auth.state().successors[key.source_obj].len()); + assert(removed_index <= latest); + }, + _ => assert(false), + } + }; + }; + }; + assert forall|edge: rcu_spec::LinkEdge| #[trigger] + state.auth.state().incoming_all[target_obj].contains(edge) implies + prior.dead_edge(edge) by { + assert(edge.0 == key.source_obj); + assert(edge.1 < index); + assert(index < state.auth.state().successors[key.source_obj].len()); + }; + assert(state.auth.state().incoming_all[target_obj].len() > 0) by { + assert(state.auth.state().successors[key.source_obj][(index - 1) as int] + == Some((target, target_obj))); + assert(rcu_spec::LinkedListTraversalSpec::link_inv( + key.source, + key.source_obj, + (index - 1) as nat, + target, + target_obj, + state.auth.state(), + )); + assert(state.auth.state().incoming_all[target_obj].contains(( + key.source_obj, + (index - 1) as nat, + ))); + }; + state.auth.lemma_has_info_for_object(target_obj); + let tracked object = state.auth.tracked_info_for(target_obj); + let tracked retire = state.auth.tracked_retire_node(target_obj, prior); + let tracked claim = state.permissions.tracked_retire(target_obj); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Retired { index, removal }, + ), + ); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + if obj == target_obj { + assert(state.lifecycle@[obj] is Retired); + assert(state.auth.removed().contains(obj)); + assert(!state.auth.has_retire_perm(obj)); + assert(state.permissions.contains(obj)); + assert(!state.permissions.has_unretired_claim(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.auth.removed().contains(obj) + == state_before.auth.removed().contains(obj)); + assert(state.auth.has_retire_perm(obj) + == state_before.auth.has_retire_perm(obj)); + assert(state.permissions.has_unretired_claim(obj) + == permissions_before.has_unretired_claim(obj)); + if !state.lifecycle@[obj].is_reclaimed() { + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(state.auth.removed().subset_of( + state.registry@.dom().remove(key.source_obj), + )); + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == permissions_before.allocations()); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + assert(state.registry@ == state.auth.state().objects); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.auth.state() == state_before.auth.state()); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle.id() == state_before.lifecycle.id()); + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(state.link.wf(state.points_to.hist(), state.auth)); + assert(state.link.native_observations_wf(state.points_to)); + assert forall|n: rcu_spec::LinkIndex| + n < state.auth.state().successors[key.source_obj].len() + && state.auth.state().successors[key.source_obj][n as int] is Some + implies #[trigger] + state.auth.state().successors[key.source_obj][n as int]->Some_0.1 + != key.source_obj by {}; + assert(match state.current@ { + None => state.auth.state().successors[key.source_obj].last() is None, + Some(obj) => { + &&& obj != key.source_obj + &&& state.registry@.contains_key(obj) + &&& state.auth.state().successors[key.source_obj].last() == Some(( + state.registry@[obj], + obj, + )) + }, + }); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(object.ptr() == target); + assert(retire.ptr() == target); + assert(claim.ptr() == target); + detached = LinkedListDetachedChild { object, retire, claim, removal }; + } + }); + Tracked(detached) + } + + /// Recovers one retired registered allocation after a completed grace + /// period has excluded all outstanding leases for that AId. + #[verifier::atomic] + pub fn reclaim_retired_target( + &mut self, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, + Tracked(claim): Tracked>, + Tracked(completed): Tracked, + Tracked(tv): Tracked<&mut ViewSeen>, + ) -> (res: Tracked) + requires + old(self).well_formed(), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == target_obj, + claim.is_pending(), + claim.ptr() == target, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == target_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).target_phase(target_obj)->Retired_removal, + ensures + old(tv)@.spec_le(final(tv)@), + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Reclaimed { + index: old(self).target_phase(target_obj)->Retired_index, + removal: old(self).target_phase(target_obj)->Retired_removal, + }, + ), + final(self).target_phase(target_obj).is_reclaimed(), + OwnPred::owns(target, res@), + no_unwind + { + proof { + use_type_invariant(&*self); + } + let raw_atomic = &self.atomic; + proof_decl! { + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked ownership; + } + vstd::invariant::open_atomic_invariant!(atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + let ghost permissions_before = state.permissions; + let _loaded = raw_atomic.load( + Ordering::Relaxed, + Tracked(tv), + Tracked(&state.points_to), + ); + proof { + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(registered_linked_list_target_inv::( + key, + state, + target_obj, + )); + assert(state.lifecycle@[target_obj] is Retired); + let ghost index = state.lifecycle@[target_obj]->Retired_index; + let ghost removal = state.lifecycle@[target_obj]->Retired_removal; + assert(completed.record().removal == removal); + assert(OwnPred::owns( + target, + permissions_before.ownership(target_obj), + )); + state.permissions.lemma_completed_excludes_active(&completed, target_obj); + assert(!state.permissions.has_active(target_obj)); + ownership = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Reclaimed { index, removal }, + ), + ); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + if obj == target_obj { + assert(state.lifecycle@[obj] is Reclaimed); + assert(!state.permissions.contains(obj)); + assert(state.permissions.reclaim_states()[obj] is None); + assert(state.permissions.reclaimed().contains_pair(obj, completed)); + assert(state.permissions.reclaimed()[obj].record().removal == removal); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.permissions.reclaim_states()[obj] + == permissions_before.reclaim_states()[obj]); + assert(state.permissions.has_unretired_claim(obj) + == permissions_before.has_unretired_claim(obj)); + if state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed()[obj] + == permissions_before.reclaimed()[obj]); + } + if !state.lifecycle@[obj].is_reclaimed() { + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() + == permissions_before.allocations()); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle.id() == state_before.lifecycle.id()); + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(ownership == permissions_before.ownership(target_obj)); + assert(OwnPred::owns(target, ownership)); + } + }); + Tracked(ownership) + } + + /// Proof-mode reclaim entry used by a type-erased callback that already + /// owns an invariant-opening credit from the monitor. + pub proof fn tracked_reclaim_retired_target( + tracked &mut self, + target: *mut rcu_spec::LinkedListNode, + target_obj: nat, + tracked claim: rcu_cpu_spec::RcuReclaimClaim, + tracked completed: rcu_cpu_spec::RcuReclaimedWitness, + tracked credit: vstd::invariant::OpenInvariantCredit, + ) -> (tracked ownership: O) + requires + old(self).well_formed(), + old(self).registered_targets().contains_pair(target_obj, target), + old(self).target_lifecycles().contains_key(target_obj), + old(self).target_phase(target_obj).is_retired(), + claim.registry() == old(self).constant().reclaim_registry, + claim.obj() == target_obj, + claim.is_pending(), + claim.ptr() == target, + completed.wf(), + completed.scheduler() == old(self).constant().scheduler, + completed.record().domain == old(self).constant().domain, + completed.record().obj == target_obj, + completed.record().retire_observation_registry == old( + self, + ).constant().retire_observation_registry, + completed.record().removal == old(self).target_phase(target_obj)->Retired_removal, + ensures + final(self).well_formed(), + final(self).constant() == old(self).constant(), + final(self).native_loc() == old(self).native_loc(), + final(self).registered_targets() == old(self).registered_targets(), + final(self).current_target() == old(self).current_target(), + final(self).target_lifecycles() == old(self).target_lifecycles().insert( + target_obj, + LinkedListChildPhase::Reclaimed { + index: old(self).target_phase(target_obj)->Retired_index, + removal: old(self).target_phase(target_obj)->Retired_removal, + }, + ), + final(self).target_phase(target_obj).is_reclaimed(), + OwnPred::owns(target, ownership), + opens_invariants [self.invariant_namespace()] + { + use_type_invariant(&*self); + let ghost key = self.constant(); + let ghost native_loc = self.native_loc(); + let tracked atomic_inv = self.tracked_atomic_inv.get(); + let tracked mut recovered; + vstd::invariant::open_atomic_invariant_in_proof!(credit => atomic_inv => state => { + let ghost state_before = state; + let ghost old_lifecycle = state.lifecycle@; + let ghost permissions_before = state.permissions; + state.registry.agree(self.tracked_registry.borrow()); + state.current.agree(self.tracked_current.borrow()); + state.lifecycle.agree(self.tracked_lifecycle.borrow()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(registered_linked_list_target_inv::( + key, + state, + target_obj, + )); + assert(state.lifecycle@[target_obj] is Retired); + let ghost index = state.lifecycle@[target_obj]->Retired_index; + let ghost removal = state.lifecycle@[target_obj]->Retired_removal; + assert(completed.record().removal == removal); + assert(OwnPred::owns(target, permissions_before.ownership(target_obj))); + state.permissions.lemma_completed_excludes_active(&completed, target_obj); + assert(!state.permissions.has_active(target_obj)); + recovered = state.permissions.tracked_reclaim(claim, completed); + state.lifecycle.update( + self.tracked_lifecycle.borrow_mut(), + old_lifecycle.insert( + target_obj, + LinkedListChildPhase::Reclaimed { index, removal }, + ), + ); + state.permissions.lemma_all_contains_iff_keys(); + state.permissions.lemma_all_allocations_have_reclaim_states(); + state.permissions.lemma_all_unretired_domains(); + assert forall|obj: nat| #[trigger] + state.registry@.dom().remove(key.source_obj).contains(obj) implies + registered_linked_list_target_inv::(key, state, obj) by { + assert(registered_linked_list_target_inv::( + key, + state_before, + obj, + )); + if obj == target_obj { + assert(state.lifecycle@[obj] is Reclaimed); + assert(!state.permissions.contains(obj)); + assert(state.permissions.reclaim_states()[obj] is None); + assert(state.permissions.reclaimed().contains_pair(obj, completed)); + assert(state.permissions.reclaimed()[obj].record().removal == removal); + assert(registered_linked_list_target_inv::(key, state, obj)); + } else { + assert(state.lifecycle@[obj] == old_lifecycle[obj]); + assert(state.permissions.reclaim_states()[obj] + == permissions_before.reclaim_states()[obj]); + assert(state.permissions.has_unretired_claim(obj) + == permissions_before.has_unretired_claim(obj)); + if state.lifecycle@[obj].is_reclaimed() { + assert(permissions_before.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed().contains_key(obj)); + assert(state.permissions.reclaimed()[obj] + == permissions_before.reclaimed()[obj]); + } + if !state.lifecycle@[obj].is_reclaimed() { + assert(state.permissions.contains(obj)); + assert(state.permissions.ownership(obj) + == permissions_before.ownership(obj)); + } + assert(registered_linked_list_target_inv::(key, state, obj)); + } + }; + assert(old_lifecycle.contains_key(target_obj)); + assert(state.lifecycle@.dom() == old_lifecycle.dom()); + assert(state.lifecycle@.dom() + == state.registry@.dom().remove(key.source_obj)); + assert(state.permissions.allocations() == permissions_before.allocations()); + assert(state.permissions.allocations() + == state.registry@.dom().remove(key.source_obj)); + assert(state.points_to == state_before.points_to); + assert(state.link == state_before.link); + assert(state.auth == state_before.auth); + assert(state.registry == state_before.registry); + assert(state.current == state_before.current); + assert(state.lifecycle.id() == state_before.lifecycle.id()); + assert(state.permissions.wf()); + assert(state.permissions.scheduler() == permissions_before.scheduler()); + assert(state.permissions.domain() == permissions_before.domain()); + assert(state.permissions.root() == permissions_before.root()); + assert(state.permissions.retire_observation_registry() + == permissions_before.retire_observation_registry()); + assert(state.permissions.reclaim_registry() + == permissions_before.reclaim_registry()); + assert(state.permissions.active_lease_registry() + == permissions_before.active_lease_registry()); + assert(RegisteredLinkedListAtomicInv::::inv((key, native_loc), state)); + assert(recovered == permissions_before.ownership(target_obj)); + assert(OwnPred::owns(target, recovered)); + }); + recovered + } } /// Immutable identities carried by the native atomic invariant for the diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 94389b291..09891bfa1 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -149,14 +149,15 @@ use crate::{ sync::{ rcu as rcu_spec, rcu_cpu as rcu_cpu_spec, weak_memory::{ - LinkedListRetiredChild, LinkedListWeakAtomicLink, RcuRetiredRootObject, - RcuRootAtomicInv, RcuRootAtomicInvariant, RcuRootAtomicState, RcuWeakAtomicPtr, + LinkedListRetiredChild, RcuRetiredRootObject, RcuRootAtomicInv, + RcuRootAtomicInvariant, RcuRootAtomicState, RcuWeakAtomicPtr, + RegisteredLinkedListWeakAtomicLink, }, }, task::InAtomicMode, }, sync::Once, - task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, + task::{disable_preempt_in_context, DisabledPreemptGuard, RunningTaskContext}, }; use vstd_extra::atomic_irc11::{ThreadViewOrder, ViewSeen}; @@ -201,7 +202,7 @@ impl rcu_spec::RcuRootOwnershipPredicate< /// Concrete linked-list atomic whose pointee ownership is a real smart-pointer /// permission understood by [`NonNullPtrRef`]. -type RcuLinkedListAtomicLink

= LinkedListWeakAtomicLink< +type RcuLinkedListAtomicLink

= RegisteredLinkedListWeakAtomicLink<

::Permission, RcuPointerOwnership

, >; @@ -225,6 +226,7 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr bool { &&& self.link.well_formed() + &&& self.link.no_reclaimed_targets() &&& self.proof_active ==> { &&& self.tracked_guard@ is Some &&& self.tracked_observation@ is Some @@ -247,8 +249,9 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr self.obj_ptr.addr() == 0, (Some(child), Some(lease)) => { &&& equal(child.ptr(), self.obj_ptr) - &&& child.ptr() == self.link.constant().child - &&& child.obj() == self.link.constant().child_obj + &&& self.link.registered_targets().contains_pair(child.obj(), child.ptr()) + &&& self.link.target_lifecycles().contains_key(child.obj()) + &&& !self.link.target_phase(child.obj()).is_reclaimed() &&& child.protected_by(self.tracked_guard@->Some_0.paper_guard()) &&& lease.key() == child.obj() &&& lease.active_registry() == self.link.constant().active_lease_registry @@ -281,7 +284,7 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr (res: Self) requires link.well_formed(), - !link.child_phase().is_reclaimed(), + link.no_reclaimed_targets(), guard.wf(), guard.scheduler() == link.constant().scheduler, guard.domain() == link.constant().domain, @@ -324,6 +327,19 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr { + assert(loaded_child.obj() != link.constant().source_obj); + assert(link.registered_targets().dom().remove( + link.constant().source_obj, + ).contains(loaded_child.obj())); + assert(!link.target_phase(loaded_child.obj()).is_reclaimed()); + assert(false); + }, + _ => {}, + } + } let res = Self { obj_ptr, link, @@ -385,7 +401,7 @@ impl<'a, P> LinkedListChildReadGuard<'a, P> where P: NonNullPtr LinkedListChildReadGuard<'a, P> where P: NonNullPtr where { pointer: NonNull, link: RcuLinkedListAtomicLink

, + ghost_target_obj: Ghost, tracked_object: Tracked>, tracked_claim: Tracked>, ghost_removal: Ghost, @@ -537,14 +554,20 @@ impl

LinkedListDropCallbackContext

where #[verifier::type_invariant] closed spec fn type_inv(self) -> bool { &&& self.link.well_formed() - &&& (self.link.child_phase().is_retired() || self.link.child_phase().is_reclaimed()) + &&& self.link.registered_targets().contains_pair( + self.ghost_target_obj@, + self.pointer.view_ptr_mut(), + ) + &&& self.link.target_lifecycles().contains_key(self.ghost_target_obj@) + &&& (self.link.target_phase(self.ghost_target_obj@).is_retired() || self.link.target_phase( + self.ghost_target_obj@, + ).is_reclaimed()) &&& self.tracked_object@.wf() &&& equal(self.tracked_object@.ptr(), self.pointer.view_ptr_mut()) &&& self.tracked_object@.domain() == self.link.constant().domain - &&& self.tracked_object@.obj() == self.link.constant().child_obj - &&& equal(self.tracked_object@.ptr(), self.link.constant().child) + &&& self.tracked_object@.obj() == self.ghost_target_obj@ &&& self.ghost_scheduler@ == self.link.constant().scheduler - &&& match self.link.child_phase() { + &&& match self.link.target_phase(self.ghost_target_obj@) { crate::specs::sync::weak_memory::LinkedListChildPhase::Retired { index: _, removal } | crate::specs::sync::weak_memory::LinkedListChildPhase::Reclaimed { index: _, @@ -555,7 +578,7 @@ impl

LinkedListDropCallbackContext

where &&& self.ghost_removal@.root == self.link.constant().root &&& self.ghost_retire_observation_registry@ == self.link.constant().retire_observation_registry - &&& self.link.child_phase().is_retired() + &&& self.link.target_phase(self.ghost_target_obj@).is_retired() &&& self.tracked_claim@.registry() == self.link.constant().reclaim_registry &&& self.tracked_claim@.obj() == self.tracked_object@.obj() &&& self.tracked_claim@.is_pending() @@ -584,20 +607,23 @@ impl

RawCallbackContextWithProof for LinkedListDro let ghost callback = permit.callback(); assert(self.permit_matches(permit)); assert(permit.authorizes(callback)); - assert(self.link.child_phase().is_retired()); + assert(self.link.target_phase(self.ghost_target_obj@).is_retired()); completed = permit.tracked_into_reclaimed_witness(callback); assert(completed.wf()); assert(completed.scheduler() == self.link.constant().scheduler); assert(completed.record() == callback.retired_record()); assert(completed.record().domain == self.link.constant().domain); - assert(completed.record().obj == self.link.constant().child_obj); + assert(completed.record().obj == self.ghost_target_obj@); assert(completed.record().retire_observation_registry == self.link.constant().retire_observation_registry); - assert(completed.record().removal == self.link.child_phase()->Retired_removal); + assert(completed.record().removal == self.link.target_phase( + self.ghost_target_obj@, + )->Retired_removal); } let LinkedListDropCallbackContext { pointer, mut link, + ghost_target_obj, tracked_object: _, tracked_claim, ghost_removal: _, @@ -605,7 +631,14 @@ impl

RawCallbackContextWithProof for LinkedListDro ghost_scheduler: _, } = self; proof { - permission = link.tracked_reclaim_retired_child(tracked_claim.get(), completed, credit); + permission = + link.tracked_reclaim_retired_target( + pointer.as_ptr(), + ghost_target_obj@, + tracked_claim.get(), + completed, + credit, + ); assert(RcuPointerOwnership::

::owns(pointer.as_ptr(), permission)); assert(P::ptr_perm_match(pointer.as_ptr(), permission)); assert(permission.inv()); @@ -948,6 +981,8 @@ fn callback_from_detached( /// physical permission. fn callback_from_linked_list_child

( link: RcuLinkedListAtomicLink

, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, Tracked(retired): Tracked, ) -> (res: ( RawCallbackWithProof, @@ -955,23 +990,25 @@ fn callback_from_linked_list_child

( )) where P: NonNullPtr + Send requires link.well_formed(), - link.child_phase().is_retired(), + link.registered_targets().contains_pair(target_obj, target), + link.target_lifecycles().contains_key(target_obj), + link.target_phase(target_obj).is_retired(), retired.object().wf(), retired.object().domain() == link.constant().domain, - retired.object().obj() == link.constant().child_obj, - equal(retired.object().ptr(), link.constant().child), + retired.object().obj() == target_obj, + equal(retired.object().ptr(), target), retired.claim().registry() == link.constant().reclaim_registry, - retired.claim().obj() == link.constant().child_obj, + retired.claim().obj() == target_obj, retired.claim().is_pending(), - equal(retired.claim().ptr(), link.constant().child), - retired.retired().removal() == link.child_phase()->Retired_removal, - link.child_phase()->Retired_removal.root == link.constant().root, + equal(retired.claim().ptr(), target), + retired.retired().removal() == link.target_phase(target_obj)->Retired_removal, + link.target_phase(target_obj)->Retired_removal.root == link.constant().root, retired.retired().retire_observation_registry() == link.constant().retire_observation_registry, ensures res.1@.domain() == link.constant().domain, - res.1@.obj() == link.constant().child_obj, - res.1@.removal() == link.child_phase()->Retired_removal, + res.1@.obj() == target_obj, + res.1@.removal() == link.target_phase(target_obj)->Retired_removal, res.1@.retire_observation_registry() == link.constant().retire_observation_registry, forall|permit: monitor::RcuReclaimPermit| permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj @@ -983,29 +1020,24 @@ fn callback_from_linked_list_child

( proof_decl! { let tracked (object, cert, claim) = retired.tracked_certify_callback(); } - let child_ptr = link.child_raw(); proof { - link.lemma_well_formed_facts(); + object.lemma_wf_facts(); assert(link.well_formed()); - assert(link.child_ptr().addr() != 0); - assert(equal(child_ptr, link.child_ptr())); - assert(child_ptr.addr() != 0); + assert(target.addr() != 0); } - let pointer = unsafe { NonNull::new_unchecked(child_ptr) }; + let pointer = unsafe { NonNull::new_unchecked(target) }; proof { - assert(equal(pointer.view_ptr_mut(), child_ptr)); - assert(link.child_ptr().addr() != 0); + assert(equal(pointer.view_ptr_mut(), target)); assert(object.wf()); - assert(link.child_phase() is Retired); + assert(link.target_phase(target_obj) is Retired); assert(equal(object.ptr(), pointer.view_ptr_mut())); - assert(equal(object.ptr(), link.constant().child)); assert(object.domain() == link.constant().domain); - assert(object.obj() == link.constant().child_obj); + assert(object.obj() == target_obj); assert(claim.registry() == link.constant().reclaim_registry); assert(claim.obj() == object.obj()); assert(claim.is_pending()); assert(equal(claim.ptr(), pointer.view_ptr_mut())); - assert(cert.removal() == link.child_phase()->Retired_removal); + assert(cert.removal() == link.target_phase(target_obj)->Retired_removal); assert(cert.removal().root == link.constant().root); assert(cert.retire_observation_registry() == link.constant().retire_observation_registry); } @@ -1013,6 +1045,7 @@ fn callback_from_linked_list_child

( let context = LinkedListDropCallbackContext::

{ pointer, link, + ghost_target_obj: Ghost(target_obj), tracked_object: Tracked(object), tracked_claim: Tracked(claim), ghost_removal: Ghost(cert.removal()), @@ -1030,6 +1063,8 @@ fn callback_from_linked_list_child

( /// case until a production data-structure adapter chooses its public API. fn after_grace_period_linked_list_child

( link: RcuLinkedListAtomicLink

, + target: *mut rcu_spec::LinkedListNode, + Ghost(target_obj): Ghost, Tracked(retired): Tracked, Tracked(session): Tracked<&mut RunningTaskContext>, ) where P: NonNullPtr + Send @@ -1038,17 +1073,19 @@ fn after_grace_period_linked_list_child

( old(session).scheduler() == rcu_spec::rcu_scheduler(), link.well_formed(), link.constant().scheduler == old(session).scheduler(), - link.child_phase().is_retired(), + link.registered_targets().contains_pair(target_obj, target), + link.target_lifecycles().contains_key(target_obj), + link.target_phase(target_obj).is_retired(), retired.object().wf(), retired.object().domain() == link.constant().domain, - retired.object().obj() == link.constant().child_obj, - equal(retired.object().ptr(), link.constant().child), + retired.object().obj() == target_obj, + equal(retired.object().ptr(), target), retired.claim().registry() == link.constant().reclaim_registry, - retired.claim().obj() == link.constant().child_obj, + retired.claim().obj() == target_obj, retired.claim().is_pending(), - equal(retired.claim().ptr(), link.constant().child), - retired.retired().removal() == link.child_phase()->Retired_removal, - link.child_phase()->Retired_removal.root == link.constant().root, + equal(retired.claim().ptr(), target), + retired.retired().removal() == link.target_phase(target_obj)->Retired_removal, + link.target_phase(target_obj)->Retired_removal.root == link.constant().root, retired.retired().removal().observed_by(old(session).irc11_view()), retired.retired().retire_observation_registry() == link.constant().retire_observation_registry, @@ -1066,7 +1103,12 @@ fn after_grace_period_linked_list_child

( final(session).rcu_participant_view() == old(session).rcu_participant_view(), final(session).rcu_fraction() == old(session).rcu_fraction(), { - let (callback, cert) = callback_from_linked_list_child::

(link, Tracked(retired)); + let (callback, cert) = callback_from_linked_list_child::

( + link, + target, + Ghost(target_obj), + Tracked(retired), + ); if let Some(monitor) = RCU_MONITOR.get() { #[verus_spec(with Tracked(session))] monitor.after_grace_period(callback, cert); diff --git a/ostd/src/task/preempt/guard.rs b/ostd/src/task/preempt/guard.rs index a2178ec76..a58e4f19d 100644 --- a/ostd/src/task/preempt/guard.rs +++ b/ostd/src/task/preempt/guard.rs @@ -20,7 +20,7 @@ verus! { broadcast use vstd::thread_view::group_thread_view_axioms; -pub const PREEMPT_SESSION_FRACTIONS: u64 = 1 << 31; +pub const PREEMPT_SESSION_FRACTIONS: usize = 1 << 31; /// Proof token carried by a nested preemption-disable guard. /// @@ -71,7 +71,7 @@ impl PreemptSessionToken { ensures res.wf(), { - assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); let tracked mut tokens = CountGhostResource::< PreemptSessionState, @@ -144,7 +144,7 @@ impl PreemptThreadViewSession { res.wf_session_resource(), res.wf(sched_view), { - assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); assert(PREEMPT_SESSION_FRACTIONS > 1) by (compute); let task = task_view.task(); let ghost state = PreemptSessionState { task, quiescent_generation: 0 }; @@ -258,7 +258,7 @@ impl PreemptThreadViewSession { final(self).available_fractions() == old(self).available_fractions() + token.frac(), final(self).wf_session_resource(), { - assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); let ghost old_frac = self.tokens.frac(); let tracked PreemptSessionToken { token } = token; let ghost returned_frac = token.frac(); @@ -496,7 +496,7 @@ impl RunningTaskContext { preempt_depth: Ghost(0), cpu: Ghost(cpu), }; - assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); assert(res.wf()); assert(res.session.wf(sched_view)); assert(res.wf_scheduler(sched_view)); @@ -1134,7 +1134,7 @@ impl RunningTaskContext { PreemptGuardResource::Nested { session: token, nested } }; self.preempt_depth = Ghost(depth_before + 1); - assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); assert(self.wf()); resource } @@ -1165,7 +1165,7 @@ impl RunningTaskContext { let ghost old_depth = self.preempt_depth@; resource.tracked_return_to_session(&mut self.session); self.preempt_depth = Ghost((old_depth - 1) as nat); - assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000u64) by (compute); + assert(PREEMPT_SESSION_FRACTIONS == 0x8000_0000usize) by (compute); assert(self.wf()); } } diff --git a/verified_libs/vstd_extra/src/rcu_read_pool.rs b/verified_libs/vstd_extra/src/rcu_read_pool.rs index a82882622..8ea1dec15 100644 --- a/verified_libs/vstd_extra/src/rcu_read_pool.rs +++ b/verified_libs/vstd_extra/src/rcu_read_pool.rs @@ -7,7 +7,7 @@ //! only after all leases have been returned and the pool fraction is whole. use vstd::{ prelude::*, - resource::{Loc, frac_opt::Frac}, + resource::{frac_opt::Frac, Loc}, }; verus! { @@ -421,7 +421,7 @@ impl RcuReadPoolRegistry { /// Relates registry membership to the complete key set for all keys. pub proof fn lemma_all_contains_iff_keys(tracked &self) ensures - forall|key: K| #[trigger] self.keys().contains(key) ==> self.contains(key), + forall|key: K| #[trigger] self.contains(key) <==> self.keys().contains(key), { } @@ -544,7 +544,7 @@ impl RcuTrackedReadPoolRegistry { /// Relates registry membership to the complete key set for all keys. pub proof fn lemma_all_contains_iff_keys(tracked &self) ensures - forall|key: K| #[trigger] self.keys().contains(key) ==> self.contains(key), + forall|key: K| #[trigger] self.contains(key) <==> self.keys().contains(key), { } From e3c620c29e9a21750232b9148a14c4e8b4a5cf7b Mon Sep 17 00:00:00 2001 From: Hiroki Date: Mon, 10 Aug 2026 23:31:54 -0400 Subject: [PATCH 45/47] Add lease-retaining RCU read API --- ostd/src/sync/rcu/mod.rs | 42 +++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 09891bfa1..1d2ab38c4 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -128,12 +128,13 @@ //! Before invoking a callback, its reclaim permit excludes every active lease //! for the retired allocation. Reclamation then recovers the complete //! `P::Permission` from the root invariant and passes it to the typed callback. -//! Two language-integration boundaries remain explicit: `read_with()` uses -//! `assume_shared_ref` until external atomic-mode guards expose the same lease -//! protocol, and verified callers use the consuming guard `drop()` method -//! because Verus cannot yet attach this invariant-opening transition to Rust's -//! implicit `Drop::drop(&mut self)`. Runtime destruction still restores the -//! executable preemption counter through `DisabledPreemptGuard`. +//! Two language-integration boundaries remain explicit: the legacy +//! `read_with()` compatibility API uses `assume_shared_ref`, while new callers +//! can use `read_with_guard()` to retain the physical lease in a verified read +//! guard. Verified callers use the consuming guard `drop()` method because +//! Verus cannot yet attach this invariant-opening transition to Rust's implicit +//! `Drop::drop(&mut self)`. Runtime destruction still restores the executable +//! preemption counter through `DisabledPreemptGuard`. use alloc::boxed::Box; use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; @@ -157,7 +158,7 @@ use crate::{ task::InAtomicMode, }, sync::Once, - task::{disable_preempt_in_context, DisabledPreemptGuard, RunningTaskContext}, + task::{DisabledPreemptGuard, RunningTaskContext, disable_preempt_in_context}, }; use vstd_extra::atomic_irc11::{ThreadViewOrder, ViewSeen}; @@ -2072,6 +2073,33 @@ impl RcuOption

{ RcuOptionReadGuard(self.0.read(Tracked(session))) } + /// Acquires the current pointer while an external atomic-mode guard is live. + /// + /// Unlike the legacy [`Self::read_with`] compatibility API, this method + /// returns an RCU read guard that retains the loaded allocation's physical + /// read lease. Call [`RcuOptionReadGuard::get`] to borrow the pointer and + /// consume [`RcuOptionReadGuard::drop`] before the external guard expires. + /// The returned guard owns a nested preemption-disable scope, so its lease + /// protocol does not rely on an unverified projection from `InAtomicMode`. + #[inline] + #[verus_spec(res => + with + Tracked(session): Tracked<&'a mut RunningTaskContext>, + requires + old(session).wf(), + old(session).scheduler() == rcu_spec::rcu_scheduler(), + old(session).available_fractions() > 1, + )] + pub fn read_with_guard<'a, A: InAtomicMode>(&'a self, _guard: &'a A) -> RcuOptionReadGuard< + 'a, + P, + > { + proof { + use_type_invariant(self); + } + RcuOptionReadGuard(self.0.read(Tracked(session))) + } + #[inline] #[verus_spec( with From c90ab27e6992c83941b0d8ec0dbb0085f58d4135 Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 13 Aug 2026 01:57:27 -0400 Subject: [PATCH 46/47] Fix weak-memory CI bootstrap --- .github/workflows/ci-macos.yml | 4 +- .github/workflows/ci-upstream-verus.yml | 2 +- .github/workflows/ci.yml | 8 ++-- .github/workflows/doc.yml | 8 ++-- ostd/specs/sync/rcu.rs | 4 +- ostd/specs/sync/rcu_cpu.rs | 6 ++- ostd/specs/sync/weak_memory.rs | 2 +- ostd/src/sync/rcu/mod.rs | 6 ++- ostd/src/sync/rcu/monitor.rs | 7 +++- tools/bootstrap-verus-irc11.sh | 42 +++++++++++++++++++ verified_libs/vstd_extra/src/raw_callback.rs | 2 +- verified_libs/vstd_extra/src/rcu_read_pool.rs | 29 ++++++++----- 12 files changed, 91 insertions(+), 29 deletions(-) create mode 100644 tools/bootstrap-verus-irc11.sh diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index d5cba5ac9..bc3f49726 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -76,7 +76,7 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-irc11-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} + key: ${{ runner.os }}-verus-irc11-weak-memory-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/bootstrap-verus-irc11.sh', 'tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Bootstrap Verus (if needed) shell: bash @@ -92,7 +92,7 @@ jobs: git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" - cargo dv bootstrap + bash tools/bootstrap-verus-irc11.sh fi test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" test -f tools/verus/source/vstd/atomic_weak.rs diff --git a/.github/workflows/ci-upstream-verus.yml b/.github/workflows/ci-upstream-verus.yml index 36f338f20..3a821ce35 100644 --- a/.github/workflows/ci-upstream-verus.yml +++ b/.github/workflows/ci-upstream-verus.yml @@ -80,7 +80,7 @@ jobs: git -C tools/verus fetch --depth=1 origin "$VERUS_BASE_COMMIT" git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" - cargo dv bootstrap + bash tools/bootstrap-verus-irc11.sh test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2871da9b9..e6c3abe89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-irc11-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} + key: ${{ runner.os }}-verus-irc11-weak-memory-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/bootstrap-verus-irc11.sh', 'tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt @@ -106,12 +106,12 @@ jobs: git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" - cargo dv bootstrap + bash tools/bootstrap-verus-irc11.sh fi if ! command -v verusfmt >/dev/null 2>&1; then - echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap + echo "verusfmt not found, installing..." + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/verus-lang/verusfmt/releases/latest/download/verusfmt-installer.sh | sh fi test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" test -f tools/verus/source/vstd/atomic_weak.rs diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index dc0b38588..93a497ae9 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -86,7 +86,7 @@ jobs: uses: actions/cache@v6 with: path: tools/verus - key: ${{ runner.os }}-verus-irc11-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} + key: ${{ runner.os }}-verus-irc11-weak-memory-${{ env.VERUS_BASE_COMMIT }}-${{ hashFiles('tools/bootstrap-verus-irc11.sh', 'tools/patches/verus-irc11.patch', 'tools/patches/verus-irc11-vstd.patch') }} - name: Cache verusfmt id: cache-verusfmt @@ -108,12 +108,12 @@ jobs: git -C tools/verus checkout --detach FETCH_HEAD git -C tools/verus apply "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" git -C tools/verus apply --reverse --check "$GITHUB_WORKSPACE/$VERUS_IRC11_PATCH" - cargo dv bootstrap + bash tools/bootstrap-verus-irc11.sh fi if ! command -v verusfmt >/dev/null 2>&1; then - echo "verusfmt not found, installing via cargo dv bootstrap..." - cargo dv bootstrap + echo "verusfmt not found, installing..." + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/verus-lang/verusfmt/releases/latest/download/verusfmt-installer.sh | sh fi test "$(git -C tools/verus rev-parse HEAD)" = "$VERUS_BASE_COMMIT" test -f tools/verus/source/vstd/atomic_weak.rs diff --git a/ostd/specs/sync/rcu.rs b/ostd/specs/sync/rcu.rs index d4f69e4c8..738bccea2 100644 --- a/ostd/specs/sync/rcu.rs +++ b/ostd/specs/sync/rcu.rs @@ -36,8 +36,8 @@ use crate::specs::mm::cpu::CpuId; use vstd::invariant::InvariantPredicate; use vstd::prelude::*; -use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; use vstd::resource::Loc; +use vstd::resource::map::{GhostMapAuth, GhostPersistentPointsTo, GhostPointsTo}; use vstd::thread_view::Objective; use vstd_extra::atomic_irc11::{ AtomicHistory as Irc11History, AtomicId as Irc11AtomicId, AtomicPointsTo, @@ -1666,6 +1666,7 @@ pub proof fn rcu_monitor_flag_initial_inv( { assert(history.dom() == Set::empty().insert(timestamp)) by { assert forall|ts: nat| + #![auto] history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { if history.dom().contains(ts) { assert(history.contains_timestamp(ts)); @@ -4617,6 +4618,7 @@ impl LinkedListAtomicLinkGhost { assert(history.is_max_timestamp(timestamp)); assert(history.dom() == Set::empty().insert(timestamp)) by { assert forall|ts: nat| + #![auto] history.dom().contains(ts) <==> Set::empty().insert(timestamp).contains(ts) by { if history.dom().contains(ts) { assert(history.contains_timestamp(ts)); diff --git a/ostd/specs/sync/rcu_cpu.rs b/ostd/specs/sync/rcu_cpu.rs index 08a9906ad..c3e339322 100644 --- a/ostd/specs/sync/rcu_cpu.rs +++ b/ostd/specs/sync/rcu_cpu.rs @@ -46,20 +46,20 @@ //! counter and is not an authority for this persistent CPU generation. Reader //! contexts obtain their CPU generation from [`CpuRcuReaderFragment`]. use crate::specs::{ - mm::cpu::{online_cpus, CpuId}, + mm::cpu::{CpuId, online_cpus}, task::cpu_core::{CpuCoreLocalState, CpuCoreOwner, CpuCoreOwnerBinding, CpuCoreRegistration}, }; use vstd::{ modes::tracked_swap, prelude::*, resource::{ + Loc, agree::AgreementRA, algebra::{Resource, ResourceAlgebra}, frac::FractionRA, map::{GhostMapAuth, GhostPointsTo}, product::ProductRA, relations::frame_preserving_update_opt, - Loc, }, }; @@ -2783,8 +2783,10 @@ impl RcuRootPermissionState { { if self.has_active(obj) { assert(exists|lease_id: nat| + #![auto] self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == obj); let ghost lease_id = choose|lease_id: nat| + #![auto] self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == obj; let ghost record = self.active_record(lease_id); assert(record.key() == obj); diff --git a/ostd/specs/sync/weak_memory.rs b/ostd/specs/sync/weak_memory.rs index a153470e8..5689aabaa 100644 --- a/ostd/specs/sync/weak_memory.rs +++ b/ostd/specs/sync/weak_memory.rs @@ -12,8 +12,8 @@ use crate::specs::mm::cpu::online_cpus; use vstd::invariant::{AtomicInvariant, InvariantPredicate}; use vstd::modes::tracked_static_ref; use vstd::prelude::*; -use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; use vstd::resource::Loc; +use vstd::resource::ghost_var::{GhostVar, GhostVarAuth}; use vstd::thread_view::Objective; use vstd_extra::atomic_irc11::{ AtomicId as Irc11AtomicId, AtomicPointsTo, PAtomicWeakBool as Irc11AtomicBool, PAtomicWeakPtr, diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 1d2ab38c4..5a1aa1d71 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -699,7 +699,7 @@ impl RawCallbackContextWithProof< state.permissions.lemma_active_registry_projection(); let ghost permissions_before_exclusion = state.permissions; let ghost registry_before_exclusion = state.permissions.registry(); - assert forall|lease_id: nat| + assert forall|lease_id: nat| #![auto] state.permissions.active_ids().contains(lease_id) && state.permissions.active_record(lease_id).key() == callback.obj implies { @@ -734,7 +734,7 @@ impl RawCallbackContextWithProof< { let tracked registry = state.permissions.tracked_registry_mut(); assert(*registry == registry_before_exclusion); - assert forall|lease_id: nat| + assert forall|lease_id: nat| #![auto] (*registry).active_ids().contains(lease_id) && (*registry).active_record(lease_id).key() == callback.obj implies { @@ -927,6 +927,7 @@ fn callback_from_detached( ensures res.1@.removal() == owned.retired().removal(), forall|permit: monitor::RcuReclaimPermit| + #![auto] permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj == res.1@.obj() && permit.callback().removal == res.1@.removal() && permit.callback().retire_observation_registry @@ -1012,6 +1013,7 @@ fn callback_from_linked_list_child

( res.1@.removal() == link.target_phase(target_obj)->Retired_removal, res.1@.retire_observation_registry() == link.constant().retire_observation_registry, forall|permit: monitor::RcuReclaimPermit| + #![auto] permit.wf() && permit.callback().domain == res.1@.domain() && permit.callback().obj == res.1@.obj() && permit.callback().removal == res.1@.removal() && permit.callback().retire_observation_registry diff --git a/ostd/src/sync/rcu/monitor.rs b/ostd/src/sync/rcu/monitor.rs index 5e7ac210f..9d8cad538 100644 --- a/ostd/src/sync/rcu/monitor.rs +++ b/ostd/src/sync/rcu/monitor.rs @@ -221,6 +221,7 @@ impl RcuCallback { requires cert.removal().observed_by(retire_view), forall|permit: RcuReclaimPermit| + #![auto] permit.wf() && permit.callback().domain == cert.domain() && permit.callback().obj == cert.obj() && permit.callback().removal == cert.removal() && permit.callback().retire_observation_registry @@ -508,6 +509,7 @@ impl RcuReclaimPermit { self.authorizes(callback), old(registry).wf(), forall|lease_id: nat| + #![auto] old(registry).active_ids().contains(lease_id) && old(registry).active_record( lease_id, ).key() == callback.obj ==> { @@ -529,6 +531,7 @@ impl RcuReclaimPermit { { if old(registry).has_active(callback.obj) { let ghost lease_id = choose|lease_id: nat| + #![auto] old(registry).active_ids().contains(lease_id) && old(registry).active_record( lease_id, ).key() == callback.obj; @@ -1025,6 +1028,7 @@ impl GracePeriod { assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); }; assert forall|i: int| + #![auto] 0 <= i < self.callback_summaries().len() implies self.tracked_closed_generations@[cpu].known_retired().contains( self.callback_summaries()[i].retired_record()) by { @@ -1049,6 +1053,7 @@ impl GracePeriod { assert(self.callback_summaries()[i] == old(self).callback_summaries()[i]); }; assert forall|i: int| + #![auto] 0 <= i < self.callback_summaries().len() implies self.tracked_closed_generations@[cpu].known_retired().contains( self.callback_summaries()[i].retired_record()) by { @@ -1840,7 +1845,7 @@ impl RcuMonitor { old(session).wf(), old(session).scheduler() == rcu_spec::rcu_scheduler(), cert@.removal().observed_by(old(session).irc11_view()), - forall|permit: RcuReclaimPermit| + forall|permit: RcuReclaimPermit| #![auto] permit.wf() && permit.callback().domain == cert@.domain() && permit.callback().obj == cert@.obj() diff --git a/tools/bootstrap-verus-irc11.sh b/tools/bootstrap-verus-irc11.sh new file mode 100644 index 000000000..be7a7cd17 --- /dev/null +++ b/tools/bootstrap-verus-irc11.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +verus_dir="$repo_root/tools/verus" +source_dir="$verus_dir/source" +vargo_manifest="$verus_dir/tools/vargo/Cargo.toml" +vargo_bin="$verus_dir/tools/vargo/target/release/vargo" + +if [[ ! -f "$source_dir/vstd/atomic_weak.rs" ]]; then + echo "The pinned Verus checkout does not contain IRC11 support" >&2 + exit 1 +fi + +if [[ ! -x "$source_dir/z3" ]]; then + ( + cd "$source_dir" + ./tools/get-z3.sh + ) +fi + +cargo build --release --manifest-path "$vargo_manifest" + +# The IRC11 patch predates weak-memory in Vargo's build fingerprint, so force +# vstd to rebuild whenever this bootstrap path is invoked. +rm -f "$source_dir/target-verus/release/.vstd-fingerprint" + +( + cd "$source_dir" + "$vargo_bin" build --release --features singular --vstd-weak-memory + "$vargo_bin" build --release -p verusdoc +) + +# This Verus revision places standalone package builds in Cargo's default +# target directory, while the current dv searches beside the Verus binary. +cp "$source_dir/target/release/verusdoc" "$source_dir/target-verus/release/verusdoc" + +test -x "$source_dir/target-verus/release/verus" +test -x "$source_dir/target-verus/release/verusdoc" +test -f "$source_dir/target-verus/release/verus-root" +test -f "$source_dir/target-verus/release/vstd.vir" diff --git a/verified_libs/vstd_extra/src/raw_callback.rs b/verified_libs/vstd_extra/src/raw_callback.rs index d6b1d1901..2fcf5c39e 100644 --- a/verified_libs/vstd_extra/src/raw_callback.rs +++ b/verified_libs/vstd_extra/src/raw_callback.rs @@ -122,7 +122,7 @@ impl RawCallbackWithProof { #[verifier::external_body] pub fn new>(context: C) -> (res: Self) ensures - forall|proof: S| res.call_requires(proof) == context.call_requires(proof), + forall|proof: S| #![auto] res.call_requires(proof) == context.call_requires(proof), { let payload = Box::new(RawCallbackPayloadWithProof { context, _proof: PhantomData }); Self { diff --git a/verified_libs/vstd_extra/src/rcu_read_pool.rs b/verified_libs/vstd_extra/src/rcu_read_pool.rs index 8ea1dec15..0d068cfb9 100644 --- a/verified_libs/vstd_extra/src/rcu_read_pool.rs +++ b/verified_libs/vstd_extra/src/rcu_read_pool.rs @@ -7,7 +7,7 @@ //! only after all leases have been returned and the pool fraction is whole. use vstd::{ prelude::*, - resource::{frac_opt::Frac, Loc}, + resource::{Loc, frac_opt::Frac}, }; verus! { @@ -276,7 +276,7 @@ proof fn lemma_active_fraction_zero( upto: nat, ) requires - forall|id: nat| id < upto && active.contains_key(id) ==> active[id].key() != key, + forall|id: nat| #![auto] id < upto && active.contains_key(id) ==> active[id].key() != key, ensures active_lease_fraction(active, key, upto) == 0real, decreases upto, @@ -615,6 +615,7 @@ impl RcuTrackedReadPoolRegistry { ).fraction(), final(self).active_record(lease_id).witness() == *final(witness), forall|other: nat| + #![auto] other != lease_id && old(self).active_ids().contains(other) ==> final(self).active_record(other) == old(self).active_record(other), { @@ -624,6 +625,7 @@ impl RcuTrackedReadPoolRegistry { pub open spec fn has_active(self, key: K) -> bool { exists|lease_id: nat| + #![auto] self.active_ids().contains(lease_id) && self.active_record(lease_id).key() == key } @@ -655,6 +657,7 @@ impl RcuTrackedReadPoolRegistry { final(self).active_ids() == old(self).active_ids(), final(self).next_lease() == old(self).next_lease(), forall|lease_id: nat| + #![auto] old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) == old(self).active_record(lease_id), final(self).contains(key), @@ -682,7 +685,7 @@ impl RcuTrackedReadPoolRegistry { ) == 1real by {}; let tracked pool = RcuReadPool::new(resource); self.pools.tracked_insert(key, pool); - assert forall|lease_id: nat| self.active_ids().contains(lease_id) implies { + assert forall|lease_id: nat| #![auto] self.active_ids().contains(lease_id) implies { &&& lease_id < self.next_lease() &&& self.contains(self.active_record(lease_id).key()) &&& self.active_record(lease_id).pool_id() == self.pool( @@ -694,6 +697,7 @@ impl RcuTrackedReadPoolRegistry { assert(old(self).active_record(lease_id).key() != key); }; assert forall|lease_id: nat| + #![auto] lease_id < self.next_lease() && self.active_records().contains_key( lease_id, ) implies self.active_records()[lease_id].key() != key by { @@ -703,7 +707,7 @@ impl RcuTrackedReadPoolRegistry { assert(active_lease_fraction(self.active_records(), key, self.next_lease()) == 0real) by { lemma_active_fraction_zero(self.active_records(), key, self.next_lease()); }; - assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { if other == key { assert(self.pool(key).fraction() == 1real); @@ -734,6 +738,7 @@ impl RcuTrackedReadPoolRegistry { final(self).active_record(lease.lease_id()).fraction() == lease.fraction(), final(self).active_record(lease.lease_id()).witness() == witness, forall|lease_id: nat| + #![auto] old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) == old(self).active_record(lease_id), lease.pool_id() == old(self).pool(key).id(), @@ -767,7 +772,7 @@ impl RcuTrackedReadPoolRegistry { self.active.tracked_insert(lease_id, record); self.next_lease = lease_id + 1; - assert forall|active_id: nat| self.active_ids().contains(active_id) implies { + assert forall|active_id: nat| #![auto] self.active_ids().contains(active_id) implies { &&& active_id < self.next_lease() &&& self.contains(self.active_record(active_id).key()) &&& self.active_record(active_id).pool_id() == self.pool( @@ -783,7 +788,7 @@ impl RcuTrackedReadPoolRegistry { } }; - assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { lemma_active_fraction_insert_next( old(self).active_records(), @@ -830,6 +835,7 @@ impl RcuTrackedReadPoolRegistry { final(self).active_ids() == old(self).active_ids().remove(lease.lease_id()), witness == old(self).active_record(lease.lease_id()).witness(), forall|lease_id: nat| + #![auto] lease_id != lease.lease_id() && old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) == old(self).active_record(lease_id), final(self).pool(lease.key()).id() == old(self).pool(lease.key()).id(), @@ -856,7 +862,7 @@ impl RcuTrackedReadPoolRegistry { let tracked pool = self.pools.tracked_borrow_mut(key); pool.return_lease(lease.lease); - assert forall|active_id: nat| self.active_ids().contains(active_id) implies { + assert forall|active_id: nat| #![auto] self.active_ids().contains(active_id) implies { &&& active_id < self.next_lease() &&& self.contains(self.active_record(active_id).key()) &&& self.active_record(active_id).pool_id() == self.pool( @@ -869,7 +875,7 @@ impl RcuTrackedReadPoolRegistry { assert(self.active_record(active_id) == old(self).active_record(active_id)); }; - assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { lemma_active_fraction_remove( old(self).active_records(), @@ -909,6 +915,7 @@ impl RcuTrackedReadPoolRegistry { final(self).active_records() == old(self).active_records(), final(self).next_lease() == old(self).next_lease(), forall|lease_id: nat| + #![auto] old(self).active_ids().contains(lease_id) ==> final(self).active_record(lease_id) == old(self).active_record(lease_id), !final(self).contains(key), @@ -929,12 +936,14 @@ impl RcuTrackedReadPoolRegistry { old(self).next_lease(), ) == 1real by {}; assert forall|lease_id: nat| + #![auto] lease_id < self.next_lease() && self.active_records().contains_key( lease_id, ) implies self.active_records()[lease_id].key() != key by { if self.active_records()[lease_id].key() == key { assert(self.active_ids().contains(lease_id)); assert(exists|candidate: nat| + #![auto] self.active_ids().contains(candidate) && self.active_record(candidate).key() == key) by { assert(self.active_record(lease_id).key() == key); @@ -946,7 +955,7 @@ impl RcuTrackedReadPoolRegistry { assert(self.pool(key).fraction() == 1real); let tracked pool = self.pools.tracked_remove(key); let tracked resource = pool.reclaim(); - assert forall|lease_id: nat| self.active_ids().contains(lease_id) implies { + assert forall|lease_id: nat| #![auto] self.active_ids().contains(lease_id) implies { &&& lease_id < self.next_lease() &&& self.contains(self.active_record(lease_id).key()) &&& self.active_record(lease_id).pool_id() == self.pool( @@ -957,7 +966,7 @@ impl RcuTrackedReadPoolRegistry { assert(old(self).active_ids().contains(lease_id)); assert(old(self).active_record(lease_id).key() != key); }; - assert forall|other: K| self.contains(other) implies self.pool(other).fraction() + assert forall|other: K| #![auto] self.contains(other) implies self.pool(other).fraction() + active_lease_fraction(self.active_records(), other, self.next_lease()) == 1real by { assert(other != key); assert(old(self).contains(other)); From e47182cd1331f2f4ed9fd0e83ac070fbcd2460fe Mon Sep 17 00:00:00 2001 From: Hiroki Date: Thu, 13 Aug 2026 02:26:31 -0400 Subject: [PATCH 47/47] Align Cargo lock with weak-memory Verus --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92c2f90e6..f1945994d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -595,11 +595,11 @@ dependencies = [ [[package]] name = "verus_builtin" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" [[package]] name = "verus_builtin_macros" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" dependencies = [ "convert_case", "proc-macro2", @@ -612,7 +612,7 @@ dependencies = [ [[package]] name = "verus_prettyplease" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" dependencies = [ "proc-macro2", "verus_syn", @@ -651,7 +651,7 @@ checksum = "af8ca9a5d4debca0633e697c88269395493cebf2e10db21ca2dbde37c1356452" [[package]] name = "vstd" -version = "0.0.0-2026-08-09-0044" +version = "0.0.0-2026-08-02-0125" dependencies = [ "verus_builtin", "verus_builtin_macros",