diff --git a/crates/nokv-client/src/file_client.rs b/crates/nokv-client/src/file_client.rs index 68885b956..6701fd6cc 100644 --- a/crates/nokv-client/src/file_client.rs +++ b/crates/nokv-client/src/file_client.rs @@ -1,5 +1,5 @@ use std::collections::HashSet; -use std::io::Read; +use std::io::{Read, Seek, SeekFrom}; use std::net::SocketAddr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -31,6 +31,7 @@ const MAX_READ_PIPELINES: usize = 1024; const MAX_READ_PLAN_CACHE_ENTRIES: usize = 4096; const MAX_BATCH_READ_WORKERS: usize = 32; const MIN_RANGE_READAHEAD_BYTES: usize = DEFAULT_BLOCK_SIZE / 4; +const MAX_ARTIFACT_OBJECT_GC_REFRESHES: usize = 3; struct BatchReadTask { index: usize, @@ -1728,48 +1729,19 @@ where bytes: Vec, metadata: ArtifactMetadata, ) -> Result { - let prepared = self.metadata.prepare_artifact_path(path, false)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_body(&prepared, &bytes, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result.entry), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_bytes(path, &bytes, metadata, false, None) + .map(|result| result.entry) } - pub fn put_artifact_from_reader( + pub fn put_artifact_from_reader( &self, path: &str, - reader: R, + mut reader: R, metadata: ArtifactMetadata, ) -> Result { let prepared = self.metadata.prepare_artifact_path(path, false)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_reader(&prepared, reader, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result.entry), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_reader(prepared, &mut reader, metadata) + .map(|result| result.entry) } pub fn put_artifact_replace( @@ -1778,48 +1750,17 @@ where bytes: Vec, metadata: ArtifactMetadata, ) -> Result { - let prepared = self.metadata.prepare_artifact_path(path, true)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_body(&prepared, &bytes, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_bytes(path, &bytes, metadata, true, None) } - pub fn put_artifact_replace_from_reader( + pub fn put_artifact_replace_from_reader( &self, path: &str, - reader: R, + mut reader: R, metadata: ArtifactMetadata, ) -> Result { let prepared = self.metadata.prepare_artifact_path(path, true)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_reader(&prepared, reader, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_reader(prepared, &mut reader, metadata) } /// Publish `delta` at the current end of `path` as a delta generation: only @@ -1863,7 +1804,7 @@ where let new_size = base_size .checked_add(delta.len() as u64) .ok_or_else(|| ClientError::Protocol("append size exceeds u64".to_owned()))?; - let prepared = self + let mut prepared = self .metadata .prepare_artifact_path(path, true) .map_err(|err| fold_append_prepare_not_found(err, expected_generation))?; @@ -1872,63 +1813,92 @@ where // would be wrong, so fail before writing any blocks. return Err(artifact_write_conflict()); } - let dirty_ranges = if delta.is_empty() { - Vec::new() - } else { - vec![ChunkWriteRange { - logical_offset: base_size, - bytes: delta.into(), - }] - }; - let written = match self.objects.write_ranges_with_block_index_base( - dirty_ranges, - ChunkWriteOptions { - manifest_id: metadata.manifest_id.clone(), - mount: prepared.mount, - inode: prepared.inode.get(), - generation: prepared.generation, - chunk_size: DEFAULT_CHUNK_SIZE, - block_size: DEFAULT_BLOCK_SIZE, - }, - 0, - ) { - Ok(written) => written, - Err(err) => { - cleanup_staged_write_error(&self.objects, &err)?; - return Err(ClientError::Object(err)); - } - }; - self.record_staged_write_stats(&written); - let staged = written.staged_objects()?; - let session = PublishArtifactStagedSession { - parent: prepared.parent, - name: prepared.name.clone(), - producer: metadata.producer, - digest_uri: metadata.digest_uri, - content_type: metadata.content_type, - manifest_id: written.manifest_id.clone(), - size: new_size, - chunks: written.chunk_manifests(), - staged: staged.clone(), - mode: metadata.mode, - uid: metadata.uid, - gid: metadata.gid, - }; - match self - .metadata - .publish_prepared_artifact_staged_session(prepared, session) - { - Ok(result) => Ok(AppendOutcome { - new_size: result.entry.attr.size, - generation: artifact_body_generation(&result.entry), - created: false, - }), - Err(err) => { - // Best-effort cleanup: a failed staged-object delete must not - // replace the publish error, whose classification - // (is_artifact_write_conflict) drives the caller's retry loop. - let _ = self.objects.delete_staged(&staged); + let mut refreshes = 0; + loop { + let dirty_ranges = if delta.is_empty() { + Vec::new() + } else { + vec![ChunkWriteRange { + logical_offset: base_size, + bytes: delta.clone().into(), + }] + }; + let written = match self.objects.write_ranges_with_block_index_base( + dirty_ranges, + ChunkWriteOptions { + manifest_id: metadata.manifest_id.clone(), + mount: prepared.mount, + inode: prepared.inode.get(), + generation: prepared.generation, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE, + }, + 0, + ) { + Ok(written) => written, + Err(err) => { + cleanup_staged_write_error(&self.objects, &err)?; + return Err(ClientError::Object(err)); + } + }; + self.record_staged_write_stats(&written); + let staged = written.staged_objects()?; + let session = PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: metadata.producer.clone(), + digest_uri: metadata.digest_uri.clone(), + content_type: metadata.content_type.clone(), + manifest_id: written.manifest_id.clone(), + size: new_size, + chunks: written.chunk_manifests(), + staged: staged.clone(), + mode: metadata.mode, + uid: metadata.uid, + gid: metadata.gid, + }; + match self + .metadata + .publish_prepared_artifact_staged_session(prepared.clone(), session) + { + Ok(result) => { + return Ok(AppendOutcome { + new_size: result.entry.attr.size, + generation: artifact_body_generation(&result.entry), + created: false, + }); + } Err(err) + if is_stale_prepared_object_gc_epoch(&err) + && refreshes < MAX_ARTIFACT_OBJECT_GC_REFRESHES => + { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + prepared = self + .metadata + .refresh_prepared_artifact_object_gc_epoch(prepared)?; + refreshes += 1; + } + Err(err) if is_stale_prepared_object_gc_epoch(&err) => { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + return Err(err); + } + Err(err) => { + // A missing/undecodable response can arrive after the + // metadata transaction committed. Deleting in that case + // would turn a successful publish into a dangling object + // reference, so only clean up explicit pre-commit rejects. + if artifact_publish_was_definitively_rejected(&err) { + // Keep the metadata error that drives the caller's + // conflict classifier; terminal cleanup remains + // best-effort after no new generation will be minted. + let _ = self.objects.delete_staged(&staged); + } + return Err(err); + } } } } @@ -1944,25 +1914,115 @@ where metadata: ArtifactMetadata, expected_generation: u64, ) -> Result { - let prepared = self.metadata.prepare_artifact_path(path, true)?; - if prepared.old_generation != Some(expected_generation) { + self.put_artifact_bytes(path, &bytes, metadata, true, Some(expected_generation)) + } + + /// Execute a replayable artifact transaction. When object GC advances + /// after prepare, the failed generation remains unreachable: remove it, + /// refresh the same namespace proof, and fully restage under the newly + /// minted generation before publishing again. + fn put_artifact_bytes( + &self, + path: &str, + bytes: &[u8], + metadata: ArtifactMetadata, + replace: bool, + expected_generation: Option, + ) -> Result { + let mut prepared = self.metadata.prepare_artifact_path(path, replace)?; + if expected_generation.is_some() && prepared.old_generation != expected_generation { return Err(artifact_write_conflict()); } - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_body(&prepared, &bytes, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result), - Err(err) => { - // Best-effort cleanup: a failed staged-object delete must not - // replace the publish error, whose classification - // (is_artifact_write_conflict) drives the caller's retry loop. - let _ = self.objects.delete_staged(&staged); - Err(err) + let mut refreshes = 0; + loop { + let (body, chunks, staged) = + self.stage_artifact_body(&prepared, bytes, metadata.clone())?; + match self.metadata.publish_prepared_artifact( + prepared.clone(), + body, + chunks, + metadata.mode, + metadata.uid, + metadata.gid, + ) { + Ok(result) => return Ok(result), + Err(err) if is_stale_prepared_object_gc_epoch(&err) => { + // A refreshed generation is safe only after proving the + // old unattached generation no longer has staged objects. + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + if refreshes >= MAX_ARTIFACT_OBJECT_GC_REFRESHES { + return Err(err); + } + prepared = self + .metadata + .refresh_prepared_artifact_object_gc_epoch(prepared)?; + refreshes += 1; + } + Err(err) => { + if artifact_publish_was_definitively_rejected(&err) { + if expected_generation.is_some() { + // Preserve the compare-and-swap failure classification. + let _ = self.objects.delete_staged(&staged); + } else { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + } + } + return Err(err); + } + } + } + } + + fn put_artifact_reader( + &self, + mut prepared: ClientPreparedArtifact, + reader: &mut R, + metadata: ArtifactMetadata, + ) -> Result { + let initial_position = reader + .stream_position() + .map_err(|err| ClientError::Io(err.to_string()))?; + let mut refreshes = 0; + loop { + reader + .seek(SeekFrom::Start(initial_position)) + .map_err(|err| ClientError::Io(err.to_string()))?; + let (body, chunks, staged) = + self.stage_artifact_reader(&prepared, &mut *reader, metadata.clone())?; + match self.metadata.publish_prepared_artifact( + prepared.clone(), + body, + chunks, + metadata.mode, + metadata.uid, + metadata.gid, + ) { + Ok(result) => return Ok(result), + Err(err) if is_stale_prepared_object_gc_epoch(&err) => { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + if refreshes < MAX_ARTIFACT_OBJECT_GC_REFRESHES { + prepared = self + .metadata + .refresh_prepared_artifact_object_gc_epoch(prepared)?; + refreshes += 1; + continue; + } + return Err(err); + } + Err(err) => { + if artifact_publish_was_definitively_rejected(&err) { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + } + return Err(err); + } } } } @@ -2244,6 +2304,63 @@ pub fn is_artifact_write_conflict(err: &ClientError) -> bool { ) } +/// True when restarting an artifact write from its read/prepare boundary is +/// safe. A stale object-GC epoch is retryable because the file client removes +/// the unattached generation before returning that error. +pub fn is_artifact_write_retryable(err: &ClientError) -> bool { + is_artifact_write_conflict(err) || is_stale_prepared_object_gc_epoch(err) +} + +fn is_stale_prepared_object_gc_epoch(err: &ClientError) -> bool { + matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { .. }) + ) +} + +/// Return true only when the server has authoritatively rejected the publish +/// before committing metadata. Transport/protocol failures and post-commit +/// sync-log failures are deliberately absent: keeping an unreachable staged +/// generation is safe, while deleting a possibly live generation is not. +fn artifact_publish_was_definitively_rejected(err: &ClientError) -> bool { + match err { + ClientError::Metadata(err) => metad_publish_was_definitively_rejected(err), + ClientError::LockConflict(_) | ClientError::Object(_) => true, + _ => false, + } +} + +fn metad_publish_was_definitively_rejected(err: &nokv_meta::MetadError) -> bool { + match err { + nokv_meta::MetadError::Metadata(err) => matches!( + err, + nokv_meta::MetadataError::ZeroVersion + | nokv_meta::MetadataError::CommitBeforeRead + | nokv_meta::MetadataError::EmptyRequestId + | nokv_meta::MetadataError::EmptyPrimaryKey + | nokv_meta::MetadataError::PutWithoutValue + | nokv_meta::MetadataError::DeleteWithValue + | nokv_meta::MetadataError::PredicateFailed + ), + nokv_meta::MetadError::SyncLogArchiveFailed { committed, .. } => !committed, + nokv_meta::MetadError::PublishArtifactFailed { source, .. } => { + metad_publish_was_definitively_rejected(source) + } + nokv_meta::MetadError::BodySizeMismatch { .. } + | nokv_meta::MetadError::InvalidPreparedArtifact(_) + | nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { .. } + | nokv_meta::MetadError::StaleBodyGeneration { .. } + | nokv_meta::MetadError::LockConflict(_) + | nokv_meta::MetadError::InvalidPath(_) + | nokv_meta::MetadError::NotFound + | nokv_meta::MetadError::NotFile + | nokv_meta::MetadError::NotDirectory + | nokv_meta::MetadError::MissingBodyDescriptor + | nokv_meta::MetadError::InvalidOwnerEpoch => true, + _ => false, + } +} + // Client-side guard failures surface as the same predicate error a server-side // publish CAS loss decodes to, so one discriminant covers both. fn artifact_write_conflict() -> ClientError { @@ -2676,6 +2793,83 @@ mod tests { ))); } + #[test] + fn artifact_write_retryable_includes_stale_object_gc_epoch() { + let stale = + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { + expected: 7, + current: 9, + }); + assert!(!is_artifact_write_conflict(&stale)); + assert!(is_artifact_write_retryable(&stale)); + assert!(is_artifact_write_retryable(&ClientError::Metadata( + nokv_meta::MetadError::Metadata(nokv_meta::MetadataError::PredicateFailed), + ))); + assert!(!is_artifact_write_retryable(&ClientError::Metadata( + nokv_meta::MetadError::NotFile, + ))); + } + + #[test] + fn artifact_publish_cleanup_classifier_fails_closed_for_uncertain_outcomes() { + let predicate = ClientError::Metadata(nokv_meta::MetadError::Metadata( + nokv_meta::MetadataError::PredicateFailed, + )); + let stale = + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { + expected: 7, + current: 9, + }); + let rejected_before_commit = + ClientError::Metadata(nokv_meta::MetadError::SyncLogArchiveFailed { + committed: false, + message: "injected".to_owned(), + }); + assert!(artifact_publish_was_definitively_rejected(&predicate)); + assert!(artifact_publish_was_definitively_rejected(&stale)); + assert!(artifact_publish_was_definitively_rejected( + &rejected_before_commit + )); + + let backend = ClientError::Metadata(nokv_meta::MetadError::Metadata( + nokv_meta::MetadataError::Backend("outcome unknown".to_owned()), + )); + let failed_after_commit = + ClientError::Metadata(nokv_meta::MetadError::SyncLogArchiveFailed { + committed: true, + message: "injected".to_owned(), + }); + assert!(!artifact_publish_was_definitively_rejected(&backend)); + assert!(!artifact_publish_was_definitively_rejected( + &failed_after_commit + )); + assert!(!artifact_publish_was_definitively_rejected( + &ClientError::Io("response lost".to_owned()) + )); + assert!(!artifact_publish_was_definitively_rejected( + &ClientError::Protocol("response undecodable".to_owned()) + )); + for ownership_error in [ + ClientError::Metadata(nokv_meta::MetadError::StaleOwnerEpoch { + owner_epoch: 7, + required_epoch: 8, + }), + ClientError::Metadata(nokv_meta::MetadError::LeaseExpired { + now_ms: 101, + deadline_ms: 100, + }), + ClientError::Metadata(nokv_meta::MetadError::NotOwner { + shard_id: "mount-1:/".to_owned(), + endpoint: None, + }), + ] { + assert!( + !artifact_publish_was_definitively_rejected(&ownership_error), + "ownership errors can surface after metadata commit: {ownership_error}" + ); + } + } + /// A file deleted between `lookup` and `prepare` inside `append_artifact` /// must classify as a retryable conflict when no generation is pinned, so /// the caller's retry loop re-reads and takes the create branch. diff --git a/crates/nokv-client/src/lib.rs b/crates/nokv-client/src/lib.rs index fd5a17458..99c3b3268 100644 --- a/crates/nokv-client/src/lib.rs +++ b/crates/nokv-client/src/lib.rs @@ -26,8 +26,8 @@ pub use artifact::{ ArtifactRepositoryOptions, }; pub use file_client::{ - is_artifact_write_conflict, AppendOutcome, NoKvFsClient, PathRangeReadRequest, PathReadRange, - PreparedPathRangeBatch, + is_artifact_write_conflict, is_artifact_write_retryable, AppendOutcome, NoKvFsClient, + PathRangeReadRequest, PathReadRange, PreparedPathRangeBatch, }; pub use nokv_object::{DataFabricReadStats, ObjectReadPlan}; pub use nokv_protocol::{decode_name_cursor, encode_name_cursor}; diff --git a/crates/nokv-client/src/service.rs b/crates/nokv-client/src/service.rs index 8c6ecad53..6c04fa234 100644 --- a/crates/nokv-client/src/service.rs +++ b/crates/nokv-client/src/service.rs @@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; -use nokv_control::{ControlStore, ShardId, ShardRecord}; +use nokv_control::{ControlStore, ShardId, ShardRecord, ShardState}; use nokv_meta::{ DentryWithAttr, NamespaceAggregateGroup, NamespaceAggregateMeasure, NamespaceAggregateOp, NamespaceAggregateOutputMeasure, NamespaceAggregateRequest, NamespaceAggregateResult, @@ -117,12 +117,13 @@ pub struct ClientPreparedArtifact { pub replace: bool, pub dentry_version: Option, pub old_generation: Option, + pub object_gc_claim_version: u64, } /// Stable reconstruction fields persisted by clients that journal a prepared -/// artifact outside `nokv-client`. Future publish-fencing fields belong on -/// [`ClientPreparedArtifact`] so external journals can adopt them explicitly -/// without breaking compilation in the protocol change that introduces them. +/// artifact outside `nokv-client`. New publish-fencing fields belong on +/// [`ClientPreparedArtifact`] and default to a fail-closed value here until the +/// external journal explicitly adopts them. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ClientPreparedArtifactRecovery { pub mount: u64, @@ -152,8 +153,14 @@ impl ClientPreparedArtifact { replace: recovery.replace, dentry_version: recovery.dentry_version, old_generation: recovery.old_generation, + object_gc_claim_version: 0, } } + + pub fn with_object_gc_claim_version(mut self, object_gc_claim_version: u64) -> Self { + self.object_gc_claim_version = object_gc_claim_version; + self + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -183,8 +190,9 @@ pub struct PathLayoutOpenRequest { pub expected_generation: Option, } -/// Result of a copy-on-write subtree clone: the fork's namespace root inode and the -/// retained snapshot pin that protects the shared base blocks. +/// Result of a copy-on-write subtree clone: the fork's namespace root inode and +/// the durable retention identity that protects its shared base blocks until +/// explicit retirement. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct CloneOutcome { pub root: InodeId, @@ -2003,6 +2011,18 @@ impl MetadataClient { } } + pub fn refresh_prepared_artifact_object_gc_epoch( + &self, + prepared: ClientPreparedArtifact, + ) -> Result { + match self.call(MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { + prepared: prepared_artifact_to_wire(&prepared)?, + })? { + MetadataRpcResult::PreparedArtifact { prepared } => wire_prepared_artifact(prepared), + other => Err(unexpected_result(other)), + } + } + pub fn publish_prepared_artifact( &self, prepared: ClientPreparedArtifact, @@ -2232,9 +2252,10 @@ impl FleetRouter { /// Build the `(shard_map, endpoints)` pair for fleet routing from the control /// store. Each registered shard contributes a longest-prefix route (non-default /// shards only — the default shard owns `/` implicitly) and a -/// `shard_index -> endpoint` entry. Shards without an endpoint (currently -/// unowned) are skipped; a request that routes to one fails resolution and is -/// retried after the next refresh. +/// `shard_index -> endpoint` entry. Only an owner that has completed recovery +/// and reached `Serving` is routable. Registered, unowned, and `Recovering` +/// shards still contribute their prefix so requests fail resolution instead of +/// falling through to the default shard, then retry after a later refresh. fn resolve_fleet_routes( control: &dyn ControlStore, mount: MountId, @@ -2245,14 +2266,16 @@ fn resolve_fleet_routes( let mut routes = Vec::new(); let mut endpoints = HashMap::new(); for record in records { - if let Some(endpoint) = record.endpoint.as_deref() { - let address = endpoint.parse::().map_err(|err| { - ClientError::Protocol(format!( - "control shard {} has unparseable endpoint {endpoint:?}: {err}", - record.shard_id - )) - })?; - endpoints.insert(record.shard_index, address); + if record.state == ShardState::Serving && record.owner.is_some() { + if let Some(endpoint) = record.endpoint.as_deref() { + let address = endpoint.parse::().map_err(|err| { + ClientError::Protocol(format!( + "control shard {} has unparseable endpoint {endpoint:?}: {err}", + record.shard_id + )) + })?; + endpoints.insert(record.shard_index, address); + } } // The default shard owns "/" implicitly; only non-default subtree shards // are entered as longest-prefix routes (mirrors the server's ShardMap). diff --git a/crates/nokv-client/src/service_tests.rs b/crates/nokv-client/src/service_tests.rs index 70358931e..6014e43ad 100644 --- a/crates/nokv-client/src/service_tests.rs +++ b/crates/nokv-client/src/service_tests.rs @@ -76,6 +76,30 @@ fn serve_request_sequence( addr } +fn serve_request_sequence_then_drop( + handlers: Vec Vec + Send>>, + dropped_request: Box, +) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut magic = [0_u8; FRAMED_RPC_MAGIC.len()]; + stream.read_exact(&mut magic).unwrap(); + assert_eq!(&magic, FRAMED_RPC_MAGIC); + for handler in handlers { + let (request_id, flags, request) = read_frame(&mut stream).unwrap(); + let request = decode_request(&request).unwrap(); + let response = handler(request); + write_frame(&mut stream, request_id, flags, &response).unwrap(); + } + let (_, _, request) = read_frame(&mut stream).unwrap(); + dropped_request(decode_request(&request).unwrap()); + // Simulate a response lost after the server may have committed. + }); + addr +} + fn response_body(json: &str) -> Vec { let envelope: MetadataRpcEnvelope = serde_json::from_str(json).unwrap(); encode_envelope(&envelope).unwrap() @@ -608,7 +632,7 @@ fn service_create_file_prepared_uses_single_frame() { other => panic!("unexpected request: {other:?}"), } response_body( - r#"{"ok":true,"result":{"type":"created_prepared_artifact","entry":{"dentry":{"parent":2,"name_hex":"636865636b706f696e742e62696e","child":40,"child_type":"file","attr_generation":7},"attr":{"inode":40,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":0,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":null},"prepared":{"mount":1,"parent":2,"name":"checkpoint.bin","inode":40,"generation":8,"mtime_ms":8,"ctime_ms":8,"replace":true,"dentry_version":7,"old_generation":null}}}"#, + r#"{"ok":true,"result":{"type":"created_prepared_artifact","entry":{"dentry":{"parent":2,"name_hex":"636865636b706f696e742e62696e","child":40,"child_type":"file","attr_generation":7},"attr":{"inode":40,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":0,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":null},"prepared":{"mount":1,"parent":2,"name":"checkpoint.bin","inode":40,"generation":8,"mtime_ms":8,"ctime_ms":8,"replace":true,"dentry_version":7,"old_generation":null,"object_gc_claim_version":12}}}"#, ) }); let client = MetadataClient::connect(addr); @@ -625,6 +649,43 @@ fn service_create_file_prepared_uses_single_frame() { assert_eq!(created.prepared.inode.get(), 40); assert_eq!(created.prepared.generation, 8); assert!(created.prepared.replace); + assert_eq!(created.prepared.object_gc_claim_version, 12); +} + +#[test] +fn service_refresh_prepared_upload_mints_generation_and_object_gc_epoch() { + let addr = serve_one_request(|request| { + match request { + MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared } => { + assert_eq!(prepared.inode, 42); + assert_eq!(prepared.generation, 7); + assert_eq!(prepared.object_gc_claim_version, 12); + } + other => panic!("unexpected request: {other:?}"), + } + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":2,"name":"artifact.bin","inode":42,"generation":13,"mtime_ms":1700000000001,"ctime_ms":1700000000001,"replace":true,"dentry_version":7,"old_generation":6,"object_gc_claim_version":14}}}"#, + ) + }); + let client = MetadataClient::connect(addr); + let refreshed = client + .refresh_prepared_artifact_object_gc_epoch(ClientPreparedArtifact { + mount: 1, + parent: InodeId::new(2).unwrap(), + name: DentryName::new(b"artifact.bin".to_vec()).unwrap(), + path: None, + inode: InodeId::new(42).unwrap(), + generation: 7, + mtime_ms: 1_700_000_000_000, + ctime_ms: 1_700_000_000_000, + replace: true, + dentry_version: Some(7), + old_generation: Some(6), + object_gc_claim_version: 12, + }) + .unwrap(); + assert_eq!(refreshed.generation, 13); + assert_eq!(refreshed.object_gc_claim_version, 14); } #[test] @@ -2456,7 +2517,7 @@ fn service_file_client_uploads_blocks_then_publishes_metadata() { let store = MemoryObjectStore::new(); let addr = serve_many(vec![ response_body( - r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null}}}"#, + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, ), response_body( r#"{"ok":true,"result":{"type":"rename_replace","entry":{"dentry":{"parent":1,"name_hex":"61727469666163742e62696e","child":42,"child_type":"file","attr_generation":7},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":11,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":{"producer":"unit-test","digest_uri":"sha256:demo","size":11,"content_type":"application/octet-stream","manifest_id":"artifact.bin","generation":7,"base_generation":0,"chunk_size":67108864,"block_size":4194304}},"replaced":null}}"#, @@ -2480,12 +2541,120 @@ fn service_file_client_uploads_blocks_then_publishes_metadata() { ); } +#[test] +fn service_file_client_refreshes_and_restages_after_object_gc_epoch_change() { + let store = MemoryObjectStore::new(); + let addr = serve_request_sequence(vec![ + Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::PrepareArtifactPath { + path: "/artifact.bin".to_owned(), + replace: false, + } + ); + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","path":"/artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::PublishPreparedArtifact { prepared, body, .. } = request else { + panic!("expected initial prepared publish") + }; + assert_eq!(prepared.generation, 7); + assert_eq!(prepared.object_gc_claim_version, 12); + assert_eq!(body.generation, 7); + response_body( + r#"{"ok":false,"error":"prepared artifact object-GC epoch 12 is stale; current epoch is 14","error_kind":{"type":"stale_prepared_artifact_object_gc_epoch","expected":12,"current":14}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared } = request + else { + panic!("expected prepared object-GC refresh") + }; + assert_eq!(prepared.path.as_deref(), Some("/artifact.bin")); + assert_eq!(prepared.generation, 7); + assert_eq!(prepared.object_gc_claim_version, 12); + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","path":"/artifact.bin","inode":42,"generation":13,"mtime_ms":1700000000001,"ctime_ms":1700000000001,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":14}}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::PublishPreparedArtifact { prepared, body, .. } = request else { + panic!("expected refreshed prepared publish") + }; + assert_eq!(prepared.generation, 13); + assert_eq!(prepared.object_gc_claim_version, 14); + assert_eq!(body.generation, 13); + response_body( + r#"{"ok":true,"result":{"type":"rename_replace","entry":{"dentry":{"parent":1,"name_hex":"61727469666163742e62696e","child":42,"child_type":"file","attr_generation":13},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":11,"generation":13,"mtime_ms":13,"ctime_ms":13},"body":{"producer":"unit-test","digest_uri":"sha256:artifact.bin","size":11,"content_type":"application/octet-stream","manifest_id":"artifact.bin","generation":13,"base_generation":0,"chunk_size":67108864,"block_size":4194304}},"replaced":null}}"#, + ) + }), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let entry = client + .put_artifact( + "/artifact.bin", + b"hello world".to_vec(), + artifact_metadata("artifact.bin"), + ) + .unwrap(); + + assert_eq!(entry.attr.generation, 13); + assert!(store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_none()); + assert!(store + .head(&ObjectKey::new("blocks/1/42/13/0/0").unwrap()) + .unwrap() + .is_some()); +} + +#[test] +fn service_file_client_replays_seekable_reader_from_initial_position_after_refresh() { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ), + response_body( + r#"{"ok":false,"error":"prepared artifact object-GC epoch 12 is stale; current epoch is 14","error_kind":{"type":"stale_prepared_artifact_object_gc_epoch","expected":12,"current":14}}"#, + ), + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":13,"mtime_ms":1700000000001,"ctime_ms":1700000000001,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":14}}}"#, + ), + response_body( + r#"{"ok":true,"result":{"type":"rename_replace","entry":{"dentry":{"parent":1,"name_hex":"61727469666163742e62696e","child":42,"child_type":"file","attr_generation":13},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":11,"generation":13,"mtime_ms":13,"ctime_ms":13},"body":{"producer":"unit-test","digest_uri":"sha256:artifact.bin","size":11,"content_type":"application/octet-stream","manifest_id":"artifact.bin","generation":13,"base_generation":0,"chunk_size":67108864,"block_size":4194304}},"replaced":null}}"#, + ), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + let mut reader = std::io::Cursor::new(b"skip--hello world".to_vec()); + reader.set_position(6); + + let entry = client + .put_artifact_from_reader("/artifact.bin", reader, artifact_metadata("artifact.bin")) + .unwrap(); + + assert_eq!(entry.attr.size, 11); + assert!(store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_none()); + assert!(store + .head(&ObjectKey::new("blocks/1/42/13/0/0").unwrap()) + .unwrap() + .is_some()); +} + #[test] fn service_file_client_cleans_staged_blocks_after_publish_failure() { let store = MemoryObjectStore::new(); let addr = serve_many(vec![ response_body( - r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null}}}"#, + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, ), response_body( r#"{"ok":false,"error":"metadata command predicate failed","error_kind":{"type":"predicate_failed"}}"#, @@ -2514,6 +2683,327 @@ fn service_file_client_cleans_staged_blocks_after_publish_failure() { ); } +#[test] +fn service_file_client_keeps_bytes_staged_when_publish_reports_committed() { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ), + response_body( + r#"{"ok":false,"error":"metadata sync log archive failed after commit","error_kind":{"type":"sync_log_archive_failed","committed":true,"message":"injected"}}"#, + ), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .put_artifact( + "/artifact.bin", + b"hello world".to_vec(), + artifact_metadata("artifact.bin"), + ) + .unwrap_err(); + + assert!(matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_some(), + "a committed publish may already reference the staged object" + ); +} + +#[test] +fn service_file_client_keeps_bytes_staged_when_metadata_backend_publish_is_uncertain() { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ), + response_body( + r#"{"ok":false,"error":"metadata backend unavailable after publish attempt","error_kind":{"type":"metadata","message":"metadata backend unavailable after publish attempt"}}"#, + ), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .put_artifact( + "/artifact.bin", + b"hello world".to_vec(), + artifact_metadata("artifact.bin"), + ) + .unwrap_err(); + + assert!(matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::Metadata( + nokv_meta::MetadataError::Backend(_) + )) + )); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_some(), + "a metadata backend failure does not prove the publish was rejected" + ); +} + +#[test] +fn service_file_client_keeps_bytes_staged_for_ownership_errors_with_unknown_commit_phase() { + let ownership_errors = [ + ( + r#"{"ok":false,"error":"owner epoch 7 is stale; required owner epoch is 8","error_kind":{"type":"stale_owner_epoch","owner_epoch":7,"required_epoch":8}}"#, + "stale owner epoch", + ), + ( + r#"{"ok":false,"error":"owner lease expired","error_kind":{"type":"lease_expired","now_ms":101,"deadline_ms":100}}"#, + "expired owner lease", + ), + ( + r#"{"ok":false,"error":"not owner","error_kind":{"type":"not_owner","shard_id":"mount-1:/","endpoint":null}}"#, + "owner handoff", + ), + ]; + + for (wire_error, context) in ownership_errors { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ), + response_body(wire_error), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .put_artifact( + "/artifact.bin", + b"hello world".to_vec(), + artifact_metadata("artifact.bin"), + ) + .unwrap_err(); + + assert!(matches!( + err, + ClientError::Metadata( + nokv_meta::MetadError::StaleOwnerEpoch { .. } + | nokv_meta::MetadError::LeaseExpired { .. } + | nokv_meta::MetadError::NotOwner { .. } + ) + )); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_some(), + "{context} does not prove the publish was rejected before commit" + ); + } +} + +#[test] +fn service_file_client_keeps_reader_staged_when_publish_response_is_lost() { + let store = MemoryObjectStore::new(); + let addr = serve_request_sequence_then_drop( + vec![Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::PrepareArtifactPath { + path: "/artifact.bin".to_owned(), + replace: false, + } + ); + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ) + })], + Box::new(|request| { + let MetadataRpcRequest::PublishPreparedArtifact { prepared, body, .. } = request else { + panic!("expected prepared artifact publish") + }; + assert_eq!(prepared.generation, 7); + assert_eq!(body.generation, 7); + }), + ); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .put_artifact_from_reader( + "/artifact.bin", + std::io::Cursor::new(b"hello world".to_vec()), + artifact_metadata("artifact.bin"), + ) + .unwrap_err(); + + assert!(matches!(err, ClientError::Io(_))); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_some(), + "a lost response leaves the publish outcome unknown" + ); +} + +#[test] +fn service_file_client_keeps_reader_staged_when_metadata_backend_publish_is_uncertain() { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ), + response_body( + r#"{"ok":false,"error":"metadata backend unavailable after publish attempt","error_kind":{"type":"metadata","message":"metadata backend unavailable after publish attempt"}}"#, + ), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .put_artifact_from_reader( + "/artifact.bin", + std::io::Cursor::new(b"hello world".to_vec()), + artifact_metadata("artifact.bin"), + ) + .unwrap_err(); + + assert!(matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::Metadata( + nokv_meta::MetadataError::Backend(_) + )) + )); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_some(), + "a metadata backend failure does not prove the reader publish was rejected" + ); +} + +#[test] +fn service_file_client_keeps_append_staged_when_publish_reports_committed() { + let store = MemoryObjectStore::new(); + let addr = serve_request_sequence(vec![ + Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::LookupPath { + path: "/log.txt".to_owned(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"dentry","entry":{"dentry":{"parent":1,"name_hex":"6c6f672e747874","child":42,"child_type":"file","attr_generation":7},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":5,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":{"producer":"unit-test","digest_uri":"sha256:base","size":5,"content_type":"application/octet-stream","manifest_id":"base.log","generation":7,"base_generation":0,"chunk_size":67108864,"block_size":4194304}}}}"#, + ) + }), + Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::PrepareArtifactPath { + path: "/log.txt".to_owned(), + replace: true, + } + ); + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"log.txt","inode":42,"generation":9,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":true,"dentry_version":7,"old_generation":7,"object_gc_claim_version":12}}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::PublishPreparedArtifactStagedSession { + prepared, + size, + chunks, + staged, + .. + } = request + else { + panic!("expected staged-session artifact publish") + }; + assert_eq!(prepared.generation, 9); + assert_eq!(prepared.old_generation, Some(7)); + assert_eq!(size, 11); + assert!(!chunks.is_empty()); + assert!(!staged.objects.is_empty()); + response_body( + r#"{"ok":false,"error":"metadata sync log archive failed after commit","error_kind":{"type":"sync_log_archive_failed","committed":true,"message":"injected"}}"#, + ) + }), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .append_artifact( + "/log.txt", + b" world".to_vec(), + artifact_metadata("delta.log"), + None, + ) + .unwrap_err(); + + assert!(matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/9/0/0").unwrap()) + .unwrap() + .is_some(), + "a committed append may already reference the staged delta" + ); +} + +#[test] +fn service_file_client_keeps_append_staged_when_metadata_backend_publish_is_uncertain() { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"dentry","entry":{"dentry":{"parent":1,"name_hex":"6c6f672e747874","child":42,"child_type":"file","attr_generation":7},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":5,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":{"producer":"unit-test","digest_uri":"sha256:base","size":5,"content_type":"application/octet-stream","manifest_id":"base.log","generation":7,"base_generation":0,"chunk_size":67108864,"block_size":4194304}}}}"#, + ), + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"log.txt","inode":42,"generation":9,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":true,"dentry_version":7,"old_generation":7,"object_gc_claim_version":12}}}"#, + ), + response_body( + r#"{"ok":false,"error":"metadata backend unavailable after publish attempt","error_kind":{"type":"metadata","message":"metadata backend unavailable after publish attempt"}}"#, + ), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let err = client + .append_artifact( + "/log.txt", + b" world".to_vec(), + artifact_metadata("delta.log"), + None, + ) + .unwrap_err(); + + assert!(matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::Metadata( + nokv_meta::MetadataError::Backend(_) + )) + )); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/9/0/0").unwrap()) + .unwrap() + .is_some(), + "a metadata backend failure does not prove the append was rejected" + ); +} + #[test] fn service_clone_subtree_path_sends_typed_rpc_and_maps_outcome() { let addr = serve_one_request(|request| { @@ -2654,6 +3144,7 @@ fn register_owned_shard( let lease = store .acquire_unassigned(shard_id, NodeId::new(endpoint)) .unwrap(); + store.mark_serving(&lease, None, None, 0).unwrap(); lease.epoch } @@ -2702,6 +3193,35 @@ fn fleet_resolves_each_request_to_its_shard_owner_endpoint() { ); } +#[test] +fn fleet_keeps_recovering_owner_unroutable_until_mark_serving() { + let store = Arc::new(InMemoryControlStore::new()); + let shard_id = ShardId::new("mount-1:/dataset"); + store + .register_shard(shard_id.clone(), "/dataset".to_owned(), 1) + .unwrap(); + let lease = store + .acquire_unassigned(shard_id, NodeId::new("127.0.0.1:7002")) + .unwrap(); + let control: Arc = store.clone(); + let client = MetadataClient::fleet(control, mount_one()).unwrap(); + let request = MetadataRpcRequest::StatPath { + path: "/dataset/recovering.bin".to_owned(), + }; + + assert_eq!(client.resolve_target_for_test(&request).unwrap(), None); + + store.mark_serving(&lease, None, None, 0).unwrap(); + let RoutingMode::Fleet(router) = &client.mode else { + panic!("expected fleet router"); + }; + router.refresh(&client).unwrap(); + assert_eq!( + client.resolve_target_for_test(&request).unwrap(), + Some("127.0.0.1:7002".parse().unwrap()) + ); +} + #[test] fn fleet_refreshes_and_retries_against_new_owner_on_not_owner() { // Two tiny one-shot servers: the stale owner replies NotOwner, the new owner @@ -2741,6 +3261,7 @@ fn fleet_refreshes_and_retries_against_new_owner_on_not_owner() { let lease = store .acquire_unassigned(dataset_id.clone(), NodeId::new(stale_owner.to_string())) .unwrap(); + store.mark_serving(&lease, None, None, 0).unwrap(); // Build the client BEFORE the handoff so it caches the stale endpoint. let client = MetadataClient::fleet(Arc::clone(&store), mount_one()).unwrap(); @@ -2755,9 +3276,10 @@ fn fleet_refreshes_and_retries_against_new_owner_on_not_owner() { // Hand the shard off to the new owner in the control plane (bumps the epoch // and rewrites the endpoint). The client's cache is now stale. - store + let successor = store .acquire_after_failure(dataset_id, NodeId::new(new_owner.to_string()), lease.epoch) .unwrap(); + store.mark_serving(&successor, None, None, 0).unwrap(); // The call hits the stale owner (NotOwner), refreshes from control, and // retries against the new owner, which succeeds. diff --git a/crates/nokv-client/src/wire.rs b/crates/nokv-client/src/wire.rs index 32bc93c76..722daf2dd 100644 --- a/crates/nokv-client/src/wire.rs +++ b/crates/nokv-client/src/wire.rs @@ -86,6 +86,7 @@ pub(crate) fn wire_prepared_artifact( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, }) } @@ -104,6 +105,7 @@ pub(crate) fn prepared_artifact_to_wire( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, }) } @@ -262,6 +264,12 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr dest_shard, }), WireMetadataError::GraftPoint => ClientError::Metadata(nokv_meta::MetadError::GraftPoint), + WireMetadataError::StalePreparedArtifactObjectGcEpoch { expected, current } => { + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current, + }) + } WireMetadataError::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, @@ -293,6 +301,20 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr WireMetadataError::SnapshotBindingChanged { root_path } => { ClientError::Metadata(nokv_meta::MetadError::SnapshotBindingChanged { root_path }) } + WireMetadataError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + } => match (InodeId::new(fork_root), InodeId::new(borrower)) { + (Ok(fork_root), Ok(borrower)) => { + ClientError::Metadata(nokv_meta::MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) + } + _ => ClientError::Protocol("fork retention error carries an invalid inode".to_owned()), + }, WireMetadataError::SnapshotRenewContended { snapshot_id, attempts, diff --git a/crates/nokv-client/tests/object_gc_fence_live.rs b/crates/nokv-client/tests/object_gc_fence_live.rs new file mode 100644 index 000000000..07136f28c --- /dev/null +++ b/crates/nokv-client/tests/object_gc_fence_live.rs @@ -0,0 +1,618 @@ +//! Live RustFS acceptance for the durable object-reference/GC fence. +//! +//! This test is ignored by default because it needs a real S3-compatible +//! endpoint. `scripts/run-object-gc-fence-live-e2e.sh` owns that environment. + +use std::collections::BTreeSet; +use std::env; +use std::net::{SocketAddr, TcpListener}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use nokv_client::{ArtifactMetadata, ClientError, ClientPreparedArtifact, NoKvFsClient}; +use nokv_meta::{HistoryGcOptions, ObjectGcOptions}; +use nokv_object::{ + ChunkStore, ChunkWriteOptions, ObjectBytes, ObjectCapabilities, ObjectError, ObjectInfo, + ObjectKey, ObjectRange, ObjectStore, ObjectStoreConfig, S3ObjectStore, S3ObjectStoreOptions, + StagedObjectSet, DEFAULT_BLOCK_SIZE, DEFAULT_CHUNK_SIZE, +}; +use nokv_server::ServerOptions; +use nokv_types::{BodyDescriptor, ChunkManifest, MountId}; + +type LiveClient = NoKvFsClient; + +#[derive(Clone)] +struct PausingPutStore { + inner: S3ObjectStore, + gate: Arc<(Mutex, Condvar)>, +} + +#[derive(Default)] +struct PausingPutState { + armed: bool, + expected_puts: usize, + reached: bool, + released: bool, + staged_keys: BTreeSet, +} + +impl PausingPutStore { + fn new(inner: S3ObjectStore) -> Self { + Self { + inner, + gate: Arc::new((Mutex::new(PausingPutState::default()), Condvar::new())), + } + } + + fn arm(&self, expected_puts: usize) { + assert!( + expected_puts > 1, + "the paused write must span multiple blocks" + ); + let (lock, _) = &*self.gate; + *lock.lock().expect("lock put pause") = PausingPutState { + armed: true, + expected_puts, + ..PausingPutState::default() + }; + } + + fn wait_until_staged(&self) -> BTreeSet { + let (lock, changed) = &*self.gate; + let deadline = Instant::now() + Duration::from_secs(15); + let mut state = lock.lock().expect("lock put pause"); + while !state.reached { + let now = Instant::now(); + assert!(now < deadline, "timed out waiting for paused RustFS PUT"); + let (next, timed_out) = changed + .wait_timeout(state, deadline - now) + .expect("wait for paused RustFS PUT"); + state = next; + assert!( + !timed_out.timed_out() || state.reached, + "timed out waiting for paused RustFS PUT" + ); + } + assert_eq!(state.staged_keys.len(), state.expected_puts); + state.staged_keys.clone() + } + + fn release(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().expect("lock put pause"); + state.released = true; + changed.notify_all(); + } + + fn pause_after_put(&self, key: &ObjectKey) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().expect("lock put pause"); + if !state.armed { + return; + } + state.staged_keys.insert(key.as_str().to_owned()); + if state.staged_keys.len() < state.expected_puts { + return; + } + state.armed = false; + state.reached = true; + changed.notify_all(); + while !state.released { + state = changed.wait(state).expect("wait for PUT release"); + } + } +} + +impl ObjectStore for PausingPutStore { + fn capabilities(&self) -> ObjectCapabilities { + self.inner.capabilities() + } + + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + let result = self.inner.put(key, bytes)?; + self.pause_after_put(key); + Ok(result) + } + + fn get(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { + self.inner.get(key, range) + } + + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } + + fn delete(&self, key: &ObjectKey) -> Result { + self.inner.delete(key) + } +} + +fn live_options(test_namespace: &str) -> S3ObjectStoreOptions { + let run_prefix = env::var("NOKV_OBJECT_GC_LIVE_RUN_PREFIX") + .expect("NOKV_OBJECT_GC_LIVE_RUN_PREFIX must isolate this live run"); + let run_prefix = run_prefix.trim_matches('/'); + assert!(!run_prefix.is_empty(), "live run prefix must not be empty"); + let mut options = S3ObjectStoreOptions::rustfs( + env::var("NOKV_OBJECT_GC_LIVE_BUCKET") + .expect("NOKV_OBJECT_GC_LIVE_BUCKET must name the live test bucket"), + env::var("NOKV_OBJECT_GC_LIVE_ENDPOINT") + .expect("NOKV_OBJECT_GC_LIVE_ENDPOINT must name the RustFS endpoint"), + env::var("NOKV_OBJECT_GC_LIVE_ACCESS_KEY") + .expect("NOKV_OBJECT_GC_LIVE_ACCESS_KEY must be set"), + env::var("NOKV_OBJECT_GC_LIVE_SECRET_KEY") + .expect("NOKV_OBJECT_GC_LIVE_SECRET_KEY must be set"), + ); + options.root = format!("/{run_prefix}/{test_namespace}"); + options +} + +fn spawn_live_server(options: S3ObjectStoreOptions, mount: MountId) -> SocketAddr { + let metadata_dir = tempfile::tempdir().expect("create metadata directory"); + let listener = TcpListener::bind("127.0.0.1:0").expect("bind metadata server"); + let bind = listener.local_addr().expect("read metadata server address"); + let server = nokv_server::Server::open(ServerOptions { + bind, + mount, + meta_path: metadata_dir.path().join("meta"), + metadata_checkpoint_archive_prefix: None, + object: ObjectStoreConfig::s3(options), + uid: 1000, + gid: 1000, + object_gc: ObjectGcOptions { + interval: Duration::from_millis(10), + limit: 128, + run_immediately: true, + read_lease_grace: Duration::ZERO, + }, + history_gc: HistoryGcOptions { + interval: Duration::from_secs(3600), + limit: 128, + run_immediately: false, + }, + control: None, + }) + .expect("open metadata server"); + thread::spawn(move || { + let _metadata_dir = metadata_dir; + server.serve(listener).expect("serve metadata requests"); + }); + bind +} + +fn artifact_metadata(manifest_id: &str) -> ArtifactMetadata { + ArtifactMetadata { + producer: "object-gc-fence-live-e2e".to_owned(), + digest_uri: format!("sha256:{manifest_id}"), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + } +} + +fn stage( + objects: &S3ObjectStore, + prepared: &ClientPreparedArtifact, + bytes: &[u8], + manifest_id: &str, +) -> (BodyDescriptor, Vec, StagedObjectSet) { + let written = objects + .write_bytes( + bytes, + ChunkWriteOptions { + manifest_id: manifest_id.to_owned(), + mount: prepared.mount, + inode: prepared.inode.get(), + generation: prepared.generation, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE, + }, + ) + .expect("stage artifact body in RustFS"); + let staged = written.staged_objects().expect("collect staged objects"); + let chunks = written.chunk_manifests(); + let body = BodyDescriptor { + producer: "object-gc-fence-live-e2e".to_owned(), + digest_uri: format!("sha256:{manifest_id}"), + size: written.size, + content_type: "application/octet-stream".to_owned(), + manifest_id: written.manifest_id, + generation: prepared.generation, + base_generation: 0, + chunk_size: written.chunk_size, + block_size: written.block_size, + }; + (body, chunks, staged) +} + +fn staged_keys(staged: &StagedObjectSet) -> BTreeSet { + staged + .objects() + .iter() + .map(|object| object.key.as_str().to_owned()) + .collect() +} + +fn multi_block_payload(seed: u8) -> Vec { + (0..DEFAULT_BLOCK_SIZE + 257) + .map(|index| seed.wrapping_add((index as u8).wrapping_mul(31))) + .collect() +} + +fn expected_block_count(payload_len: usize) -> usize { + payload_len.div_ceil(DEFAULT_BLOCK_SIZE) +} + +fn path_object_keys(client: &LiveClient, path: &str) -> BTreeSet { + let entry = client + .metadata() + .lookup(path) + .expect("lookup object-backed path") + .expect("object-backed path exists"); + let body = entry.body.expect("path has a body descriptor"); + client + .metadata() + .read_body_plan( + entry.attr.inode, + body.generation, + 0, + usize::try_from(entry.attr.size).expect("live test artifact fits usize"), + ) + .expect("read path object plan") + .blocks + .into_iter() + .map(|block| block.object_key) + .collect() +} + +fn wait_until_objects_are_deleted(objects: &S3ObjectStore, keys: &BTreeSet) { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let remaining = keys + .iter() + .filter(|key| { + objects + .head(&ObjectKey::new((*key).clone()).expect("valid object key")) + .expect("query RustFS object") + .is_some() + }) + .count(); + if remaining == 0 { + return; + } + assert!( + Instant::now() < deadline, + "timed out waiting for object GC to delete {remaining} RustFS objects" + ); + thread::sleep(Duration::from_millis(25)); + } +} + +fn assert_objects_survive_gc_window( + objects: &S3ObjectStore, + keys: &BTreeSet, + window: Duration, +) { + assert!(!keys.is_empty(), "retention check requires object keys"); + let deadline = Instant::now() + window; + loop { + for key in keys { + assert!( + objects + .head(&ObjectKey::new(key.clone()).expect("valid object key")) + .expect("query retained RustFS object") + .is_some(), + "object GC deleted retained RustFS object {key}" + ); + } + if Instant::now() >= deadline { + return; + } + thread::sleep(Duration::from_millis(25)); + } +} + +#[test] +#[ignore = "requires a live RustFS endpoint; run scripts/run-object-gc-fence-live-e2e.sh"] +fn rustfs_stale_prepare_is_rejected_then_refreshed_and_restaged() { + let options = live_options("stale-restage"); + let objects = S3ObjectStore::new(options.clone()).expect("open RustFS object store"); + let address = spawn_live_server(options, MountId::new(1).expect("non-zero mount")); + let client = LiveClient::connect(address, objects.clone()); + + client + .put_artifact( + "/gc-source.bin", + b"old generation queued for collection".to_vec(), + artifact_metadata("gc-source-old"), + ) + .expect("publish source generation"); + let source = client + .metadata() + .lookup("/gc-source.bin") + .expect("lookup source") + .expect("source exists"); + let source_body = source.body.expect("source body descriptor"); + let source_plan = client + .metadata() + .read_body_plan( + source.attr.inode, + source_body.generation, + 0, + source.attr.size as usize, + ) + .expect("read source object plan"); + let old_source_keys = source_plan + .blocks + .into_iter() + .map(|block| block.object_key) + .collect::>(); + assert!(!old_source_keys.is_empty(), "source must stage an object"); + + let stale = client + .metadata() + .prepare_artifact_path("/gc-target.bin", false) + .expect("prepare target before the delete epoch"); + client + .put_artifact_replace( + "/gc-source.bin", + b"replacement generation remains live".to_vec(), + artifact_metadata("gc-source-new"), + ) + .expect("replace source and enqueue its old objects"); + wait_until_objects_are_deleted(&objects, &old_source_keys); + + let payload = multi_block_payload(0x31); + let (stale_body, stale_chunks, stale_staged) = + stage(&objects, &stale, &payload, "gc-target-stale"); + let stale_keys = staged_keys(&stale_staged); + assert_eq!(stale_keys.len(), expected_block_count(payload.len())); + assert!( + stale_keys.len() > 1, + "manual stale stage must be multi-block" + ); + let err = client + .metadata() + .publish_prepared_artifact(stale.clone(), stale_body, stale_chunks, 0o644, 1000, 1000) + .expect_err("publish with a pre-GC epoch must fail closed"); + assert!( + matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { .. }) + ), + "expected typed stale object-GC epoch, got {err:?}" + ); + assert!( + client + .metadata() + .lookup("/gc-target.bin") + .expect("lookup target after rejected publish") + .is_none(), + "stale publish must not attach namespace metadata" + ); + objects + .delete_staged(&stale_staged) + .expect("delete stale staged generation"); + wait_until_objects_are_deleted(&objects, &stale_keys); + + let refreshed = client + .metadata() + .refresh_prepared_artifact_object_gc_epoch(stale.clone()) + .expect("refresh stale prepared identity"); + assert_ne!(refreshed.generation, stale.generation); + assert_ne!( + refreshed.object_gc_claim_version, + stale.object_gc_claim_version + ); + let (body, chunks, staged) = stage(&objects, &refreshed, &payload, "gc-target-refreshed"); + let refreshed_keys = staged_keys(&staged); + assert_eq!(refreshed_keys.len(), expected_block_count(payload.len())); + assert!( + refreshed_keys.is_disjoint(&stale_keys), + "refreshed generation must not reuse stale object keys" + ); + client + .metadata() + .publish_prepared_artifact(refreshed, body, chunks, 0o644, 1000, 1000) + .expect("publish fully restaged generation"); + assert_eq!( + path_object_keys(&client, "/gc-target.bin"), + refreshed_keys, + "published target must reference the complete refreshed object set" + ); + assert_eq!( + client + .cat("/gc-target.bin") + .expect("read target from RustFS"), + payload + ); + + // Exercise the public file-client retry path against the same live + // endpoint. Pause its first object PUT after prepare/stage, advance GC from + // another client, then prove the writer cleans that stale generation and + // transparently performs a full restage under a fresh generation. + client + .put_artifact( + "/auto-gc-source.bin", + b"old source for automatic restage".to_vec(), + artifact_metadata("auto-gc-source-old"), + ) + .expect("publish automatic-restage source generation"); + let auto_source = client + .metadata() + .lookup("/auto-gc-source.bin") + .expect("lookup automatic-restage source") + .expect("automatic-restage source exists"); + let auto_source_body = auto_source.body.expect("automatic-restage source body"); + let auto_source_keys = client + .metadata() + .read_body_plan( + auto_source.attr.inode, + auto_source_body.generation, + 0, + auto_source.attr.size as usize, + ) + .expect("read automatic-restage source object plan") + .blocks + .into_iter() + .map(|block| block.object_key) + .collect::>(); + + let pausing_objects = PausingPutStore::new(objects.clone()); + let auto_payload = multi_block_payload(0xa7); + let auto_block_count = expected_block_count(auto_payload.len()); + pausing_objects.arm(auto_block_count); + let writer_objects = pausing_objects.clone(); + let expected_auto_payload = auto_payload.clone(); + let writer = thread::spawn(move || { + let writer = NoKvFsClient::connect(address, writer_objects); + writer.put_artifact( + "/auto-gc-target.bin", + auto_payload, + artifact_metadata("auto-gc-target"), + ) + }); + let stale_auto_keys = pausing_objects.wait_until_staged(); + assert_eq!(stale_auto_keys.len(), auto_block_count); + + client + .put_artifact_replace( + "/auto-gc-source.bin", + b"new source survives automatic restage".to_vec(), + artifact_metadata("auto-gc-source-new"), + ) + .expect("advance object GC while the file client is paused after staging"); + wait_until_objects_are_deleted(&objects, &auto_source_keys); + pausing_objects.release(); + + writer + .join() + .expect("automatic-restage writer thread") + .expect("file client refreshes and fully restages after stale epoch"); + wait_until_objects_are_deleted(&objects, &stale_auto_keys); + let fresh_auto_keys = path_object_keys(&client, "/auto-gc-target.bin"); + assert_eq!(fresh_auto_keys.len(), auto_block_count); + assert!( + fresh_auto_keys.is_disjoint(&stale_auto_keys), + "automatic restage must mint a fully disjoint object-key set" + ); + assert_eq!( + client + .cat("/auto-gc-target.bin") + .expect("read automatically restaged target from RustFS"), + expected_auto_payload + ); +} + +#[test] +#[ignore = "requires a live RustFS endpoint; run scripts/run-object-gc-fence-live-e2e.sh"] +fn rustfs_fork_binding_retains_then_releases_source_objects() { + let options = live_options("fork-retention"); + let objects = S3ObjectStore::new(options.clone()).expect("open RustFS object store"); + let address = spawn_live_server(options, MountId::new(2).expect("non-zero mount")); + let client = LiveClient::connect(address, objects.clone()); + + client + .metadata() + .mkdir("/base", 0o755, 1000, 1000) + .expect("create clone source root"); + let source_payload = multi_block_payload(0x52); + client + .put_artifact( + "/base/data.bin", + source_payload.clone(), + artifact_metadata("fork-retention-source"), + ) + .expect("publish clone source"); + let source_keys = path_object_keys(&client, "/base/data.bin"); + assert_eq!( + source_keys.len(), + expected_block_count(source_payload.len()) + ); + + let fork = client + .metadata() + .clone_subtree_path("/base", "/fork") + .expect("create object-sharing fork"); + let borrower = client + .metadata() + .lookup("/fork/data.bin") + .expect("lookup fork borrower") + .expect("fork borrower exists") + .attr + .inode; + assert_eq!( + path_object_keys(&client, "/fork/data.bin"), + source_keys, + "fresh fork must borrow the source object set" + ); + + client + .metadata() + .remove("/base/data.bin") + .expect("remove source namespace reference"); + // The server has a 10 ms object-GC interval and zero read-lease grace. + // Observe many background passes before opening the fork for the first time. + assert_objects_survive_gc_window(&objects, &source_keys, Duration::from_millis(500)); + assert_eq!( + client + .cat("/fork/data.bin") + .expect("freshly read retained fork from RustFS"), + source_payload + ); + + let error = client + .metadata() + .retire_snapshot("/base", fork.snapshot_id) + .expect_err("borrowed blocks must prevent fork-retention retirement"); + assert!( + matches!( + error, + ClientError::Metadata(nokv_meta::MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower: active_borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && active_borrower == borrower + ), + "expected typed ForkRetentionActive, got {error:?}" + ); + assert_objects_survive_gc_window(&objects, &source_keys, Duration::from_millis(250)); + + let replacement_payload = multi_block_payload(0xd3); + client + .put_artifact_replace( + "/fork/data.bin", + replacement_payload.clone(), + artifact_metadata("fork-retention-rewritten"), + ) + .expect("rewrite borrower onto self-owned objects"); + let replacement_keys = path_object_keys(&client, "/fork/data.bin"); + assert!( + replacement_keys.is_disjoint(&source_keys), + "rewritten borrower must no longer reference source objects" + ); + assert!( + client + .metadata() + .retire_snapshot("/base", fork.snapshot_id) + .expect("retire releasable fork binding"), + "fork binding must retire exactly once" + ); + + wait_until_objects_are_deleted(&objects, &source_keys); + assert_eq!( + client + .cat("/fork/data.bin") + .expect("read rewritten fork after source GC"), + replacement_payload + ); + assert_objects_survive_gc_window(&objects, &replacement_keys, Duration::from_millis(100)); +} diff --git a/crates/nokv-control/src/codec.rs b/crates/nokv-control/src/codec.rs index 1c8c46f19..568c1abd3 100644 --- a/crates/nokv-control/src/codec.rs +++ b/crates/nokv-control/src/codec.rs @@ -47,6 +47,7 @@ mod tests { object_key: "meta/checkpoints/7".to_owned(), lsn: 128, image_bytes: 4096, + image_digest: "sha256:image".to_owned(), digest: "ckpt-digest".to_owned(), }), log: Some(LogRef { @@ -60,6 +61,7 @@ mod tests { digest: "log-digest".to_owned(), }), durable_lsn: 144, + ever_served: true, endpoint: Some("10.0.0.1:7000".to_owned()), prefix: "/dataset".to_owned(), shard_index: 4, @@ -80,4 +82,13 @@ mod tests { matches!(err, ControlError::Codec(message) if message.contains("unsupported shard record codec version")) ); } + + #[test] + fn legacy_shard_record_defaults_to_ever_served_fail_closed() { + let bytes = br#"{"version":1,"record":{"shard_id":"mount-1:/","owner":null,"epoch":3,"lease_id":7,"state":"unassigned","checkpoint":null,"log":null,"durable_lsn":0}}"#; + + let record = decode_shard_record(bytes).unwrap(); + + assert!(record.ever_served); + } } diff --git a/crates/nokv-control/src/errors.rs b/crates/nokv-control/src/errors.rs index bd83e3c9b..c7512c55a 100644 --- a/crates/nokv-control/src/errors.rs +++ b/crates/nokv-control/src/errors.rs @@ -31,11 +31,25 @@ pub enum ControlError { ShardNotRegistered { shard_id: ShardId, }, + /// A server requested a fresh start for a shard that has already held an + /// owner epoch. Its local disk cannot be assumed authoritative; recovery + /// must use the explicit failover path and a checkpoint archive. + FreshAcquireRequiresFailover { + shard_id: ShardId, + epoch: u64, + }, StaleLease { shard_id: ShardId, epoch: u64, lease_id: u64, }, + /// A checkpoint/log publication would make the durable recovery identity + /// ambiguous or move one of its references behind the recorded durable + /// boundary. Reject it instead of allowing the last writer to win. + RecoveryPublicationConflict { + shard_id: ShardId, + reason: String, + }, InvalidOptions(String), Codec(String), Backend(String), @@ -80,6 +94,12 @@ impl fmt::Display for ControlError { "shard {} must be registered before it can be acquired", shard_id.as_str() ), + Self::FreshAcquireRequiresFailover { shard_id, epoch } => write!( + f, + "shard {} has prior owner epoch {}; use explicit failover recovery", + shard_id.as_str(), + epoch + ), Self::StaleLease { shard_id, epoch, @@ -91,6 +111,11 @@ impl fmt::Display for ControlError { epoch, lease_id ), + Self::RecoveryPublicationConflict { shard_id, reason } => write!( + f, + "recovery publication for shard {} conflicts with durable state: {reason}", + shard_id.as_str() + ), Self::InvalidOptions(err) => write!(f, "invalid control store options: {err}"), Self::Codec(err) => write!(f, "control store codec error: {err}"), Self::Backend(err) => write!(f, "control store backend error: {err}"), diff --git a/crates/nokv-control/src/etcd.rs b/crates/nokv-control/src/etcd.rs index 5c3c5f534..3a5ec98ad 100644 --- a/crates/nokv-control/src/etcd.rs +++ b/crates/nokv-control/src/etcd.rs @@ -3,7 +3,7 @@ use std::future::Future; use etcd_client::{Client, Compare, CompareOp, GetOptions, PutOptions, Txn, TxnOp}; use tokio::runtime::{Builder, Runtime}; -use crate::store::ControlStore; +use crate::store::{apply_recovery_publication, ControlStore}; use crate::{ decode_shard_record, encode_shard_record, CheckpointRef, ControlError, EtcdControlStoreOptions, LogRef, NodeId, ShardId, ShardLease, ShardRecord, ShardState, @@ -200,11 +200,24 @@ impl ControlStore for EtcdControlStore { } }; if let Some(existing_owner) = current.record.owner.clone() { - return Err(ControlError::ShardAlreadyOwned { - shard_id, - owner: existing_owner, - epoch: current.record.epoch, - }); + // A crashed owner leaves its durable record behind after the + // lease-attached session key expires. Fresh may take over that + // stale record only when durable control state proves this shard + // never reached Serving and has no recovery identity. The txn's + // session-key create_revision guard below is the atomic liveness + // proof; an active prior session still rejects the takeover. + let never_served_empty = !current.record.ever_served + && current.record.state == ShardState::Recovering + && current.record.checkpoint.is_none() + && current.record.log.is_none() + && current.record.durable_lsn == 0; + if !never_served_empty { + return Err(ControlError::ShardAlreadyOwned { + shard_id, + owner: existing_owner, + epoch: current.record.epoch, + }); + } } let lease_id = grant_lease(&mut client, options.lease_ttl_seconds()).await?; @@ -213,7 +226,10 @@ impl ControlStore for EtcdControlStore { next.endpoint = Some(owner.as_str().to_owned()); next.epoch = next.epoch.saturating_add(1).max(1); next.lease_id = lease_id; - next.state = ShardState::Serving; + // A fresh lease grants recovery authority, not readiness. Only the + // exact-lease CAS in `mark_serving` may make the shard routable after + // restore and checkpoint publication have completed. + next.state = ShardState::Recovering; let lease = ShardLease { shard_id: shard_id.clone(), @@ -407,14 +423,9 @@ impl ControlStore for EtcdControlStore { validate_record_lease(¤t.record, &lease)?; let mut next = current.record.clone(); - if let Some(checkpoint) = checkpoint { - next.checkpoint = Some(checkpoint); - } - if let Some(log) = log { - next.log = Some(log); - } - next.durable_lsn = next.durable_lsn.max(durable_lsn); + apply_recovery_publication(&mut next, checkpoint, log, durable_lsn)?; next.state = ShardState::Serving; + next.ever_served = true; // `Compare::lease(session_key) == my lease_id` proves I am still the // live owner: the stable session key exists AND is attached to my @@ -709,6 +720,57 @@ mod etcd_session_tests { cleanup_keys(&endpoints, &key_prefix); } + #[test] + fn fresh_resurrects_only_a_never_served_owner_after_session_expiry() { + let Some(endpoints) = live_etcd_endpoints() else { + return; + }; + let key_prefix = unique_key_prefix(); + let options = EtcdControlStoreOptions::new(endpoints.clone()) + .with_key_prefix(key_prefix.clone()) + .with_lease_ttl_seconds(30); + let store = EtcdControlStore::connect(options).expect("connect etcd control store"); + let shard_id = ShardId::new(format!("{key_prefix}:/runs")); + store + .register_shard(shard_id.clone(), "/runs".to_owned(), 2) + .expect("register shard"); + + let first = store + .acquire_unassigned(shard_id.clone(), NodeId::new("node-a")) + .expect("first Fresh acquire"); + assert!(!store.get_shard(&shard_id).unwrap().ever_served); + assert!(matches!( + store.acquire_unassigned(shard_id.clone(), NodeId::new("node-b")), + Err(ControlError::ShardAlreadyOwned { .. }) + )); + + // Crash before mark_serving: the durable record keeps owner A, while + // revoking its lease removes the only liveness proof. + revoke_lease_via_raw_client(&endpoints, first.lease_id); + let second = await_fresh_success(&store, &shard_id, &NodeId::new("node-b")); + assert_eq!(second.epoch, 2); + let resurrected = store.get_shard(&shard_id).unwrap(); + assert_eq!(resurrected.owner, Some(NodeId::new("node-b"))); + assert!(!resurrected.ever_served); + assert!(resurrected.checkpoint.is_none()); + assert!(resurrected.log.is_none()); + assert_eq!(resurrected.durable_lsn, 0); + + // Once any generation reaches Serving, an expired session can only use + // explicit Failover; Fresh must never reinterpret it as empty. + store + .mark_serving(&second, None, None, 0) + .expect("mark resurrected shard serving"); + revoke_lease_via_raw_client(&endpoints, second.lease_id); + let served_fresh = store.acquire_unassigned(shard_id.clone(), NodeId::new("node-c")); + assert!(matches!( + served_fresh, + Err(ControlError::ShardAlreadyOwned { .. }) + )); + + cleanup_keys(&endpoints, &key_prefix); + } + /// Retry the failover until the revoked lease's session key has been reaped, /// so the create_revision==0 guard can pass. Bounded so a real failure still /// surfaces rather than hanging. @@ -733,6 +795,28 @@ mod etcd_session_tests { } } + fn await_fresh_success( + store: &EtcdControlStore, + shard_id: &ShardId, + owner: &NodeId, + ) -> crate::ShardLease { + use std::thread::sleep; + use std::time::{Duration, Instant}; + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match store.acquire_unassigned(shard_id.clone(), owner.clone()) { + Ok(lease) => return lease, + Err(err) if Instant::now() < deadline => { + sleep(Duration::from_millis(100)); + let _ = err; + } + Err(err) => { + panic!("Fresh resurrection did not succeed after lease revoke: {err:?}") + } + } + } + } + fn revoke_lease_via_raw_client(endpoints: &[String], lease_id: u64) { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -843,6 +927,54 @@ mod etcd_session_tests { cleanup_keys(&endpoints, &key_prefix); } + #[test] + fn etcd_rejects_malformed_log_without_overwriting_durable_ref() { + let Some(endpoints) = live_etcd_endpoints() else { + return; + }; + let key_prefix = unique_key_prefix(); + let options = EtcdControlStoreOptions::new(endpoints.clone()) + .with_key_prefix(key_prefix.clone()) + .with_lease_ttl_seconds(30); + let store = EtcdControlStore::connect(options).expect("connect etcd control store"); + let shard_id = ShardId::new(format!("{key_prefix}:/runs")); + store + .register_shard(shard_id.clone(), "/runs".to_owned(), 2) + .expect("register shard"); + let lease = store + .acquire_unassigned(shard_id.clone(), NodeId::new("node-a")) + .expect("acquire shard"); + let durable = LogRef { + segments: vec![crate::LogSegmentRef { + segment_key: "meta/log/segment-1".to_owned(), + first_lsn: 1, + last_lsn: 1, + digest: "digest-1".to_owned(), + }], + durable_lsn: 1, + digest: "digest-1".to_owned(), + }; + store + .mark_serving(&lease, None, Some(durable.clone()), 1) + .expect("publish durable log"); + + let malformed = LogRef { + segments: Vec::new(), + durable_lsn: 2, + digest: "digest-2".to_owned(), + }; + assert!(matches!( + store.mark_serving(&lease, None, Some(malformed), 2), + Err(ControlError::RecoveryPublicationConflict { .. }) + )); + let record = store.get_shard(&shard_id).unwrap(); + assert_eq!(record.log, Some(durable)); + assert_eq!(record.durable_lsn, 1); + + let _ = store.release(&lease); + cleanup_keys(&endpoints, &key_prefix); + } + fn cleanup_keys(endpoints: &[String], key_prefix: &str) { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() diff --git a/crates/nokv-control/src/store.rs b/crates/nokv-control/src/store.rs index e63224d6c..b5b711532 100644 --- a/crates/nokv-control/src/store.rs +++ b/crates/nokv-control/src/store.rs @@ -43,6 +43,15 @@ pub trait ControlStore: Send + Sync { previous_epoch: u64, ) -> Result; fn renew(&self, lease: &ShardLease) -> Result; + /// Publish recovery references for the current lease. + /// + /// This API is owner-fenced but deliberately does not infer capture order + /// between concurrent calls made with the same lease. A caller that can + /// publish from multiple tasks must provide one single-writer critical + /// section spanning capture/prepare, this CAS, and pruning of the superseded + /// object(s). The server's recovery-publication gate is the production + /// implementation of that contract. Pruning before a successful CAS, or + /// allowing a later capture to overtake an earlier one, is unsafe. fn mark_serving( &self, lease: &ShardLease, @@ -92,6 +101,251 @@ pub(crate) fn is_default_shard(shard_id: &ShardId) -> bool { == "/" } +/// Merge one caller-serialized, owner-fenced recovery publication. +/// +/// This rejects durable-LSN rollback and conflicting comparable log identities, +/// but it is not a concurrency sequencer: in particular, checkpoint-only images +/// at LSN zero may legitimately replace one another. Capture order must already +/// be serialized by the `mark_serving` caller as documented on [`ControlStore`]. +/// +/// Checkpoints dominate logs they fully cover: once a checkpoint at the +/// durable tail is published, its covered log chain can be pruned and must not +/// be reattached by a delayed same-LSN log publication. A later log may advance +/// beyond that checkpoint and retains it as its replay base. +pub(crate) fn apply_recovery_publication( + record: &mut ShardRecord, + checkpoint: Option, + log: Option, + durable_lsn: u64, +) -> Result<(), ControlError> { + let shard_id = record.shard_id.clone(); + let conflict = |reason: String| ControlError::RecoveryPublicationConflict { + shard_id: shard_id.clone(), + reason, + }; + + let reference_lsn = match (checkpoint.as_ref(), log.as_ref()) { + (Some(checkpoint), Some(log)) => checkpoint.lsn.max(log.durable_lsn), + (Some(checkpoint), None) => checkpoint.lsn, + (None, Some(log)) => log.durable_lsn, + (None, None) => { + if durable_lsn > record.durable_lsn { + return Err(conflict(format!( + "durable LSN {durable_lsn} has no checkpoint or log identity" + ))); + } + return Ok(()); + } + }; + if durable_lsn != reference_lsn { + return Err(conflict(format!( + "publication durable LSN {durable_lsn} does not match reference tail {reference_lsn}" + ))); + } + + if let Some(checkpoint) = checkpoint.as_ref() { + if checkpoint.lsn < record.durable_lsn { + return Err(conflict(format!( + "checkpoint LSN {} is behind durable LSN {}", + checkpoint.lsn, record.durable_lsn + ))); + } + // Checkpoint-only deployments have no logical log allocator, so every + // image legitimately carries LSN 0 and the zero digest even as metadata + // changes. The owner publication gate serializes those images; at the + // durable-store boundary only their logical tail digest is comparable. + } + if let Some(log) = log.as_ref() { + if log.durable_lsn < record.durable_lsn { + return Err(conflict(format!( + "log LSN {} is behind durable LSN {}", + log.durable_lsn, record.durable_lsn + ))); + } + validate_log_ref(record, checkpoint.as_ref(), log).map_err(conflict)?; + if let Some(current) = record + .log + .as_ref() + .filter(|current| current.durable_lsn == log.durable_lsn) + { + if current != log { + return Err(conflict(format!( + "log identity differs at LSN {}", + log.durable_lsn + ))); + } + } + } + + if let (Some(checkpoint), Some(log)) = (checkpoint.as_ref(), log.as_ref()) { + if checkpoint.lsn == log.durable_lsn && checkpoint.digest != log.digest { + return Err(conflict(format!( + "checkpoint and log digests differ at LSN {}", + checkpoint.lsn + ))); + } + } + + let current_tail_digest = durable_tail_digest(record).map_err(conflict)?; + if let Some(expected) = current_tail_digest.as_deref() { + if let Some(checkpoint) = checkpoint + .as_ref() + .filter(|checkpoint| checkpoint.lsn == record.durable_lsn) + { + if checkpoint.digest != expected { + return Err(conflict(format!( + "checkpoint digest differs from durable tail at LSN {}", + checkpoint.lsn + ))); + } + } + if let Some(log) = log + .as_ref() + .filter(|log| log.durable_lsn == record.durable_lsn) + { + if log.digest != expected { + return Err(conflict(format!( + "log digest differs from durable tail at LSN {}", + log.durable_lsn + ))); + } + } + } + + if let Some(checkpoint) = checkpoint { + record.checkpoint = Some(checkpoint); + } + if let Some(log) = log { + record.log = Some(log); + } + if let Some(checkpoint) = record.checkpoint.as_ref() { + if record + .log + .as_ref() + .is_some_and(|log| log.durable_lsn <= checkpoint.lsn) + { + record.log = None; + } + } + record.durable_lsn = durable_lsn; + Ok(()) +} + +fn validate_log_ref( + record: &ShardRecord, + incoming_checkpoint: Option<&CheckpointRef>, + log: &LogRef, +) -> Result<(), String> { + if log.segments.is_empty() { + return Err("log segment chain is empty".to_owned()); + } + + let checkpoint_lsn = incoming_checkpoint + .or(record.checkpoint.as_ref()) + .map_or(0, |checkpoint| checkpoint.lsn); + if log.durable_lsn < checkpoint_lsn { + return Err(format!( + "log durable LSN {} is behind checkpoint LSN {checkpoint_lsn}", + log.durable_lsn + )); + } + + for (index, segment) in log.segments.iter().enumerate() { + if segment.segment_key.is_empty() { + return Err(format!("log segment {index} has an empty object key")); + } + if segment.digest.is_empty() { + return Err(format!("log segment {index} has an empty digest")); + } + if segment.first_lsn == 0 || segment.first_lsn > segment.last_lsn { + return Err(format!( + "log segment {index} has invalid LSN range {}..{}", + segment.first_lsn, segment.last_lsn + )); + } + if let Some(previous) = index.checked_sub(1).map(|previous| &log.segments[previous]) { + let expected = previous + .last_lsn + .checked_add(1) + .ok_or_else(|| "log segment LSN range is exhausted".to_owned())?; + if segment.first_lsn != expected { + return Err(format!( + "log segment {index} starts at {}, expected {expected}", + segment.first_lsn + )); + } + } + } + + let first = log + .segments + .first() + .expect("non-empty segment chain has a first segment"); + // A log at the checkpoint tail is a delayed, fully-covered publication. It + // is still validated internally and then discarded by the checkpoint-cover + // rule, but its historical first LSN need not follow the new checkpoint. + if log.durable_lsn > checkpoint_lsn { + let expected_first = checkpoint_lsn + .checked_add(1) + .ok_or_else(|| "checkpoint LSN is exhausted".to_owned())?; + if first.first_lsn != expected_first { + return Err(format!( + "log chain starts at LSN {}, expected {expected_first} after checkpoint", + first.first_lsn + )); + } + } + + let last = log + .segments + .last() + .expect("non-empty segment chain has a last segment"); + if last.last_lsn != log.durable_lsn { + return Err(format!( + "log tail segment ends at LSN {}, durable LSN is {}", + last.last_lsn, log.durable_lsn + )); + } + if last.digest != log.digest { + return Err("log tail digest does not match the final segment digest".to_owned()); + } + + if let Some(current) = record.log.as_ref().filter(|current| { + current.durable_lsn > checkpoint_lsn && log.durable_lsn > current.durable_lsn + }) { + if log.segments.len() < current.segments.len() + || log.segments[..current.segments.len()] != current.segments + { + return Err( + "advanced log chain does not preserve the durable segment prefix".to_owned(), + ); + } + } + Ok(()) +} + +fn durable_tail_digest(record: &ShardRecord) -> Result, String> { + let checkpoint = record + .checkpoint + .as_ref() + .filter(|checkpoint| checkpoint.lsn == record.durable_lsn); + let log = record + .log + .as_ref() + .filter(|log| log.durable_lsn == record.durable_lsn); + if let (Some(checkpoint), Some(log)) = (checkpoint, log) { + if checkpoint.digest != log.digest { + return Err(format!( + "stored checkpoint and log digests differ at durable LSN {}", + record.durable_lsn + )); + } + } + Ok(log + .map(|log| log.digest.clone()) + .or_else(|| checkpoint.map(|checkpoint| checkpoint.digest.clone()))) +} + #[derive(Default)] pub struct InMemoryControlStore { shards: Mutex>, @@ -202,7 +456,11 @@ impl ControlStore for InMemoryControlStore { record.endpoint = Some(owner.as_str().to_owned()); record.epoch = record.epoch.saturating_add(1).max(1); record.lease_id = Self::next_lease(record); - record.state = ShardState::Serving; + // Acquisition only establishes exclusive recovery ownership. The shard + // must remain unroutable until the owner has restored local metadata, + // installed its durability fences, and published an exact recovery + // checkpoint through `mark_serving`. + record.state = ShardState::Recovering; Ok(ShardLease { shard_id, owner, @@ -262,14 +520,9 @@ impl ControlStore for InMemoryControlStore { .get_mut(&lease.shard_id) .ok_or_else(|| ControlError::ShardNotFound(lease.shard_id.clone()))?; Self::validate_lease(record, lease)?; - if checkpoint.is_some() { - record.checkpoint = checkpoint; - } - if log.is_some() { - record.log = log; - } - record.durable_lsn = record.durable_lsn.max(durable_lsn); + apply_recovery_publication(record, checkpoint, log, durable_lsn)?; record.state = ShardState::Serving; + record.ever_served = true; Ok(record.clone()) } @@ -310,6 +563,48 @@ mod tests { store } + fn checkpoint_ref(lsn: u64, digest: &str, identity: &str) -> CheckpointRef { + CheckpointRef { + object_key: format!("meta/checkpoints/{identity}"), + lsn, + image_bytes: 1024, + image_digest: format!("sha256:{identity}"), + digest: digest.to_owned(), + } + } + + fn log_ref(lsn: u64, digest: &str, identity: &str) -> LogRef { + log_ref_range(1, lsn, digest, identity) + } + + fn log_ref_range(first_lsn: u64, last_lsn: u64, digest: &str, identity: &str) -> LogRef { + LogRef { + segments: vec![LogSegmentRef { + segment_key: format!("meta/log/{identity}"), + first_lsn, + last_lsn, + digest: digest.to_owned(), + }], + durable_lsn: last_lsn, + digest: digest.to_owned(), + } + } + + fn extend_log_ref(current: &LogRef, lsn: u64, digest: &str, identity: &str) -> LogRef { + let mut segments = current.segments.clone(); + segments.push(LogSegmentRef { + segment_key: format!("meta/log/{identity}"), + first_lsn: current.durable_lsn + 1, + last_lsn: lsn, + digest: digest.to_owned(), + }); + LogRef { + segments, + durable_lsn: lsn, + digest: digest.to_owned(), + } + } + #[test] fn fresh_acquire_sets_owner_epoch_and_lease() { let store = registered_store(); @@ -320,7 +615,8 @@ mod tests { let record = store.get_shard(&lease.shard_id).unwrap(); assert_eq!(record.owner, Some(node("node-a"))); - assert_eq!(record.state, ShardState::Serving); + assert_eq!(record.state, ShardState::Recovering); + assert!(!record.ever_served); // Registered identity survives acquisition. assert_eq!(record.shard_index, 2); assert_eq!(record.prefix, "/runs"); @@ -370,42 +666,185 @@ mod tests { .acquire_after_failure(shard(), node("node-b"), old.epoch) .unwrap(); - assert!(store.mark_serving(&old, None, None, 7).is_err()); + assert!(store.mark_serving(&old, None, None, 0).is_err()); - let record = store.mark_serving(&new, None, None, 7).unwrap(); + let record = store.mark_serving(&new, None, None, 0).unwrap(); assert_eq!(record.state, ShardState::Serving); - assert_eq!(record.durable_lsn, 7); + assert_eq!(record.durable_lsn, 0); + assert!(record.ever_served); } #[test] fn mark_serving_preserves_recovery_refs_when_not_replaced() { let store = registered_store(); let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); - let log = LogRef { - segments: vec![LogSegmentRef { - segment_key: "meta/log/segment".to_owned(), - first_lsn: 8, - last_lsn: 9, - digest: "abc123".to_owned(), - }], - durable_lsn: 9, - digest: "abc123".to_owned(), - }; - let checkpoint = CheckpointRef { - object_key: "meta/checkpoints/ckpt".to_owned(), - lsn: 9, - image_bytes: 1024, - digest: "def456".to_owned(), - }; + let checkpoint = checkpoint_ref(9, "abc123", "ckpt-9"); + let log = log_ref_range(10, 10, "def456", "segment-10"); + store + .mark_serving(&lease, Some(checkpoint.clone()), None, 9) + .unwrap(); store - .mark_serving(&lease, Some(checkpoint.clone()), Some(log.clone()), 9) + .mark_serving(&lease, None, Some(log.clone()), 10) .unwrap(); let record = store.mark_serving(&lease, None, None, 0).unwrap(); assert_eq!(record.checkpoint, Some(checkpoint)); assert_eq!(record.log, Some(log)); - assert_eq!(record.durable_lsn, 9); + assert_eq!(record.durable_lsn, 10); + } + + #[test] + fn mark_serving_rejects_a_log_behind_the_durable_tail() { + let store = registered_store(); + let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); + let older = log_ref(9, "digest-9", "segment-9"); + let newer = extend_log_ref(&older, 10, "digest-10", "segment-10"); + store + .mark_serving(&lease, None, Some(older.clone()), 9) + .unwrap(); + store + .mark_serving(&lease, None, Some(newer.clone()), 10) + .unwrap(); + + let err = store + .mark_serving(&lease, None, Some(older), 9) + .unwrap_err(); + assert!(matches!( + err, + ControlError::RecoveryPublicationConflict { .. } + )); + let record = store.get_shard(&lease.shard_id).unwrap(); + assert_eq!(record.durable_lsn, 10); + assert_eq!(record.log, Some(newer)); + } + + #[test] + fn mark_serving_rejects_conflicting_log_identity_at_the_same_lsn() { + let store = registered_store(); + let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); + let published = log_ref(9, "digest-9", "segment-a"); + store + .mark_serving(&lease, None, Some(published.clone()), 9) + .unwrap(); + + for conflicting in [ + log_ref(9, "digest-9", "segment-b"), + log_ref(9, "different-digest", "segment-a"), + ] { + assert!(matches!( + store.mark_serving(&lease, None, Some(conflicting), 9), + Err(ControlError::RecoveryPublicationConflict { .. }) + )); + } + assert_eq!( + store.get_shard(&lease.shard_id).unwrap().log, + Some(published) + ); + } + + #[test] + fn malformed_log_publication_cannot_replace_the_durable_chain() { + let store = registered_store(); + let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); + let durable = log_ref(2, "digest-2", "segment-1-2"); + store + .mark_serving(&lease, None, Some(durable.clone()), 2) + .unwrap(); + + let truncated = log_ref_range(3, 3, "digest-3", "segment-3"); + let mut gap = durable.clone(); + gap.segments.push(LogSegmentRef { + segment_key: "meta/log/segment-4".to_owned(), + first_lsn: 4, + last_lsn: 4, + digest: "digest-4".to_owned(), + }); + gap.durable_lsn = 4; + gap.digest = "digest-4".to_owned(); + let mut tail_mismatch = extend_log_ref(&durable, 3, "digest-3", "segment-3"); + tail_mismatch.durable_lsn = 4; + let mut digest_mismatch = extend_log_ref(&durable, 3, "digest-3", "segment-3"); + digest_mismatch.digest = "different-tail-digest".to_owned(); + + for malformed in [ + LogRef { + segments: Vec::new(), + durable_lsn: 3, + digest: "digest-3".to_owned(), + }, + truncated, + gap, + tail_mismatch, + digest_mismatch, + ] { + assert!(matches!( + store.mark_serving(&lease, None, Some(malformed.clone()), malformed.durable_lsn,), + Err(ControlError::RecoveryPublicationConflict { .. }) + )); + let record = store.get_shard(&lease.shard_id).unwrap(); + assert_eq!(record.log, Some(durable.clone())); + assert_eq!(record.durable_lsn, 2); + } + } + + #[test] + fn checkpoint_covers_log_and_only_a_newer_log_reopens_the_chain() { + let store = registered_store(); + let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); + let covered_log = log_ref(9, "digest-9", "segment-9"); + store + .mark_serving(&lease, None, Some(covered_log.clone()), 9) + .unwrap(); + let checkpoint = checkpoint_ref(9, "digest-9", "ckpt-9"); + let covered = store + .mark_serving(&lease, Some(checkpoint.clone()), None, 9) + .unwrap(); + assert_eq!(covered.checkpoint, Some(checkpoint.clone())); + assert_eq!(covered.log, None); + + let delayed = store + .mark_serving(&lease, None, Some(covered_log), 9) + .unwrap(); + assert_eq!(delayed.checkpoint, Some(checkpoint.clone())); + assert_eq!(delayed.log, None); + + let advanced_log = log_ref_range(10, 10, "digest-10", "segment-10"); + let advanced = store + .mark_serving(&lease, None, Some(advanced_log.clone()), 10) + .unwrap(); + assert_eq!(advanced.checkpoint, Some(checkpoint)); + assert_eq!(advanced.log, Some(advanced_log)); + assert_eq!(advanced.durable_lsn, 10); + } + + #[test] + fn checkpoint_only_publication_allows_a_new_image_at_lsn_zero() { + let store = registered_store(); + let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); + let first = checkpoint_ref(0, "zero-digest", "image-a"); + let second = checkpoint_ref(0, "zero-digest", "image-b"); + store.mark_serving(&lease, Some(first), None, 0).unwrap(); + + let record = store + .mark_serving(&lease, Some(second.clone()), None, 0) + .unwrap(); + assert_eq!(record.checkpoint, Some(second)); + assert_eq!(record.durable_lsn, 0); + } + + #[test] + fn mark_serving_cannot_advance_without_a_recovery_identity() { + let store = registered_store(); + let lease = store.acquire_unassigned(shard(), node("node-a")).unwrap(); + + assert!(matches!( + store.mark_serving(&lease, None, None, 1), + Err(ControlError::RecoveryPublicationConflict { .. }) + )); + let record = store.get_shard(&lease.shard_id).unwrap(); + assert_eq!(record.durable_lsn, 0); + assert_eq!(record.state, ShardState::Recovering); } #[test] diff --git a/crates/nokv-control/src/types.rs b/crates/nokv-control/src/types.rs index c36fa270a..1de39e9e1 100644 --- a/crates/nokv-control/src/types.rs +++ b/crates/nokv-control/src/types.rs @@ -25,6 +25,11 @@ pub struct CheckpointRef { pub object_key: String, pub lsn: u64, pub image_bytes: u64, + /// SHA-256 identity of the exact checkpoint image. Empty only when decoding + /// a pre-fence control record; recovery must reject those legacy refs. + #[serde(default)] + pub image_digest: String, + /// Digest of the logical metadata log at `lsn` (not the image digest). pub digest: String, } @@ -60,6 +65,12 @@ pub struct ShardRecord { pub checkpoint: Option, pub log: Option, pub durable_lsn: u64, + /// Durable proof that this shard has completed `mark_serving` at least once. + /// A never-served shard may safely retry Fresh acquisition after a startup + /// failure; every legacy record defaults to `true` on decode so historical + /// data is never mistaken for an empty, resurrectable shard. + #[serde(default = "legacy_record_ever_served")] + pub ever_served: bool, /// Reachable endpoint (host:port) of the current owner, so a client can /// route to it. `None` when unowned. (In this deployment the `NodeId` is the /// bind address, so this mirrors `owner` while owned.) @@ -89,6 +100,10 @@ fn default_shard_prefix() -> String { "/".to_owned() } +fn legacy_record_ever_served() -> bool { + true +} + /// Derive the path prefix from a `mount-:` shard id, defaulting to `/`. fn shard_prefix_from_id(shard_id: &ShardId) -> String { shard_id @@ -140,6 +155,7 @@ impl ShardRecord { checkpoint: None, log: None, durable_lsn: 0, + ever_served: false, endpoint: None, prefix, shard_index: 0, diff --git a/crates/nokv-fuse/src/backend.rs b/crates/nokv-fuse/src/backend.rs index 298b15a40..f30223685 100644 --- a/crates/nokv-fuse/src/backend.rs +++ b/crates/nokv-fuse/src/backend.rs @@ -1321,6 +1321,10 @@ where replace: fields.replace, dentry_version: fields.dentry_version, old_generation: fields.old_generation, + // The legacy FUSE journal has no durable object-GC token. Keep + // recovery fail-closed until FUSE can refresh and fully restage a + // new generation in its own scoped change. + object_gc_claim_version: 0, } } @@ -1455,6 +1459,7 @@ mod tests { replace: true, dentry_version: Some(1), old_generation: None, + object_gc_claim_version: 2, }; let pending = backend @@ -1520,6 +1525,7 @@ mod tests { replace: true, dentry_version: Some(1), old_generation: None, + object_gc_claim_version: 2, }; let pending = backend @@ -1583,6 +1589,7 @@ mod tests { replace: true, dentry_version: Some(1), old_generation: None, + object_gc_claim_version: 2, }; let err = backend diff --git a/crates/nokv-meta/src/command.rs b/crates/nokv-meta/src/command.rs index c421980e6..f5a7d8949 100644 --- a/crates/nokv-meta/src/command.rs +++ b/crates/nokv-meta/src/command.rs @@ -304,12 +304,19 @@ pub trait MetadataStore { .collect() } + /// Authoritatively read the current atomic-apply marker for `request_id`. + /// + /// `Some(result)` proves that the command's apply is visible in this + /// metadata engine with that exact result. `None` must authoritatively prove + /// that the preceding commit call did not make this request id visible. An + /// error means the outcome remains unknown; callers must not interpret it + /// as absence or allow a later ordered commit to overtake the unresolved + /// command. This marker alone does not establish cross-machine durability; + /// synchronous log archival provides that separate guarantee. fn committed_request_result( &self, - _request_id: &[u8], - ) -> Result, MetadataError> { - Ok(None) - } + request_id: &[u8], + ) -> Result, MetadataError>; fn history_retention_epoch(&self) -> Result { Ok(0) @@ -423,6 +430,16 @@ impl Version { } impl MetadataCommand { + /// The only successful result this command is allowed to persist in the + /// request-id deduplication record. + pub fn expected_commit_result(&self) -> CommitResult { + CommitResult { + commit_version: self.commit_version, + applied_mutations: self.mutations.len(), + watch_events: self.watch.len(), + } + } + pub fn validate(&self) -> Result<(), MetadataError> { if self.request_id.is_empty() { return Err(MetadataError::EmptyRequestId); @@ -566,6 +583,13 @@ mod tests { unimplemented!("not needed by scan_delimited tests") } + fn committed_request_result( + &self, + _request_id: &[u8], + ) -> Result, MetadataError> { + unimplemented!("not needed by scan_delimited tests") + } + fn prune_history( &self, _request: HistoryPruneRequest, diff --git a/crates/nokv-meta/src/layout/mod.rs b/crates/nokv-meta/src/layout/mod.rs index 77110106b..efc9eb13e 100644 --- a/crates/nokv-meta/src/layout/mod.rs +++ b/crates/nokv-meta/src/layout/mod.rs @@ -30,6 +30,40 @@ pub fn allocator_key(mount: MountId) -> Vec { out } +pub fn object_gc_claim_key(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 16); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"object-gc-claim\0"); + out +} + +pub fn object_gc_scan_cursor_key(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 22); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"object-gc-scan-cursor\0"); + out +} + +pub fn failover_durability_required_key(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 29); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"failover-durability-required\0"); + out +} + +pub fn object_gc_quarantine_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 21); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"object-gc-quarantine\0"); + out +} + +pub fn object_gc_quarantine_key(mount: MountId, record_digest: &[u8; 32]) -> Vec { + let mut out = object_gc_quarantine_prefix(mount); + out.extend_from_slice(record_digest); + out +} + pub fn inode_key(mount: MountId, inode: InodeId) -> Vec { let mut out = inode_prefix(mount); push_u64(&mut out, inode.get()); @@ -417,6 +451,21 @@ mod tests { assert!(key < later); } + #[test] + fn object_gc_control_keys_are_mount_scoped_and_distinct() { + let claim = object_gc_claim_key(mount()); + let cursor = object_gc_scan_cursor_key(mount()); + let failover = failover_durability_required_key(mount()); + let quarantine = object_gc_quarantine_key(mount(), &[7; 32]); + let other_claim = object_gc_claim_key(MountId::new(8).unwrap()); + + assert_ne!(claim, cursor); + assert_ne!(claim, failover); + assert_ne!(cursor, failover); + assert_ne!(claim, other_claim); + assert!(quarantine.starts_with(&object_gc_quarantine_prefix(mount()))); + } + #[test] fn snapshot_pin_keys_are_mount_scoped() { let key = snapshot_pin_key(mount(), 10); diff --git a/crates/nokv-meta/src/lib.rs b/crates/nokv-meta/src/lib.rs index 504bdaa47..5d5cd44e1 100644 --- a/crates/nokv-meta/src/lib.rs +++ b/crates/nokv-meta/src/lib.rs @@ -32,7 +32,8 @@ pub use log::{ pub use service::{ BodyReadPlan, CheckpointHandle, CheckpointShard, CloneHandle, CreateInDirPathBatch, CreatedPreparedArtifact, DanglingBlock, DentryWithAttr, FsckReport, MetadError, - MetadataArchiveConfig, MetadataBackupOutcome, MetadataLogArchiveConfig, + MetadataArchiveConfig, MetadataBackupOutcome, MetadataCheckpointIdentity, + MetadataLogArchiveConfig, MetadataLogPruneOutcome, MetadataLogPublicationState, MetadataLogRestoreOutcome, MetadataLogSegmentArchiveOutcome, MetadataLogSegmentPointer, MetadataLogSyncConfig, MetadataLogSyncSnapshot, MetadataRestoreOutcome, MetadataServiceStats, NamespaceAggregateGroup, NamespaceAggregateMeasure, NamespaceAggregateOp, diff --git a/crates/nokv-meta/src/log.rs b/crates/nokv-meta/src/log.rs index 3e0d1c04c..6a6afd6c1 100644 --- a/crates/nokv-meta/src/log.rs +++ b/crates/nokv-meta/src/log.rs @@ -56,6 +56,7 @@ pub enum MetadataLogError { DigestMismatch, SegmentDigestMismatch, SegmentHeaderMismatch, + UnexpectedShard { expected: String, actual: String }, ChainShardMismatch { previous: String, next: String }, ChainLsnGap { previous: u64, next: u64 }, ChainDigestMismatch, @@ -282,15 +283,25 @@ impl MetadataLogSegment { } pub fn metadata_log_replay_entries( + expected_shard_id: &str, segments: &[MetadataLogSegment], checkpoint_lsn: u64, checkpoint_digest: [u8; 32], ) -> Result, MetadataLogError> { + if expected_shard_id.is_empty() { + return Err(MetadataLogError::EmptyShardId); + } let mut previous_lsn = checkpoint_lsn; let mut previous_digest = checkpoint_digest; let mut out = Vec::new(); for segment in segments { segment.verify_follows(previous_lsn, previous_digest)?; + if segment.shard_id != expected_shard_id { + return Err(MetadataLogError::UnexpectedShard { + expected: expected_shard_id.to_owned(), + actual: segment.shard_id.clone(), + }); + } out.extend(segment.entries.iter().cloned()); previous_lsn = segment.last_lsn; previous_digest = segment.last_digest; @@ -835,6 +846,10 @@ impl fmt::Display for MetadataLogError { Self::DigestMismatch => write!(f, "metadata log digest mismatch"), Self::SegmentDigestMismatch => write!(f, "metadata log segment digest mismatch"), Self::SegmentHeaderMismatch => write!(f, "metadata log segment header mismatch"), + Self::UnexpectedShard { expected, actual } => write!( + f, + "metadata log belongs to shard {actual}, expected {expected}" + ), Self::ChainShardMismatch { previous, next } => { write!(f, "metadata log shard changed from {previous} to {next}") } @@ -1147,8 +1162,13 @@ mod tests { let left = MetadataLogSegment::seal(vec![first.clone(), second.clone()]).unwrap(); let right = MetadataLogSegment::seal(vec![third.clone()]).unwrap(); - let replay = - metadata_log_replay_entries(&[left, right], 0, METADATA_LOG_ZERO_DIGEST).unwrap(); + let replay = metadata_log_replay_entries( + "mount-1:/runs", + &[left, right], + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap(); assert_eq!(replay, vec![first, second, third]); } @@ -1159,11 +1179,28 @@ mod tests { let segment = MetadataLogSegment::seal(vec![first]).unwrap(); assert!(matches!( - metadata_log_replay_entries(&[segment], 0, METADATA_LOG_ZERO_DIGEST), + metadata_log_replay_entries("mount-1:/runs", &[segment], 0, METADATA_LOG_ZERO_DIGEST,), Err(MetadataLogError::ChainLsnGap { previous: 0, next: 2 }) )); } + + #[test] + fn replay_entries_reject_a_self_consistent_foreign_shard_chain() { + let first = entry(b"req-1", 7, 1, 11, METADATA_LOG_ZERO_DIGEST); + let segment = MetadataLogSegment::seal(vec![first]).unwrap(); + + assert!(matches!( + metadata_log_replay_entries( + "mount-1:/other", + &[segment], + 0, + METADATA_LOG_ZERO_DIGEST, + ), + Err(MetadataLogError::UnexpectedShard { expected, actual }) + if expected == "mount-1:/other" && actual == "mount-1:/runs" + )); + } } diff --git a/crates/nokv-meta/src/service.rs b/crates/nokv-meta/src/service.rs index 4850b9657..6d2444661 100644 --- a/crates/nokv-meta/src/service.rs +++ b/crates/nokv-meta/src/service.rs @@ -26,14 +26,18 @@ mod snapshot; mod watch; mod xattr; -pub use self::backup::{MetadataArchiveConfig, MetadataBackupOutcome, MetadataRestoreOutcome}; +pub use self::backup::{ + MetadataArchiveConfig, MetadataBackupOutcome, MetadataCheckpointIdentity, + MetadataRestoreOutcome, +}; pub use self::checkpoint::{CheckpointHandle, CheckpointShard}; pub use self::fsck::{DanglingBlock, FsckReport}; pub use self::log_archive::{ MetadataLogArchiveConfig, MetadataLogRestoreOutcome, MetadataLogSegmentArchiveOutcome, }; pub use self::log_sync::{ - MetadataLogSegmentPointer, MetadataLogSyncConfig, MetadataLogSyncSnapshot, + MetadataLogPruneOutcome, MetadataLogPublicationState, MetadataLogSegmentPointer, + MetadataLogSyncConfig, MetadataLogSyncSnapshot, }; pub use self::snapshot::DEFAULT_SNAPSHOT_LEASE_MS; @@ -57,10 +61,12 @@ use crate::layout::{ decode_body_descriptor, decode_chunk_manifest, decode_dentry_projection, decode_inode_attr, decode_object_gc_record, decode_path_index_catalog, decode_path_index_row, decode_snapshot_pin, decode_watch_event, dentry_key, dentry_mount_prefix, dentry_prefix, encode_allocator_state, - encode_body_descriptor, encode_chunk_manifest, encode_dentry_projection, encode_inode_attr, - encode_object_gc_record, encode_path_index_catalog, encode_path_index_row, encode_snapshot_pin, - encode_watch_event, gc_object_key, gc_queue_prefix, inode_key, path_index_catalog_key, - path_index_key, path_index_prefix, path_index_row_key, path_index_row_prefix, snapshot_pin_key, + encode_body_descriptor, encode_chunk_manifest, encode_dentry_projection, encode_fork_binding, + encode_inode_attr, encode_object_gc_record, encode_path_index_catalog, encode_path_index_row, + encode_snapshot_pin, encode_watch_event, failover_durability_required_key, fork_binding_key, + gc_object_key, gc_queue_prefix, inode_key, inode_prefix, object_gc_claim_key, + object_gc_quarantine_key, object_gc_scan_cursor_key, path_index_catalog_key, path_index_key, + path_index_prefix, path_index_row_key, path_index_row_prefix, snapshot_pin_key, snapshot_pin_prefix, watch_log_key, watch_log_prefix, xattr_key, xattr_prefix, PathIndexCatalogRecord, PathIndexFieldRecord, PathIndexRowRecord, PathIndexValueRecord, PATH_INDEX_DELIMITER, @@ -72,7 +78,7 @@ use nokv_object::{ }; use nokv_types::{ parse_absolute_path, AdvisoryLock, BlockDescriptor, BodyDescriptor, ChunkManifest, DentryName, - DentryProjection, DentryRecord, FileType, InodeAttr, InodeId, ModelError, MountId, + DentryProjection, DentryRecord, FileType, ForkBinding, InodeAttr, InodeId, ModelError, MountId, ObjectGcRecord, PathError, PathMetadata, ReadLease, RecordFamily, SliceManifest, SnapshotPin, SpecialNodeSpec, WatchCursor, WatchEvent, WatchEventKind, WatchRecord, }; @@ -109,6 +115,231 @@ const PATH_INDEX_LOOKUP_CACHE_MAX_ENTRIES_PER_SHARD: usize = const PATH_INDEX_VALIDATION_CACHE_MAX_ENTRIES_PER_SHARD: usize = PATH_INDEX_VALIDATION_CACHE_MAX_ENTRIES / PATH_CACHE_SHARD_COUNT; pub(crate) const DEFAULT_READ_LEASE_MS: u64 = 3_600_000; +const FAILOVER_DURABILITY_REQUIRED_MARKER: &[u8] = &[1]; + +fn decode_failover_durability_required_marker(bytes: &[u8]) -> Result<(), MetadError> { + if bytes == FAILOVER_DURABILITY_REQUIRED_MARKER { + Ok(()) + } else { + Err(MetadError::Codec( + "invalid failover durability requirement marker".to_owned(), + )) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum ObjectGcClaim { + Open, + Deleting { + owner_epoch: u64, + operation_token: u64, + gc_record_key: Vec, + gc_record_version: u64, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ObjectGcRecordKey { + enqueue_version: u64, + inode: InodeId, + generation: u64, + chunk_index: u64, + block_index: u64, +} + +fn decode_canonical_block_object_owner( + object_key: &str, +) -> Result<(u64, u64, u64, u64, u64), MetadError> { + fn number(parts: &mut std::str::Split<'_, char>) -> Result { + let raw = parts + .next() + .ok_or_else(|| MetadError::Codec("block object key is truncated".to_owned()))?; + let value = raw.parse::().map_err(|_| { + MetadError::Codec("block object key contains an invalid number".to_owned()) + })?; + if value.to_string() != raw { + return Err(MetadError::Codec( + "block object key is not canonical".to_owned(), + )); + } + Ok(value) + } + + let mut parts = object_key.split('/'); + if parts.next() != Some("blocks") { + return Err(MetadError::Codec( + "object does not name a canonical block".to_owned(), + )); + } + let mount = number(&mut parts)?; + let inode = number(&mut parts)?; + let generation = number(&mut parts)?; + let chunk_index = number(&mut parts)?; + let block_index = number(&mut parts)?; + if parts.next().is_some() { + return Err(MetadError::Codec( + "block object key has trailing components".to_owned(), + )); + } + Ok((mount, inode, generation, chunk_index, block_index)) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ObjectReferenceMutation { + claim_version: Version, +} + +impl ObjectReferenceMutation { + pub(super) fn from_version(claim_version: Version) -> Self { + Self { claim_version } + } + + pub(super) fn version(self) -> Version { + self.claim_version + } + + pub(super) fn predicate(self, mount: MountId) -> PredicateRef { + PredicateRef { + family: RecordFamily::System, + key: object_gc_claim_key(mount), + predicate: Predicate::VersionEquals(self.claim_version), + } + } +} + +fn encode_object_gc_claim(claim: &ObjectGcClaim) -> Result, MetadError> { + match claim { + ObjectGcClaim::Open => Ok(vec![1]), + ObjectGcClaim::Deleting { + owner_epoch, + operation_token, + gc_record_key, + gc_record_version, + } => { + let key_len = u32::try_from(gc_record_key.len()) + .map_err(|_| MetadError::Codec("object GC record key is too long".to_owned()))?; + let mut out = Vec::with_capacity(29 + gc_record_key.len()); + out.push(2); + out.extend_from_slice(&owner_epoch.to_be_bytes()); + out.extend_from_slice(&operation_token.to_be_bytes()); + out.extend_from_slice(&gc_record_version.to_be_bytes()); + out.extend_from_slice(&key_len.to_be_bytes()); + out.extend_from_slice(gc_record_key); + Ok(out) + } + } +} + +fn decode_object_gc_claim(mount: MountId, bytes: &[u8]) -> Result { + match bytes { + [1] => Ok(ObjectGcClaim::Open), + [2, rest @ ..] if rest.len() >= 28 => { + let owner_epoch = u64::from_be_bytes(rest[..8].try_into().expect("u64 width")); + let operation_token = u64::from_be_bytes(rest[8..16].try_into().expect("u64 width")); + let gc_record_version = u64::from_be_bytes(rest[16..24].try_into().expect("u64 width")); + let key_len = u32::from_be_bytes(rest[24..28].try_into().expect("u32 width")) as usize; + if rest.len() != 28 + key_len { + return Err(MetadError::Codec( + "object GC claim record length mismatch".to_owned(), + )); + } + if owner_epoch == 0 { + return Err(MetadError::Codec( + "object GC claim owner epoch must be non-zero".to_owned(), + )); + } + if operation_token == 0 { + return Err(MetadError::Codec( + "object GC claim operation token must be non-zero".to_owned(), + )); + } + if gc_record_version == 0 { + return Err(MetadError::Codec( + "object GC claim record version must be non-zero".to_owned(), + )); + } + let gc_record_key = &rest[28..]; + decode_object_gc_record_key(mount, gc_record_key)?; + Ok(ObjectGcClaim::Deleting { + owner_epoch, + operation_token, + gc_record_key: gc_record_key.to_vec(), + gc_record_version, + }) + } + _ => Err(MetadError::Codec( + "invalid durable object GC claim record".to_owned(), + )), + } +} + +fn decode_object_gc_record_key( + mount: MountId, + gc_record_key: &[u8], +) -> Result { + const OBJECT_GC_KEY_FIELDS: usize = 6; + const U64_BYTES: usize = std::mem::size_of::(); + const OBJECT_GC_KEY_BYTES: usize = OBJECT_GC_KEY_FIELDS * U64_BYTES; + + if gc_record_key.len() != OBJECT_GC_KEY_BYTES { + return Err(MetadError::Codec( + "object GC claim record key has an invalid shape".to_owned(), + )); + } + + let field = |index: usize| { + let offset = index * U64_BYTES; + u64::from_be_bytes( + gc_record_key[offset..offset + U64_BYTES] + .try_into() + .expect("validated object GC key field width"), + ) + }; + let encoded_mount = field(0); + let enqueue_version = field(1); + let inode = field(2); + let generation = field(3); + let chunk_index = field(4); + let block_index = field(5); + + if encoded_mount != mount.get() { + return Err(MetadError::Codec( + "object GC claim record key belongs to another mount".to_owned(), + )); + } + if enqueue_version == 0 { + return Err(MetadError::Codec( + "object GC claim enqueue version must be non-zero".to_owned(), + )); + } + let inode = InodeId::new(inode) + .map_err(|err| MetadError::Codec(format!("invalid object GC claim inode: {err}")))?; + if generation == 0 { + return Err(MetadError::Codec( + "object GC claim generation must be non-zero".to_owned(), + )); + } + if gc_object_key( + mount, + enqueue_version, + inode, + generation, + chunk_index, + block_index, + ) != gc_record_key + { + return Err(MetadError::Codec( + "object GC claim record key is not canonical".to_owned(), + )); + } + Ok(ObjectGcRecordKey { + enqueue_version, + inode, + generation, + chunk_index, + block_index, + }) +} // Families folded into the fallback allocator rebuild when the durable // `allocator_key` System record is absent. Each row contributes its commit @@ -131,7 +362,7 @@ pub(crate) const DEFAULT_READ_LEASE_MS: u64 = 3_600_000; // `CommandDedupe` is the ONLY family with a non-standard value encoding; every // other family below writes through `encode_current_value`, so the scan can // decode them and recovery stays correct. -const ALLOCATOR_RECOVERY_FAMILIES: [RecordFamily; 12] = [ +const ALLOCATOR_RECOVERY_FAMILIES: [RecordFamily; 13] = [ RecordFamily::System, RecordFamily::Mount, RecordFamily::Inode, @@ -143,6 +374,7 @@ const ALLOCATOR_RECOVERY_FAMILIES: [RecordFamily; 12] = [ RecordFamily::PathIndex, RecordFamily::Watch, RecordFamily::Snapshot, + RecordFamily::ForkBinding, RecordFamily::Gc, ]; @@ -201,6 +433,7 @@ struct ReplaceProjectionCommit<'a> { old_generation: Option, version: Version, path_index: Option>, + object_reference: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -309,6 +542,10 @@ pub struct PreparedArtifact { pub replace: bool, pub dentry_version: Option, pub old_generation: Option, + /// Durable object-GC Open epoch captured before upload/planning. Publish + /// must CAS this exact version so an intervening delete cycle cannot make a + /// newly committed manifest point at an object that GC already removed. + pub object_gc_claim_version: u64, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -381,6 +618,7 @@ pub struct PendingObjectCleanupOutcome { pub scanned: usize, pub blocked_by_snapshots: usize, pub blocked_by_read_leases: usize, + pub blocked_by_failover_durability: usize, pub attempted: usize, pub deleted: usize, pub missing: usize, @@ -438,11 +676,11 @@ pub struct RenameReplaceResult { /// /// `root` is the new namespace root: it sees every file and directory the source /// subtree had at clone time and shares the source's object blocks (no data copy) -/// until the fork diverges on write. `snapshot_id` is a retained snapshot pin on -/// the source that holds the GC retention floor down so the shared base blocks are -/// protected while the fork references them. Retire it with -/// [`NoKvFs::retire_snapshot`] once the fork's own divergent state no longer needs -/// the source's base objects (typically when the fork is deleted). +/// until the fork diverges on write. `snapshot_id` identifies the durable +/// fork-retention binding that holds the GC retention floor after the temporary +/// construction snapshot expires. Retire it with [`NoKvFs::retire_snapshot`] +/// only after no fork reference can reach borrowed source blocks, including +/// hardlinks moved outside the original fork root. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct CloneHandle { pub root: InodeId, @@ -499,6 +737,26 @@ pub enum MetadError { bytes: u64, }, InvalidPreparedArtifact(String), + /// A prepared upload crossed an object-GC delete epoch. The caller may + /// refresh the prepared upload identity, but must mint a new generation and + /// fully restage the body before retrying the metadata publish. + StalePreparedArtifactObjectGcEpoch { + expected: u64, + current: u64, + }, + /// A crash interrupted a claimed external object deletion in a mode where + /// metadata alone cannot prove whether the delete took effect. Serving must + /// remain fenced until an operator completes controlled recovery. + ObjectGcRecoveryRequiresIntervention { + owner_epoch: u64, + operation_token: u64, + }, + /// The selected metadata checkpoint predates the durable object-GC + /// failover fence. It may reference objects deleted after that image was + /// produced, so installing it as a recovery source is unsafe. + MetadataArchiveMissingObjectGcFence { + checkpoint_key: String, + }, InvalidQuery(String), StaleBodyGeneration { expected: u64, @@ -563,6 +821,15 @@ pub enum MetadError { SnapshotBindingChanged { root_path: String, }, + /// The mount still has at least one current file whose effective manifest + /// borrows object blocks minted by another inode. Because history retention + /// uses one mount-global floor, no fork binding can be released until that + /// borrower is removed or rewritten onto self-owned blocks. + ForkRetentionActive { + snapshot_id: u64, + fork_root: InodeId, + borrower: InodeId, + }, SnapshotRenewContended { snapshot_id: u64, attempts: usize, @@ -587,12 +854,33 @@ pub struct NoKvFs { objects: O, allocator_gate: Mutex<()>, backup_gate: Mutex<()>, + /// Prevents a live worker/manual GC call from mistaking another in-process + /// worker's durable Deleting claim for crash recovery. + object_gc_gate: Mutex<()>, + /// Fail-closed process-local hint that an interrupted clone/rollback + /// materialization may have left an unbound inode tree. `link`/`rename` + /// perform the expensive mount-wide reachability proof only while this is + /// set. It is raised before the first materialization write and cleared only + /// after a proof sees every current inode/dentry under the mount root or a + /// live ForkBinding root. Reopen/restore reconstruct it from one such scan. + materialization_orphan_possible: AtomicBool, + #[cfg(test)] + namespace_reachability_scans: AtomicU64, /// Serializes owner-epoch installs/observes against in-flight commits. /// Commits hold a read guard across their fence check + durable apply; /// epoch changes take the write guard. This closes the TOCTOU where a /// failover epoch bump could land between a commit's check and its apply, /// letting a stale owner commit one more time. epoch_fence: RwLock<()>, + /// Lets ordinary commits run concurrently while sync logging is disabled, + /// but gives log enable/disable an exclusive linearization boundary against + /// every in-flight commit. + metadata_log_enable_fence: RwLock<()>, + /// While sync logging is enabled, preserves one total order from metadata + /// apply through archival. Without this gate, a thread paused after applying + /// command A can be overtaken by B, archiving B at LSN N and A at LSN N+1; + /// failover replay would then differ from the live metadata engine. + metadata_commit_log_gate: Mutex<()>, path_resolution_cache: Vec>>, path_index_lookup_cache: Vec>>, @@ -941,10 +1229,16 @@ fn create_watch_kind(kind: CommandKind) -> WatchEventKind { } fn validate_prepared_artifact( + mount: MountId, prepared: &PreparedArtifact, body: &BodyDescriptor, chunks: &[ChunkManifest], ) -> Result<(), MetadError> { + if prepared.object_gc_claim_version == 0 { + return Err(MetadError::InvalidPreparedArtifact( + "prepared artifact is missing a durable mutation epoch".to_owned(), + )); + } if body.generation != prepared.generation { return Err(MetadError::InvalidPreparedArtifact(format!( "body generation {} does not match prepared generation {}", @@ -1028,12 +1322,76 @@ fn validate_prepared_artifact( "slice descriptor is outside chunk range".to_owned(), )); } + for block in &slice.blocks { + let (block_mount, _, _, block_chunk_index, _) = + decode_canonical_block_object_owner(&block.object_key).map_err(|err| { + MetadError::InvalidPreparedArtifact(format!( + "invalid block object key {}: {err}", + block.object_key + )) + })?; + if block_mount != mount.get() { + return Err(MetadError::InvalidPreparedArtifact(format!( + "block object {} belongs to mount {block_mount}, expected {}", + block.object_key, + mount.get() + ))); + } + if block_chunk_index != chunk_index { + return Err(MetadError::InvalidPreparedArtifact(format!( + "block object {} belongs to chunk {block_chunk_index}, expected {chunk_index}", + block.object_key + ))); + } + } validate_slice_block_coverage(chunk.chunk_index, body.block_size, slice, slice_end)?; } } Ok(()) } +fn validate_new_prepared_block_identities( + mount: MountId, + prepared: &PreparedArtifact, + chunks: &[ChunkManifest], +) -> Result<(), MetadError> { + let mut seen_objects = HashSet::new(); + for chunk in chunks { + for slice in &chunk.slices { + for block in &slice.blocks { + let (block_mount, block_inode, block_generation, block_chunk_index, _) = + decode_canonical_block_object_owner(&block.object_key).map_err(|err| { + MetadError::InvalidPreparedArtifact(format!( + "invalid staged block object key {}: {err}", + block.object_key + )) + })?; + if block_mount != mount.get() + || block_inode != prepared.inode.get() + || block_generation != prepared.generation + || block_chunk_index != chunk.chunk_index + { + return Err(MetadError::InvalidPreparedArtifact(format!( + "staged block object {} does not match prepared mount/inode/generation/chunk {}/{}/{}/{}", + block.object_key, + mount.get(), + prepared.inode.get(), + prepared.generation, + chunk.chunk_index + ))); + } + if !seen_objects.insert(block.object_key.as_str()) { + return Err(MetadError::InvalidPreparedArtifact(format!( + "duplicate staged block object {}", + block.object_key + ))); + } + } + } + } + Ok(()) +} + fn validate_slice_block_coverage( chunk_index: u64, block_size: u64, @@ -1498,6 +1856,21 @@ impl fmt::Display for MetadError { Self::InvalidPreparedArtifact(err) => { write!(f, "invalid prepared artifact: {err}") } + Self::StalePreparedArtifactObjectGcEpoch { expected, current } => write!( + f, + "prepared artifact object-GC epoch {expected} is stale; current epoch is {current}" + ), + Self::ObjectGcRecoveryRequiresIntervention { + owner_epoch, + operation_token, + } => write!( + f, + "object GC deletion from owner epoch {owner_epoch} with operation token {operation_token} has an uncertain outcome and requires controlled recovery" + ), + Self::MetadataArchiveMissingObjectGcFence { checkpoint_key } => write!( + f, + "metadata checkpoint {checkpoint_key} predates the durable object-GC failover fence and cannot be restored safely" + ), Self::InvalidQuery(err) => write!(f, "invalid namespace query: {err}"), Self::StaleBodyGeneration { expected, current } => write!( f, @@ -1582,6 +1955,16 @@ impl fmt::Display for MetadError { Self::SnapshotBindingChanged { root_path } => { write!(f, "snapshot root binding changed while resolving {root_path}") } + Self::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + } => write!( + f, + "fork retention {snapshot_id} for root inode {} is still required by borrower inode {}", + fork_root.get(), + borrower.get() + ), Self::SnapshotRenewContended { snapshot_id, attempts, diff --git a/crates/nokv-meta/src/service/allocator.rs b/crates/nokv-meta/src/service/allocator.rs index 74d74939e..bad477b49 100644 --- a/crates/nokv-meta/src/service/allocator.rs +++ b/crates/nokv-meta/src/service/allocator.rs @@ -50,14 +50,6 @@ where { return Ok(()); } - // Read guard held across the fence check and the reservation commit so a - // failover epoch bump cannot land between them. - let _fence = self - .epoch_fence - .read() - .unwrap_or_else(|err| err.into_inner()); - self.ensure_owner_epoch_current()?; - let reserved_version = current_reserved_version.max(reservation_upper_bound( required_version, ALLOCATOR_VERSION_RESERVATION, @@ -74,8 +66,8 @@ where .max(2), )?; let key = allocator_key(self.mount); - self.metadata - .commit_metadata(MetadataCommand { + let reservation = self.commit_metadata_from_factory(|| { + Ok(MetadataCommand { request_id: allocator_reservation_request_id( self.mount, commit_version, @@ -95,17 +87,37 @@ where value: Some(Value(encode_allocator_state( reserved_version, reserved_next_inode, + // The factory executes under the epoch read fence held + // through metadata apply. self.epoch.load(Ordering::Relaxed), ))), }], watch: Vec::new(), }) - .map_err(MetadError::from)?; - self.reserved_version - .store(reserved_version, Ordering::Relaxed); - self.reserved_next_inode - .store(reserved_next_inode, Ordering::Relaxed); - Ok(()) + }); + match reservation { + Ok(_) => { + // The reservation is durable in the metadata engine. Preserve + // that knowledge locally. + self.reserved_version + .store(reserved_version, Ordering::Relaxed); + self.reserved_next_inode + .store(reserved_next_inode, Ordering::Relaxed); + Ok(()) + } + Err( + err @ MetadError::SyncLogArchiveFailed { + committed: true, .. + }, + ) => { + // Keep the old process-local watermark even though Holt applied + // the reservation. The next prepare must re-enter this slow path + // so the pending segment is flushed before it can acknowledge a + // newly allocated generation or inode. + Err(err) + } + Err(err) => Err(err), + } } pub(super) fn next_version(&self) -> Result { diff --git a/crates/nokv-meta/src/service/backup.rs b/crates/nokv-meta/src/service/backup.rs index a8189f17e..92d0cafc1 100644 --- a/crates/nokv-meta/src/service/backup.rs +++ b/crates/nokv-meta/src/service/backup.rs @@ -17,7 +17,9 @@ use super::*; use crate::command::MetadataCheckpointStore; const ARCHIVE_MAGIC: &str = "nokv-metadata-archive"; -const ARCHIVE_FORMAT: u32 = 1; +const ARCHIVE_FORMAT: u32 = 2; +const CHECKPOINT_PROOF_MAGIC: &str = "nokv-metadata-checkpoint-proof"; +const CHECKPOINT_PROOF_FORMAT: u32 = 1; /// Where (and how many) metadata checkpoints to keep in the object store. #[derive(Clone, Debug, PartialEq, Eq)] @@ -35,6 +37,16 @@ impl MetadataArchiveConfig { keep_last: keep_last.max(1), } } + + /// Validate a control-plane checkpoint identity without touching the object + /// store. Startup uses this before acquiring a failover lease so a malformed + /// or cross-shard reference cannot create local state or perform object I/O. + pub fn validate_controlled_checkpoint_identity( + &self, + identity: &MetadataCheckpointIdentity, + ) -> Result<(), MetadError> { + validate_controlled_checkpoint_identity(self, identity) + } } /// Result of publishing a metadata checkpoint to the archive. @@ -42,8 +54,20 @@ impl MetadataArchiveConfig { pub struct MetadataBackupOutcome { pub checkpoint_key: String, pub image_bytes: u64, + /// SHA-256 identity of the exact checkpoint image. + pub image_digest: String, pub commit_version: u64, pub pruned: usize, + /// Shared-log pointers retired after this checkpoint won the control CAS. + /// Standalone backups leave this at zero because they have no control-plane + /// publication boundary authorizing shared-log object deletion. + pub log_segments_pruned: usize, + /// Covered shared-log objects physically deleted after publication. + pub log_segment_objects_deleted: usize, + /// Covered shared-log objects that were already absent during cleanup. + pub log_segment_objects_missing: usize, + /// Covered shared-log objects left as safe leaks because cleanup failed. + pub log_segment_delete_failures: usize, /// Logical-log position the image is consistent with, captured atomically /// with the image so a published `CheckpointRef` never claims an LSN beyond /// the image content (which would silently drop an acknowledged write on @@ -54,6 +78,15 @@ pub struct MetadataBackupOutcome { pub log_digest: [u8; 32], } +/// Exact immutable checkpoint identity carried by a control-plane reference. +/// Controlled restore never follows the mutable standalone `CURRENT` pointer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MetadataCheckpointIdentity { + pub checkpoint_key: String, + pub image_bytes: u64, + pub image_digest: String, +} + /// Result of restoring the namespace from the archive. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MetadataRestoreOutcome { @@ -70,9 +103,32 @@ struct ArchiveManifest { current: String, version: u64, size: u64, + /// True only when the checkpoint image itself contains the durable fence + /// that prevents object GC from outrunning off-node metadata recovery. + object_gc_failover_fenced: bool, recent: Vec, } +/// Immutable proof stored beside a controlled checkpoint image. Its key is +/// derived only after validating `checkpoint_key` against the shard archive +/// prefix and content digest. +#[derive(Clone, Debug, PartialEq, Eq)] +struct CheckpointProof { + checkpoint_key: String, + image_bytes: u64, + image_digest: String, + object_gc_failover_fenced: bool, +} + +struct ExportedMetadataCheckpoint { + image: Vec, + image_bytes: u64, + image_digest: String, + commit_version: u64, + log_lsn: u64, + log_digest: [u8; 32], +} + impl NoKvFs where M: MetadataStore + MetadataCheckpointStore, @@ -84,6 +140,11 @@ where &self, config: &MetadataArchiveConfig, ) -> Result { + // Every published recovery image must contain the durable fence. Doing + // this before checkpoint export also upgrades a live pre-fence store + // safely: current namespace references are intact, then future object + // deletion remains disabled before CURRENT can name the new image. + self.require_failover_durability()?; let keep_last = config.keep_last.max(1); // The manifest is a read-modify-write of the `recent` window plus a // sequence-derived checkpoint key, so two backups must not interleave. @@ -92,21 +153,7 @@ where .lock() .unwrap_or_else(|err| err.into_inner()); - // Capture the logical-log boundary BEFORE exporting the image. Commits - // apply to the engine before they archive (so durable_lsn <= applied - // set), which makes this a safe lower bound: the image is guaranteed to - // contain at least everything up to (log_lsn, log_digest). Reading it - // after the export (as the server used to) could capture an LSN beyond - // the image and lose that write on restore. - let (log_lsn, log_digest) = self - .sync_metadata_log_snapshot() - .map(|snapshot| (snapshot.durable_lsn, snapshot.last_digest)) - .unwrap_or((0, crate::METADATA_LOG_ZERO_DIGEST)); - - // Fold the WAL into a durable checkpoint, then export that image. - self.metadata.checkpoint()?; - let image = self.metadata.export_checkpoint_image()?; - let commit_version = self.clock.load(Ordering::SeqCst); + let exported = self.export_metadata_checkpoint()?; let manifest_key = archive_manifest_key(&config.prefix); let prior = self.read_archive_manifest(&manifest_key)?; @@ -115,8 +162,7 @@ where // Object-first: write the checkpoint image before anything references it. let object_key = ObjectKey::new(checkpoint_key.clone())?; - let image_size = image.len() as u64; - self.objects.put(&object_key, image)?; + self.objects.put(&object_key, exported.image)?; // Compute the retained window; everything older than keep_last is pruned. let mut recent = prior.map(|m| m.recent).unwrap_or_default(); @@ -130,8 +176,9 @@ where let manifest = ArchiveManifest { seq: next_seq, current: checkpoint_key.clone(), - version: commit_version, - size: image_size, + version: exported.commit_version, + size: exported.image_bytes, + object_gc_failover_fenced: true, recent, }; let manifest_object = ObjectKey::new(manifest_key)?; @@ -153,9 +200,188 @@ where Ok(MetadataBackupOutcome { checkpoint_key, - image_bytes: image_size, - commit_version, + image_bytes: exported.image_bytes, + image_digest: exported.image_digest, + commit_version: exported.commit_version, pruned, + log_segments_pruned: 0, + log_segment_objects_deleted: 0, + log_segment_objects_missing: 0, + log_segment_delete_failures: 0, + log_lsn: exported.log_lsn, + log_digest: exported.log_digest, + }) + } + + /// Export one immutable, content-addressed checkpoint plus an exact proof, + /// without reading or mutating the standalone `CURRENT` pointer. The caller + /// must publish the returned identity through its own durable authority + /// (the control-plane lease CAS) before pruning any prior checkpoint. + pub fn prepare_immutable_metadata_backup( + &self, + config: &MetadataArchiveConfig, + ) -> Result { + self.require_failover_durability()?; + self.ensure_owner_epoch_current()?; + let _guard = self + .backup_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let exported = self.export_metadata_checkpoint()?; + self.ensure_owner_epoch_current()?; + + let checkpoint_key = controlled_checkpoint_key(&config.prefix, &exported.image_digest)?; + let identity = MetadataCheckpointIdentity { + checkpoint_key: checkpoint_key.clone(), + image_bytes: exported.image_bytes, + image_digest: exported.image_digest.clone(), + }; + validate_controlled_checkpoint_identity(config, &identity)?; + let image_key = ObjectKey::new(checkpoint_key.clone())?; + put_immutable_object(&self.objects, &image_key, &exported.image)?; + + // If the local owner fence expired during the image PUT, stop before + // publishing a proof. The content-addressed image is only an orphan. + self.ensure_owner_epoch_current()?; + let proof = CheckpointProof { + checkpoint_key: checkpoint_key.clone(), + image_bytes: exported.image_bytes, + image_digest: exported.image_digest.clone(), + object_gc_failover_fenced: true, + }; + let proof_key = ObjectKey::new(checkpoint_proof_key(&identity))?; + let encoded_proof = serialize_checkpoint_proof(&proof).into_bytes(); + put_immutable_object(&self.objects, &proof_key, &encoded_proof)?; + + Ok(MetadataBackupOutcome { + checkpoint_key, + image_bytes: exported.image_bytes, + image_digest: exported.image_digest, + commit_version: exported.commit_version, + pruned: 0, + log_segments_pruned: 0, + log_segment_objects_deleted: 0, + log_segment_objects_missing: 0, + log_segment_delete_failures: 0, + log_lsn: exported.log_lsn, + log_digest: exported.log_digest, + }) + } + + /// Delete one no-longer-authoritative immutable controlled checkpoint. + /// Callers must first replace its exact control-plane reference by lease CAS. + pub fn prune_immutable_metadata_backup( + &self, + config: &MetadataArchiveConfig, + identity: &MetadataCheckpointIdentity, + ) -> Result { + validate_controlled_checkpoint_identity(config, identity)?; + let proof_key = ObjectKey::new(checkpoint_proof_key(identity))?; + let image_key = ObjectKey::new(identity.checkpoint_key.clone())?; + let _ = self.objects.delete(&proof_key)?; + Ok(usize::from(self.objects.delete(&image_key)?)) + } + + /// Restore one exact immutable controlled checkpoint. Proof is loaded and + /// matched to the control identity before the image object is fetched, so a + /// legacy/misdirected ref cannot mutate the target metadata store. + pub fn restore_metadata_checkpoint( + &self, + config: &MetadataArchiveConfig, + identity: &MetadataCheckpointIdentity, + ) -> Result { + validate_controlled_checkpoint_identity(config, identity)?; + let proof_key = ObjectKey::new(checkpoint_proof_key(identity))?; + if self.objects.head(&proof_key)?.is_none() { + return Err(MetadError::MetadataArchiveMissingObjectGcFence { + checkpoint_key: identity.checkpoint_key.clone(), + }); + } + let proof_bytes = self.objects.get(&proof_key, None)?; + let proof_text = String::from_utf8(proof_bytes) + .map_err(|_| MetadError::Codec("checkpoint proof is not valid UTF-8".to_owned()))?; + let proof = parse_checkpoint_proof(&proof_text)?; + if !proof.object_gc_failover_fenced { + return Err(MetadError::MetadataArchiveMissingObjectGcFence { + checkpoint_key: identity.checkpoint_key.clone(), + }); + } + if proof.checkpoint_key != identity.checkpoint_key + || proof.image_bytes != identity.image_bytes + || proof.image_digest != identity.image_digest + { + return Err(MetadError::Codec(format!( + "checkpoint proof does not match control identity for {}", + identity.checkpoint_key + ))); + } + + let image_key = ObjectKey::new(identity.checkpoint_key.clone())?; + let image = self.objects.get(&image_key, None)?; + if image.len() as u64 != identity.image_bytes { + return Err(MetadError::Codec(format!( + "checkpoint image size mismatch for {}", + identity.checkpoint_key + ))); + } + if checkpoint_image_digest(&image) != identity.image_digest { + return Err(MetadError::Codec(format!( + "checkpoint image digest mismatch for {}", + identity.checkpoint_key + ))); + } + + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.mark_materialization_orphan_possible_under_gc_gate(); + self.metadata.install_checkpoint_image(&image)?; + self.purge_path_caches_after_write(); + self.refresh_allocator_state()?; + self.verify_failover_durability_required()?; + self.reconcile_materialization_orphan_state_under_gc_gate()?; + Ok(MetadataRestoreOutcome { + checkpoint_key: identity.checkpoint_key.clone(), + image_bytes: identity.image_bytes, + commit_version: self.clock.load(Ordering::SeqCst), + }) + } + + fn export_metadata_checkpoint(&self) -> Result { + let _log_enable_fence = self + .metadata_log_enable_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + let _commit_log_guard = self + .metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + // An image may not pass an ambiguous apply or an unarchived committed + // segment. Resolve and flush while holding the same total-order gate + // used to capture both its logical-log boundary and engine image. + self.resolve_unresolved_metadata_commit_group_locked()?; + self.flush_pending_metadata_log_segment_locked() + .map_err(|err| MetadError::SyncLogArchiveFailed { + committed: true, + message: err.to_string(), + })?; + // Capture the logical-log boundary BEFORE exporting the image while the + // commit/log gate is still held. Commits apply before archive, making + // this a safe lower bound for replay. + let (log_lsn, log_digest) = self + .sync_metadata_log_snapshot() + .map(|snapshot| (snapshot.durable_lsn, snapshot.last_digest)) + .unwrap_or((0, crate::METADATA_LOG_ZERO_DIGEST)); + self.metadata.checkpoint()?; + let image = self.metadata.export_checkpoint_image()?; + let image_bytes = image.len() as u64; + let image_digest = checkpoint_image_digest(&image); + Ok(ExportedMetadataCheckpoint { + image, + image_bytes, + image_digest, + commit_version: self.clock.load(Ordering::SeqCst), log_lsn, log_digest, }) @@ -175,15 +401,30 @@ where let Some(manifest) = self.read_archive_manifest(&manifest_key)? else { return Ok(None); }; + if !manifest.object_gc_failover_fenced { + return Err(MetadError::MetadataArchiveMissingObjectGcFence { + checkpoint_key: manifest.current, + }); + } let object_key = ObjectKey::new(manifest.current.clone())?; let image = self.objects.get(&object_key, None)?; // install_checkpoint_image validates the image and rejects truncation or // corruption, so a torn archive surfaces as a metadata error here. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.mark_materialization_orphan_possible_under_gc_gate(); self.metadata.install_checkpoint_image(&image)?; // The image replaced the engine state wholesale, bypassing the commit // funnel; entries cached before the install may not exist in it at all. self.purge_path_caches_after_write(); self.refresh_allocator_state()?; + // The manifest proof is written only after exporting a fenced image; + // verify the installed state as defense against a mismatched/corrupt + // archive assembled outside the supported backup path. + self.verify_failover_durability_required()?; + self.reconcile_materialization_orphan_state_under_gc_gate()?; Ok(Some(MetadataRestoreOutcome { checkpoint_key: manifest.current, image_bytes: image.len() as u64, @@ -206,6 +447,171 @@ where } } +fn checkpoint_image_digest(image: &[u8]) -> String { + format!("sha256:{:x}", Sha256::digest(image)) +} + +fn controlled_checkpoint_key(prefix: &str, image_digest: &str) -> Result { + let digest = canonical_sha256_hex(image_digest)?; + Ok(format!( + "{}/controlled/sha256/{digest}.image", + normalize_prefix(prefix) + )) +} + +fn validate_controlled_checkpoint_identity( + config: &MetadataArchiveConfig, + identity: &MetadataCheckpointIdentity, +) -> Result<(), MetadError> { + let expected = controlled_checkpoint_key(&config.prefix, &identity.image_digest)?; + if identity.checkpoint_key != expected { + return Err(MetadError::Codec(format!( + "controlled checkpoint key {} does not match archive prefix and image digest", + identity.checkpoint_key + ))); + } + if identity.image_bytes == 0 { + return Err(MetadError::Codec( + "controlled checkpoint image size must be non-zero".to_owned(), + )); + } + Ok(()) +} + +fn canonical_sha256_hex(image_digest: &str) -> Result<&str, MetadError> { + let Some(hex) = image_digest.strip_prefix("sha256:") else { + return Err(MetadError::Codec( + "checkpoint image digest must use sha256".to_owned(), + )); + }; + if hex.len() != 64 + || !hex + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + { + return Err(MetadError::Codec( + "checkpoint image digest is not canonical SHA-256 hex".to_owned(), + )); + } + Ok(hex) +} + +fn checkpoint_proof_key(identity: &MetadataCheckpointIdentity) -> String { + format!("{}.proof", identity.checkpoint_key) +} + +fn put_immutable_object( + objects: &O, + key: &ObjectKey, + expected: &[u8], +) -> Result<(), MetadError> { + if objects.head(key)?.is_some() { + let current = objects.get(key, None)?; + if current != expected { + return Err(MetadError::Codec(format!( + "immutable metadata archive object {} already has different bytes", + key.as_str() + ))); + } + return Ok(()); + } + objects.put(key, expected.to_vec())?; + if objects.get(key, None)? != expected { + return Err(MetadError::Codec(format!( + "immutable metadata archive object {} failed read-after-write verification", + key.as_str() + ))); + } + Ok(()) +} + +fn serialize_checkpoint_proof(proof: &CheckpointProof) -> String { + format!( + "{CHECKPOINT_PROOF_MAGIC}\t{CHECKPOINT_PROOF_FORMAT}\ncheckpoint_key\t{}\nimage_bytes\t{}\nimage_digest\t{}\nobject_gc_failover_fenced\t{}\n", + proof.checkpoint_key, + proof.image_bytes, + proof.image_digest, + u8::from(proof.object_gc_failover_fenced), + ) +} + +fn parse_checkpoint_proof(text: &str) -> Result { + let mut lines = text.lines(); + let header = lines + .next() + .ok_or_else(|| MetadError::Codec("checkpoint proof is empty".to_owned()))?; + let expected_header = format!("{CHECKPOINT_PROOF_MAGIC}\t{CHECKPOINT_PROOF_FORMAT}"); + if header != expected_header { + return Err(MetadError::Codec( + "checkpoint proof header is unsupported".to_owned(), + )); + } + let mut checkpoint_key = None; + let mut image_bytes = None; + let mut image_digest = None; + let mut object_gc_failover_fenced = None; + for line in lines { + if line.is_empty() { + continue; + } + let (tag, value) = line + .split_once('\t') + .ok_or_else(|| MetadError::Codec("checkpoint proof line is malformed".to_owned()))?; + let duplicate = |name: &str| MetadError::Codec(format!("checkpoint proof repeats {name}")); + match tag { + "checkpoint_key" => { + if checkpoint_key.replace(value.to_owned()).is_some() { + return Err(duplicate("checkpoint key")); + } + } + "image_bytes" => { + if image_bytes.replace(parse_u64(value)?).is_some() { + return Err(duplicate("image bytes")); + } + } + "image_digest" => { + canonical_sha256_hex(value)?; + if image_digest.replace(value.to_owned()).is_some() { + return Err(duplicate("image digest")); + } + } + "object_gc_failover_fenced" => { + let fenced = match value { + "0" => false, + "1" => true, + _ => { + return Err(MetadError::Codec( + "checkpoint proof failover fence must be 0 or 1".to_owned(), + )); + } + }; + if object_gc_failover_fenced.replace(fenced).is_some() { + return Err(duplicate("failover fence")); + } + } + _ => { + return Err(MetadError::Codec(format!( + "checkpoint proof has unknown tag: {tag}" + ))); + } + } + } + Ok(CheckpointProof { + checkpoint_key: checkpoint_key + .filter(|value| !value.is_empty()) + .ok_or_else(|| MetadError::Codec("checkpoint proof has no key".to_owned()))?, + image_bytes: image_bytes + .filter(|value| *value > 0) + .ok_or_else(|| MetadError::Codec("checkpoint proof has no image bytes".to_owned()))?, + image_digest: image_digest + .ok_or_else(|| MetadError::Codec("checkpoint proof has no image digest".to_owned()))?, + object_gc_failover_fenced: object_gc_failover_fenced.ok_or_else(|| { + MetadError::Codec("checkpoint proof has no failover fence".to_owned()) + })?, + }) +} + fn normalize_prefix(prefix: &str) -> &str { prefix.trim_end_matches('/') } @@ -225,6 +631,10 @@ fn serialize_archive_manifest(manifest: &ArchiveManifest) -> String { out.push_str(&format!("current\t{}\n", manifest.current)); out.push_str(&format!("version\t{}\n", manifest.version)); out.push_str(&format!("size\t{}\n", manifest.size)); + out.push_str(&format!( + "object_gc_failover_fenced\t{}\n", + u8::from(manifest.object_gc_failover_fenced) + )); for key in &manifest.recent { out.push_str(&format!("recent\t{key}\n")); } @@ -236,7 +646,7 @@ fn parse_archive_manifest(text: &str) -> Result { let header = lines .next() .ok_or_else(|| MetadError::Codec("archive manifest is empty".to_owned()))?; - let (magic, _format) = header + let (magic, format) = header .split_once('\t') .ok_or_else(|| MetadError::Codec("archive manifest header is malformed".to_owned()))?; if magic != ARCHIVE_MAGIC { @@ -244,8 +654,17 @@ fn parse_archive_manifest(text: &str) -> Result { "unexpected archive manifest magic: {magic}" ))); } + let format = format + .parse::() + .map_err(|_| MetadError::Codec("archive manifest format is not a number".to_owned()))?; + if !(1..=ARCHIVE_FORMAT).contains(&format) { + return Err(MetadError::Codec(format!( + "unsupported archive manifest format: {format}" + ))); + } let mut manifest = ArchiveManifest::default(); let mut saw_current = false; + let mut saw_object_gc_fence = false; for line in lines { if line.is_empty() { continue; @@ -261,6 +680,23 @@ fn parse_archive_manifest(text: &str) -> Result { } "version" => manifest.version = parse_u64(value)?, "size" => manifest.size = parse_u64(value)?, + "object_gc_failover_fenced" => { + if saw_object_gc_fence { + return Err(MetadError::Codec( + "archive manifest repeats object-GC failover fence".to_owned(), + )); + } + manifest.object_gc_failover_fenced = match value { + "0" => false, + "1" => true, + _ => { + return Err(MetadError::Codec( + "archive manifest object-GC failover fence must be 0 or 1".to_owned(), + )); + } + }; + saw_object_gc_fence = true; + } "recent" => manifest.recent.push(value.to_owned()), _ => { return Err(MetadError::Codec(format!( @@ -274,6 +710,11 @@ fn parse_archive_manifest(text: &str) -> Result { "archive manifest has no current checkpoint".to_owned(), )); } + if format >= 2 && !saw_object_gc_fence { + return Err(MetadError::Codec( + "archive manifest has no object-GC failover fence proof".to_owned(), + )); + } Ok(manifest) } @@ -286,6 +727,48 @@ fn parse_u64(value: &str) -> Result { #[cfg(test)] mod tests { use super::*; + use nokv_object::MemoryObjectStore; + + #[test] + fn archive_manifest_v2_round_trips_failover_fence_proof() { + let manifest = ArchiveManifest { + seq: 7, + current: "meta/ckpt/7.image".to_owned(), + version: 11, + size: 13, + object_gc_failover_fenced: true, + recent: vec!["meta/ckpt/7.image".to_owned()], + }; + + assert_eq!( + parse_archive_manifest(&serialize_archive_manifest(&manifest)).unwrap(), + manifest + ); + } + + #[test] + fn archive_manifest_v1_has_no_failover_fence_proof() { + let manifest = parse_archive_manifest( + "nokv-metadata-archive\t1\nseq\t1\ncurrent\tmeta/ckpt/1.image\nversion\t2\nsize\t3\nrecent\tmeta/ckpt/1.image\n", + ) + .unwrap(); + + assert!(!manifest.object_gc_failover_fenced); + } + + #[test] + fn archive_manifest_v2_requires_failover_fence_proof() { + let err = parse_archive_manifest( + "nokv-metadata-archive\t2\nseq\t1\ncurrent\tmeta/ckpt/1.image\nversion\t2\nsize\t3\n", + ) + .unwrap_err(); + + assert!(matches!( + err, + MetadError::Codec(message) + if message == "archive manifest has no object-GC failover fence proof" + )); + } #[test] fn archive_manifest_rejects_unknown_tags() { @@ -299,4 +782,49 @@ mod tests { MetadError::Codec(message) if message == "archive manifest has unknown tag: extra" )); } + + #[test] + fn checkpoint_proof_round_trips_exact_image_identity() { + let proof = CheckpointProof { + checkpoint_key: format!("meta/controlled/sha256/{}.image", "a".repeat(64)), + image_bytes: 4096, + image_digest: format!("sha256:{}", "a".repeat(64)), + object_gc_failover_fenced: true, + }; + + assert_eq!( + parse_checkpoint_proof(&serialize_checkpoint_proof(&proof)).unwrap(), + proof + ); + } + + #[test] + fn immutable_archive_object_rejects_different_bytes_at_same_key() { + let objects = MemoryObjectStore::new(); + let key = ObjectKey::new("meta/controlled/proof").unwrap(); + put_immutable_object(&objects, &key, b"first").unwrap(); + put_immutable_object(&objects, &key, b"first").unwrap(); + + let err = put_immutable_object(&objects, &key, b"second").unwrap_err(); + assert!(matches!( + err, + MetadError::Codec(message) if message.contains("already has different bytes") + )); + assert_eq!(objects.get(&key, None).unwrap(), b"first"); + } + + #[test] + fn controlled_identity_binds_prefix_key_and_digest() { + let config = MetadataArchiveConfig::new("meta/shard", 4); + let digest = format!("sha256:{}", "b".repeat(64)); + let identity = MetadataCheckpointIdentity { + checkpoint_key: controlled_checkpoint_key(&config.prefix, &digest).unwrap(), + image_bytes: 1024, + image_digest: digest, + }; + validate_controlled_checkpoint_identity(&config, &identity).unwrap(); + + let cross_prefix = MetadataArchiveConfig::new("meta/other", 4); + assert!(validate_controlled_checkpoint_identity(&cross_prefix, &identity).is_err()); + } } diff --git a/crates/nokv-meta/src/service/checkpoint.rs b/crates/nokv-meta/src/service/checkpoint.rs index ab22433d2..4e8fa819a 100644 --- a/crates/nokv-meta/src/service/checkpoint.rs +++ b/crates/nokv-meta/src/service/checkpoint.rs @@ -52,6 +52,7 @@ where "checkpoint requires at least one shard".to_owned(), )); } + let object_reference = self.begin_object_reference_mutation()?; // Stage every shard's objects + build its (projection, chunks). Track the // staged object sets so a mid-stage failure reclaims them instead of @@ -70,6 +71,10 @@ where } }; let version = Version::new(prepared.generation)?; + if prepared.object_gc_claim_version != object_reference.version().get() { + self.abort_staged_checkpoint(&staged_sets); + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } let request = PublishArtifact { parent, name: shard.name.clone(), @@ -116,8 +121,29 @@ where // One atomic commit makes every shard visible together. let commit_version = self.next_version()?; - if let Err(err) = self.commit_checkpoint_shards(parent, &staged_artifacts, commit_version) { - self.abort_staged_checkpoint(&staged_sets); + if let Err(err) = self.commit_checkpoint_shards( + parent, + &staged_artifacts, + commit_version, + object_reference, + ) { + // A synchronous-log failure with `committed=true` happens after the + // atomic metadata command made every staged object reachable. A + // backend error may likewise be an apply-then-journal-ACK failure. + // In either uncertain case the objects are no longer abortable: + // deleting them could corrupt a now-visible checkpoint. Propagate + // the acknowledgement failure while preserving referenced data (or + // a safe orphan when the backend ultimately did not apply). + let commit_may_have_applied = matches!( + &err, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } | MetadError::Metadata(MetadataError::Backend(_)) + ); + if !commit_may_have_applied { + self.abort_staged_checkpoint(&staged_sets); + } return Err(err); } Ok(CheckpointHandle { @@ -137,12 +163,14 @@ where parent: InodeId, shards: &[(DentryProjection, Vec)], commit_version: Version, + object_reference: ObjectReferenceMutation, ) -> Result<(), MetadError> { let mut predicates = vec![PredicateRef { family: RecordFamily::Inode, key: inode_key(self.mount, parent), predicate: Predicate::Exists, }]; + predicates.push(object_reference.predicate(self.mount)); let mut mutations = Vec::new(); for (proj, chunks) in shards { let inode = proj.attr.inode; diff --git a/crates/nokv-meta/src/service/clone.rs b/crates/nokv-meta/src/service/clone.rs index 54a142560..942b331d5 100644 --- a/crates/nokv-meta/src/service/clone.rs +++ b/crates/nokv-meta/src/service/clone.rs @@ -30,22 +30,136 @@ where /// fork's own inode, producing new object keys; the borrowed source blocks are /// left untouched (a borrower never GCs another namespace's blocks). /// - /// The clone retains a snapshot pin on the source ([`CloneHandle::snapshot_id`]) - /// so the GC retention floor protects the shared base blocks while the fork - /// references them. Retire it with [`NoKvFs::retire_snapshot`] when the fork no - /// longer needs the source's base objects (typically when the fork is deleted). + /// The construction snapshot is converted into a durable fork-retention + /// binding identified by [`CloneHandle::snapshot_id`]. Unlike the snapshot + /// lease, that binding does not expire: it holds the GC retention floor while + /// any fork reference can still reach borrowed source blocks. Retire it with + /// [`NoKvFs::retire_snapshot`] only after every such reference is gone, + /// including hardlinks moved outside the original fork root. pub fn clone_subtree(&self, src_root: InodeId) -> Result { // Pin the source first so the read version we copy from is stable and the // shared base objects are GC-protected from the moment the fork exists. let pin = self.snapshot_subtree(src_root)?; + // Materialization publishes borrowed object references over multiple + // metadata commits. Keep this owner's GC out until the complete fork is + // built, then atomically replace the construction lease with a durable + // fork binding before exposing the handle. HA object GC is fail-closed + // separately. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let handle = self.materialize_clone_under_gc_gate(src_root, &pin)?; + self.commit_detached_clone_binding(&handle, src_root, pin.read_version)?; + // The binding commit made this detached tree a legal root. A preexisting + // orphan (or a proof error) keeps the marker set; otherwise later + // link/rename calls return to their O(1) reachability fast path. + let _ = self.reconcile_materialization_orphan_state_under_gc_gate(); + Ok(handle) + } + + fn materialize_clone_under_gc_gate( + &self, + src_root: InodeId, + pin: &SnapshotPin, + ) -> Result { + let versioned = self + .versioned_snapshot_pin_at( + pin.snapshot_id, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + self.ensure_snapshot_pin_live(&versioned.pin)?; let version = Version::new(pin.read_version)?; let dst_root = self.materialize_subtree_at(src_root, version)?; + let retained = self + .versioned_snapshot_pin_at( + pin.snapshot_id, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if retained.pin.root != src_root || retained.pin.read_version != pin.read_version { + return Err(MetadError::Codec(format!( + "snapshot {} changed identity during clone materialization", + pin.snapshot_id + ))); + } + self.ensure_snapshot_pin_live(&retained.pin)?; Ok(CloneHandle { root: dst_root, snapshot_id: pin.snapshot_id, }) } + fn commit_detached_clone_binding( + &self, + handle: &CloneHandle, + src_root: InodeId, + snapshot_read_version: u64, + ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let retained = self + .versioned_snapshot_pin_at( + handle.snapshot_id, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if retained.pin.root != src_root || retained.pin.read_version != snapshot_read_version { + return Err(MetadError::Codec(format!( + "snapshot {} changed identity before clone retention commit", + handle.snapshot_id + ))); + } + self.ensure_snapshot_pin_live(&retained.pin)?; + + let version = self.next_version()?; + let binding_key = fork_binding_key(self.mount, handle.root); + let binding = ForkBinding { + fork_root: handle.root, + source_root: src_root, + pinned_read_version: snapshot_read_version, + snapshot_id: handle.snapshot_id, + created_version: version.get(), + }; + self.commit_metadata(MetadataCommand { + request_id: request_id(b"clone-subtree-retention", self.mount, handle.root, version), + kind: CommandKind::SnapshotSubtree, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::ForkBinding, + primary_key: binding_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, handle.root), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Snapshot, + key: snapshot_pin_key(self.mount, handle.snapshot_id), + predicate: Predicate::VersionEquals(retained.version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::NotExists, + }, + object_reference.predicate(self.mount), + ], + mutations: vec![Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&binding))), + }], + watch: Vec::new(), + })?; + Ok(()) + } + /// Materialize the directory subtree rooted at `src_root`, **as seen at /// `read_version`**, into a brand-new detached namespace root and return that /// root inode. @@ -73,6 +187,10 @@ where src_root: InodeId, read_version: Version, ) -> Result { + // Callers hold object_gc_gate. Raise the process-local slow-path marker + // before even validating the source so every partial metadata commit (or + // an error after one) remains fail-closed for inode-addressed exposure. + self.mark_materialization_orphan_possible_under_gc_gate(); let Some(src_attr) = self.get_attr_at_version_for_purpose(src_root, read_version, ReadPurpose::Snapshot)? else { @@ -135,9 +253,9 @@ where /// This is [`NoKvFs::clone_subtree_path`] plus a single dentry that attaches the /// detached fork root under `dst_path`'s parent. The fork shares the source's /// object blocks until it diverges on write (see [`NoKvFs::clone_subtree`]) and - /// the returned [`CloneHandle`] carries the retained snapshot pin exactly as the - /// detached clone does. `dst_path`'s parent must already exist and `dst_path` - /// itself must be free. + /// the returned [`CloneHandle`] carries the durable retention identity exactly + /// as the detached clone does. `dst_path`'s parent must already exist and + /// `dst_path` itself must be free. pub fn clone_subtree_path_into( &self, src_path: &str, @@ -145,20 +263,56 @@ where ) -> Result { let src_root = self.resolve_directory_path(src_path)?; let (dst_parent, dst_name) = self.resolve_parent_path(dst_path)?; - let handle = self.clone_subtree(src_root)?; - self.link_clone_root(handle.root, dst_parent, dst_name)?; + let pin = self.snapshot_subtree(src_root)?; + // Keep GC excluded through both materialization and the commit that + // exposes the fork. Releasing the gate between those phases would let a + // near-expiry pin admit DELETE before the detached tree became live. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let handle = self.materialize_clone_under_gc_gate(src_root, &pin)?; + self.link_clone_root( + handle.root, + dst_parent, + dst_name, + src_root, + pin.snapshot_id, + pin.read_version, + )?; + let _ = self.reconcile_materialization_orphan_state_under_gc_gate(); Ok(handle) } /// Attach an existing (detached) fork root inode under `dst_parent` as - /// `dst_name`, committing only the directory dentry that names it. The inode - /// already exists from [`NoKvFs::clone_subtree`]; this just makes it reachable. + /// `dst_name`. The same atomic commit exact-version fences the live + /// construction pin, installs the non-expiring fork binding, and joins the + /// durable object-GC Open epoch, so borrowed objects cannot be deleted + /// between materialization and exposure or after the snapshot lease expires. fn link_clone_root( &self, root: InodeId, dst_parent: InodeId, dst_name: DentryName, + src_root: InodeId, + snapshot_id: u64, + snapshot_read_version: u64, ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let retained = self + .versioned_snapshot_pin_at( + snapshot_id, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if retained.pin.root != src_root || retained.pin.read_version != snapshot_read_version { + return Err(MetadError::Codec(format!( + "snapshot {snapshot_id} changed identity before clone link" + ))); + } + self.ensure_snapshot_pin_live(&retained.pin)?; + let version = self.next_version()?; let read_version = predecessor(version)?; let Some(mut attr) = @@ -169,6 +323,14 @@ where attr.ctime_ms = current_time_ms(); let projection = projection(dst_parent, dst_name, attr, None); let dentry = dentry_key(self.mount, dst_parent, &projection.dentry.name); + let binding_key = fork_binding_key(self.mount, root); + let binding = ForkBinding { + fork_root: root, + source_root: src_root, + pinned_read_version: snapshot_read_version, + snapshot_id, + created_version: version.get(), + }; self.commit_metadata(MetadataCommand { request_id: request_id(b"clone-subtree-link", self.mount, root, version), kind: CommandKind::CreateDir, @@ -187,12 +349,27 @@ where key: dentry.clone(), predicate: Predicate::NotExists, }, + PredicateRef { + family: RecordFamily::Snapshot, + key: snapshot_pin_key(self.mount, snapshot_id), + predicate: Predicate::VersionEquals(retained.version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::NotExists, + }, + object_reference.predicate(self.mount), + ], + mutations: vec![ + put_projection_mutation(RecordFamily::Dentry, dentry, &projection), + Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&binding))), + }, ], - mutations: vec![put_projection_mutation( - RecordFamily::Dentry, - dentry, - &projection, - )], watch: Vec::new(), })?; Ok(()) diff --git a/crates/nokv-meta/src/service/command.rs b/crates/nokv-meta/src/service/command.rs index 14fb34d38..6101c4fd2 100644 --- a/crates/nokv-meta/src/service/command.rs +++ b/crates/nokv-meta/src/service/command.rs @@ -110,6 +110,13 @@ where self.lease_deadline_ms.load(Ordering::Relaxed) } + /// Verify that this service still holds a current, unexpired owner fence + /// before performing an external side effect such as publishing recovery + /// metadata. The control-plane CAS remains the final authority. + pub fn verify_owner_lease(&self) -> Result<(), MetadError> { + self.ensure_owner_epoch_current() + } + /// Single source of truth for "may this owner commit?": epoch fence first, /// then the wall-clock lease deadline (the partition-safe self-fence). pub(super) fn check_owner_lease(&self) -> Result<(), OwnerLeaseFault> { @@ -146,7 +153,92 @@ where &self, command: MetadataCommand, ) -> Result { - let result = self.commit_metadata_without_sync_log(command.clone())?; + self.commit_metadata_from_factory(|| Ok(command)) + } + + /// Build and commit one command while holding the same epoch fence used by + /// the apply. Allocator reservations use this entry point because their + /// persisted owner epoch must be read inside that fence, and because every + /// reservation must join the synchronous recovery log when it is enabled. + pub(super) fn commit_metadata_from_factory( + &self, + build: F, + ) -> Result + where + F: FnOnce() -> Result, + { + let _log_enable_fence = self + .metadata_log_enable_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + let log_enabled = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .is_some(); + let _commit_log_guard = log_enabled.then(|| { + self.metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()) + }); + if log_enabled { + self.resolve_unresolved_metadata_commit_group_locked() + .map_err(blocked_before_apply)?; + self.flush_pending_metadata_log_segment_locked() + .map_err(|err| MetadError::SyncLogArchiveFailed { + committed: false, + message: err.to_string(), + })?; + } + + let (command, result) = { + // The command factory, owner check, and apply share one epoch read + // fence. In particular, a reservation cannot encode an old owner + // epoch and then apply after a concurrent failover bump. Release the + // fence before object-store archive I/O so lease renewal can still + // install the current epoch while an upload is slow. + let _epoch_fence = self + .epoch_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + self.ensure_owner_epoch_current()?; + let command = build()?; + if log_enabled { + self.preflight_sync_metadata_log_locked(std::slice::from_ref(&command)) + .map_err(|err| MetadError::SyncLogArchiveFailed { + committed: false, + message: err.to_string(), + })?; + } + match self.metadata.commit_metadata(command.clone()) { + Ok(result) => { + self.purge_path_caches_after_write(); + (command, result) + } + Err(backend @ MetadataError::Backend(_)) if log_enabled => { + // A backend error may be an acknowledgement lost after the + // atomic apply became visible. Invalidate conservatively + // before readback so blocked writers cannot leave readers + // observing a stale pre-apply path cache. + self.purge_path_caches_after_write(); + let group = log_sync::UnresolvedMetadataCommitGroup::new( + vec![command.clone()], + vec![Err(backend)], + )?; + match self.reconcile_metadata_commit_group_locked(&group) { + Ok(mut resolved) => { + let result = resolved.pop().expect("single commit result")?; + (command, result) + } + Err(err) => { + self.defer_unresolved_metadata_commit_group_locked(group)?; + return Err(err); + } + } + } + Err(err) => return Err(err.into()), + } + }; // The command is already durably applied; if the sync-log segment fails // to archive we report committed=true so the caller reconciles rather // than blindly retrying data that actually landed. @@ -179,19 +271,109 @@ where &self, commands: &[MetadataCommand], ) -> Vec> { - // Read guard held across the fence check and the whole batch apply, so a - // failover epoch bump cannot interleave with an accepted batch. - let _fence = self - .epoch_fence + let _log_enable_fence = self + .metadata_log_enable_fence .read() .unwrap_or_else(|err| err.into_inner()); - if let Err(fault) = self.check_owner_lease() { - return commands.iter().map(|_| Err(fault.into_error())).collect(); + let log_enabled = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .is_some(); + let _commit_log_guard = log_enabled.then(|| { + self.metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()) + }); + if log_enabled { + if let Err(err) = self.resolve_unresolved_metadata_commit_group_locked() { + return blocked_batch_before_apply(commands.len(), err); + } + if let Err(err) = self + .flush_pending_metadata_log_segment_locked() + .and_then(|()| self.preflight_sync_metadata_log_locked(commands)) + { + let message = err.to_string(); + return commands + .iter() + .map(|_| { + Err(MetadError::SyncLogArchiveFailed { + committed: false, + message: message.clone(), + }) + }) + .collect(); + } } + let raw_results = { + // Read guard held across the fence check and the whole batch apply, + // so a failover epoch bump cannot interleave with an accepted batch. + let _fence = self + .epoch_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + if let Err(fault) = self.check_owner_lease() { + return commands.iter().map(|_| Err(fault.into_error())).collect(); + } + self.metadata.commit_independent_batch(commands) + }; + + let may_have_committed = raw_results + .iter() + .any(|result| matches!(result, Ok(_) | Err(MetadataError::Backend(_)))); + if may_have_committed { + // Backend outcomes are ambiguous and successful subgroup outcomes + // are definitely visible. Purge once for the whole batch before + // any possible unresolved state blocks subsequent writes. + self.purge_path_caches_after_write(); + } + + let metadata_results = if log_enabled + && raw_results + .iter() + .any(|result| matches!(result, Err(MetadataError::Backend(_)))) + { + let group = match log_sync::UnresolvedMetadataCommitGroup::new( + commands.to_vec(), + raw_results, + ) { + Ok(group) => group, + Err(err) => { + let message = err.to_string(); + return commands + .iter() + .map(|_| Err(MetadError::Codec(message.clone()))) + .collect(); + } + }; + match self.reconcile_metadata_commit_group_locked(&group) { + Ok(resolved) => resolved, + Err(err) => { + let message = err.to_string(); + if let Err(store_err) = + self.defer_unresolved_metadata_commit_group_locked(group) + { + let message = store_err.to_string(); + return commands + .iter() + .map(|_| Err(MetadError::Codec(message.clone()))) + .collect(); + } + // Do not acknowledge even later successful Holt subgroups: + // the exact whole batch remains frozen until every earlier + // ambiguous outcome can be resolved and archived in order. + return commands + .iter() + .map(|_| Err(MetadError::Codec(message.clone()))) + .collect(); + } + } + } else { + raw_results + }; + let mut successful = Vec::new(); - let mut results = self - .metadata - .commit_independent_batch(commands) + let mut results = metadata_results .into_iter() .zip(commands) .enumerate() @@ -203,10 +385,6 @@ where .map_err(MetadError::from) }) .collect::>(); - if !successful.is_empty() { - self.purge_path_caches_after_write(); - } - let log_commands = successful .iter() .map(|(_, command, result)| (*command, result)) @@ -227,6 +405,38 @@ where } } +fn blocked_before_apply(err: MetadError) -> MetadError { + match err { + MetadError::SyncLogArchiveFailed { message, .. } => MetadError::SyncLogArchiveFailed { + committed: false, + message, + }, + err => err, + } +} + +fn blocked_batch_before_apply( + command_count: usize, + err: MetadError, +) -> Vec> { + match err { + MetadError::SyncLogArchiveFailed { message, .. } => (0..command_count) + .map(|_| { + Err(MetadError::SyncLogArchiveFailed { + committed: false, + message: message.clone(), + }) + }) + .collect(), + err => { + let message = err.to_string(); + (0..command_count) + .map(|_| Err(MetadError::Codec(message.clone()))) + .collect() + } + } +} + fn validate_owner_epoch(epoch: u64) -> Result<(), MetadError> { if epoch == 0 { return Err(MetadError::InvalidOwnerEpoch); diff --git a/crates/nokv-meta/src/service/gc.rs b/crates/nokv-meta/src/service/gc.rs index edb38e283..c3fadaeda 100644 --- a/crates/nokv-meta/src/service/gc.rs +++ b/crates/nokv-meta/src/service/gc.rs @@ -1,11 +1,694 @@ use super::*; use std::time::Duration; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ObjectGcClaimProgress { + Open, + Pending, + BlockedByFailoverDurability, + UncertainDelete { + owner_epoch: u64, + operation_token: u64, + }, +} + +fn encode_object_gc_quarantine_record(row: &crate::command::ScanItem, reason: &str) -> Vec { + let mut value = Vec::new(); + let reason = reason.as_bytes(); + value.extend_from_slice(&(row.key.len() as u32).to_be_bytes()); + value.extend_from_slice(&row.key); + value.extend_from_slice(&(row.value.0.len() as u32).to_be_bytes()); + value.extend_from_slice(&row.value.0); + let reason_len = reason.len().min(4096); + value.extend_from_slice(&(reason_len as u32).to_be_bytes()); + value.extend_from_slice(&reason[..reason_len]); + value +} + +fn decode_validated_object_gc_row( + mount: MountId, + row: &crate::command::ScanItem, +) -> Result { + let key = decode_object_gc_record_key(mount, &row.key)?; + let record = + decode_object_gc_record(&row.value.0).map_err(|err| MetadError::Codec(err.to_string()))?; + if record.enqueue_version != key.enqueue_version { + return Err(MetadError::Codec( + "object GC row enqueue version does not match its key".to_owned(), + )); + } + if record.inode != key.inode { + return Err(MetadError::Codec( + "object GC row inode does not match its key".to_owned(), + )); + } + if record.generation != key.generation { + return Err(MetadError::Codec( + "object GC row generation does not match its key".to_owned(), + )); + } + let (object_mount, object_inode, object_generation, object_chunk_index, object_block_index) = + decode_canonical_block_object_owner(&record.object_key)?; + if object_mount != mount.get() + || object_inode != key.inode.get() + || object_generation != key.generation + || object_chunk_index != key.chunk_index + || object_block_index != key.block_index + { + return Err(MetadError::Codec( + "object GC row object identity does not match its key".to_owned(), + )); + } + Ok(record) +} + impl NoKvFs where M: MetadataStore, O: ObjectStore, { + pub(super) fn ensure_object_gc_claim_record(&self) -> Result<(), MetadError> { + let key = object_gc_claim_key(self.mount); + if let Some(value) = self.metadata.get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + decode_object_gc_claim(self.mount, &value.0)?; + return Ok(()); + } + let version = self.next_version()?; + let result = self.commit_metadata(MetadataCommand { + request_id: request_id( + b"initialize-object-gc-claim", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_claim(&ObjectGcClaim::Open)?)), + }], + watch: Vec::new(), + }); + match result { + Ok(_) + | Err(MetadError::Metadata(MetadataError::PredicateFailed)) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => { + let value = self + .metadata + .get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "object GC claim initialization was not durable".to_owned(), + ) + })?; + decode_object_gc_claim(self.mount, &value.0)?; + Ok(()) + } + Err(err) => Err(err), + } + } + + /// Capture the durable Open epoch before object upload or historical + /// planning. The exact record version must join the metadata commit that + /// makes the reference durable. + pub(super) fn begin_object_reference_mutation( + &self, + ) -> Result { + self.ensure_object_gc_claim_record()?; + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &object_gc_claim_key(self.mount), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + let claim = decode_object_gc_claim(self.mount, &item.value.0)?; + let current_epoch = self.epoch.load(Ordering::Relaxed); + let claim_owner = match &claim { + ObjectGcClaim::Open => None, + ObjectGcClaim::Deleting { owner_epoch, .. } => Some(*owner_epoch), + }; + if claim_owner.is_some_and(|owner| owner > current_epoch) { + return Err(MetadError::StaleOwnerEpoch { + owner_epoch: current_epoch, + required_epoch: claim_owner.expect("future claim owner exists"), + }); + } + match claim { + ObjectGcClaim::Open => Ok(ObjectReferenceMutation::from_version(item.version)), + ObjectGcClaim::Deleting { .. } => { + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + } + } + } + + fn failover_durability_required(&self) -> Result { + let Some(value) = self.metadata.get( + RecordFamily::System, + &failover_durability_required_key(self.mount), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(false); + }; + decode_failover_durability_required_marker(&value.0)?; + Ok(true) + } + + fn update_object_gc_scan_cursor( + &self, + cursor: Option<&crate::command::ReadItem>, + next: Option>, + ) -> Result { + if cursor.is_none() && next.is_none() { + return Ok(true); + } + let key = object_gc_scan_cursor_key(self.mount); + let version = self.next_version()?; + let cursor_predicate = cursor.map_or(Predicate::NotExists, |item| { + Predicate::VersionEquals(item.version) + }); + let mutation = match next { + Some(next) => Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(next)), + }, + None => delete_mutation(RecordFamily::System, key.clone()), + }; + let command = MetadataCommand { + request_id: request_id( + b"advance-object-gc-scan-cursor", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key, + predicate: cursor_predicate, + }, + PredicateRef { + family: RecordFamily::System, + key: failover_durability_required_key(self.mount), + predicate: Predicate::PrefixEmpty, + }, + ], + mutations: vec![mutation], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) => Ok(true), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => Ok(false), + Err(err) => Err(err), + } + } + + fn recover_object_gc_claim_locked( + &self, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result { + self.ensure_object_gc_claim_record()?; + let key = object_gc_claim_key(self.mount); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + match decode_object_gc_claim(self.mount, &item.value.0)? { + ObjectGcClaim::Open => Ok(ObjectGcClaimProgress::Open), + ObjectGcClaim::Deleting { + owner_epoch, + operation_token, + gc_record_key, + gc_record_version, + } => { + let current_epoch = self.epoch.load(Ordering::Relaxed); + if owner_epoch > current_epoch { + return Err(MetadError::StaleOwnerEpoch { + owner_epoch: current_epoch, + required_epoch: owner_epoch, + }); + } + if self.failover_durability_required()? { + // The durable claim cannot tell whether the external DELETE + // completed before the old owner crashed, or is still in + // flight. Keep the namespace closed: reopening would admit + // references that a late DELETE could invalidate. Startup + // treats this outcome as requiring controlled operator + // recovery; the worker preserves both claim and queue row. + outcome.blocked_by_failover_durability += 1; + return Ok(ObjectGcClaimProgress::UncertainDelete { + owner_epoch, + operation_token, + }); + } + let Some(row) = self.metadata.get_versioned( + RecordFamily::Gc, + &gc_record_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + else { + self.release_object_gc_claim(item.version, None)?; + return Ok(ObjectGcClaimProgress::Open); + }; + if row.version.get() != gc_record_version { + self.release_object_gc_claim(item.version, None)?; + return Ok(ObjectGcClaimProgress::Open); + } + let row = crate::command::ScanItem { + key: gc_record_key, + value: row.value, + version: row.version, + }; + let record = match decode_validated_object_gc_row(self.mount, &row) { + Ok(record) => record, + Err(err) => { + self.quarantine_claimed_gc_row( + item.version, + &row, + &err.to_string(), + outcome, + )?; + return Ok(ObjectGcClaimProgress::Open); + } + }; + self.finish_claimed_gc_row(item.version, &row, &record, outcome) + } + } + } + + fn acquire_object_gc_claim( + &self, + row: &crate::command::ScanItem, + ) -> Result, MetadError> { + if self.failover_durability_required()? { + return Ok(None); + } + let open = self.begin_object_reference_mutation()?; + match self.transition_object_gc_claim( + open.version(), + &ObjectGcClaim::Deleting { + owner_epoch: self.epoch.load(Ordering::Relaxed), + operation_token: open.version().get(), + gc_record_key: row.key.clone(), + gc_record_version: row.version.get(), + }, + Some((&row.key, row.version)), + b"claim-object-delete", + ) { + Ok(version) => Ok(Some(version)), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + if self.failover_durability_required()? => + { + Ok(None) + } + Err(err) => Err(err), + } + } + + fn transition_object_gc_claim( + &self, + expected_claim_version: Version, + next: &ObjectGcClaim, + gc_record: Option<(&[u8], Version)>, + request_domain: &[u8], + ) -> Result { + let key = object_gc_claim_key(self.mount); + let version = self.next_version()?; + let encoded = encode_object_gc_claim(next)?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(expected_claim_version), + }]; + predicates.push(PredicateRef { + family: RecordFamily::System, + key: failover_durability_required_key(self.mount), + predicate: Predicate::PrefixEmpty, + }); + if let Some((gc_key, gc_version)) = gc_record { + predicates.push(PredicateRef { + family: RecordFamily::Gc, + key: gc_key.to_vec(), + predicate: Predicate::VersionEquals(gc_version), + }); + } + let command = MetadataCommand { + request_id: request_id(request_domain, self.mount, InodeId::root(), version), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates, + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encoded.clone())), + }], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + result @ (Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + })) => { + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("object GC claim transition was lost".to_owned()) + })?; + if item.version != version || item.value.0 != encoded { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + match result { + Ok(_) => Ok(item.version), + Err(err) => Err(err), + } + } + Err(err) => Err(err), + } + } + + fn release_object_gc_claim( + &self, + expected_claim_version: Version, + delete_gc_record: Option<(&[u8], Version)>, + ) -> Result<(), MetadError> { + let key = object_gc_claim_key(self.mount); + let version = self.next_version()?; + let encoded = encode_object_gc_claim(&ObjectGcClaim::Open)?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(expected_claim_version), + }]; + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encoded.clone())), + }]; + if let Some((gc_key, gc_version)) = delete_gc_record { + predicates.push(PredicateRef { + family: RecordFamily::Gc, + key: gc_key.to_vec(), + predicate: Predicate::VersionEquals(gc_version), + }); + mutations.push(delete_mutation(RecordFamily::Gc, gc_key.to_vec())); + } + let command = MetadataCommand { + request_id: request_id( + b"release-object-gc-claim", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + match self.commit_metadata(command) { + result @ (Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + })) => { + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("object GC claim was lost".to_owned()))?; + if item.version != version + || decode_object_gc_claim(self.mount, &item.value.0)? != ObjectGcClaim::Open + { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + match result { + Ok(_) => Ok(()), + Err(err) => Err(err), + } + } + Err(err) => Err(err), + } + } + + fn delete_gc_row_under_durable_claim( + &self, + row: &crate::command::ScanItem, + record: &ObjectGcRecord, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result { + let claim_version = match self.acquire_object_gc_claim(row) { + Ok(Some(version)) => version, + Ok(None) => { + outcome.blocked_by_failover_durability += 1; + return Ok(ObjectGcClaimProgress::BlockedByFailoverDurability); + } + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + return Ok(ObjectGcClaimProgress::Pending); + } + Err(err) => return Err(err), + }; + self.finish_claimed_gc_row(claim_version, row, record, outcome) + } + + fn finish_claimed_gc_row( + &self, + claim_version: Version, + row: &crate::command::ScanItem, + record: &ObjectGcRecord, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result { + let protected = match self.object_delete_is_protected(record) { + Ok(protected) => protected, + Err(err) => { + self.quarantine_claimed_gc_row(claim_version, row, &err.to_string(), outcome)?; + return Ok(ObjectGcClaimProgress::Open); + } + }; + if protected { + outcome.blocked_by_snapshots += 1; + self.release_object_gc_claim(claim_version, None)?; + return Ok(ObjectGcClaimProgress::Open); + } + self.ensure_owner_epoch_current()?; + let object_key = match ObjectKey::new(record.object_key.clone()) { + Ok(key) => key, + Err(err) => { + self.quarantine_claimed_gc_row(claim_version, row, &err.to_string(), outcome)?; + return Ok(ObjectGcClaimProgress::Open); + } + }; + outcome.attempted += 1; + match self.objects.delete(&object_key) { + Ok(true) => outcome.deleted += 1, + Ok(false) => outcome.missing += 1, + // An error does not prove the remote DELETE failed: the object may + // already be gone, its acknowledgement may have been lost, or the + // request may still be in flight. Keep the durable Deleting claim + // closed so no writer can restage this generation until recovery + // retries the same idempotent delete under the same claim. + Err(err) => return Err(err.into()), + } + self.release_object_gc_claim(claim_version, Some((&row.key, row.version)))?; + outcome.records_removed += 1; + Ok(ObjectGcClaimProgress::Open) + } + + fn quarantine_claimed_gc_row( + &self, + claim_version: Version, + row: &crate::command::ScanItem, + reason: &str, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result<(), MetadError> { + let claim_key = object_gc_claim_key(self.mount); + let digest: [u8; 32] = Sha256::digest(&row.key).into(); + let quarantine_key = object_gc_quarantine_key(self.mount, &digest); + let version = self.next_version()?; + let value = encode_object_gc_quarantine_record(row, reason); + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"quarantine-object-gc-row", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: claim_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim_version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: quarantine_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::System, + key: claim_key, + op: MutationOp::Put, + value: Some(Value(encode_object_gc_claim(&ObjectGcClaim::Open)?)), + }, + delete_mutation(RecordFamily::Gc, row.key.clone()), + Mutation { + family: RecordFamily::System, + key: quarantine_key, + op: MutationOp::Put, + value: Some(Value(value)), + }, + ], + watch: Vec::new(), + })?; + outcome.records_removed += 1; + Ok(()) + } + + /// Quarantine a row whose key/value cannot name a valid object deletion. + /// No external DELETE is possible, so an exact-version metadata move is + /// sufficient and deliberately avoids persisting an invalid Deleting claim. + fn quarantine_unclaimed_gc_row( + &self, + row: &crate::command::ScanItem, + reason: &str, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result<(), MetadError> { + let digest: [u8; 32] = Sha256::digest(&row.key).into(); + let quarantine_key = object_gc_quarantine_key(self.mount, &digest); + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"quarantine-invalid-object-gc-row", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: row.key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::Gc, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: quarantine_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + delete_mutation(RecordFamily::Gc, row.key.clone()), + Mutation { + family: RecordFamily::System, + key: quarantine_key, + op: MutationOp::Put, + value: Some(Value(encode_object_gc_quarantine_record(row, reason))), + }, + ], + watch: Vec::new(), + })?; + outcome.records_removed += 1; + Ok(()) + } + + fn object_delete_is_protected(&self, record: &ObjectGcRecord) -> Result { + if self + .history_retention_floor()? + .is_some_and(|floor| floor.get() < record.enqueue_version) + { + return Ok(true); + } + let Some(body) = self.body_descriptor(record.inode)? else { + return Ok(false); + }; + let manifests = self.chunk_manifests_for_body_at_version( + record.inode, + &body, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + Ok(manifests + .iter() + .flat_map(|manifest| &manifest.slices) + .flat_map(|slice| &slice.blocks) + .any(|block| block.object_key == record.object_key)) + } + pub fn cleanup_staged_objects( &self, staged: &StagedObjectSet, @@ -13,6 +696,127 @@ where self.objects.delete_staged(staged).map_err(Into::into) } + /// Resume only a crash-left durable object-GC claim. This does not scan or + /// start new GC work, so a server can call it after installing durability + /// policy and before admitting writers or starting the background worker. + pub fn recover_object_gc_claim(&self) -> Result { + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let mut outcome = PendingObjectCleanupOutcome::default(); + match self.recover_object_gc_claim_locked(&mut outcome)? { + ObjectGcClaimProgress::UncertainDelete { + owner_epoch, + operation_token, + } => Err(MetadError::ObjectGcRecoveryRequiresIntervention { + owner_epoch, + operation_token, + }), + _ => Ok(outcome), + } + } + + /// Invalidate every prepared object reference minted by a previous owner. + /// + /// Failover calls this only after crash-left claim recovery has proved the + /// durable claim is Open. Rewriting Open at a fresh metadata version makes + /// the claim version an incarnation fence: a prepared upload from the old + /// owner can no longer publish a manifest after takeover. + pub fn rotate_object_gc_claim_for_failover(&self) -> Result<(), MetadError> { + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.ensure_object_gc_claim_record()?; + let key = object_gc_claim_key(self.mount); + let old = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + if !matches!( + decode_object_gc_claim(self.mount, &old.value.0)?, + ObjectGcClaim::Open + ) { + // Recovery must never reopen or overwrite a deletion claim whose + // external DELETE outcome is not known. + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + + let version = self.next_version()?; + if version <= old.version { + return Err(MetadError::Codec( + "object GC claim rotation did not advance its record version".to_owned(), + )); + } + let command = MetadataCommand { + request_id: request_id( + b"rotate-failover-object-gc-claim", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(old.version), + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_claim(&ObjectGcClaim::Open)?)), + }], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) => Ok(()), + Err(err) + if matches!( + &err, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } | MetadError::Metadata(MetadataError::Backend(_)) + ) => + { + // A backend may apply durably and lose only its acknowledgement. + // Accept that uncertain result solely when an exact read-back + // proves this command's new Open version is present. + let current = self.metadata.get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + if current.as_ref().is_some_and(|item| { + item.version == version + && matches!( + decode_object_gc_claim(self.mount, &item.value.0), + Ok(ObjectGcClaim::Open) + ) + }) { + Ok(()) + } else { + Err(err) + } + } + Err(err) => Err(err), + } + } + pub fn cleanup_pending_objects( &self, limit: usize, @@ -25,86 +829,95 @@ where limit: usize, read_lease_grace: Duration, ) -> Result { - // Reap expired snapshot pins first so the retention floor reflects only - // live snapshots before deciding what is reclaimable. - let snapshot_reap = self.reclaim_expired_snapshot_pins(limit)?; - let now_ms = self.now_ms(); - let grace_ms = duration_millis_u64(read_lease_grace); - let version = self.read_version()?; + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let mut outcome = PendingObjectCleanupOutcome::default(); + if !matches!( + self.recover_object_gc_claim_locked(&mut outcome)?, + ObjectGcClaimProgress::Open + ) { + return Ok(outcome); + } + let page_size = limit.max(1); + let cursor_key = object_gc_scan_cursor_key(self.mount); + let read_version = self.read_version()?; + let cursor = self.metadata.get_versioned( + RecordFamily::System, + &cursor_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let queue_prefix = gc_queue_prefix(self.mount); + if cursor.as_ref().is_some_and(|item| { + !item.value.0.starts_with(&queue_prefix) || item.value.0.len() <= queue_prefix.len() + }) { + return Err(MetadError::Codec( + "durable object GC scan cursor is outside the GC queue".to_owned(), + )); + } let rows = self.metadata.scan(ScanRequest { family: RecordFamily::Gc, - prefix: gc_queue_prefix(self.mount), - start_after: None, - version, - limit, + prefix: queue_prefix, + start_after: cursor.as_ref().map(|item| item.value.0.clone()), + version: read_version, + limit: page_size, purpose: ReadPurpose::UserStrong, })?; + outcome.scanned += rows.len(); + if self.failover_durability_required()? { + outcome.blocked_by_failover_durability = + outcome.blocked_by_failover_durability.max(rows.len()); + return Ok(outcome); + } + // Reap expired snapshot pins only after the failover marker check, so + // HA fail-closed mode performs no unrelated cleanup mutations. + outcome.snapshot_reap = self.reclaim_expired_snapshot_pins(page_size)?; if rows.is_empty() { - return Ok(PendingObjectCleanupOutcome { - snapshot_reap, - ..PendingObjectCleanupOutcome::default() - }); + self.update_object_gc_scan_cursor(cursor.as_ref(), None)?; + return Ok(outcome); } - let retention_floor = self.history_retention_floor()?; - - let mut outcome = PendingObjectCleanupOutcome { - snapshot_reap, - scanned: rows.len(), - blocked_by_snapshots: 0, - blocked_by_read_leases: 0, - attempted: 0, - deleted: 0, - missing: 0, - records_removed: 0, - }; - let mut cleaned_keys = Vec::with_capacity(rows.len()); + + let now_ms = self.now_ms(); + let grace_ms = duration_millis_u64(read_lease_grace); + let reached_tail = rows.len() < page_size; + let last_scanned_key = rows.last().expect("non-empty GC page").key.clone(); + let mut page_completed = true; for row in rows { - let record = decode_object_gc_record(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - if retention_floor.is_some_and(|floor| floor.get() < record.enqueue_version) { - outcome.blocked_by_snapshots += 1; - continue; - } + let record = match decode_validated_object_gc_row(self.mount, &row) { + Ok(record) => record, + Err(err) => { + self.quarantine_unclaimed_gc_row(&row, &err.to_string(), &mut outcome)?; + continue; + } + }; if now_ms < record.enqueue_unix_ms.saturating_add(grace_ms) { outcome.blocked_by_read_leases += 1; continue; } - let key = ObjectKey::new(record.object_key)?; - outcome.attempted += 1; - if self.objects.delete(&key)? { - outcome.deleted += 1; - } else { - outcome.missing += 1; + // Avoid rotating the global reference epoch for a candidate that is + // already known to be protected. This first check is advisory: a + // snapshot/reference can race it, so every actual delete still + // acquires the durable claim and repeats the protection check. + if matches!(self.object_delete_is_protected(&record), Ok(true)) { + outcome.blocked_by_snapshots += 1; + continue; + } + match self.delete_gc_row_under_durable_claim(&row, &record, &mut outcome)? { + ObjectGcClaimProgress::Open => {} + ObjectGcClaimProgress::Pending + | ObjectGcClaimProgress::BlockedByFailoverDurability + | ObjectGcClaimProgress::UncertainDelete { .. } => { + page_completed = false; + break; + } } - cleaned_keys.push(row.key); } - - if cleaned_keys.is_empty() { - return Ok(outcome); + if page_completed { + let next = (!reached_tail).then_some(last_scanned_key); + self.update_object_gc_scan_cursor(cursor.as_ref(), next)?; } - - let commit_version = self.next_version()?; - let records_removed = cleaned_keys.len(); - self.commit_metadata(MetadataCommand { - request_id: request_id( - b"cleanup-objects", - self.mount, - InodeId::root(), - commit_version, - ), - kind: CommandKind::CleanupObjects, - read_version: predecessor(commit_version)?, - commit_version, - primary_family: RecordFamily::Gc, - primary_key: gc_queue_prefix(self.mount), - predicates: Vec::new(), - mutations: cleaned_keys - .into_iter() - .map(|key| delete_mutation(RecordFamily::Gc, key)) - .collect(), - watch: Vec::new(), - })?; - outcome.records_removed = records_removed; Ok(outcome) } @@ -151,12 +964,43 @@ where object_key.starts_with(&owner_prefix) } + /// Whether a canonical block key was minted by `inode`, irrespective of + /// generation. Fork retirement uses this across the effective generation + /// chain: blocks owned by the fork inode are self-contained, while a block + /// owned by any other inode is still borrowed and keeps the durable fork + /// retention binding live. + pub(super) fn block_object_is_owned_by_inode( + &self, + inode: InodeId, + object_key: &str, + ) -> Result { + let (mount, owner, _, _, _) = decode_canonical_block_object_owner(object_key)?; + Ok(mount == self.mount.get() && owner == inode.get()) + } + + pub(super) fn canonical_block_object_identity( + &self, + object_key: &str, + ) -> Result<(InodeId, u64, u64, u64), MetadError> { + let (mount, inode, generation, chunk_index, block_index) = + decode_canonical_block_object_owner(object_key)?; + if mount != self.mount.get() { + return Err(MetadError::Codec( + "block object belongs to another mount".to_owned(), + )); + } + let inode = InodeId::new(inode) + .map_err(|err| MetadError::Codec(format!("invalid block owner inode: {err}")))?; + Ok((inode, generation, chunk_index, block_index)) + } + pub(super) fn history_retention_floor(&self) -> Result, MetadError> { + let read_version = self.read_version()?; let rows = self.metadata.scan(ScanRequest { family: RecordFamily::Snapshot, prefix: snapshot_pin_prefix(self.mount), start_after: None, - version: self.read_version()?, + version: read_version, limit: 0, purpose: ReadPurpose::UserStrong, })?; @@ -174,6 +1018,14 @@ where let version = Version::new(pin.read_version)?; floor = Some(floor.map_or(version, |floor| floor.min(version))); } + // A clone's snapshot lease is only the construction-time fence. Once + // the eager fork is exposed, its durable ForkBinding is the retention + // root for borrowed source blocks and deliberately has no wall-clock + // expiry. Only explicit snapshot retirement may delete the binding. + for versioned in self.versioned_fork_bindings_at(read_version, ReadPurpose::UserStrong)? { + let version = Version::new(versioned.binding.pinned_read_version)?; + floor = Some(floor.map_or(version, |floor| floor.min(version))); + } Ok(floor) } @@ -353,16 +1205,12 @@ where purpose: ReadPurpose::WritePlanLocal, })?; let mut mutations = Vec::new(); + let mut queued_object_keys = HashSet::new(); for row in rows { if chunk_index_from_manifest_key(&row.key)? != BODY_SUMMARY_CHUNK_INDEX { let manifest = decode_chunk_manifest(&row.value.0) .map_err(|err| MetadError::Codec(err.to_string()))?; - for (block_index, block) in manifest - .slices - .iter() - .flat_map(|slice| slice.blocks.iter()) - .enumerate() - { + for block in manifest.slices.iter().flat_map(|slice| slice.blocks.iter()) { if retained_object_keys.contains(&block.object_key) { continue; } @@ -372,6 +1220,19 @@ where // must never enqueue another namespace's blocks for GC. continue; } + let (owner, object_generation, chunk_index, block_index) = + self.canonical_block_object_identity(&block.object_key)?; + if owner != inode + || object_generation != generation + || chunk_index != manifest.chunk_index + { + return Err(MetadError::Codec( + "owned block object identity does not match its manifest".to_owned(), + )); + } + if !queued_object_keys.insert(block.object_key.clone()) { + continue; + } let record = ObjectGcRecord { inode, generation, @@ -388,8 +1249,8 @@ where enqueue_version.get(), inode, generation, - manifest.chunk_index, - block_index as u64, + chunk_index, + block_index, ), op: MutationOp::Put, value: Some(Value(encode_object_gc_record(&record))), @@ -437,13 +1298,13 @@ where // Fast path: a self-contained old generation whose manifests the caller // already resolved is reclaimed without a metadata scan. if chain.len() == 1 && !old_chunks.is_empty() { - return Ok(self.chunk_manifest_delete_and_gc_mutations_from_manifests( + return self.chunk_manifest_delete_and_gc_mutations_from_manifests( inode, old_top_generation, old_chunks, enqueue_version, retained_object_keys, - )); + ); } let mut mutations = Vec::new(); for generation in chain { @@ -464,19 +1325,15 @@ where manifests: &[ChunkManifest], enqueue_version: Version, retained_object_keys: &HashSet, - ) -> Vec { + ) -> Result, MetadError> { let enqueue_unix_ms = current_time_ms(); let mut mutations = vec![delete_mutation( RecordFamily::ChunkManifest, chunk_manifest_key(self.mount, inode, generation, BODY_SUMMARY_CHUNK_INDEX), )]; + let mut queued_object_keys = HashSet::new(); for manifest in manifests { - for (block_index, block) in manifest - .slices - .iter() - .flat_map(|slice| slice.blocks.iter()) - .enumerate() - { + for block in manifest.slices.iter().flat_map(|slice| slice.blocks.iter()) { if retained_object_keys.contains(&block.object_key) { continue; } @@ -485,6 +1342,19 @@ where // that minted it, so this borrower must not enqueue it for GC. continue; } + let (owner, object_generation, chunk_index, block_index) = + self.canonical_block_object_identity(&block.object_key)?; + if owner != inode + || object_generation != generation + || chunk_index != manifest.chunk_index + { + return Err(MetadError::Codec( + "owned block object identity does not match its manifest".to_owned(), + )); + } + if !queued_object_keys.insert(block.object_key.clone()) { + continue; + } let record = ObjectGcRecord { inode, generation, @@ -501,8 +1371,8 @@ where enqueue_version.get(), inode, generation, - manifest.chunk_index, - block_index as u64, + chunk_index, + block_index, ), op: MutationOp::Put, value: Some(Value(encode_object_gc_record(&record))), @@ -513,7 +1383,7 @@ where chunk_manifest_key(self.mount, inode, generation, manifest.chunk_index), )); } - mutations + Ok(mutations) } } diff --git a/crates/nokv-meta/src/service/lifecycle.rs b/crates/nokv-meta/src/service/lifecycle.rs index 8597e8370..0d90fb49b 100644 --- a/crates/nokv-meta/src/service/lifecycle.rs +++ b/crates/nokv-meta/src/service/lifecycle.rs @@ -13,7 +13,13 @@ where objects, allocator_gate: Mutex::new(()), backup_gate: Mutex::new(()), + object_gc_gate: Mutex::new(()), + materialization_orphan_possible: AtomicBool::new(true), + #[cfg(test)] + namespace_reachability_scans: AtomicU64::new(0), epoch_fence: RwLock::new(()), + metadata_log_enable_fence: RwLock::new(()), + metadata_commit_log_gate: Mutex::new(()), path_resolution_cache: new_path_resolution_cache_shards(), path_index_lookup_cache: new_path_index_lookup_cache_shards(), path_index_validation_cache: new_path_index_validation_cache_shards(), @@ -88,6 +94,10 @@ where /// and the allocator floor is then seeded into this shard's high-bit subspace /// exactly like [`Self::with_shard_index`]. So callers must NOT chain /// `.with_shard_index(...)` after `open_existing`; pass the index here. + /// + /// This constructor only reconstructs process-local state. A durable object + /// GC claim is resumed by the ordinary cleanup worker after ownership and + /// durability policy have been installed. pub fn open_existing( mount: MountId, metadata: M, @@ -103,14 +113,20 @@ where .expect("composed allocator base inode is valid") .get(); let next_inode = allocator.next_inode.max(allocator_floor); - Ok(Self { + let service = Self { mount, shard_index, metadata, objects, allocator_gate: Mutex::new(()), backup_gate: Mutex::new(()), + object_gc_gate: Mutex::new(()), + materialization_orphan_possible: AtomicBool::new(true), + #[cfg(test)] + namespace_reachability_scans: AtomicU64::new(0), epoch_fence: RwLock::new(()), + metadata_log_enable_fence: RwLock::new(()), + metadata_commit_log_gate: Mutex::new(()), path_resolution_cache: new_path_resolution_cache_shards(), path_index_lookup_cache: new_path_index_lookup_cache_shards(), path_index_validation_cache: new_path_index_validation_cache_shards(), @@ -154,7 +170,9 @@ where read_dir_plus_total: AtomicU64::new(0), read_dir_plus_entry_total: AtomicU64::new(0), read_dir_plus_projection_hit_total: AtomicU64::new(0), - }) + }; + service.recover_materialization_orphan_state()?; + Ok(service) } pub fn object_stats(&self) -> ObjectTransferStats { diff --git a/crates/nokv-meta/src/service/log_archive.rs b/crates/nokv-meta/src/service/log_archive.rs index 5d6cfc0e2..0726c493a 100644 --- a/crates/nokv-meta/src/service/log_archive.rs +++ b/crates/nokv-meta/src/service/log_archive.rs @@ -5,7 +5,9 @@ //! control-plane pointer publication or restore-time command application. use super::*; -use crate::{metadata_log_replay_entries, MetadataCheckpointStore, MetadataLogSegment}; +use crate::{ + metadata_log_replay_entries, MetadataCheckpointStore, MetadataLogEntry, MetadataLogSegment, +}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct MetadataLogArchiveConfig { @@ -37,6 +39,30 @@ impl MetadataLogArchiveConfig { prefix: prefix.into(), } } + + /// Reject a control-plane pointer that escapes this shard's exact shared + /// log namespace before restore performs object I/O. + pub fn validate_segment_key(&self, segment_key: &str) -> Result<(), MetadError> { + let prefix = normalize_log_prefix(&self.prefix); + if prefix.is_empty() { + return Err(MetadError::Codec( + "metadata log archive prefix is empty".to_owned(), + )); + } + let expected_prefix = format!("{prefix}/log/"); + let Some(filename) = segment_key.strip_prefix(&expected_prefix) else { + return Err(MetadError::Codec(format!( + "metadata log segment key {segment_key} is outside archive prefix {prefix}" + ))); + }; + if filename.is_empty() || filename.contains('/') { + return Err(MetadError::Codec(format!( + "metadata log segment key {segment_key} is not a direct archive object" + ))); + } + ObjectKey::new(segment_key.to_owned())?; + Ok(()) + } } impl NoKvFs @@ -71,6 +97,7 @@ where pub fn restore_metadata_with_archived_log_segments( &self, checkpoint_config: &MetadataArchiveConfig, + expected_shard_id: &str, segment_keys: &[String], checkpoint_lsn: u64, checkpoint_digest: [u8; 32], @@ -81,6 +108,7 @@ where .collect::, _>>()?; self.restore_metadata_with_log_segments( checkpoint_config, + expected_shard_id, &segments, checkpoint_lsn, checkpoint_digest, @@ -90,20 +118,92 @@ where pub fn restore_metadata_with_log_segments( &self, checkpoint_config: &MetadataArchiveConfig, + expected_shard_id: &str, segments: &[MetadataLogSegment], checkpoint_lsn: u64, checkpoint_digest: [u8; 32], ) -> Result, MetadError> { - let entries = metadata_log_replay_entries(segments, checkpoint_lsn, checkpoint_digest) - .map_err(|err| MetadError::Codec(format!("metadata log replay failed: {err}")))?; + let entries = metadata_log_replay_entries( + expected_shard_id, + segments, + checkpoint_lsn, + checkpoint_digest, + ) + .map_err(|err| MetadError::Codec(format!("metadata log replay failed: {err}")))?; let Some(checkpoint) = self.restore_metadata(checkpoint_config)? else { return Ok(None); }; + self.replay_metadata_log_entries(checkpoint, &entries, checkpoint_lsn, checkpoint_digest) + .map(Some) + } + + /// Restore a controlled shard from the exact immutable checkpoint named by + /// its control record, then replay the archived log tail above it. Mutable + /// standalone `CURRENT` is deliberately ignored. + pub fn restore_metadata_checkpoint_with_archived_log_segments( + &self, + checkpoint_config: &MetadataArchiveConfig, + expected_shard_id: &str, + checkpoint: &MetadataCheckpointIdentity, + segment_keys: &[String], + checkpoint_lsn: u64, + checkpoint_digest: [u8; 32], + ) -> Result { + let segments = segment_keys + .iter() + .map(|key| self.load_metadata_log_segment(key)) + .collect::, _>>()?; + self.restore_metadata_checkpoint_with_log_segments( + checkpoint_config, + expected_shard_id, + checkpoint, + &segments, + checkpoint_lsn, + checkpoint_digest, + ) + } + + pub fn restore_metadata_checkpoint_with_log_segments( + &self, + checkpoint_config: &MetadataArchiveConfig, + expected_shard_id: &str, + checkpoint: &MetadataCheckpointIdentity, + segments: &[MetadataLogSegment], + checkpoint_lsn: u64, + checkpoint_digest: [u8; 32], + ) -> Result { + let entries = metadata_log_replay_entries( + expected_shard_id, + segments, + checkpoint_lsn, + checkpoint_digest, + ) + .map_err(|err| MetadError::Codec(format!("metadata log replay failed: {err}")))?; + let checkpoint = self.restore_metadata_checkpoint(checkpoint_config, checkpoint)?; + self.replay_metadata_log_entries(checkpoint, &entries, checkpoint_lsn, checkpoint_digest) + } + + fn replay_metadata_log_entries( + &self, + checkpoint: MetadataRestoreOutcome, + entries: &[MetadataLogEntry], + checkpoint_lsn: u64, + checkpoint_digest: [u8; 32], + ) -> Result { + // Replay may reproduce only the prefix of a multi-command detached + // materialization if the archived tail ended at a crash boundary. Keep + // inode-addressed exposure fail-closed for the whole replay, then derive + // the fast-path state from the final installed namespace. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.mark_materialization_orphan_possible_under_gc_gate(); let mut durable_lsn = checkpoint_lsn; let mut last_digest = checkpoint_digest; let mut replay_next_inode = None; - for entry in &entries { + for entry in entries { let result = self.commit_metadata_without_sync_log(entry.command.clone())?; if result != entry.result { return Err(MetadError::Codec(format!( @@ -115,7 +215,7 @@ where .fetch_max(result.commit_version.get(), Ordering::Relaxed); replay_next_inode = max_optional_u64( replay_next_inode, - command_replay_next_inode(&entry.command)?, + command_replay_next_inode(&entry.command, self.shard_index())?, ); durable_lsn = entry.lsn; last_digest = entry.digest; @@ -126,13 +226,14 @@ where .fetch_max(next_inode, Ordering::Relaxed); } self.refresh_allocator_state()?; + self.reconcile_materialization_orphan_state_under_gc_gate()?; - Ok(Some(MetadataLogRestoreOutcome { + Ok(MetadataLogRestoreOutcome { checkpoint, replayed_entries: entries.len(), durable_lsn, last_digest, - })) + }) } } @@ -178,7 +279,10 @@ pub(super) fn archive_metadata_log_segment_to_store( }) } -fn command_replay_next_inode(command: &MetadataCommand) -> Result, MetadError> { +fn command_replay_next_inode( + command: &MetadataCommand, + local_shard_index: u16, +) -> Result, MetadError> { let mut max_inode = None; for mutation in &command.mutations { if mutation.op != MutationOp::Put { @@ -191,20 +295,19 @@ fn command_replay_next_inode(command: &MetadataCommand) -> Result, M RecordFamily::Inode => { let attr = decode_inode_attr(&value.0) .map_err(|err| MetadError::Codec(err.to_string()))?; - max_inode = max_optional_u64(max_inode, Some(attr.inode.get())); + max_inode = + max_optional_u64(max_inode, local_inode_raw(attr.inode, local_shard_index)); } RecordFamily::Dentry => { let projection = decode_dentry_projection(&value.0) .map_err(|err| MetadError::Codec(err.to_string()))?; max_inode = max_optional_u64( max_inode, - Some( - projection - .attr - .inode - .get() - .max(projection.dentry.child.get()), - ), + local_inode_raw(projection.attr.inode, local_shard_index), + ); + max_inode = max_optional_u64( + max_inode, + local_inode_raw(projection.dentry.child, local_shard_index), ); } _ => {} @@ -215,6 +318,10 @@ fn command_replay_next_inode(command: &MetadataCommand) -> Result, M .transpose() } +fn local_inode_raw(inode: InodeId, local_shard_index: u16) -> Option { + (inode.shard_index() == local_shard_index).then_some(inode.get()) +} + fn max_optional_u64(left: Option, right: Option) -> Option { match (left, right) { (Some(left), Some(right)) => Some(left.max(right)), diff --git a/crates/nokv-meta/src/service/log_sync.rs b/crates/nokv-meta/src/service/log_sync.rs index 97132312c..648c24977 100644 --- a/crates/nokv-meta/src/service/log_sync.rs +++ b/crates/nokv-meta/src/service/log_sync.rs @@ -39,6 +39,32 @@ pub struct MetadataLogSyncSnapshot { pub segments: Vec, } +/// Atomic local state used to decide whether a control-published recovery tail +/// still exactly covers every metadata mutation visible on this server. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MetadataLogPublicationState { + pub snapshot: MetadataLogSyncSnapshot, + pub has_pending_segment: bool, + pub has_unresolved_commit_group: bool, +} + +/// Result of retiring shared-log segment objects covered by an authoritative +/// checkpoint. Object deletion is best-effort because the checkpoint remains +/// published even when cleanup fails. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MetadataLogPruneOutcome { + /// Exact pointers removed from the live chain because their tail is at or + /// before the checkpoint LSN. + pub pointers_pruned: usize, + /// Covered segment objects physically deleted from the object store. + pub objects_deleted: usize, + /// Covered segment objects that were already absent. + pub objects_missing: usize, + /// Covered segment objects whose key validation or deletion failed and + /// therefore remain as safe, unreferenced leaks. + pub delete_failures: usize, +} + pub(super) struct MetadataLogSyncState { config: MetadataLogArchiveConfig, shard_id: String, @@ -46,18 +72,59 @@ pub(super) struct MetadataLogSyncState { next_lsn: u64, prev_digest: [u8; 32], segments: Vec, + /// A locally committed segment whose object-store archive has not yet + /// completed. No later metadata command may apply until this exact segment + /// is durably retried, otherwise its LSN could be reused and recovery would + /// permanently omit the earlier commit. + pending_segment: Option, + /// Exact input and engine results for a commit group containing at least + /// one `Backend` outcome whose durable apply status could not be read back. + /// No later command may apply, and no checkpoint may be exported, until the + /// whole group is resolved and its actually committed subset is archived in + /// original input order. + unresolved_commit_group: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct UnresolvedMetadataCommitGroup { + commands: Vec, + results: Vec>, +} + +impl UnresolvedMetadataCommitGroup { + pub(super) fn new( + commands: Vec, + results: Vec>, + ) -> Result { + if commands.len() != results.len() { + return Err(MetadError::Codec( + "metadata commit resolution group length mismatch".to_owned(), + )); + } + if !results + .iter() + .any(|result| matches!(result, Err(MetadataError::Backend(_)))) + { + return Err(MetadError::Codec( + "metadata commit resolution group has no backend outcome".to_owned(), + )); + } + Ok(Self { commands, results }) + } } impl MetadataLogSyncState { - fn snapshot(&self) -> Option { - let last = self.segments.last()?; - Some(MetadataLogSyncSnapshot { + fn snapshot(&self) -> MetadataLogSyncSnapshot { + MetadataLogSyncSnapshot { shard_id: self.shard_id.clone(), epoch: self.epoch, - durable_lsn: last.last_lsn, - last_digest: last.last_digest, + // `segments` contains only the live chain above the latest + // checkpoint and can therefore be empty immediately after pruning. + // The allocator tail remains the authoritative durable boundary. + durable_lsn: self.next_lsn - 1, + last_digest: self.prev_digest, segments: self.segments.clone(), - }) + } } } @@ -92,6 +159,37 @@ where M: MetadataStore, O: ObjectStore, { + /// Verify that an installed/restored metadata image already carries the + /// failover fence. This never creates or repairs the marker: callers use it + /// to reject checkpoints produced before failover-safe object GC existed. + pub fn verify_failover_durability_required(&self) -> Result<(), MetadError> { + if self.failover_durability_is_required()? { + return Ok(()); + } + Err(MetadError::Codec( + "failover durability requirement marker is missing; the checkpoint predates failover-safe object GC" + .to_owned(), + )) + } + + /// Report whether the persisted metadata image carries the failover + /// durability fence. A present marker is decoded strictly so corrupt or + /// unknown marker values are never reported as a safe deployment. + pub fn failover_durability_is_required(&self) -> Result { + let key = failover_durability_required_key(self.mount); + let Some(value) = self.metadata.get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(false); + }; + decode_failover_durability_required_marker(&value.0)?; + Ok(true) + } + pub fn enable_sync_metadata_log( &self, config: MetadataLogSyncConfig, @@ -108,6 +206,20 @@ where .durable_lsn .checked_add(1) .ok_or_else(|| MetadError::Codec("metadata log LSN is exhausted".to_owned()))?; + self.require_failover_durability()?; + let _log_enable_fence = self + .metadata_log_enable_fence + .write() + .unwrap_or_else(|err| err.into_inner()); + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + if guard.is_some() { + return Err(MetadError::Codec( + "synchronous metadata log is already enabled".to_owned(), + )); + } let state = MetadataLogSyncState { config: config.archive, shard_id: config.shard_id, @@ -115,34 +227,193 @@ where next_lsn, prev_digest: config.last_digest, segments: config.segments, + pending_segment: None, + unresolved_commit_group: None, }; - *self - .metadata_log_sync - .lock() - .unwrap_or_else(|err| err.into_inner()) = Some(state); + *guard = Some(state); Ok(()) } - /// Drop archived segments fully covered by a new checkpoint at - /// `checkpoint_lsn` (their effects now live in the checkpoint image), keeping - /// the live chain bounded. Called after a checkpoint is published. - pub fn prune_sync_metadata_log_segments(&self, checkpoint_lsn: u64) { - let mut guard = self - .metadata_log_sync + /// Persist the policy fence required whenever this metadata state can be + /// restored on another server. Object deletion stays fail-closed until a + /// future control-published durability watermark can prove completion. + /// + /// This is intentionally independent of synchronous logical-log archiving: + /// checkpoint-only and control-owned deployments also cross server failure + /// boundaries and must install the same fence before recovery or serving. + pub fn require_failover_durability(&self) -> Result<(), MetadError> { + const MAX_ATTEMPTS: usize = 8; + let _gate = self + .object_gc_gate .lock() .unwrap_or_else(|err| err.into_inner()); - if let Some(state) = guard.as_mut() { - state + let key = failover_durability_required_key(self.mount); + for attempt in 0..MAX_ATTEMPTS { + if let Some(value) = self.metadata.get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + decode_failover_durability_required_marker(&value.0)?; + return Ok(()); + } + + self.ensure_object_gc_claim_record()?; + let claim_key = object_gc_claim_key(self.mount); + let claim = self + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + let claim_state = decode_object_gc_claim(self.mount, &claim.value.0)?; + if let ObjectGcClaim::Deleting { owner_epoch, .. } = claim_state { + let current_epoch = self.epoch.load(Ordering::Relaxed); + if owner_epoch > current_epoch { + return Err(MetadError::StaleOwnerEpoch { + owner_epoch: current_epoch, + required_epoch: owner_epoch, + }); + } + } + let version = self.next_version()?; + let command = MetadataCommand { + request_id: request_id( + b"require-failover-durability", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key, + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(FAILOVER_DURABILITY_REQUIRED_MARKER.to_vec())), + }], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => { + let value = self + .metadata + .get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "failover durability requirement marker was not durable".to_owned(), + ) + })?; + decode_failover_durability_required_marker(&value.0)?; + return Ok(()); + } + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + if attempt + 1 < MAX_ATTEMPTS => + { + continue; + } + Err(err) => return Err(err), + } + } + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + } + + /// Retire archived segments fully covered by an authoritative checkpoint at + /// `checkpoint_lsn` (their effects now live in the checkpoint image), while + /// preserving every tail segment above the checkpoint boundary. + /// + /// Callers must invoke this only after the checkpoint publication CAS wins. + /// Deletion uses the exact retired pointers rather than listing an object + /// prefix. Failures are reported as safe leaks and never roll back the + /// already-published checkpoint. + pub fn prune_sync_metadata_log_segments(&self, checkpoint_lsn: u64) -> MetadataLogPruneOutcome { + let (covered, archive) = { + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let Some(state) = guard.as_mut() else { + return MetadataLogPruneOutcome::default(); + }; + let (covered, tail): (Vec<_>, Vec<_>) = state .segments - .retain(|segment| segment.last_lsn > checkpoint_lsn); + .drain(..) + .partition(|segment| segment.last_lsn <= checkpoint_lsn); + state.segments = tail; + (covered, state.config.clone()) + }; + + let mut outcome = MetadataLogPruneOutcome { + pointers_pruned: covered.len(), + ..MetadataLogPruneOutcome::default() + }; + for segment in covered { + if archive.validate_segment_key(&segment.segment_key).is_err() { + outcome.delete_failures += 1; + continue; + } + let Ok(key) = ObjectKey::new(segment.segment_key) else { + outcome.delete_failures += 1; + continue; + }; + match self.objects.delete(&key) { + Ok(true) => outcome.objects_deleted += 1, + Ok(false) => outcome.objects_missing += 1, + Err(_) => outcome.delete_failures += 1, + } } + outcome } - pub fn disable_sync_metadata_log(&self) { + pub fn disable_sync_metadata_log(&self) -> Result<(), MetadError> { + let _log_enable_fence = self + .metadata_log_enable_fence + .write() + .unwrap_or_else(|err| err.into_inner()); + let _commit_log_guard = self + .metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.resolve_unresolved_metadata_commit_group_locked()?; + self.flush_pending_metadata_log_segment_locked() + .map_err(|err| MetadError::SyncLogArchiveFailed { + committed: true, + message: err.to_string(), + })?; *self .metadata_log_sync .lock() .unwrap_or_else(|err| err.into_inner()) = None; + Ok(()) } pub fn sync_metadata_log_snapshot(&self) -> Option { @@ -150,7 +421,316 @@ where .lock() .unwrap_or_else(|err| err.into_inner()) .as_ref() - .and_then(|state| state.snapshot()) + .map(MetadataLogSyncState::snapshot) + } + + /// Read the logical tail and its unpublishable local states under one lock. + /// A caller may treat a cached control tail as clean only when both flags + /// are false and the snapshot identity matches exactly. + pub fn sync_metadata_log_publication_state(&self) -> Option { + self.metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .as_ref() + .map(|state| MetadataLogPublicationState { + snapshot: state.snapshot(), + has_pending_segment: state.pending_segment.is_some(), + has_unresolved_commit_group: state.unresolved_commit_group.is_some(), + }) + } + + /// Resolve every ambiguous apply and flush the exact committed segment + /// before a control-plane recovery reference is selected. This uses the + /// same lock order as metadata commit and checkpoint export. + pub fn flush_sync_metadata_log_for_publication(&self) -> Result<(), MetadError> { + fn committed_failure(err: MetadError) -> MetadError { + match err { + MetadError::SyncLogArchiveFailed { message, .. } => { + MetadError::SyncLogArchiveFailed { + committed: true, + message, + } + } + err => MetadError::SyncLogArchiveFailed { + committed: true, + message: err.to_string(), + }, + } + } + + let _log_enable_fence = self + .metadata_log_enable_fence + .read() + .unwrap_or_else(|err| err.into_inner()); + let enabled = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .is_some(); + if !enabled { + return Ok(()); + } + let _commit_log_guard = self + .metadata_commit_log_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.resolve_unresolved_metadata_commit_group_locked() + .map_err(committed_failure)?; + self.flush_pending_metadata_log_segment_locked() + .map_err(committed_failure) + } + + /// Retry a segment left pending by a prior committed metadata command. + /// The caller must hold `metadata_commit_log_gate`, keeping the retry and + /// any following metadata apply in one total order. + pub(super) fn flush_pending_metadata_log_segment_locked(&self) -> Result<(), MetadError> { + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let Some(state) = guard.as_mut() else { + return Ok(()); + }; + let Some(segment) = state.pending_segment.clone() else { + return Ok(()); + }; + self.archive_and_advance_metadata_log_state(state, &segment)?; + state.pending_segment = None; + Ok(()) + } + + /// Resolve one exact commit group using the metadata engine's authoritative + /// request-id lookup. The caller must hold `metadata_commit_log_gate`. + /// + /// Only `Backend` outcomes are ambiguous. `Some` is accepted solely when it + /// equals the result derived from the original command; `None` is the + /// authoritative proof that the command did not apply. Any lookup error or + /// mismatch leaves the whole group unresolved. + pub(super) fn reconcile_metadata_commit_group_locked( + &self, + group: &UnresolvedMetadataCommitGroup, + ) -> Result>, MetadError> { + let mut resolved = group.results.clone(); + for (index, (command, original_result)) in + group.commands.iter().zip(&group.results).enumerate() + { + if !matches!(original_result, Err(MetadataError::Backend(_))) { + continue; + } + let expected = command.expected_commit_result(); + match self.metadata.committed_request_result(&command.request_id) { + Ok(Some(actual)) if actual == expected => resolved[index] = Ok(actual), + Ok(Some(actual)) => { + return Err(MetadError::Codec(format!( + "metadata commit readback result mismatch: expected {expected:?}, got {actual:?}" + ))); + } + Ok(None) => { + // Keep the original Backend error: authoritative lookup + // proved that this command did not apply. + } + Err(err) => { + return Err(MetadError::Codec(format!( + "metadata commit readback failed: {err}" + ))); + } + } + } + Ok(resolved) + } + + /// Retain an unresolved group before returning its uncertainty to the + /// caller. The caller must hold `metadata_commit_log_gate`. + pub(super) fn defer_unresolved_metadata_commit_group_locked( + &self, + group: UnresolvedMetadataCommitGroup, + ) -> Result<(), MetadError> { + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let state = guard.as_mut().ok_or_else(|| { + MetadError::Codec( + "synchronous metadata log was disabled during commit resolution".to_owned(), + ) + })?; + if state.pending_segment.is_some() { + return Err(MetadError::Codec( + "metadata log has a pending segment before unresolved commit state".to_owned(), + )); + } + if state.unresolved_commit_group.is_some() { + return Err(MetadError::Codec( + "metadata log already has an unresolved commit group".to_owned(), + )); + } + state.unresolved_commit_group = Some(group); + Ok(()) + } + + /// Resolve and archive the exact group left by a prior ambiguous backend + /// outcome. No later apply or checkpoint capture may pass this method while + /// the authoritative lookup still fails or disagrees. The caller must hold + /// `metadata_commit_log_gate`. + pub(super) fn resolve_unresolved_metadata_commit_group_locked(&self) -> Result<(), MetadError> { + let group = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()) + .as_ref() + .and_then(|state| state.unresolved_commit_group.clone()); + let Some(group) = group else { + return Ok(()); + }; + + let resolved = self.reconcile_metadata_commit_group_locked(&group)?; + let committed = group + .commands + .iter() + .zip(&resolved) + .filter_map(|(command, result)| { + result + .as_ref() + .ok() + .map(|result| (command.clone(), result.clone())) + }) + .collect::>(); + + { + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let state = guard.as_mut().ok_or_else(|| { + MetadError::Codec( + "synchronous metadata log was disabled during commit resolution".to_owned(), + ) + })?; + if state.unresolved_commit_group.as_ref() != Some(&group) { + return Err(MetadError::Codec( + "metadata commit resolution group changed unexpectedly".to_owned(), + )); + } + state.unresolved_commit_group = None; + } + + let log_commands = committed + .iter() + .map(|(command, result)| (command, result)) + .collect::>(); + if let Err(err) = self.record_committed_metadata_commands(&log_commands) { + let mut guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let pending = guard + .as_ref() + .is_some_and(|state| state.pending_segment.is_some()); + if !pending { + if let Some(state) = guard.as_mut() { + state.unresolved_commit_group = Some(group); + } + return Err(err); + } + return Err(MetadError::SyncLogArchiveFailed { + committed: true, + message: err.to_string(), + }); + } + Ok(()) + } + + /// Prove before metadata apply that every command which can succeed has a + /// representable LSN and can be sealed into the logical log. Predicate + /// failures may reduce the actual successful subset, so reserving for all + /// structurally valid commands is conservative and fail-closed. + /// The caller must hold `metadata_commit_log_gate`. + pub(super) fn preflight_sync_metadata_log_locked( + &self, + commands: &[MetadataCommand], + ) -> Result<(), MetadError> { + let guard = self + .metadata_log_sync + .lock() + .unwrap_or_else(|err| err.into_inner()); + let Some(state) = guard.as_ref() else { + return Ok(()); + }; + if state.unresolved_commit_group.is_some() { + return Err(MetadError::Codec( + "metadata log has an unresolved commit group".to_owned(), + )); + } + if state.pending_segment.is_some() { + return Err(MetadError::Codec( + "metadata log has an unflushed committed segment".to_owned(), + )); + } + + let mut entries = Vec::new(); + let mut next_lsn = state.next_lsn; + let mut prev_digest = state.prev_digest; + for command in commands { + // Structurally invalid commands are rejected independently by the + // metadata engine and therefore consume no logical-log LSN. + if command.validate().is_err() { + continue; + } + let result = command.expected_commit_result(); + let entry = MetadataLogEntry::seal( + state.shard_id.clone(), + state.epoch, + next_lsn, + command.clone(), + result, + prev_digest, + ) + .map_err(|err| MetadError::Codec(format!("metadata log preflight failed: {err}")))?; + next_lsn = next_lsn.checked_add(1).ok_or_else(|| { + MetadError::Codec("metadata log LSN is exhausted before commit".to_owned()) + })?; + prev_digest = entry.digest; + entries.push(entry); + } + if !entries.is_empty() { + MetadataLogSegment::seal(entries).map_err(|err| { + MetadError::Codec(format!("metadata log segment preflight failed: {err}")) + })?; + } + Ok(()) + } + + fn archive_and_advance_metadata_log_state( + &self, + state: &mut MetadataLogSyncState, + segment: &MetadataLogSegment, + ) -> Result<(), MetadError> { + if segment.first_lsn != state.next_lsn || segment.prev_digest != state.prev_digest { + return Err(MetadError::Codec( + "pending metadata log segment does not follow the durable sync-log tail".to_owned(), + )); + } + let next_lsn = segment + .last_lsn + .checked_add(1) + .ok_or_else(|| MetadError::Codec("metadata log LSN is exhausted".to_owned()))?; + let archived = + archive_metadata_log_segment_to_store(&self.objects, &state.config, segment)?; + self.metadata_log_segments_archived_total + .fetch_add(1, Ordering::Relaxed); + self.metadata_log_entries_archived_total + .fetch_add(segment.entries.len() as u64, Ordering::Relaxed); + self.metadata_log_archive_bytes_total + .fetch_add(archived.encoded_bytes, Ordering::Relaxed); + state.next_lsn = next_lsn; + state.prev_digest = segment.last_digest; + state.segments.push(MetadataLogSegmentPointer { + segment_key: archived.segment_key, + first_lsn: segment.first_lsn, + last_lsn: segment.last_lsn, + last_digest: segment.last_digest, + }); + Ok(()) } pub(super) fn record_committed_metadata_command( @@ -175,6 +755,16 @@ where let Some(state) = guard.as_mut() else { return Ok(None); }; + if state.unresolved_commit_group.is_some() { + return Err(MetadError::Codec( + "metadata log has an unresolved commit group".to_owned(), + )); + } + if state.pending_segment.is_some() { + return Err(MetadError::Codec( + "metadata log has an unflushed committed segment".to_owned(), + )); + } let mut entries = Vec::with_capacity(commands.len()); let mut next_lsn = state.next_lsn; @@ -196,35 +786,15 @@ where entries.push(entry); } - let first_lsn = entries - .first() - .expect("non-empty command list yields non-empty log entries") - .lsn; - let last = entries - .last() - .expect("non-empty command list yields non-empty log entries") - .clone(); let segment = MetadataLogSegment::seal(entries) .map_err(|err| MetadError::Codec(format!("metadata log segment seal failed: {err}")))?; - let archived = - archive_metadata_log_segment_to_store(&self.objects, &state.config, &segment)?; - self.metadata_log_segments_archived_total - .fetch_add(1, Ordering::Relaxed); - self.metadata_log_entries_archived_total - .fetch_add(segment.entries.len() as u64, Ordering::Relaxed); - self.metadata_log_archive_bytes_total - .fetch_add(archived.encoded_bytes, Ordering::Relaxed); - // Advance the chain only after the segment is durable in object storage; - // on archive failure `?` returns above and next_lsn/prev_digest/segments - // are left untouched so the next command re-chains from the same prev. - state.next_lsn = next_lsn; - state.prev_digest = last.digest; - state.segments.push(MetadataLogSegmentPointer { - segment_key: archived.segment_key, - first_lsn, - last_lsn: last.lsn, - last_digest: last.digest, - }); - Ok(state.snapshot()) + // Retain the exact deterministic segment before the first archive + // attempt. Failure leaves it pending and prevents every subsequent + // command from applying until this same content-addressed object is + // durably retried. + state.pending_segment = Some(segment.clone()); + self.archive_and_advance_metadata_log_state(state, &segment)?; + state.pending_segment = None; + Ok(Some(state.snapshot())) } } diff --git a/crates/nokv-meta/src/service/namespace.rs b/crates/nokv-meta/src/service/namespace.rs index 17d9c31a0..8843b260c 100644 --- a/crates/nokv-meta/src/service/namespace.rs +++ b/crates/nokv-meta/src/service/namespace.rs @@ -22,12 +22,20 @@ struct LinkedDentryProjection { version: Version, } +#[derive(Default)] +struct NamespaceReachability { + directories: HashSet, + inodes: HashSet, + dentries: HashSet>, +} + impl NoKvFs where M: MetadataStore, O: ObjectStore, { pub fn bootstrap_root(&self, mode: u32, uid: u32, gid: u32) -> Result { + self.ensure_object_gc_claim_record()?; let version = self.next_version()?; let root = directory_attr(InodeId::root(), mode, uid, gid, version.get()); let command = MetadataCommand { @@ -51,7 +59,13 @@ where watch: Vec::new(), }; match self.commit_metadata(command) { - Ok(_) | Err(MetadError::Metadata(MetadataError::PredicateFailed)) => Ok(root), + Ok(_) | Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + // `new` starts fail-closed because callers may hand it a + // pre-populated store. Once a root exists, one recovery proof + // establishes the healthy fast-path or keeps slow checking on. + self.recover_materialization_orphan_state()?; + Ok(root) + } Err(err) => Err(err), } } @@ -259,6 +273,7 @@ where "symlink target must be non-empty and must not contain NUL".to_owned(), )); } + let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; let inode = self.next_inode()?; let digest_uri = body_digest_uri(&target); @@ -300,6 +315,7 @@ where &projection, &chunks, version, + object_reference, ) { return Err(MetadError::PublishArtifactFailed { source: Box::new(err), @@ -353,6 +369,15 @@ where // `ensure_same_shard`). A hardlink across shards would name an inode from a // foreign namespace, which this shard cannot own or GC. self.ensure_same_shard(inode, new_parent)?; + // A hardlink can turn a previously unreachable borrowed body into a live + // object reference. Serialize its reachability proof + commit with fork + // retirement and local object GC, and join the exact durable Open claim + // so a remote/failover GC transition cannot race the exposure. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; let read_version = predecessor(version)?; let source_inode_key = inode_key(self.mount, inode); @@ -381,8 +406,22 @@ where if parent_attr.file_type != FileType::Directory { return Err(MetadError::NotDirectory); } + let reachable = self.namespace_reachability_for_exposure(read_version)?; + if reachable.as_ref().is_some_and(|reachable| { + !reachable.directories.contains(&new_parent) || !reachable.inodes.contains(&inode) + }) { + // Inode-addressed APIs must not provide an escape hatch for partial + // clone/rollback materializations which have no durable binding. + return Err(MetadError::NotFound); + } let linked = self.linked_dentry_projections_for_inode(inode, read_version)?; - let Some(first_link) = linked.first() else { + let first_link = match &reachable { + Some(reachable) => linked + .iter() + .find(|linked| reachable.dentries.contains(&linked.key)), + None => linked.first(), + }; + let Some(first_link) = first_link else { return Err(MetadError::NotFound); }; attr.nlink = attr @@ -415,6 +454,7 @@ where key: destination_key.clone(), predicate: Predicate::NotExists, }, + object_reference.predicate(self.mount), ]; let mut mutations = vec![Mutation { family: RecordFamily::Inode, @@ -480,6 +520,10 @@ where if changes.is_empty() { return Ok(entry); } + let object_reference = changes + .size + .map(|_| self.begin_object_reference_mutation()) + .transpose()?; let version = self.next_version()?; let mut attr = entry.attr.clone(); if let Some(mode) = changes.mode { @@ -568,6 +612,7 @@ where old_generation, version, path_index: None, + object_reference, })?; Ok(projection.into()) } @@ -1399,6 +1444,14 @@ where // `ensure_same_shard`): `parent`/`new_parent` are the source and // destination directory inodes. self.ensure_same_shard(parent, new_parent)?; + // Rename can expose a dentry that was left behind by a failed detached + // materialization. Hold the same gate as fork retirement from the + // reachability proof through the commit; the durable Open-claim CAS + // below also fences object GC on another owner. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); let (source, source_version) = self .lookup_plus_for_write_plan(parent, name)? .ok_or(MetadError::NotFound)?; @@ -1438,9 +1491,19 @@ where } } + let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; + let read_version = predecessor(version)?; let source_key = dentry_key(self.mount, parent, name); let destination_key = dentry_key(self.mount, new_parent, &new_name); + let reachable = self.namespace_reachability_for_exposure(read_version)?; + if reachable.as_ref().is_some_and(|reachable| { + !reachable.directories.contains(&parent) + || !reachable.directories.contains(&new_parent) + || !reachable.dentries.contains(&source_key) + }) { + return Err(MetadError::NotFound); + } let projection = projection( new_parent, new_name, @@ -1458,6 +1521,7 @@ where key: source_key.clone(), predicate: Predicate::VersionEquals(source_version), }, + object_reference.predicate(self.mount), ]; let replaced = if let Some((entry, destination_version)) = destination { predicates.push(PredicateRef { @@ -1510,7 +1574,7 @@ where let Some(replaced_inode_item) = self.metadata.get_versioned( RecordFamily::Inode, &replaced_inode_key, - predecessor(version)?, + read_version, ReadPurpose::WritePlanLocal, )? else { @@ -1548,10 +1612,9 @@ where op: MutationOp::Put, value: Some(Value(encode_inode_attr(&replaced_attr))), }); - for linked in self.linked_dentry_projections_for_inode( - replaced.attr.inode, - predecessor(version)?, - )? { + for linked in + self.linked_dentry_projections_for_inode(replaced.attr.inode, read_version)? + { if linked.key == destination_key { continue; } @@ -1627,7 +1690,7 @@ where } else { CommandKind::Rename }, - read_version: predecessor(version)?, + read_version, commit_version: version, primary_family: RecordFamily::Dentry, primary_key: destination_key, @@ -1701,13 +1764,267 @@ where Ok(linked) } + /// Enter fail-closed mode before the first metadata write of a detached + /// materialization. Callers hold `object_gc_gate`, so a link/rename cannot + /// observe the old healthy state and then race the first orphan-producing + /// commit. + pub(super) fn mark_materialization_orphan_possible_under_gc_gate(&self) { + self.materialization_orphan_possible + .store(true, Ordering::Release); + } + + /// Reconstruct the process-local fast-path state after bootstrap, reopen, or + /// checkpoint installation. An error leaves the marker set, so callers never + /// mistake an unproven namespace for the healthy state. + pub(super) fn recover_materialization_orphan_state(&self) -> Result<(), MetadError> { + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + self.mark_materialization_orphan_possible_under_gc_gate(); + + // `open_existing` is also the server's construction path for a brand-new + // shard: it opens an empty store first and bootstraps the mount root only + // after the service exists. Keep that pristine case fail-closed until + // `bootstrap_root` creates the root and runs the full proof. A missing + // root in any non-empty namespace is different: serving it would hide + // corruption (including a detached materialization whose binding was + // never committed), so startup must fail. + let version = self.read_version()?; + if self + .get_attr_at_version_for_purpose(InodeId::root(), version, ReadPurpose::WritePlanLocal)? + .is_none() + { + if self.root_dependent_namespace_records_exist_at(version)? { + return Err(MetadError::Codec( + "mount root is missing while namespace records still exist".to_owned(), + )); + } + return Ok(()); + } + + self.reconcile_materialization_orphan_state_under_gc_gate()?; + Ok(()) + } + + /// Return whether the mount has any current record whose existence depends + /// on a bootstrapped namespace. System/Mount state, command dedupe, and GC + /// control records are deliberately excluded: those can be written before + /// the root commit and must not make a retryable bootstrap look corrupt. + fn root_dependent_namespace_records_exist_at( + &self, + version: Version, + ) -> Result { + const ROOT_DEPENDENT_FAMILIES: [RecordFamily; 11] = [ + RecordFamily::Inode, + RecordFamily::Dentry, + RecordFamily::Parent, + RecordFamily::Xattr, + RecordFamily::ChunkManifest, + RecordFamily::Session, + RecordFamily::PathIndex, + RecordFamily::Watch, + RecordFamily::Snapshot, + RecordFamily::ForkBinding, + RecordFamily::ForkShadow, + ]; + let mount_prefix = inode_prefix(self.mount); + for family in ROOT_DEPENDENT_FAMILIES { + if !self + .metadata + .scan(ScanRequest { + family, + prefix: mount_prefix.clone(), + start_after: None, + version, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty() + { + return Ok(true); + } + } + Ok(false) + } + + /// Clear the slow-path marker only after proving that every current inode + /// and dentry belongs to the mount-root/ForkBinding-root reachability graph. + /// Callers hold `object_gc_gate`; failures leave the marker unchanged. + pub(super) fn reconcile_materialization_orphan_state_under_gc_gate( + &self, + ) -> Result { + let version = self.read_version()?; + let reachable = self.namespace_reachability_at(version)?; + let proven_safe = self.all_current_namespace_records_reachable(&reachable, version)?; + self.materialization_orphan_possible + .store(!proven_safe, Ordering::Release); + Ok(proven_safe) + } + + /// Healthy services skip the mount-wide scan entirely. The slow path is + /// entered only after materialization began or startup/restore has not yet + /// proved the installed state. A harmless failed preflight can self-heal: + /// when the slow scan finds no unbound records it clears the marker for later + /// operations, while this operation still uses the proof it just computed. + fn namespace_reachability_for_exposure( + &self, + version: Version, + ) -> Result, MetadError> { + if !self.materialization_orphan_possible.load(Ordering::Acquire) { + return Ok(None); + } + let reachable = self.namespace_reachability_at(version)?; + if self.all_current_namespace_records_reachable(&reachable, version)? { + self.materialization_orphan_possible + .store(false, Ordering::Release); + } + Ok(Some(reachable)) + } + + fn all_current_namespace_records_reachable( + &self, + reachable: &NamespaceReachability, + version: Version, + ) -> Result { + let purpose = ReadPurpose::WritePlanLocal; + let dentry_rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_mount_prefix(self.mount), + start_after: None, + version, + limit: 0, + purpose, + })?; + if dentry_rows + .iter() + .any(|row| !reachable.dentries.contains(&row.key)) + { + return Ok(false); + } + + for row in self.metadata.scan(ScanRequest { + family: RecordFamily::Inode, + prefix: inode_prefix(self.mount), + start_after: None, + version, + limit: 0, + purpose, + })? { + let attr = decode_inode_attr(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if row.key != inode_key(self.mount, attr.inode) { + return Err(MetadError::Codec(format!( + "inode row key does not match inode {} during materialization recovery", + attr.inode.get() + ))); + } + if !reachable.inodes.contains(&attr.inode) { + return Ok(false); + } + } + Ok(true) + } + + #[cfg(test)] + pub(super) fn materialization_orphan_slow_path_enabled(&self) -> bool { + self.materialization_orphan_possible.load(Ordering::Acquire) + } + + #[cfg(test)] + pub(super) fn namespace_reachability_scan_count(&self) -> u64 { + self.namespace_reachability_scans.load(Ordering::Relaxed) + } + + /// Enumerate the namespace roots from which an inode-addressed mutation may + /// expose a dentry. Besides the mount root, a detached clone root is legal + /// only while its validated durable ForkBinding exists and its root inode is + /// still present. Rollback bindings deliberately survive deletion of their + /// materialization root; those missing roots are not reachability anchors. + /// + /// Callers hold `object_gc_gate` across this walk and their metadata commit. + /// Fork retirement holds the same gate across its mount-wide proof and CAS, + /// so neither side can change whether an unbound orphan is reachable in the + /// middle of the other's decision. + fn namespace_reachability_at( + &self, + version: Version, + ) -> Result { + #[cfg(test)] + self.namespace_reachability_scans + .fetch_add(1, Ordering::Relaxed); + let purpose = ReadPurpose::WritePlanLocal; + let Some(root_attr) = + self.get_attr_at_version_for_purpose(InodeId::root(), version, purpose)? + else { + return Err(MetadError::Codec( + "mount root is missing during namespace reachability proof".to_owned(), + )); + }; + if root_attr.file_type != FileType::Directory { + return Err(MetadError::Codec( + "mount root is not a directory during namespace reachability proof".to_owned(), + )); + } + + let mut pending = vec![InodeId::root()]; + for versioned in self.versioned_fork_bindings_at(version, purpose)? { + let root = versioned.binding.fork_root; + let Some(attr) = self.get_attr_at_version_for_purpose(root, version, purpose)? else { + continue; + }; + if attr.file_type != FileType::Directory { + return Err(MetadError::Codec(format!( + "fork binding root {} is not a directory", + root.get() + ))); + } + pending.push(root); + } + + let mut reachable = NamespaceReachability::default(); + while let Some(parent) = pending.pop() { + if !reachable.directories.insert(parent) { + continue; + } + reachable.inodes.insert(parent); + for row in self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_prefix(self.mount, parent), + start_after: None, + version, + limit: 0, + purpose, + })? { + let projection = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + self.validate_current_dentry_projection(&row.key, &projection, version)?; + reachable.dentries.insert(row.key); + reachable.inodes.insert(projection.attr.inode); + if projection.attr.inode.shard_index() == self.shard_index() + && projection.attr.file_type == FileType::Directory + { + pending.push(projection.attr.inode); + } + } + } + Ok(reachable) + } + pub(super) fn commit_create_projection( &self, kind: CommandKind, projection: &DentryProjection, version: Version, ) -> Result<(), MetadError> { - self.commit_create_projection_with_chunks(kind, projection, &[], version) + self.commit_create_projection_with_chunks_and_path_index( + kind, + projection, + &[], + version, + None, + None, + ) } pub(super) fn commit_create_projections_with_path_indexes( @@ -1877,12 +2194,19 @@ where projection: &DentryProjection, chunks: &[ChunkManifest], version: Version, + object_reference: ObjectReferenceMutation, ) -> Result<(), MetadError> { self.commit_create_projection_with_chunks_and_path_index( - kind, projection, chunks, version, None, + kind, + projection, + chunks, + version, + None, + Some(object_reference), ) } + #[allow(clippy::too_many_arguments)] pub(super) fn commit_create_projection_with_chunks_and_path_index( &self, kind: CommandKind, @@ -1890,6 +2214,7 @@ where chunks: &[ChunkManifest], version: Version, path_index: Option>, + object_reference: Option, ) -> Result<(), MetadError> { let inode = projection.attr.inode; let dentry = dentry_key( @@ -1934,6 +2259,21 @@ where }); } } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, projection.dentry.parent), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + ]; + if let Some(object_reference) = object_reference { + predicates.push(object_reference.predicate(self.mount)); + } self.commit_metadata(MetadataCommand { request_id: request_id(kind_name(kind), self.mount, inode, version), kind, @@ -1941,18 +2281,7 @@ where commit_version: version, primary_family: RecordFamily::Dentry, primary_key: dentry.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, projection.dentry.parent), - predicate: Predicate::Exists, - }, - PredicateRef { - family: RecordFamily::Dentry, - key: dentry, - predicate: Predicate::NotExists, - }, - ], + predicates, mutations, watch: self .watch_projection( @@ -1984,6 +2313,7 @@ where old_generation, version, path_index, + object_reference, } = commit; let inode = projection.attr.inode; let dentry = dentry_key( @@ -2013,6 +2343,9 @@ where predicate: Predicate::Exists, }, ]; + if let Some(object_reference) = object_reference { + predicates.push(object_reference.predicate(self.mount)); + } let mut mutations = vec![Mutation { family: RecordFamily::Inode, key: inode_key(self.mount, inode), diff --git a/crates/nokv-meta/src/service/publish.rs b/crates/nokv-meta/src/service/publish.rs index a7744de6b..ac8822f77 100644 --- a/crates/nokv-meta/src/service/publish.rs +++ b/crates/nokv-meta/src/service/publish.rs @@ -16,6 +16,7 @@ where O: ObjectStore, { pub fn publish_artifact(&self, request: PublishArtifact) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; let inode = self.next_inode()?; let StagedArtifactBody { @@ -44,6 +45,7 @@ where &projection, &chunks, version, + object_reference, ) { return Err(MetadError::PublishArtifactFailed { source: Box::new(err), @@ -57,6 +59,7 @@ where &self, request: PublishArtifact, ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let (existing, dentry_version) = self .lookup_plus_for_write_plan(request.parent, &request.name)? .ok_or(MetadError::NotFound)?; @@ -95,6 +98,7 @@ where old_generation, version, path_index: None, + object_reference: Some(object_reference), }) { return Err(MetadError::PublishArtifactFailed { source: Box::new(err), @@ -111,6 +115,16 @@ where &self, parent: InodeId, name: DentryName, + ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; + self.prepare_artifact_create_with_object_reference(parent, name, object_reference) + } + + fn prepare_artifact_create_with_object_reference( + &self, + parent: InodeId, + name: DentryName, + object_reference: ObjectReferenceMutation, ) -> Result { let Some(parent_attr) = self.get_attr_at_version_for_purpose( parent, @@ -140,13 +154,16 @@ where replace: false, dentry_version: None, old_generation: None, + object_gc_claim_version: object_reference.version().get(), }) } pub fn prepare_artifact_create_path(&self, path: &str) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let components = parse_absolute_path(path)?; let (parent, name) = self.resolve_parent_path(path)?; - let mut prepared = self.prepare_artifact_create(parent, name)?; + let mut prepared = + self.prepare_artifact_create_with_object_reference(parent, name, object_reference)?; prepared.path = Some(canonical_path(&components)?); Ok(prepared) } @@ -155,6 +172,16 @@ where &self, parent: InodeId, name: DentryName, + ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; + self.prepare_artifact_replace_with_object_reference(parent, name, object_reference) + } + + fn prepare_artifact_replace_with_object_reference( + &self, + parent: InodeId, + name: DentryName, + object_reference: ObjectReferenceMutation, ) -> Result { let (existing, dentry_version) = self .lookup_plus_for_write_plan(parent, &name)? @@ -175,6 +202,7 @@ where replace: true, dentry_version: Some(dentry_version.get()), old_generation: existing.body.as_ref().map(|body| body.generation), + object_gc_claim_version: object_reference.version().get(), }) } @@ -182,13 +210,89 @@ where &self, path: &str, ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let (parent, name) = self.resolve_parent_path(path)?; let components = parse_absolute_path(path)?; - let mut prepared = self.prepare_artifact_replace(parent, name)?; + let mut prepared = + self.prepare_artifact_replace_with_object_reference(parent, name, object_reference)?; prepared.path = Some(canonical_path(&components)?); Ok(prepared) } + /// Refresh an unpublished prepared upload after an unrelated object-GC + /// epoch invalidated its object reference token. The namespace proof remains + /// fixed, while a fresh generation makes it + /// impossible to reuse objects uploaded under the stale GC epoch. Callers + /// must persist this returned identity before fully restaging the body. + pub fn refresh_prepared_artifact_object_gc_epoch( + &self, + mut prepared: PreparedArtifact, + ) -> Result { + if let Some(path) = prepared.path.as_deref() { + let components = parse_absolute_path(path)?; + let Some((name, parent_components)) = components.split_last() else { + return Err(MetadError::InvalidPreparedArtifact( + "prepared artifact path has no leaf".to_owned(), + )); + }; + if name != &prepared.name + || self.resolve_components_as_directory(parent_components)? != prepared.parent + { + return Err(MetadError::InvalidPreparedArtifact( + "prepared artifact path no longer matches its parent and name".to_owned(), + )); + } + } + + let current = self.lookup_plus_for_write_plan(prepared.parent, &prepared.name)?; + if prepared.replace { + let expected = Version::new(prepared.dentry_version.ok_or_else(|| { + MetadError::InvalidPreparedArtifact( + "replace artifact is missing dentry version".to_owned(), + ) + })?)?; + let Some((entry, version)) = current else { + return Err(MetadError::NotFound); + }; + if entry.attr.file_type != FileType::File { + return Err(MetadError::NotFile); + } + if entry.attr.inode != prepared.inode || version != expected { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + } else { + if prepared.dentry_version.is_some() || prepared.old_generation.is_some() { + return Err(MetadError::InvalidPreparedArtifact( + "create artifact must not carry replace state".to_owned(), + )); + } + if current.is_some() || self.get_attr(prepared.inode)?.is_some() { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + let parent = self + .get_attr(prepared.parent)? + .ok_or(MetadError::NotFound)?; + if parent.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + } + + match self.ensure_prepared_object_gc_epoch(prepared.object_gc_claim_version) { + Ok(()) => return Ok(prepared), + Err(MetadError::StalePreparedArtifactObjectGcEpoch { .. }) => {} + Err(err) => return Err(err), + } + + let object_reference = self.begin_object_reference_mutation()?; + let generation = self.next_version()?; + let now_ms = current_time_ms(); + prepared.generation = generation.get(); + prepared.mtime_ms = now_ms; + prepared.ctime_ms = now_ms; + prepared.object_gc_claim_version = object_reference.version().get(); + Ok(prepared) + } + pub fn publish_prepared_artifact( &self, prepared: PreparedArtifact, @@ -198,6 +302,7 @@ where uid: u32, gid: u32, ) -> Result { + validate_new_prepared_block_identities(self.mount, &prepared, &chunks)?; self.publish_prepared_artifact_impl(PreparedArtifactPublish { prepared, body, @@ -222,7 +327,8 @@ where uid, gid, } = request; - validate_prepared_artifact(&prepared, &body, &chunks)?; + validate_prepared_artifact(self.mount, &prepared, &body, &chunks)?; + self.ensure_prepared_object_gc_epoch(prepared.object_gc_claim_version)?; let version = Version::new(prepared.generation)?; let mut attr = InodeAttr { inode: prepared.inode, @@ -276,6 +382,12 @@ where .map(|components| path_index_key(self.mount, &components)) }) .transpose()?, + object_reference: Some(ObjectReferenceMutation::from_version(Version::new( + prepared.object_gc_claim_version, + )?)), + }) + .map_err(|err| { + self.classify_prepared_object_gc_epoch_error(err, prepared.object_gc_claim_version) })?; Ok(RenameReplaceResult { entry: projection.into(), @@ -300,7 +412,13 @@ where .map(|components| path_index_key(self.mount, &components)) }) .transpose()?, - )?; + Some(ObjectReferenceMutation::from_version(Version::new( + prepared.object_gc_claim_version, + )?)), + ) + .map_err(|err| { + self.classify_prepared_object_gc_epoch_error(err, prepared.object_gc_claim_version) + })?; Ok(RenameReplaceResult { entry: projection.into(), replaced: None, @@ -308,6 +426,52 @@ where } } + fn ensure_prepared_object_gc_epoch(&self, expected: u64) -> Result<(), MetadError> { + let key = object_gc_claim_key(self.mount); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + if item.version.get() == expected + && matches!( + decode_object_gc_claim(self.mount, &item.value.0)?, + ObjectGcClaim::Open + ) + { + return Ok(()); + } + Err(MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current: item.version.get(), + }) + } + + fn classify_prepared_object_gc_epoch_error( + &self, + err: MetadError, + expected: u64, + ) -> MetadError { + if !matches!(err, MetadError::Metadata(MetadataError::PredicateFailed)) { + return err; + } + self.ensure_prepared_object_gc_epoch(expected) + .err() + .filter(|current| { + matches!( + current, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + ) + }) + .unwrap_or(err) + } + pub fn publish_prepared_artifact_session( &self, prepared: PreparedArtifact, @@ -393,6 +557,7 @@ where "prepared artifact target does not match staged publish session".to_owned(), )); } + validate_new_prepared_block_identities(self.mount, &prepared, &request.chunks)?; let version = Version::new(prepared.generation)?; let (chunks, base_generation) = self.resolve_session_chunks(&prepared, request.size, request.chunks)?; diff --git a/crates/nokv-meta/src/service/rollback.rs b/crates/nokv-meta/src/service/rollback.rs index 2d10d09ef..c16e958d6 100644 --- a/crates/nokv-meta/src/service/rollback.rs +++ b/crates/nokv-meta/src/service/rollback.rs @@ -1,3 +1,4 @@ +use super::snapshot::VersionedSnapshotPin; use super::*; /// A top-level child of the rollback target captured before the atomic swap, so it @@ -54,17 +55,18 @@ where /// borrows blocks whose keys are owned by inodes that the teardown destroys. Two /// rules keep the shared blocks alive while letting the delta's private blocks go: /// - /// * **Teardown skips shared blocks.** The detached subtree is purged with the - /// restored tree's referenced object keys passed as `retained_object_keys`, so - /// [`NoKvFs::chunk_manifest_delete_and_gc_mutations`] never enqueues a block the - /// restored tree still points at (e.g. an unchanged file's block). Only blocks - /// unique to the delta (a rewritten file's new body, a delta-only file) are - /// enqueued. - /// * **The swap cancels stale enqueues for restored blocks.** A pre-rollback - /// rewrite or delete already queued the snapshot's blocks for GC (blocked only - /// by the retention floor). The swap commit deletes any pending GC record whose - /// object key the restored tree references, resurrecting those blocks as live so - /// they survive even after the snapshot pin is retired. + /// * **The swap installs durable retention.** The same atomic commit that exposes + /// restored children creates a [`ForkBinding`] keyed by the materialized + /// root's unique inode. Unlike the construction snapshot lease, the binding + /// does not expire and its mount-wide history floor keeps every owner-side GC + /// row blocked. The materialized root may then be deleted after its children + /// escape to `target_root`, just like a clone binding survives its root being + /// unlinked while hardlinks remain. + /// * **Owner GC rows are preserved.** A pre-rollback rewrite/delete may already + /// have queued the snapshot blocks. Teardown also queues blocks still owned by + /// the discarded current tree. Those rows remain durable; once every borrowed + /// reference is rewritten or removed, explicit snapshot retirement releases the + /// binding and normal object GC reclaims them. /// /// The net effect mirrors [`NoKvFs::owns_block_object_key`]: a block reachable /// from the live namespace is never reclaimed, a block reachable only from the @@ -74,17 +76,16 @@ where target_root: InodeId, snapshot_id: u64, ) -> Result<(), MetadError> { - let pin = self - .snapshot_pin(snapshot_id)? - .ok_or(MetadError::NotFound)?; - if pin.root != target_root { - return Err(MetadError::InvalidPath(format!( - "snapshot {snapshot_id} pins inode {} but rollback target is {}", - pin.root.get(), - target_root.get() - ))); - } - let snapshot_version = Version::new(pin.read_version)?; + // Rollback is a rare administrative operation. Holding the in-process + // GC gate across materialize -> swap prevents an expiring pin from + // allowing this owner to delete a borrowed block mid-operation. HA GC + // is separately fail-closed by the durable failover marker. + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let pin = self.live_rollback_snapshot_pin(target_root, snapshot_id)?; + let snapshot_version = Version::new(pin.pin.read_version)?; if self .get_attr_at_version_for_purpose( @@ -96,12 +97,22 @@ where { return Err(MetadError::NotDirectory); } + self.ensure_rollback_tree_has_no_hardlinks( + target_root, + snapshot_version, + ReadPurpose::Snapshot, + )?; + self.ensure_rollback_tree_has_no_hardlinks( + target_root, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; // 1. Reproduce the snapshot's subtree under a fresh detached root, sharing // the snapshot's blocks. Snapshot-aware scans enumerate entries deleted // by the delta through the retained-history key index. let restored_root = self.materialize_subtree_at(target_root, snapshot_version)?; - let restored_keys = self.subtree_object_keys(restored_root)?; + let restored_blocks = self.subtree_object_blocks(restored_root)?; // 2. Capture both sides' top-level children, then graft atomically. let old_children = self.capture_top_level_children(target_root)?; @@ -113,57 +124,122 @@ where self.commit_rollback_swap( target_root, restored_root, + snapshot_id, &old_children, &restored_children, - &restored_keys, + &restored_blocks, )?; // 3. Tear down the now-detached pre-rollback subtree, reclaiming the delta's - // private blocks while leaving the restored tree's shared blocks live. - self.purge_detached_subtree(&old_children, &restored_keys)?; + // owner-side metadata and durably queueing its blocks behind the binding. + self.purge_detached_subtree(&old_children)?; + // The materialization root is bound (and intentionally deleted by the + // swap), while the discarded tree is now purged. Clear the slow-path + // marker only if a mount-wide proof finds no older unsafe orphan. + let _ = self.reconcile_materialization_orphan_state_under_gc_gate(); Ok(()) } - /// Path variant of [`NoKvFs::rollback_subtree`]. Resolves `target_path` to its - /// directory inode and reverts it to `snapshot_id`. - pub fn rollback_subtree_path( + /// Eager rollback currently materializes each dentry as a fresh inode. Until + /// hardlink groups are restored as groups, accepting a multiply-linked node + /// would either split one identity or let teardown delete an inode still named + /// outside the target subtree. Controlled restore must therefore fail closed. + fn ensure_rollback_tree_has_no_hardlinks( &self, - target_path: &str, - snapshot_id: u64, + root: InodeId, + version: Version, + purpose: ReadPurpose, ) -> Result<(), MetadError> { - let target_root = self.resolve_directory_path(target_path)?; - self.rollback_subtree(target_root, snapshot_id) + let mut queue = vec![root]; + while let Some(dir) = queue.pop() { + for child in self.read_dir_plus_at_version_for_purpose(dir, version, purpose)? { + if child.attr.file_type == FileType::Directory { + queue.push(child.attr.inode); + } else if child.attr.nlink > 1 { + return Err(MetadError::InvalidPath(format!( + "rollback requires a hardlink-free subtree; inode {} has {} links", + child.attr.inode.get(), + child.attr.nlink + ))); + } + } + } + Ok(()) } - /// Collect every object key referenced by the materialized subtree's file - /// bodies. These are the blocks the restored tree shares with the snapshot; the - /// teardown must not reclaim them and the swap must cancel any pending GC for - /// them. - fn subtree_object_keys(&self, root: InodeId) -> Result, MetadError> { + /// Capture each distinct block named by the materialized tree. The atomic + /// swap proactively ensures an owner-side GC row for these borrowed objects; + /// this covers historical base generations whose original delete path could + /// not otherwise enqueue them again. + fn subtree_object_blocks(&self, root: InodeId) -> Result, MetadError> { let version = self.read_version()?; - let mut keys = HashSet::new(); + let mut seen = HashSet::new(); + let mut blocks = Vec::new(); let mut queue = vec![root]; while let Some(dir) = queue.pop() { - for child in - self.read_dir_plus_at_version_for_purpose(dir, version, ReadPurpose::Snapshot)? - { - match child.attr.file_type { - FileType::Directory => queue.push(child.attr.inode), - _ => { - if let Some(body) = &child.body { - let manifests = self.chunk_manifests_for_body_at_version( - child.attr.inode, - body, - version, - ReadPurpose::Snapshot, - )?; - keys.extend(chunk_object_keys(&manifests)); - } + for child in self.read_dir_plus_at_version_for_purpose( + dir, + version, + ReadPurpose::WritePlanLocal, + )? { + if child.attr.file_type == FileType::Directory { + queue.push(child.attr.inode); + } + let Some(body) = child.body.as_ref() else { + continue; + }; + for block in self + .chunk_manifests_for_body_at_version( + child.attr.inode, + body, + version, + ReadPurpose::WritePlanLocal, + )? + .into_iter() + .flat_map(|manifest| manifest.slices) + .flat_map(|slice| slice.blocks) + { + if seen.insert(block.object_key.clone()) { + blocks.push(block); } } } } - Ok(keys) + Ok(blocks) + } + + fn live_rollback_snapshot_pin( + &self, + target_root: InodeId, + snapshot_id: u64, + ) -> Result { + let pin = self + .versioned_snapshot_pin_at( + snapshot_id, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if pin.pin.root != target_root { + return Err(MetadError::InvalidPath(format!( + "snapshot {snapshot_id} pins inode {} but rollback target is {}", + pin.pin.root.get(), + target_root.get() + ))); + } + self.ensure_snapshot_pin_live(&pin.pin)?; + Ok(pin) + } + + /// Path variant of [`NoKvFs::rollback_subtree`]. Resolves `target_path` to its + /// directory inode and reverts it to `snapshot_id`. + pub fn rollback_subtree_path( + &self, + target_path: &str, + snapshot_id: u64, + ) -> Result<(), MetadError> { + let target_root = self.resolve_directory_path(target_path)?; + self.rollback_subtree(target_root, snapshot_id) } /// Snapshot the target's current top-level children with their dentry versions, @@ -202,15 +278,22 @@ where /// The single atomic commit that installs the restored subtree over /// `target_root`: it re-parents the materialized children onto `target_root`, /// removes delta-only children, deletes the now-empty materialized root inode, - /// and cancels pending GC for the blocks the restored tree resurrects. + /// installs durable rollback retention, and ensures owner-side GC rows for + /// every block the restored tree borrows. fn commit_rollback_swap( &self, target_root: InodeId, restored_root: InodeId, + snapshot_id: u64, old_children: &[OldChild], restored_children: &[DentryWithAttr], - restored_keys: &HashSet, + restored_blocks: &[BlockDescriptor], ) -> Result<(), MetadError> { + // Capture both retention fences before the final planning reads. The + // pin is read again under the GC gate, then its exact record version and + // the exact durable Open epoch join the atomic swap. + let object_reference = self.begin_object_reference_mutation()?; + let pin = self.live_rollback_snapshot_pin(target_root, snapshot_id)?; let version = self.next_version()?; let read_version = predecessor(version)?; @@ -219,12 +302,40 @@ where .map(|child| child.dentry.name.as_bytes()) .collect(); - let mut predicates = vec![PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, target_root), - predicate: Predicate::Exists, + let binding_key = fork_binding_key(self.mount, restored_root); + let binding = ForkBinding { + fork_root: restored_root, + source_root: target_root, + pinned_read_version: pin.pin.read_version, + snapshot_id, + created_version: version.get(), + }; + + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, target_root), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Snapshot, + key: snapshot_pin_key(self.mount, snapshot_id), + predicate: Predicate::VersionEquals(pin.version), + }, + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::NotExists, + }, + ]; + let mut mutations = vec![Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&binding))), }]; - let mut mutations = Vec::new(); + mutations.extend(self.rollback_object_gc_mutations(restored_blocks, version)?); // Guard every current child dentry, and delete the ones the restored tree // does not re-establish (same-named entries are overwritten by the puts @@ -266,10 +377,10 @@ where inode_key(self.mount, restored_root), )); - // Resurrect the snapshot's blocks the restored tree now references by - // cancelling any pending GC record an earlier rewrite/delete left for them. - mutations.extend(self.pending_gc_cancellations_for_keys(restored_keys, read_version)?); - + // Materialization and swap planning may be large enough to cross the + // lease deadline. Reject before exposing the restored references; the + // pin CAS handles concurrent renew/reap after this check. + self.ensure_snapshot_pin_live(&pin.pin)?; self.commit_metadata(MetadataCommand { request_id: request_id(b"rollback-subtree-swap", self.mount, target_root, version), kind: CommandKind::RenameReplace, @@ -284,44 +395,48 @@ where Ok(()) } - /// Delete mutations for every pending object-GC record whose block the restored - /// tree references, so those blocks stop being scheduled for deletion. - fn pending_gc_cancellations_for_keys( + fn rollback_object_gc_mutations( &self, - restored_keys: &HashSet, - version: Version, + restored_blocks: &[BlockDescriptor], + enqueue_version: Version, ) -> Result, MetadError> { - if restored_keys.is_empty() { - return Ok(Vec::new()); - } - let rows = self.metadata.scan(ScanRequest { - family: RecordFamily::Gc, - prefix: gc_queue_prefix(self.mount), - start_after: None, - version, - limit: 0, - purpose: ReadPurpose::WritePlanLocal, - })?; - let mut mutations = Vec::new(); - for row in rows { - let record = decode_object_gc_record(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - if restored_keys.contains(&record.object_key) { - mutations.push(delete_mutation(RecordFamily::Gc, row.key)); - } - } - Ok(mutations) + let enqueue_unix_ms = current_time_ms(); + restored_blocks + .iter() + .map(|block| { + let (inode, generation, chunk_index, block_index) = + self.canonical_block_object_identity(&block.object_key)?; + let record = ObjectGcRecord { + inode, + generation, + object_key: block.object_key.clone(), + size: block.len, + digest_uri: block.digest_uri.clone(), + enqueue_version: enqueue_version.get(), + enqueue_unix_ms, + }; + Ok(Mutation { + family: RecordFamily::Gc, + key: gc_object_key( + self.mount, + enqueue_version.get(), + inode, + generation, + chunk_index, + block_index, + ), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }) + }) + .collect() } /// Tear down the detached pre-rollback subtree rooted at the captured top-level /// children. Each node's metadata is deleted bottom-up and its inode-owned blocks - /// are enqueued for GC, except blocks the restored tree still references - /// (`retained_object_keys`), which stay live. - fn purge_detached_subtree( - &self, - old_children: &[OldChild], - retained_object_keys: &HashSet, - ) -> Result<(), MetadError> { + /// are enqueued for GC. The rollback binding installed by the swap holds the + /// retention floor while the restored tree still borrows any of those blocks. + fn purge_detached_subtree(&self, old_children: &[OldChild]) -> Result<(), MetadError> { let version = self.read_version()?; // Discover the full detached subtree (the top-level dentries are already // gone, but the inodes and their descendants persist until purged). @@ -339,8 +454,9 @@ where self.classify_detached_node(&child, &mut nodes, &mut dirs); } } + let retained_object_keys = HashSet::new(); for node in nodes { - self.purge_detached_node(&node, retained_object_keys)?; + self.purge_detached_node(&node, &retained_object_keys)?; } Ok(()) } diff --git a/crates/nokv-meta/src/service/snapshot.rs b/crates/nokv-meta/src/service/snapshot.rs index 7e5a0b726..5a1be6219 100644 --- a/crates/nokv-meta/src/service/snapshot.rs +++ b/crates/nokv-meta/src/service/snapshot.rs @@ -1,4 +1,5 @@ use super::*; +use crate::layout::{decode_fork_binding, fork_binding_prefix}; /// Default lease for a new snapshot pin: holders renew to keep it alive; an /// abandoned pin expires after this so a crashed client never blocks GC forever. @@ -10,9 +11,16 @@ const SNAPSHOT_ID_LOCAL_BITS: u32 = u64::BITS - SNAPSHOT_ID_SHARD_BITS; const SNAPSHOT_ID_LOCAL_MASK: u64 = (1_u64 << SNAPSHOT_ID_LOCAL_BITS) - 1; #[derive(Clone, Debug)] -struct VersionedSnapshotPin { - pin: SnapshotPin, - version: Version, +pub(super) struct VersionedSnapshotPin { + pub(super) pin: SnapshotPin, + pub(super) version: Version, +} + +#[derive(Clone, Debug)] +pub(super) struct VersionedForkBinding { + pub(super) binding: ForkBinding, + pub(super) key: Vec, + pub(super) version: Version, } impl NoKvFs @@ -30,6 +38,7 @@ where lease_ms: u64, ) -> Result { for attempt in 0..SNAPSHOT_MINT_MAX_ATTEMPTS { + let object_reference = self.begin_object_reference_mutation()?; let Some(attr) = self.get_attr_at_version_for_purpose( root, self.read_version()?, @@ -69,6 +78,7 @@ where key: key.clone(), predicate: Predicate::NotExists, }, + object_reference.predicate(self.mount), ], mutations: vec![Mutation { family: RecordFamily::Snapshot, @@ -100,6 +110,7 @@ where lease_ms: u64, ) -> Result { for attempt in 0..SNAPSHOT_MINT_MAX_ATTEMPTS { + let object_reference = self.begin_object_reference_mutation()?; let binding_version = self.read_version()?; let (root, mut predicates) = self.resolve_snapshot_root_binding(path, binding_version)?; @@ -125,6 +136,7 @@ where key: key.clone(), predicate: Predicate::NotExists, }, + object_reference.predicate(self.mount), ]); match self.commit_metadata(MetadataCommand { request_id: request_id(b"snapshot-subtree-path", self.mount, root, created_version), @@ -143,18 +155,6 @@ where watch: Vec::new(), }) { Ok(_) => return Ok(pin), - Err(MetadError::SyncLogArchiveFailed { - committed: true, .. - }) => { - let durable = self.snapshot_pin(pin.snapshot_id)?; - if durable.as_ref() == Some(&pin) { - return Ok(pin); - } - return Err(MetadError::Codec(format!( - "snapshot {} reported committed without its durable pin", - pin.snapshot_id - ))); - } Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { if !self.snapshot_binding_predicates_match(&binding_predicates)? { return Err(MetadError::SnapshotBindingChanged { @@ -173,79 +173,184 @@ where #[cfg(test)] pub(crate) fn retire_snapshot(&self, snapshot_id: u64) -> Result { - let key = snapshot_pin_key(self.mount, snapshot_id); - if self.snapshot_pin(snapshot_id)?.is_none() { + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let read_version = self.read_version()?; + let pin = + self.versioned_snapshot_pin_at(snapshot_id, read_version, ReadPurpose::UserStrong)?; + let bindings = + self.fork_bindings_for_snapshot_at(snapshot_id, read_version, ReadPurpose::UserStrong)?; + if pin.is_none() && bindings.is_empty() { return Ok(false); } + self.validate_snapshot_retention_roots(snapshot_id, pin.as_ref(), &bindings, None)?; + self.ensure_fork_bindings_releasable(&bindings, read_version)?; + + let pin_key = snapshot_pin_key(self.mount, snapshot_id); + let mut predicates = Vec::with_capacity(usize::from(pin.is_some()) + bindings.len()); + let mut mutations = Vec::with_capacity(predicates.capacity()); + if let Some(versioned) = &pin { + predicates.push(PredicateRef { + family: RecordFamily::Snapshot, + key: pin_key.clone(), + predicate: Predicate::VersionEquals(versioned.version), + }); + mutations.push(delete_mutation(RecordFamily::Snapshot, pin_key.clone())); + } + for binding in &bindings { + predicates.push(PredicateRef { + family: RecordFamily::ForkBinding, + key: binding.key.clone(), + predicate: Predicate::VersionEquals(binding.version), + }); + mutations.push(delete_mutation( + RecordFamily::ForkBinding, + binding.key.clone(), + )); + } + + let (primary_family, primary_key, request_inode) = if let Some(versioned) = &pin { + (RecordFamily::Snapshot, pin_key, versioned.pin.root) + } else { + let binding = bindings + .first() + .expect("retire has either a snapshot pin or fork binding"); + ( + RecordFamily::ForkBinding, + binding.key.clone(), + binding.binding.source_root, + ) + }; let version = self.next_version()?; + let retires_binding = !bindings.is_empty(); + if retires_binding { + // A detached root becomes unbound at this CAS. Enter fail-closed + // mode before the commit so a committed-but-unacknowledged result + // cannot leave the healthy fast path enabled. + self.mark_materialization_orphan_possible_under_gc_gate(); + } self.commit_metadata(MetadataCommand { - request_id: request_id(b"retire-snapshot", self.mount, InodeId::root(), version), + request_id: request_id(b"retire-snapshot", self.mount, request_inode, version), kind: CommandKind::RetireSnapshot, read_version: predecessor(version)?, commit_version: version, - primary_family: RecordFamily::Snapshot, - primary_key: key.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::Snapshot, - key: key.clone(), - predicate: Predicate::Exists, - }], - mutations: vec![delete_mutation(RecordFamily::Snapshot, key)], + primary_family, + primary_key, + predicates, + mutations, watch: Vec::new(), })?; + if retires_binding { + let _ = self.reconcile_materialization_orphan_state_under_gc_gate(); + } Ok(true) } + /// Retire a source snapshot and every durable fork binding derived from it. + /// + /// Clone bindings deliberately outlive the snapshot lease. Because the + /// retention floor is mount-global, retirement fails closed while any + /// reachable effective manifest under the mount root or a live detached + /// fork root still borrows blocks from another inode (including hardlinks, + /// renames, and rollback propagation). + /// `root_path` is resolved at retirement time, so a renamed source is + /// retired through its new path; if the source path has been deleted this + /// path-bound API leaves the binding retained. pub fn retire_snapshot_path( &self, root_path: &str, snapshot_id: u64, ) -> Result { + let _object_gc_gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); let read_version = self.read_version()?; let (expected_root, mut predicates) = self.resolve_snapshot_root_binding(root_path, read_version)?; + let binding_predicates = predicates.clone(); self.ensure_snapshot_id_shard(snapshot_id, expected_root)?; - let Some(versioned) = - self.versioned_snapshot_pin_at(snapshot_id, read_version, ReadPurpose::UserStrong)? - else { + let pin = + self.versioned_snapshot_pin_at(snapshot_id, read_version, ReadPurpose::UserStrong)?; + let bindings = + self.fork_bindings_for_snapshot_at(snapshot_id, read_version, ReadPurpose::UserStrong)?; + if pin.is_none() && bindings.is_empty() { return Ok(false); - }; - if versioned.pin.root != expected_root { - return Err(MetadError::SnapshotRootMismatch { - snapshot_id, - expected_root, - actual_root: Some(versioned.pin.root), - actual_shard: self.shard_index(), + } + self.validate_snapshot_retention_roots( + snapshot_id, + pin.as_ref(), + &bindings, + Some(expected_root), + )?; + self.ensure_fork_bindings_releasable(&bindings, read_version)?; + + let pin_key = snapshot_pin_key(self.mount, snapshot_id); + let mut mutations = Vec::with_capacity(usize::from(pin.is_some()) + bindings.len()); + if let Some(versioned) = &pin { + predicates.push(PredicateRef { + family: RecordFamily::Snapshot, + key: pin_key.clone(), + predicate: Predicate::VersionEquals(versioned.version), }); + mutations.push(delete_mutation(RecordFamily::Snapshot, pin_key.clone())); } - let key = snapshot_pin_key(self.mount, snapshot_id); - predicates.push(PredicateRef { - family: RecordFamily::Snapshot, - key: key.clone(), - predicate: Predicate::VersionEquals(versioned.version), - }); + for binding in &bindings { + predicates.push(PredicateRef { + family: RecordFamily::ForkBinding, + key: binding.key.clone(), + predicate: Predicate::VersionEquals(binding.version), + }); + mutations.push(delete_mutation( + RecordFamily::ForkBinding, + binding.key.clone(), + )); + } + + let (primary_family, primary_key) = if pin.is_some() { + (RecordFamily::Snapshot, pin_key) + } else { + ( + RecordFamily::ForkBinding, + bindings + .first() + .expect("retire has either a snapshot pin or fork binding") + .key + .clone(), + ) + }; let version = self.next_version()?; + let retires_binding = !bindings.is_empty(); + if retires_binding { + self.mark_materialization_orphan_possible_under_gc_gate(); + } match self.commit_metadata(MetadataCommand { - request_id: request_id( - b"retire-snapshot-path", - self.mount, - versioned.pin.root, - version, - ), + request_id: request_id(b"retire-snapshot-path", self.mount, expected_root, version), kind: CommandKind::RetireSnapshot, read_version: predecessor(version)?, commit_version: version, - primary_family: RecordFamily::Snapshot, - primary_key: key.clone(), + primary_family, + primary_key, predicates, - mutations: vec![delete_mutation(RecordFamily::Snapshot, key)], + mutations, watch: Vec::new(), }) { - Ok(_) => Ok(true), + Ok(_) => { + if retires_binding { + let _ = self.reconcile_materialization_orphan_state_under_gc_gate(); + } + Ok(true) + } Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { - Err(MetadError::SnapshotBindingChanged { - root_path: root_path.to_owned(), - }) + if !self.snapshot_binding_predicates_match(&binding_predicates)? { + Err(MetadError::SnapshotBindingChanged { + root_path: root_path.to_owned(), + }) + } else { + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + } } Err(err) => Err(err), } @@ -284,6 +389,10 @@ where let requested_expiry = self.now_ms().saturating_add(lease_ms); let key = snapshot_pin_key(self.mount, snapshot_id); for _attempt in 0..SNAPSHOT_RENEW_MAX_ATTEMPTS { + // Capture the durable Open epoch before this attempt reads the pin. + // If GC starts after the live check, the renew commit conflicts and + // the next attempt re-reads the pin and its current lease state. + let object_reference = self.begin_object_reference_mutation()?; let read_version = self.read_version()?; let (bound_root, mut binding_predicates) = match root_path { Some(root_path) => { @@ -334,6 +443,7 @@ where key: key.clone(), predicate: Predicate::VersionEquals(versioned.version), }); + binding_predicates.push(object_reference.predicate(self.mount)); let commit_version = self.next_version()?; let command = MetadataCommand { request_id: request_id(b"renew-snapshot", self.mount, renewed.root, commit_version), @@ -368,25 +478,6 @@ where } continue; } - Err(MetadError::SyncLogArchiveFailed { - committed: true, - message, - }) => { - let reconciled = self.snapshot_pin(snapshot_id)?; - if let Some(pin) = reconciled { - if pin.root == renewed.root && pin.lease_expires_unix_ms >= requested_expiry - { - return Ok(SnapshotRenewOutcome::Renewed { - pin, - extended: true, - }); - } - } - return Err(MetadError::SyncLogArchiveFailed { - committed: true, - message, - }); - } Err(err) => return Err(err), } } @@ -396,6 +487,7 @@ where }) } + #[cfg(test)] pub(crate) fn snapshot_pin(&self, snapshot_id: u64) -> Result, MetadError> { self.snapshot_pin_for_purpose(snapshot_id, ReadPurpose::UserStrong) } @@ -425,6 +517,7 @@ where Ok(Some(pin)) } + #[cfg(test)] fn snapshot_pin_for_purpose( &self, snapshot_id: u64, @@ -435,7 +528,7 @@ where .map(|versioned| versioned.pin)) } - fn versioned_snapshot_pin_at( + pub(super) fn versioned_snapshot_pin_at( &self, snapshot_id: u64, version: Version, @@ -458,6 +551,320 @@ where .transpose() } + /// Prove that raising the mount-global history floor cannot orphan any + /// reachable borrowed object reference. A binding may be the oldest hold for + /// an object borrowed by a different fork or by rollback, so source-local + /// lineage is insufficient: retirement fails closed while any effective + /// manifest reachable from the mount root or a live detached fork root names + /// a block owned by another inode. + /// + /// Callers hold `object_gc_gate` across this scan and the binding CAS. Clone + /// and rollback hold the same gate while publishing new borrowed references, + /// so the proof cannot race either producer. + pub(super) fn validate_current_dentry_projection( + &self, + row_key: &[u8], + projection: &DentryProjection, + current_version: Version, + ) -> Result<(), MetadError> { + let expected_key = dentry_key( + self.mount, + projection.dentry.parent, + &projection.dentry.name, + ); + if row_key != expected_key { + return Err(MetadError::Codec(format!( + "dentry row key does not match borrower inode {}", + projection.attr.inode.get() + ))); + } + if projection.dentry.child != projection.attr.inode + || projection.dentry.child_type != projection.attr.file_type + || projection.dentry.attr_generation != projection.attr.generation + { + return Err(MetadError::Codec(format!( + "dentry projection identity does not match borrower inode {}", + projection.attr.inode.get() + ))); + } + if projection.attr.inode.shard_index() != self.shard_index() { + if projection.attr.file_type != FileType::Directory + || projection.attr.size != 0 + || projection.attr.rdev != 0 + || projection.body.is_some() + { + return Err(MetadError::Codec(format!( + "foreign dentry projection is not a valid graft for inode {}", + projection.attr.inode.get() + ))); + } + return Ok(()); + } + + let Some(inode_value) = self.metadata.get( + RecordFamily::Inode, + &inode_key(self.mount, projection.attr.inode), + current_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Err(MetadError::Codec(format!( + "dentry projection names missing borrower inode {}", + projection.attr.inode.get() + ))); + }; + let canonical_attr = + decode_inode_attr(&inode_value.0).map_err(|err| MetadError::Codec(err.to_string()))?; + if canonical_attr.inode != projection.attr.inode + || canonical_attr.file_type != projection.attr.file_type + || canonical_attr.size != projection.attr.size + || canonical_attr.generation != projection.attr.generation + { + return Err(MetadError::Codec(format!( + "dentry projection object identity does not match borrower inode {}", + projection.attr.inode.get() + ))); + } + + if let Some(body) = projection.body.as_ref() { + if body.size != projection.attr.size + || body.generation == 0 + || body.generation > projection.attr.generation + { + return Err(MetadError::Codec(format!( + "dentry projection body does not match borrower inode {}", + projection.attr.inode.get() + ))); + } + let canonical_body = self + .body_descriptor_at_version_for_purpose( + projection.attr.inode, + body.generation, + current_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::MissingBodyDescriptor)?; + if canonical_body != *body { + return Err(MetadError::Codec(format!( + "dentry projection body descriptor does not match borrower inode {}", + projection.attr.inode.get() + ))); + } + } + Ok(()) + } + + fn ensure_fork_bindings_releasable( + &self, + bindings: &[VersionedForkBinding], + current_version: Version, + ) -> Result<(), MetadError> { + let Some(versioned) = bindings.first() else { + return Ok(()); + }; + let binding = versioned.binding; + let purpose = ReadPurpose::WritePlanLocal; + let mut roots = vec![InodeId::root()]; + + // A clone may deliberately remain detached and is then reachable through + // its durable binding rather than the mount root. A rollback binding is + // also durable, but its materialization root is deleted by the graft; in + // that case the restored children are reached through the mount root. + // Conversely, an interrupted materialization has neither a namespace link + // nor a binding and must not become an immortal retention root merely + // because its partial dentry rows survived the crash/failure. + for versioned in self.versioned_fork_bindings_at(current_version, purpose)? { + let root = versioned.binding.fork_root; + let Some(attr) = + self.get_attr_at_version_for_purpose(root, current_version, purpose)? + else { + continue; + }; + if attr.file_type != FileType::Directory { + return Err(MetadError::Codec(format!( + "fork binding root {} is not a directory", + root.get() + ))); + } + roots.push(root); + } + + let Some(root_attr) = + self.get_attr_at_version_for_purpose(InodeId::root(), current_version, purpose)? + else { + return Err(MetadError::Codec( + "mount root is missing during fork retention proof".to_owned(), + )); + }; + if root_attr.file_type != FileType::Directory { + return Err(MetadError::Codec( + "mount root is not a directory during fork retention proof".to_owned(), + )); + } + + let mut visited_directories = HashSet::new(); + let mut checked_bodies = HashSet::new(); + while let Some(parent) = roots.pop() { + if !visited_directories.insert(parent) { + continue; + } + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_prefix(self.mount, parent), + start_after: None, + version: current_version, + limit: 0, + purpose, + })?; + for row in rows { + let projection = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + self.validate_current_dentry_projection(&row.key, &projection, current_version)?; + + if projection.attr.inode.shard_index() == self.shard_index() + && projection.attr.file_type == FileType::Directory + { + roots.push(projection.attr.inode); + } + + let Some(body) = projection.body.as_ref() else { + if projection.attr.size == 0 { + continue; + } + return Err(MetadError::MissingBodyDescriptor); + }; + if !checked_bodies.insert((projection.attr.inode, body.generation)) { + continue; + } + let manifests = self.chunk_manifests_for_body_at_version( + projection.attr.inode, + body, + current_version, + purpose, + )?; + for block in manifests + .iter() + .flat_map(|manifest| &manifest.slices) + .flat_map(|slice| &slice.blocks) + { + if !self + .block_object_is_owned_by_inode(projection.attr.inode, &block.object_key)? + { + return Err(MetadError::ForkRetentionActive { + snapshot_id: binding.snapshot_id, + fork_root: binding.fork_root, + borrower: projection.attr.inode, + }); + } + } + } + } + Ok(()) + } + + pub(super) fn versioned_fork_bindings_at( + &self, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + self.metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(self.mount), + start_after: None, + version, + limit: 0, + purpose, + })? + .into_iter() + .map(|row| { + let binding = decode_fork_binding(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + let expected_key = fork_binding_key(self.mount, binding.fork_root); + if row.key != expected_key { + return Err(MetadError::Codec(format!( + "fork binding key does not match fork root {}", + binding.fork_root.get() + ))); + } + if binding.fork_root.shard_index() != self.shard_index() + || binding.source_root.shard_index() != self.shard_index() + { + return Err(MetadError::Codec(format!( + "fork binding {} crosses shard boundary", + binding.snapshot_id + ))); + } + self.ensure_snapshot_id_shard(binding.snapshot_id, binding.source_root)?; + let snapshot_created_version = binding.snapshot_id & SNAPSHOT_ID_LOCAL_MASK; + if snapshot_created_version == 0 + || binding.pinned_read_version.checked_add(1) != Some(snapshot_created_version) + || snapshot_created_version >= binding.created_version + || binding.created_version != row.version.get() + { + return Err(MetadError::Codec(format!( + "fork binding {} has invalid version identity", + binding.snapshot_id + ))); + } + Ok(VersionedForkBinding { + binding, + key: row.key, + version: row.version, + }) + }) + .collect() + } + + fn fork_bindings_for_snapshot_at( + &self, + snapshot_id: u64, + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadError> { + Ok(self + .versioned_fork_bindings_at(version, purpose)? + .into_iter() + .filter(|binding| binding.binding.snapshot_id == snapshot_id) + .collect()) + } + + fn validate_snapshot_retention_roots( + &self, + snapshot_id: u64, + pin: Option<&VersionedSnapshotPin>, + bindings: &[VersionedForkBinding], + expected_root: Option, + ) -> Result<(), MetadError> { + let actual_root = pin + .map(|versioned| versioned.pin.root) + .or_else(|| bindings.first().map(|binding| binding.binding.source_root)); + if let Some(root) = actual_root { + if pin.is_some_and(|versioned| versioned.pin.snapshot_id != snapshot_id) + || bindings.iter().any(|binding| { + binding.binding.snapshot_id != snapshot_id + || binding.binding.source_root != root + || pin.is_some_and(|versioned| { + binding.binding.pinned_read_version != versioned.pin.read_version + }) + }) + { + return Err(MetadError::Codec(format!( + "snapshot {snapshot_id} retention records disagree on identity" + ))); + } + if expected_root.is_some_and(|expected| expected != root) { + return Err(MetadError::SnapshotRootMismatch { + snapshot_id, + expected_root: expected_root.expect("checked as some"), + actual_root: Some(root), + actual_shard: self.shard_index(), + }); + } + } + Ok(()) + } + fn resolve_snapshot_root_binding( &self, root_path: &str, @@ -563,7 +970,7 @@ where Ok(()) } - fn ensure_snapshot_pin_live(&self, pin: &SnapshotPin) -> Result<(), MetadError> { + pub(super) fn ensure_snapshot_pin_live(&self, pin: &SnapshotPin) -> Result<(), MetadError> { let now_ms = self.now_ms(); if now_ms >= pin.lease_expires_unix_ms { return Err(MetadError::SnapshotLeaseExpired { diff --git a/crates/nokv-meta/src/service_tests.rs b/crates/nokv-meta/src/service_tests.rs index 29bcda36c..1c57e9885 100644 --- a/crates/nokv-meta/src/service_tests.rs +++ b/crates/nokv-meta/src/service_tests.rs @@ -1,16 +1,21 @@ use super::*; -use crate::command::{ReadItem, ScanItem}; +use crate::command::{MetadataCheckpointStore, ReadItem, ScanItem}; use crate::holtstore::HoltMetadataStore; +use crate::layout::object_gc_quarantine_prefix; use crate::{MetadataLogEntry, MetadataLogSegment, METADATA_LOG_ZERO_DIGEST}; use nokv_object::{MemoryObjectStore, ObjectBytes}; use nokv_types::{AdvisoryLockKind, AdvisoryLockRequest}; -use std::sync::{Arc, Barrier}; +use std::sync::{Arc, Barrier, Condvar}; +use std::time::{Duration, Instant}; #[derive(Clone)] struct SnapshotCommitBarrierStore { inner: HoltMetadataStore, kind: CommandKind, + request_prefix: Option>, + rejected_kind: Option, remaining: Arc, + predicate_failures: Arc, entered: Arc, release: Arc, } @@ -20,7 +25,10 @@ impl SnapshotCommitBarrierStore { Self { inner: HoltMetadataStore::open_memory().unwrap(), kind, + request_prefix: None, + rejected_kind: None, remaining: Arc::new(AtomicU64::new(blocked_commits)), + predicate_failures: Arc::new(AtomicU64::new(0)), entered: Arc::new(Barrier::new(parties)), release: Arc::new(Barrier::new(parties)), } @@ -30,6 +38,24 @@ impl SnapshotCommitBarrierStore { self.entered.wait(); } + fn arm(&self, blocked_commits: u64) { + self.remaining.store(blocked_commits, Ordering::SeqCst); + } + + fn predicate_failures(&self) -> u64 { + self.predicate_failures.load(Ordering::SeqCst) + } + + fn rejecting(mut self, kind: CommandKind) -> Self { + self.rejected_kind = Some(kind); + self + } + + fn matching_request_prefix(mut self, prefix: &[u8]) -> Self { + self.request_prefix = Some(prefix.to_vec()); + self + } + fn release_blocked(&self) { self.release.wait(); } @@ -52,6 +78,10 @@ impl MetadataStore for SnapshotCommitBarrierStore { fn commit_metadata(&self, command: MetadataCommand) -> Result { if command.kind == self.kind + && self + .request_prefix + .as_ref() + .is_none_or(|prefix| command.request_id.starts_with(prefix)) && self .remaining .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { @@ -62,7 +92,14 @@ impl MetadataStore for SnapshotCommitBarrierStore { self.entered.wait(); self.release.wait(); } - self.inner.commit_metadata(command) + if self.rejected_kind == Some(command.kind) { + return Err(MetadataError::PredicateFailed); + } + let result = self.inner.commit_metadata(command); + if matches!(&result, Err(MetadataError::PredicateFailed)) { + self.predicate_failures.fetch_add(1, Ordering::SeqCst); + } + result } fn commit_independent_batch( @@ -97,6 +134,290 @@ impl MetadataStoreStatsProvider for SnapshotCommitBarrierStore { } } +/// Metadata store wrapper that pauses one matching command only after Holt has +/// durably applied it. This deterministically exposes apply-vs-log ordering +/// races without relying on scheduler timing. +#[derive(Clone)] +struct PostCommitBarrierStore { + inner: HoltMetadataStore, + request_id: Vec, + remaining: Arc, + applied: Arc, + release: Arc, +} + +impl PostCommitBarrierStore { + fn new(request_id: &[u8]) -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + request_id: request_id.to_vec(), + remaining: Arc::new(AtomicU64::new(1)), + applied: Arc::new(Barrier::new(2)), + release: Arc::new(Barrier::new(2)), + } + } + + fn wait_until_applied(&self) { + self.applied.wait(); + } + + fn release_after_apply(&self) { + self.release.wait(); + } +} + +impl MetadataStore for PostCommitBarrierStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + let should_block = command.request_id == self.request_id + && self + .remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok(); + let result = self.inner.commit_metadata(command); + if should_block && result.is_ok() { + self.applied.wait(); + self.release.wait(); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +impl MetadataCheckpointStore for PostCommitBarrierStore { + fn checkpoint(&self) -> Result<(), MetadataError> { + self.inner.checkpoint() + } + + fn export_checkpoint_image(&self) -> Result, MetadataError> { + self.inner.export_checkpoint_image() + } + + fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { + self.inner.install_checkpoint_image(image) + } + + fn reclaim_unreachable_storage(&self) -> Result { + self.inner.reclaim_unreachable_storage() + } +} + +impl MetadataStoreStatsProvider for PostCommitBarrierStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + +#[derive(Clone)] +struct PostCommitErrorStore { + inner: HoltMetadataStore, + kind: CommandKind, + remaining: Arc, + readback_failures: Arc, + readback_mismatches: Arc, + batch_backend_indices: Arc>>>, +} + +impl PostCommitErrorStore { + fn new(kind: CommandKind) -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + kind, + remaining: Arc::new(AtomicU64::new(1)), + readback_failures: Arc::new(AtomicU64::new(0)), + readback_mismatches: Arc::new(AtomicU64::new(0)), + batch_backend_indices: Arc::new(Mutex::new(None)), + } + } + + fn new_disarmed(kind: CommandKind) -> Self { + let store = Self::new(kind); + store.remaining.store(0, Ordering::SeqCst); + store + } + + fn arm(&self) { + self.remaining.store(1, Ordering::SeqCst); + } + + fn fail_next_readbacks(&self, count: u64) { + self.readback_failures.store(count, Ordering::SeqCst); + } + + fn clear_readback_failures(&self) { + self.readback_failures.store(0, Ordering::SeqCst); + } + + fn mismatch_next_readbacks(&self, count: u64) { + self.readback_mismatches.store(count, Ordering::SeqCst); + } + + fn clear_readback_mismatches(&self) { + self.readback_mismatches.store(0, Ordering::SeqCst); + } + + fn fail_next_batch_results(&self, indices: Vec) { + *self.batch_backend_indices.lock().unwrap() = Some(indices); + } +} + +impl MetadataStore for PostCommitErrorStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + let kind = command.kind; + let result = self.inner.commit_metadata(command); + if kind == self.kind + && result.is_ok() + && self + .remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(MetadataError::Backend( + "injected journal acknowledgement failure".to_owned(), + )); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + let mut results = self.inner.commit_independent_batch(commands); + if let Some(indices) = self.batch_backend_indices.lock().unwrap().take() { + for index in indices { + if matches!(results.get(index), Some(Ok(_))) { + results[index] = Err(MetadataError::Backend( + "injected batch journal acknowledgement failure".to_owned(), + )); + } + } + } + results + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + if self + .readback_failures + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(MetadataError::Backend( + "injected authoritative readback failure".to_owned(), + )); + } + let result = self.inner.committed_request_result(request_id)?; + if self + .readback_mismatches + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Ok(result.map(|mut result| { + result.applied_mutations = result.applied_mutations.saturating_add(1); + result + })); + } + Ok(result) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +impl MetadataStoreStatsProvider for PostCommitErrorStore { + fn metadata_store_stats(&self) -> MetadataStoreStats { + self.inner.metadata_store_stats() + } +} + +impl MetadataCheckpointStore for PostCommitErrorStore { + fn checkpoint(&self) -> Result<(), MetadataError> { + self.inner.checkpoint() + } + + fn export_checkpoint_image(&self) -> Result, MetadataError> { + self.inner.export_checkpoint_image() + } + + fn install_checkpoint_image(&self, image: &[u8]) -> Result<(), MetadataError> { + self.inner.install_checkpoint_image(image) + } + + fn reclaim_unreachable_storage(&self) -> Result { + self.inner.reclaim_unreachable_storage() + } +} + #[derive(Clone)] struct SnapshotPredicateOnceStore { inner: HoltMetadataStore, @@ -298,54 +619,386 @@ impl MetadataStoreStatsProvider for PurposeTrackingStore { } } -fn service() -> NoKvFs { - service_with_objects().0 +#[derive(Clone)] +struct PausingObjectGcStore { + inner: HoltMetadataStore, + gate: Arc<(Mutex, Condvar)>, } -fn service_with_objects() -> ( - NoKvFs, - MemoryObjectStore, -) { - let objects = MemoryObjectStore::new(); - let service = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - (service, objects) +#[derive(Default)] +struct PausingObjectGcState { + armed: bool, + reached: bool, + released: bool, } -fn artifact_request(name: DentryName, manifest_id: &str, bytes: &[u8]) -> PublishArtifact { - PublishArtifact { - parent: InodeId::root(), - name, - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: manifest_id.to_owned(), - bytes: bytes.to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, +impl PausingObjectGcStore { + fn new() -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + gate: Arc::new((Mutex::new(PausingObjectGcState::default()), Condvar::new())), + } + } + + fn arm(&self) { + let (lock, _) = &*self.gate; + *lock.lock().unwrap() = PausingObjectGcState { + armed: true, + reached: false, + released: false, + }; + } + + fn wait_until_reached(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !state.reached { + let now = Instant::now(); + assert!(now < deadline, "timed out waiting for durable GC claim"); + let (next, timed_out) = changed.wait_timeout(state, deadline - now).unwrap(); + state = next; + assert!( + !timed_out.timed_out() || state.reached, + "timed out waiting for durable GC claim" + ); + } + } + + fn release(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + state.released = true; + changed.notify_all(); + } + + fn pause_after_deleting_claim(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + if !state.armed { + return; + } + state.armed = false; + state.reached = true; + changed.notify_all(); + while !state.released { + state = changed.wait(state).unwrap(); + } } } -fn publish_path_artifact( - service: &NoKvFs, - path: &str, - manifest_id: &str, - bytes: &[u8], -) -> DentryWithAttr { - let prepared = service.prepare_artifact_create_path(path).unwrap(); - service - .publish_prepared_artifact_session( - prepared.clone(), - PublishArtifactSession { - parent: prepared.parent, - name: prepared.name, - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), +impl MetadataStore for PausingObjectGcStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + let deleting_claim = command.primary_family == RecordFamily::System + && command.primary_key == object_gc_claim_key(MountId::new(1).unwrap()) + && command.mutations.iter().any(|mutation| { + mutation.family == RecordFamily::System + && mutation.key == command.primary_key + && mutation + .value + .as_ref() + .is_some_and(|value| value.0.first() == Some(&2)) + }); + let result = self.inner.commit_metadata(command); + if deleting_claim && result.is_ok() { + self.pause_after_deleting_claim(); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn history_retention_epoch(&self) -> Result { + self.inner.history_retention_epoch() + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +/// Simulates a remote DELETE that takes effect but loses its acknowledgement. +/// The retry observes the object as missing, matching an idempotent S3 DELETE. +#[derive(Clone)] +struct DeleteAckLostObjectStore { + inner: MemoryObjectStore, + lose_next_delete_ack: Arc, + delete_calls: Arc, +} + +impl DeleteAckLostObjectStore { + fn new(inner: MemoryObjectStore) -> Self { + Self { + inner, + lose_next_delete_ack: Arc::new(AtomicBool::new(true)), + delete_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + } + } + + fn delete_calls(&self) -> usize { + self.delete_calls.load(Ordering::SeqCst) + } +} + +impl ObjectStore for DeleteAckLostObjectStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + self.inner.put(key, bytes) + } + + fn get( + &self, + key: &ObjectKey, + range: Option, + ) -> Result, ObjectError> { + self.inner.get(key, range) + } + + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } + + fn delete(&self, key: &ObjectKey) -> Result { + self.delete_calls.fetch_add(1, Ordering::SeqCst); + let deleted = self.inner.delete(key)?; + if self.lose_next_delete_ack.swap(false, Ordering::SeqCst) { + return Err(ObjectError::Backend( + "injected lost DELETE acknowledgement".to_owned(), + )); + } + Ok(deleted) + } +} + +fn service() -> NoKvFs { + service_with_objects().0 +} + +fn service_with_objects() -> ( + NoKvFs, + MemoryObjectStore, +) { + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + (service, objects) +} + +fn enqueue_gc_candidate(service: &NoKvFs, mut record: ObjectGcRecord) -> Vec +where + M: MetadataStore, + O: ObjectStore, +{ + let version = service.next_version().unwrap(); + record.enqueue_version = version.get(); + record.enqueue_unix_ms = service.now_ms(); + let key = gc_object_key( + service.mount, + version.get(), + record.inode, + record.generation, + 0, + 0, + ); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-enqueue-gc-candidate", + service.mount, + record.inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }], + watch: Vec::new(), + }) + .unwrap(); + key +} + +fn leave_object_gc_deleting(service: &NoKvFs, gc_row: &ScanItem) -> Vec +where + M: MetadataStore, + O: ObjectStore, +{ + let claim_key = object_gc_claim_key(service.mount); + let claim = service + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-leave-object-gc-deleting", + service.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: claim_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: gc_row.key.clone(), + predicate: Predicate::VersionEquals(gc_row.version), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: claim_key.clone(), + op: MutationOp::Put, + value: Some(Value( + encode_object_gc_claim(&ObjectGcClaim::Deleting { + owner_epoch: service.epoch.load(Ordering::Relaxed), + operation_token: claim.version.get(), + gc_record_key: gc_row.key.clone(), + gc_record_version: gc_row.version.get(), + }) + .unwrap(), + )), + }], + watch: Vec::new(), + }) + .unwrap(); + claim_key +} + +fn delete_object_gc_claim(service: &NoKvFs) +where + M: MetadataStore, + O: ObjectStore, +{ + let key = object_gc_claim_key(service.mount); + let claim = service + .metadata + .get_versioned( + RecordFamily::System, + &key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-delete-object-gc-claim", + service.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }], + mutations: vec![delete_mutation(RecordFamily::System, key)], + watch: Vec::new(), + }) + .unwrap(); +} + +fn artifact_request(name: DentryName, manifest_id: &str, bytes: &[u8]) -> PublishArtifact { + PublishArtifact { + parent: InodeId::root(), + name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest_id.to_owned(), + bytes: bytes.to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + } +} + +fn publish_path_artifact( + service: &NoKvFs, + path: &str, + manifest_id: &str, + bytes: &[u8], +) -> DentryWithAttr { + let prepared = service.prepare_artifact_create_path(path).unwrap(); + service + .publish_prepared_artifact_session( + prepared.clone(), + PublishArtifactSession { + parent: prepared.parent, + name: prepared.name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:test".to_owned(), content_type: "application/octet-stream".to_owned(), manifest_id: manifest_id.to_owned(), size: bytes.len() as u64, @@ -2905,6 +3558,71 @@ fn prepared_artifact_publish_commits_manifest_without_object_fetch() { assert_eq!(plan.blocks[0].object_offset, 1); } +#[test] +fn prepared_artifact_publish_rejects_foreign_block_identity_before_commit() { + let service = service(); + let name = DentryName::new(b"foreign-block.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name) + .unwrap(); + let body = body_descriptor(prepared.generation, 6); + let foreign_keys = [ + format!( + "blocks/2/{}/{}/0/0", + prepared.inode.get(), + prepared.generation + ), + format!("blocks/1/999/{}/0/0", prepared.generation), + format!( + "blocks/1/{}/{}/0/0", + prepared.inode.get(), + prepared.generation + 1 + ), + format!( + "blocks/1/{}/{}/1/0", + prepared.inode.get(), + prepared.generation + ), + format!( + "blocks/1/{}/{}/0", + prepared.inode.get(), + prepared.generation + ), + ]; + + for object_key in foreign_keys { + let mut chunk = one_chunk_manifest(prepared.inode, prepared.generation, 6); + chunk.slices[0].blocks[0].object_key = object_key; + let err = service + .publish_prepared_artifact( + prepared.clone(), + body.clone(), + vec![chunk], + 0o644, + 1000, + 1000, + ) + .unwrap_err(); + assert!( + matches!(err, MetadError::InvalidPreparedArtifact(_)), + "unexpected error: {err:?}" + ); + assert!(service + .lookup_plus(InodeId::root(), &prepared.name) + .unwrap() + .is_none()); + } + + let mut valid_nonzero_block = one_chunk_manifest(prepared.inode, prepared.generation, 6); + valid_nonzero_block.slices[0].blocks[0].object_key = + block_key(prepared.inode, prepared.generation, 0, 17) + .as_str() + .to_owned(); + service + .publish_prepared_artifact(prepared, body, vec![valid_nonzero_block], 0o644, 1000, 1000) + .unwrap(); +} + #[test] fn prepared_artifact_publish_rejects_duplicate_chunk_range() { let service = service(); @@ -3047,7 +3765,7 @@ fn prepared_artifact_session_uploads_only_dirty_ranges_and_reuses_old_blocks() { prepared, PublishArtifactSession { parent: InodeId::root(), - name, + name: name.clone(), producer: "unit-test".to_owned(), digest_uri: "unknown".to_owned(), content_type: "application/octet-stream".to_owned(), @@ -3104,7 +3822,7 @@ fn prepared_artifact_session_splits_noncontiguous_dirty_blocks() { prepared, PublishArtifactSession { parent: InodeId::root(), - name, + name: name.clone(), producer: "unit-test".to_owned(), digest_uri: "unknown".to_owned(), content_type: "application/octet-stream".to_owned(), @@ -3705,7 +4423,7 @@ fn hardlink_updates_link_count_and_defers_body_gc_until_last_unlink() { assert_eq!(removed_last.attr.inode, published.attr.inode); assert!(service.get_attr(published.attr.inode).unwrap().is_none()); let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); assert!(objects.head(&object).unwrap().is_none()); } @@ -3760,1625 +4478,6400 @@ fn remove_file_queues_old_body_for_object_cleanup() { } #[test] -fn read_lease_grace_blocks_recent_object_gc_records() { +fn gc_uses_the_canonical_nonzero_object_block_index() { let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) - .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) + let name = DentryName::new(b"sparse-gc.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) .unwrap(); - let new_body = replaced.entry.body.clone().unwrap(); - let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - - let blocked = service - .cleanup_pending_objects_with_grace(100, std::time::Duration::from_secs(3_600)) + let inode = prepared.inode; + let generation = prepared.generation; + let block_index = 7_u64; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "sparse-gc", + &[PublishArtifactRange { + offset: 0, + bytes: b"sparse".to_vec(), + }], + block_index, + ) .unwrap(); - assert_eq!(blocked.scanned, 1); - assert_eq!(blocked.blocked_by_snapshots, 0); - assert_eq!(blocked.blocked_by_read_leases, 1); - assert_eq!(blocked.attempted, 0); - assert_eq!(blocked.records_removed, 0); - assert!(objects.head(&old_object).unwrap().is_some()); - assert!(objects.head(&new_object).unwrap().is_some()); - - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert!(objects.head(&new_object).unwrap().is_some()); -} - -#[test] -fn replace_artifact_cleanup_deletes_only_old_generation() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "unknown".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "sparse-gc".to_owned(), + size: 6, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) + + let object = block_key(inode, generation, 0, block_index); + assert!(objects.head(&object).unwrap().is_some()); + service.remove_file(InodeId::root(), &name).unwrap(); + + let rows = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) .unwrap(); - let new_body = replaced.entry.body.clone().unwrap(); - let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - assert!(objects.head(&old_object).unwrap().is_some()); - assert!(objects.head(&new_object).unwrap().is_some()); + assert_eq!(rows.len(), 1); + let key = decode_object_gc_record_key(service.mount, &rows[0].key).unwrap(); + assert_eq!(key.chunk_index, 0); + assert_eq!(key.block_index, block_index); - let cleanup = service.cleanup_pending_objects(100).unwrap(); + let cleanup = service.cleanup_pending_objects(10).unwrap(); assert_eq!(cleanup.deleted, 1); assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert!(objects.head(&new_object).unwrap().is_some()); + assert!(objects.head(&object).unwrap().is_none()); + assert!(service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); +} + +#[test] +fn durable_object_gc_claim_codec_round_trips_and_rejects_trailing_bytes() { + let mount = MountId::new(1).unwrap(); + let deleting = ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 11, + gc_record_key: gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5), + gc_record_version: 13, + }; assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" + decode_object_gc_claim( + mount, + &encode_object_gc_claim(&ObjectGcClaim::Open).unwrap() + ) + .unwrap(), + ObjectGcClaim::Open + ); + assert_eq!( + decode_object_gc_claim(mount, &encode_object_gc_claim(&deleting).unwrap()).unwrap(), + deleting ); + let mut malformed = encode_object_gc_claim(&deleting).unwrap(); + malformed.push(0); + assert!(matches!( + decode_object_gc_claim(mount, &malformed), + Err(MetadError::Codec(_)) + )); } #[test] -fn snapshot_after_replace_does_not_block_old_object_cleanup() { - let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) +fn durable_object_gc_claim_codec_rejects_zero_identity_fields() { + let mount = MountId::new(1).unwrap(); + let gc_record_key = gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5); + for invalid in [ + ObjectGcClaim::Deleting { + owner_epoch: 0, + operation_token: 11, + gc_record_key: gc_record_key.clone(), + gc_record_version: 13, + }, + ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 0, + gc_record_key: gc_record_key.clone(), + gc_record_version: 13, + }, + ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 11, + gc_record_key: gc_record_key.clone(), + gc_record_version: 0, + }, + ] { + assert!(matches!( + decode_object_gc_claim(mount, &encode_object_gc_claim(&invalid).unwrap()), + Err(MetadError::Codec(_)) + )); + } +} + +#[test] +fn durable_object_gc_claim_codec_rejects_non_local_or_malformed_gc_keys() { + let mount = MountId::new(1).unwrap(); + let valid_key = gc_object_key(mount, 10, InodeId::new(2).unwrap(), 3, 4, 5); + let mut invalid_keys = vec![ + gc_object_key( + MountId::new(2).unwrap(), + 10, + InodeId::new(2).unwrap(), + 3, + 4, + 5, + ), + gc_queue_prefix(mount), + ]; + for field in [1, 2, 3] { + let mut key = valid_key.clone(); + key[field * 8..(field + 1) * 8].fill(0); + invalid_keys.push(key); + } + + for gc_record_key in invalid_keys { + let invalid = ObjectGcClaim::Deleting { + owner_epoch: 7, + operation_token: 11, + gc_record_key, + gc_record_version: 13, + }; + assert!(matches!( + decode_object_gc_claim(mount, &encode_object_gc_claim(&invalid).unwrap()), + Err(MetadError::Codec(_)) + )); + } +} + +#[test] +fn durable_gc_claim_blocks_new_object_reference_planning_while_deleting() { + let metadata = PausingObjectGcStore::new(); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"old".to_vec()).unwrap(), + "old", + b"old", + )) .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + let resize_name = DentryName::new(b"resize".to_vec()).unwrap(); + service + .publish_artifact(artifact_request( + resize_name.clone(), + "resize", + b"resize-body", )) .unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + service.remove_file_path("/old").unwrap(); - assert_eq!( - service - .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") - .unwrap(), - b"new-body" - ); - assert!(objects.head(&old_object).unwrap().is_some()); + metadata.arm(); + let cleaner = { + let service = Arc::clone(&service); + std::thread::spawn(move || service.cleanup_pending_objects(1)) + }; + metadata.wait_until_reached(); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.scanned, 1); - assert_eq!(cleanup.blocked_by_snapshots, 0); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" - ); - assert_eq!( - replaced.entry.body.unwrap().generation, - snapshot.read_version - ); + assert!(matches!( + service.prepare_artifact_create(InodeId::root(), DentryName::new(b"new".to_vec()).unwrap()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.publish_artifact(artifact_request( + DentryName::new(b"new-publish".to_vec()).unwrap(), + "new-publish", + b"new" + )), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.create_symlink( + InodeId::root(), + DentryName::new(b"new-link".to_vec()).unwrap(), + b"target".to_vec(), + 0o777, + 1000, + 1000 + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.create_symlink( + InodeId::root(), + DentryName::new(b"invalid-link".to_vec()).unwrap(), + Vec::new(), + 0o777, + 1000, + 1000 + ), + Err(MetadError::InvalidPath(_)) + )); + assert!(matches!( + service.update_attrs( + InodeId::root(), + &resize_name, + UpdateAttr { + size: Some(1), + ..UpdateAttr::default() + } + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.publish_checkpoint( + InodeId::root(), + vec![CheckpointShard { + name: DentryName::new(b"checkpoint-shard".to_vec()).unwrap(), + bytes: b"checkpoint".to_vec(), + }], + 1000, + 1000 + ), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!(matches!( + service.snapshot_subtree(InodeId::root()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + let attr_only = service + .update_attrs( + InodeId::root(), + &resize_name, + UpdateAttr { + mode: Some(0o600), + ..UpdateAttr::default() + }, + ) + .unwrap(); + assert_eq!(attr_only.attr.mode, 0o600); + metadata.release(); + let cleaned = cleaner.join().unwrap().unwrap(); + assert_eq!(cleaned.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); } #[test] -fn snapshot_preserves_old_artifact_and_blocks_object_gc_until_retired() { +fn gc_recheck_preserves_an_object_referenced_by_the_current_manifest() { let (service, objects) = service_with_objects(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + let name = DentryName::new(b"live".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "live", b"live")) .unwrap(); - let old_body = first.body.clone().unwrap(); - let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: published.attr.inode, + generation: body.generation, + object_key: object.as_str().to_owned(), + size: body.size, + digest_uri: body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); - let replaced = service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) - .unwrap(); - let new_body = replaced.entry.body.clone().unwrap(); - let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); + let claim_key = object_gc_claim_key(service.mount); + let claim_version_before = service + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version; + let cleanup = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.blocked_by_snapshots, 1); + assert_eq!(cleanup.deleted, 0); + assert_eq!(cleanup.records_removed, 0); + assert!(objects.head(&object).unwrap().is_some()); assert_eq!( - service - .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") - .unwrap(), - b"old" - ); - assert_eq!( - service - .get_attr_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name)) - .unwrap(), - Some(first.attr.clone()) - ); - assert_eq!( - service - .read_file_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name), 0, 3,) - .unwrap(), - b"old" + service.read_artifact(InodeId::root(), &name).unwrap(), + b"live" ); + + // The first pass advances the durable cursor, the second reaches the tail + // and resets it, and the third sees the protected row again. None of those + // advisory scans may rotate the global object-reference epoch. + assert_eq!(service.cleanup_pending_objects(1).unwrap().scanned, 0); assert_eq!( service - .read_dir_plus_at_snapshot("/", snapshot.snapshot_id, &[]) - .unwrap(), - vec![first.clone()] - ); - assert_eq!( - service.read_artifact(InodeId::root(), &name).unwrap(), - b"new-body" + .cleanup_pending_objects(1) + .unwrap() + .blocked_by_snapshots, + 1 ); - let blocked = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(blocked.scanned, 1); - assert_eq!(blocked.blocked_by_snapshots, 1); - assert_eq!(blocked.attempted, 0); - assert!(objects.head(&old_object).unwrap().is_some()); - assert!(objects.head(&new_object).unwrap().is_some()); - - assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - assert!(!service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let cleanup = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); - assert!(objects.head(&old_object).unwrap().is_none()); - assert!(objects.head(&new_object).unwrap().is_some()); + let claim_version_after = service + .metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap() + .version; + assert_eq!(claim_version_after, claim_version_before); } #[test] -fn snapshot_path_reads_are_rooted_at_snapshot_subtree_and_support_ranges() { - let service = service(); - let scope = service - .create_dir_path("/scope", 0o755, 1000, 1000) - .unwrap(); - let nested = service - .create_dir_path("/scope/nested", 0o755, 1000, 1000) - .unwrap(); - let outside = service - .create_dir_path("/outside", 0o755, 1000, 1000) - .unwrap(); - let name = DentryName::new(b"model.bin".to_vec()).unwrap(); - let inside_old = service - .publish_artifact(PublishArtifact { - parent: nested.attr.inode, - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:inside-old".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "inside-old".to_owned(), - bytes: b"inside-old".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) - .unwrap(); - service - .publish_artifact(PublishArtifact { - parent: outside.attr.inode, - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:outside".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "outside".to_owned(), - bytes: b"outside".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) - .unwrap(); - let snapshot = service.snapshot_subtree_path("/scope").unwrap(); - service - .replace_artifact(PublishArtifact { - parent: nested.attr.inode, - name: name.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:inside-new".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "inside-new".to_owned(), - bytes: b"inside-new".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) +fn snapshot_mint_retries_after_an_intervening_object_delete_epoch() { + let store = SnapshotCommitBarrierStore::new(CommandKind::SnapshotSubtree, 1, 2); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"snapshot-race".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "snapshot-race", b"old")) .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); - let root = service - .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/") - .unwrap() - .unwrap(); - assert_eq!(root.attr.inode, scope.attr.inode); - assert_eq!( - service - .read_dir_plus_path_at_snapshot("/scope", snapshot.snapshot_id, "/") - .unwrap(), - vec![nested.clone()] - ); - let file = service - .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin") - .unwrap() - .unwrap(); - assert_eq!(file.attr.generation, inside_old.attr.generation); - assert_eq!(file.body, inside_old.body); - assert_eq!( - service - .read_file_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin", 7, 3,) - .unwrap(), - b"old" - ); + let mint_service = Arc::clone(&service); + let mint = std::thread::spawn(move || mint_service.snapshot_subtree(InodeId::root())); + store.wait_until_blocked(); + service.remove_file(InodeId::root(), &name).unwrap(); + assert_eq!(service.cleanup_pending_objects(1).unwrap().deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + store.release_blocked(); + + let snapshot = mint.join().unwrap().unwrap(); assert!(matches!( - service.read_file_path_at_snapshot( - "/scope", - snapshot.snapshot_id, - "/outside/model.bin", - 0, - 7, - ), + service.read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/snapshot-race"), Err(MetadError::NotFound) )); } #[test] -fn snapshot_path_list_pages_include_entries_deleted_after_snapshot() { - let service = service(); - for name in [ - b"a.txt".as_slice(), - b"b.txt".as_slice(), - b"c.txt".as_slice(), - ] { - service - .create_file( - InodeId::root(), - DentryName::new(name.to_vec()).unwrap(), - 0o644, - 1000, - 1000, - ) - .unwrap(); - } - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - service - .rename( - InodeId::root(), - &DentryName::new(b"b.txt".to_vec()).unwrap(), - InodeId::root(), - DentryName::new(b"z.txt".to_vec()).unwrap(), - ) +fn reopen_resumes_a_durable_object_gc_claim_after_crash() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request( + DentryName::new(b"old".to_vec()).unwrap(), + "old", + b"old", + )) .unwrap(); - service - .remove_file( - InodeId::root(), - &DentryName::new(b"c.txt".to_vec()).unwrap(), + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file_path("/old").unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!(objects.head(&object).unwrap().is_some()); + let deferred_claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, ) + .unwrap() .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &deferred_claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); - let first = service - .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", None, 2) + let cleanup = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&object).unwrap().is_none()); + let open_claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() .unwrap(); assert_eq!( - first - .entries - .iter() - .map(|entry| entry.dentry.name.as_bytes()) - .collect::>(), - vec![b"a.txt".as_slice(), b"b.txt".as_slice()] + decode_object_gc_claim(reopened.mount, &open_claim.0).unwrap(), + ObjectGcClaim::Open ); - let cursor = first.next_cursor.unwrap(); - assert_eq!(cursor.as_bytes(), b"b.txt"); - - let second = service - .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", Some(&cursor), 2) - .unwrap(); - assert_eq!(second.entries.len(), 1); - assert_eq!(second.entries[0].dentry.name.as_bytes(), b"c.txt"); - assert!(second.next_cursor.is_none()); } #[test] -fn history_cleanup_keeps_snapshot_reads_until_snapshot_retired() { - let service = service(); - let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - service - .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) - .unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); - service - .replace_artifact(artifact_request( - name.clone(), - "checkpoint/new", - b"new-body", - )) +fn history_cleanup_does_not_invalidate_a_prepared_object_upload() { + let (service, _) = service_with_objects(); + let name = DentryName::new(b"upload-during-history-prune.bin".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) .unwrap(); - let retained = service.cleanup_history(100).unwrap(); - assert!(retained.retained_by_snapshots > 0); + service.cleanup_history(100).unwrap(); assert_eq!( service - .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) .unwrap(), - b"old" + prepared ); - assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); - let pruned = service.cleanup_history(100).unwrap(); - assert!(pruned.removed > 0); + let bytes = b"history-prune-independent"; + service + .publish_prepared_artifact_session( + prepared, + PublishArtifactSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:history-prune-independent".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "history-prune-independent".to_owned(), + size: bytes.len() as u64, + ranges: vec![PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); assert_eq!( - service - .metadata - .get( - RecordFamily::Dentry, - &dentry_key(service.mount, InodeId::root(), &name), - Version::new(snapshot.read_version).unwrap(), - ReadPurpose::Snapshot, - ) - .unwrap(), - None + service.read_artifact(InodeId::root(), &name).unwrap(), + bytes ); } #[test] -fn remove_empty_dir_rejects_non_empty_directory() { - let service = service(); - let dir = DentryName::new(b"runs".to_vec()).unwrap(); - let child = DentryName::new(b"1".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) - .unwrap(); +fn prepared_publish_rejects_an_intervening_object_gc_epoch_and_refreshes_identity() { + let (service, objects) = service_with_objects(); + let victim_name = DentryName::new(b"gc-victim".to_vec()).unwrap(); service - .create_dir(created.attr.inode, child, 0o755, 1000, 1000) + .publish_artifact(artifact_request( + victim_name.clone(), + "gc-victim", + b"victim", + )) .unwrap(); + service.remove_file(InodeId::root(), &victim_name).unwrap(); - let err = service.remove_empty_dir(InodeId::root(), &dir).unwrap_err(); - assert!(matches!(err, MetadError::DirectoryNotEmpty)); - assert!(service - .lookup_plus(InodeId::root(), &dir) - .unwrap() - .is_some()); -} - -#[test] -fn remove_empty_dir_deletes_empty_directory() { - let service = service(); - let dir = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + let name = DentryName::new(b"late-after-gc".to_vec()).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) .unwrap(); - - let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); - assert_eq!(removed, created); - assert!(service - .lookup_plus(InodeId::root(), &dir) - .unwrap() - .is_none()); - assert!(service.get_attr(created.attr.inode).unwrap().is_none()); + let bytes = b"late-after-gc"; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "late-after-gc", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + let error = service + .publish_prepared_artifact_staged_session( + prepared.clone(), + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:late-after-gc".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "late-after-gc".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap_err(); + let staged = match error { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + )); + staged + } + error => panic!("unexpected prepared publish error: {error:?}"), + }; + assert!(service.lookup_path("/late-after-gc").unwrap().is_none()); + let cleanup = service.cleanup_staged_objects(&staged).unwrap(); + assert_eq!(cleanup.deleted, staged.len()); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_none()); + } + + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) + .unwrap(); + assert!(refreshed.generation > prepared.generation); + assert_ne!( + refreshed.object_gc_claim_version, + prepared.object_gc_claim_version + ); } #[test] -fn remove_empty_dir_allows_directory_after_last_child_removed() { - let service = service(); - let dir = DentryName::new(b"runs".to_vec()).unwrap(); - let child = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) +fn failover_claim_rotation_rejects_old_prepared_upload_and_allows_cleanup() { + let (source, objects) = service_with_objects(); + let name = dname(b"prepared-before-failover.bin"); + let prepared = source + .prepare_artifact_create(InodeId::root(), name.clone()) .unwrap(); - service - .publish_artifact(PublishArtifact { - parent: created.attr.inode, - name: child.clone(), - producer: "unit-test".to_owned(), - digest_uri: "sha256:test".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "runs/checkpoint.bin".to_owned(), - bytes: b"payload".to_vec(), - mode: 0o644, - uid: 1000, - gid: 1000, - }) + let bytes = b"staged by the failed owner"; + let written = source + .stage_prepared_artifact_ranges( + &prepared, + "prepared-before-failover", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) .unwrap(); + let staged_before_failover = written.staged_objects().unwrap(); + let staged_key = staged_before_failover.objects()[0].key.clone(); + assert!(objects.head(&staged_key).unwrap().is_some()); - service.remove_file(created.attr.inode, &child).unwrap(); - let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); + let checkpoint = MetadataArchiveConfig::new("meta/prepared-failover", 2); + source.backup_metadata(&checkpoint).unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + // Match controlled startup: install the acquired epoch before restoring the + // old owner's checkpoint, recover any crash-left deletion, then rotate Open. + recovered.install_owner_epoch(2).unwrap(); + recovered.restore_metadata(&checkpoint).unwrap().unwrap(); + recovered.recover_object_gc_claim().unwrap(); + recovered.rotate_object_gc_claim_for_failover().unwrap(); - assert_eq!(removed, created); - assert!(service - .lookup_plus(InodeId::root(), &dir) + let error = recovered + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:prepared-before-failover".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "prepared-before-failover".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: staged_before_failover, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap_err(); + let staged = match error { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + | MetadError::Metadata(MetadataError::PredicateFailed) + )); + staged + } + other => panic!("unexpected old prepared publish error: {other:?}"), + }; + assert!(recovered + .lookup_plus(InodeId::root(), &name) .unwrap() .is_none()); + recovered.cleanup_staged_objects(&staged).unwrap(); + assert!(objects.head(&staged_key).unwrap().is_none()); } #[test] -fn rename_moves_dentry_without_changing_inode() { - let service = service(); - let old_name = DentryName::new(b"old".to_vec()).unwrap(); - let new_name = DentryName::new(b"new".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) +fn failover_claim_rotation_confirms_a_lost_backend_ack_by_exact_readback() { + let metadata = PostCommitErrorStore::new_disarmed(CommandKind::CleanupObjects); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let prepared = service + .prepare_artifact_create(InodeId::root(), dname(b"old-token.bin")) .unwrap(); + let old_claim_version = prepared.object_gc_claim_version; - let renamed = service - .rename( - InodeId::root(), - &old_name, - InodeId::root(), - new_name.clone(), - ) + metadata.arm(); + service.rotate_object_gc_claim_for_failover().unwrap(); + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared) .unwrap(); - assert_eq!(renamed.attr.inode, created.attr.inode); + assert_ne!(refreshed.object_gc_claim_version, old_claim_version); +} + +#[test] +fn blocked_head_gc_row_does_not_starve_later_reclaimable_candidate() { + let (service, objects) = service_with_objects(); + let live_name = DentryName::new(b"live-head".to_vec()).unwrap(); + let live = service + .publish_artifact(artifact_request(live_name, "live-head", b"live")) + .unwrap(); + let live_body = live.body.as_ref().unwrap(); + let live_object = block_key(live.attr.inode, live_body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: live.attr.inode, + generation: live_body.generation, + object_key: live_object.as_str().to_owned(), + size: live_body.size, + digest_uri: live_body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let reclaimable = ObjectKey::new("blocks/1/900/1/0/0").unwrap(); + objects.put(&reclaimable, b"stale".to_vec()).unwrap(); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(900).unwrap(), + generation: 1, + object_key: reclaimable.as_str().to_owned(), + size: 5, + digest_uri: "sha256:stale".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let first = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(first.scanned, 1); + assert_eq!(first.blocked_by_snapshots, 1); + assert_eq!(first.deleted, 0); + assert!(objects.head(&live_object).unwrap().is_some()); + assert!(objects.head(&reclaimable).unwrap().is_some()); + + let second = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(second.scanned, 1); + assert_eq!(second.deleted, 1); + assert!(objects.head(&live_object).unwrap().is_some()); + assert!(objects.head(&reclaimable).unwrap().is_none()); +} + +#[test] +fn local_gc_row_key_cannot_delete_an_object_owned_by_another_mount() { + let (service, objects) = service_with_objects(); + let foreign_object = ObjectKey::new("blocks/2/902/1/0/0").unwrap(); + objects.put(&foreign_object, b"foreign".to_vec()).unwrap(); + let row_key = enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(902).unwrap(), + generation: 1, + object_key: foreign_object.as_str().to_owned(), + size: 7, + digest_uri: "sha256:foreign".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + assert_eq!(row_key.len(), 48); + assert!(row_key.starts_with(&gc_queue_prefix(service.mount))); + + let outcome = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.deleted, 0); + assert_eq!(outcome.records_removed, 1); + assert!(objects.head(&foreign_object).unwrap().is_some()); assert!(service - .lookup_plus(InodeId::root(), &old_name) + .metadata + .get( + RecordFamily::Gc, + &row_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) .unwrap() .is_none()); assert_eq!( - service.lookup_plus(InodeId::root(), &new_name).unwrap(), - Some(renamed) + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 ); } #[test] -fn rename_replace_returns_replaced_file_body() { - let service = service(); - let source_name = DentryName::new(b"stage".to_vec()).unwrap(); - let final_name = DentryName::new(b"final".to_vec()).unwrap(); - let source = service - .publish_artifact(artifact_request(source_name.clone(), "stage", b"new")) - .unwrap(); - let old = service - .publish_artifact(artifact_request(final_name.clone(), "final-old", b"old")) +fn gc_row_key_cannot_delete_a_different_block_of_the_same_generation() { + let (service, objects) = service_with_objects(); + let inode = InodeId::new(904).unwrap(); + let object = ObjectKey::new("blocks/1/904/1/0/0").unwrap(); + objects.put(&object, b"protected".to_vec()).unwrap(); + + let version = service.next_version().unwrap(); + let row_key = gc_object_key(service.mount, version.get(), inode, 1, 0, 1); + let record = ObjectGcRecord { + inode, + generation: 1, + object_key: object.as_str().to_owned(), + size: 9, + digest_uri: "sha256:protected".to_owned(), + enqueue_version: version.get(), + enqueue_unix_ms: service.now_ms(), + }; + service + .commit_metadata(MetadataCommand { + request_id: b"mismatched-block-gc-row".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: row_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: row_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: row_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }], + watch: Vec::new(), + }) .unwrap(); - let result = service - .rename_replace( - InodeId::root(), - &source_name, - InodeId::root(), - final_name.clone(), - ) - .unwrap(); - assert_eq!(result.entry.attr.inode, source.attr.inode); - assert_eq!(result.replaced, Some(old.clone())); + let outcome = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.deleted, 0); + assert_eq!(outcome.records_removed, 1); + assert!(objects.head(&object).unwrap().is_some()); assert!(service - .lookup_plus(InodeId::root(), &source_name) + .metadata + .get( + RecordFamily::Gc, + &row_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) .unwrap() .is_none()); assert_eq!( - service.lookup_plus(InodeId::root(), &final_name).unwrap(), - Some(result.entry) + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 ); - assert!(service.get_attr(old.attr.inode).unwrap().is_none()); } #[test] -fn watch_replay_returns_typed_events_after_cursor() { - let service = service(); - let cursor = service.watch_subtree(InodeId::root()).unwrap(); - let name = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) +fn crash_left_gc_claim_quarantines_a_foreign_object_key_without_deleting_it() { + let (service, objects) = service_with_objects(); + let metadata = service.metadata.clone(); + let foreign_object = ObjectKey::new("blocks/2/903/1/0/0").unwrap(); + objects.put(&foreign_object, b"foreign".to_vec()).unwrap(); + let row_key = enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(903).unwrap(), + generation: 1, + object_key: foreign_object.as_str().to_owned(), + size: 7, + digest_uri: "sha256:foreign".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + let row = metadata + .get_versioned( + RecordFamily::Gc, + &row_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() .unwrap(); + leave_object_gc_deleting( + &service, + &ScanItem { + key: row_key.clone(), + value: row.value, + version: row.version, + }, + ); + drop(service); - let events = service.replay_watch(InodeId::root(), cursor, 100).unwrap(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].event.kind, WatchEventKind::Create); - assert_eq!(events[0].event.parent, Some(InodeId::root())); - assert_eq!(events[0].event.name, Some(name.clone())); - assert_eq!(events[0].event.inode, created.attr.inode); + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let outcome = reopened.recover_object_gc_claim().unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.deleted, 0); + assert_eq!(outcome.records_removed, 1); + assert!(objects.head(&foreign_object).unwrap().is_some()); + assert!(metadata + .get( + RecordFamily::Gc, + &row_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert_eq!( + metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(reopened.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); + let claim = metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Open + ); +} - let next_name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); - service +#[test] +fn delete_error_keeps_the_durable_claim_closed_until_same_claim_recovery() { + let backing = MemoryObjectStore::new(); + let objects = DeleteAckLostObjectStore::new(backing.clone()); + let metadata = HoltMetadataStore::open_memory().unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service .publish_artifact(artifact_request( - next_name.clone(), - "checkpoint.bin", - b"body", + DentryName::new(b"lost-delete-ack.bin".to_vec()).unwrap(), + "lost-delete-ack", + b"payload", )) .unwrap(); - let resumed = service - .replay_watch(InodeId::root(), events[0].cursor, 100) + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service + .remove_file( + InodeId::root(), + &DentryName::new(b"lost-delete-ack.bin".to_vec()).unwrap(), + ) .unwrap(); - assert_eq!(resumed.len(), 1); - assert_eq!(resumed[0].event.kind, WatchEventKind::PublishArtifact); - assert_eq!(resumed[0].event.name, Some(next_name)); -} - -#[test] -fn watch_replay_resumes_from_cursor_without_scanning_old_events() { - let service = service(); - let cursor = service.watch_subtree(InodeId::root()).unwrap(); - for name in ["a", "b", "c"] { - service - .create_dir( - InodeId::root(), - DentryName::new(name.as_bytes().to_vec()).unwrap(), - 0o755, - 1000, - 1000, - ) - .unwrap(); - } - let before_first = service.metadata_store_stats(); - let first = service.replay_watch(InodeId::root(), cursor, 1).unwrap(); - let after_first = service.metadata_store_stats(); - assert_eq!(first.len(), 1); - assert_eq!( - first[0].event.name.as_ref().map(DentryName::as_bytes), - Some(b"a".as_slice()) - ); - assert_eq!( - after_first.scan_key_visited_total - before_first.scan_key_visited_total, - 1 - ); + assert!(matches!( + service.cleanup_pending_objects(1), + Err(MetadError::Object(ObjectError::Backend(message))) + if message == "injected lost DELETE acknowledgement" + )); + assert_eq!(objects.delete_calls(), 1); + assert!(backing.head(&object).unwrap().is_none()); + let claim_key = object_gc_claim_key(service.mount); + let deleting = metadata + .get( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(service.mount, &deleting.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert!(matches!( + service.begin_object_reference_mutation(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); assert_eq!( - after_first.scan_key_returned_total - before_first.scan_key_returned_total, + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), 1 ); - let before_second = service.metadata_store_stats(); - let second = service - .replay_watch(InodeId::root(), first[0].cursor, 1) + let recovered = service.recover_object_gc_claim().unwrap(); + assert_eq!(objects.delete_calls(), 2); + assert_eq!(recovered.attempted, 1); + assert_eq!(recovered.missing, 1); + assert_eq!(recovered.records_removed, 1); + let open = metadata + .get( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() .unwrap(); - let after_second = service.metadata_store_stats(); - assert_eq!(second.len(), 1); - assert_eq!( - second[0].event.name.as_ref().map(DentryName::as_bytes), - Some(b"b".as_slice()) - ); - assert_eq!( - after_second.scan_key_visited_total - before_second.scan_key_visited_total, - 1 - ); assert_eq!( - after_second.scan_key_returned_total - before_second.scan_key_returned_total, - 1 + decode_object_gc_claim(service.mount, &open.0).unwrap(), + ObjectGcClaim::Open ); + assert!(metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); } #[test] -fn rename_replay_notifies_old_and_new_parent_scopes() { - let service = service(); - let old_parent_name = DentryName::new(b"old-parent".to_vec()).unwrap(); - let new_parent_name = DentryName::new(b"new-parent".to_vec()).unwrap(); - let old_parent = service - .create_dir(InodeId::root(), old_parent_name, 0o755, 1000, 1000) - .unwrap(); - let new_parent = service - .create_dir(InodeId::root(), new_parent_name, 0o755, 1000, 1000) - .unwrap(); - let file_name = DentryName::new(b"artifact".to_vec()).unwrap(); - let source = service - .create_file(old_parent.attr.inode, file_name.clone(), 0o644, 1000, 1000) - .unwrap(); - let old_cursor = service.watch_subtree(old_parent.attr.inode).unwrap(); - let new_cursor = service.watch_subtree(new_parent.attr.inode).unwrap(); - +fn malformed_gc_row_is_quarantined_without_starving_a_valid_candidate() { + let (service, objects) = service_with_objects(); + let mut malformed_key = gc_queue_prefix(service.mount); + malformed_key.extend_from_slice(&0_u64.to_be_bytes()); + let version = service.next_version().unwrap(); service - .rename( - old_parent.attr.inode, - &file_name, - new_parent.attr.inode, - file_name.clone(), - ) + .commit_metadata(MetadataCommand { + request_id: b"malformed-gc-row".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: malformed_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: malformed_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: malformed_key, + op: MutationOp::Put, + value: Some(Value(b"not-an-object-gc-record".to_vec())), + }], + watch: Vec::new(), + }) .unwrap(); - let old_events = service - .replay_watch(old_parent.attr.inode, old_cursor, 100) - .unwrap(); - assert_eq!(old_events.len(), 1); - assert_eq!(old_events[0].event.kind, WatchEventKind::Remove); - assert_eq!(old_events[0].event.inode, source.attr.inode); + let object = ObjectKey::new("blocks/1/901/1/0/0").unwrap(); + objects.put(&object, b"stale".to_vec()).unwrap(); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(901).unwrap(), + generation: 1, + object_key: object.as_str().to_owned(), + size: 5, + digest_uri: "sha256:stale".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); - let new_events = service - .replay_watch(new_parent.attr.inode, new_cursor, 100) - .unwrap(); - assert_eq!(new_events.len(), 1); - assert_eq!(new_events[0].event.kind, WatchEventKind::Rename); - assert_eq!(new_events[0].event.name, Some(file_name)); - assert_eq!(new_events[0].event.inode, source.attr.inode); + let outcome = service.cleanup_pending_objects(10).unwrap(); + assert_eq!(outcome.scanned, 2); + assert_eq!(outcome.records_removed, 2); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); } #[test] -fn watch_replay_survives_service_reopen() { - let dir = tempfile::tempdir().unwrap(); +fn failover_durability_marker_survives_disable_and_reopen_and_blocks_object_gc() { + let metadata = HoltMetadataStore::open_memory().unwrap(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let cursor = service.watch_subtree(InodeId::root()).unwrap(); - let name = DentryName::new(b"runs".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) + let name = DentryName::new(b"ha-victim".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "ha-victim", b"victim")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); + service.disable_sync_metadata_log().unwrap(); + let marker_key = failover_durability_required_key(service.mount); + assert_eq!( + metadata + .get( + RecordFamily::System, + &marker_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + FAILOVER_DURABILITY_REQUIRED_MARKER + ); drop(service); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let events = reopened.replay_watch(InodeId::root(), cursor, 100).unwrap(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].event.kind, WatchEventKind::Create); - assert_eq!(events[0].event.name, Some(name)); - assert_eq!(events[0].event.inode, created.attr.inode); + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let outcome = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.scanned, 1); + assert_eq!(outcome.blocked_by_failover_durability, 1); + assert_eq!(outcome.attempted, 0); + assert_eq!(outcome.records_removed, 0); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); } #[test] -fn open_existing_recovers_inode_and_version_allocators() { - let dir = tempfile::tempdir().unwrap(); - let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let first = service - .create_dir( - InodeId::root(), - DentryName::new(b"first".to_vec()).unwrap(), - 0o755, - 1000, - 1000, - ) +fn failover_marker_committing_first_invalidates_shared_gc_claim_without_delete() { + let metadata = SnapshotCommitBarrierStore::new(CommandKind::CleanupObjects, 0, 2); + let memory = MemoryObjectStore::new(); + let objects = DeleteAckLostObjectStore::new(memory.clone()); + let cleaner = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + cleaner.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"marker-first".to_vec()).unwrap(); + let published = cleaner + .publish_artifact(artifact_request(name.clone(), "marker-first", b"victim")) .unwrap(); - drop(service); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + cleaner.remove_file(InodeId::root(), &name).unwrap(); + let marker = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let second = reopened - .create_dir( - InodeId::root(), - DentryName::new(b"second".to_vec()).unwrap(), - 0o755, - 1000, - 1000, - ) - .unwrap(); - assert!(second.attr.inode > first.attr.inode); - assert!(second.attr.generation > first.attr.generation); + metadata.arm(1); + let cleanup = { + let cleaner = Arc::clone(&cleaner); + std::thread::spawn(move || cleaner.cleanup_pending_objects(1)) + }; + metadata.wait_until_blocked(); + marker.require_failover_durability().unwrap(); + metadata.release_blocked(); + + let outcome = cleanup.join().unwrap().unwrap(); + assert_eq!(outcome.attempted, 0); + assert_eq!(objects.delete_calls(), 0); + assert!(memory.head(&object).unwrap().is_some()); + assert_eq!(metadata.predicate_failures(), 1); + cleaner.refresh_allocator_state().unwrap(); + let blocked = cleaner.cleanup_pending_objects(1).unwrap(); + assert_eq!(blocked.blocked_by_failover_durability, 1); + assert_eq!(objects.delete_calls(), 0); + let marker_key = failover_durability_required_key(cleaner.mount); + assert_eq!( + metadata + .get( + RecordFamily::System, + &marker_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + FAILOVER_DURABILITY_REQUIRED_MARKER + ); } #[test] -fn refresh_allocator_state_advances_live_read_version_after_external_commit() { - let dir = tempfile::tempdir().unwrap(); +fn shared_gc_claim_committing_first_invalidates_preplanned_marker_cas() { + let metadata = SnapshotCommitBarrierStore::new(CommandKind::CleanupObjects, 0, 2); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); - let original = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); - original.bootstrap_root(0o755, 1000, 1000).unwrap(); - - let external = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let external_file = external - .create_file_path("/external.bin", 0o644, 1000, 1000) + let cleaner = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + cleaner.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"claim-first".to_vec()).unwrap(); + let published = cleaner + .publish_artifact(artifact_request(name.clone(), "claim-first", b"victim")) .unwrap(); - assert!(original.stat_path("/external.bin").unwrap().is_none()); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + cleaner.remove_file(InodeId::root(), &name).unwrap(); + let marker = Arc::new( + NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(), + ); - original.refresh_allocator_state().unwrap(); - let visible = original - .stat_path("/external.bin") - .unwrap() - .expect("external commit should be visible after refresh"); - assert_eq!(visible.attr, external_file.attr); - let local_file = original - .create_file_path("/after-refresh.bin", 0o644, 1000, 1000) - .unwrap(); - assert!(local_file.attr.inode > external_file.attr.inode); - assert!(local_file.attr.generation > external_file.attr.generation); + metadata.arm(1); + let marker_commit = { + let marker = Arc::clone(&marker); + std::thread::spawn(move || marker.require_failover_durability()) + }; + metadata.wait_until_blocked(); + let outcome = cleaner.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + metadata.release_blocked(); + marker_commit.join().unwrap().unwrap(); + + assert_eq!( + metadata.predicate_failures(), + 1, + "the marker command planned against the old Open claim must not cross the Deleting transition" + ); + assert_eq!( + metadata + .get( + RecordFamily::System, + &failover_durability_required_key(cleaner.mount), + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + FAILOVER_DURABILITY_REQUIRED_MARKER + ); } #[test] -fn open_existing_recovers_after_dentry_only_rename() { - let dir = tempfile::tempdir().unwrap(); +fn startup_recovery_keeps_interrupted_claim_closed_under_failover_marker() { + let metadata = HoltMetadataStore::open_memory().unwrap(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let old_name = DentryName::new(b"old".to_vec()).unwrap(); - let new_name = DentryName::new(b"new".to_vec()).unwrap(); - let created = service - .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) + let name = DentryName::new(b"interrupted-ha-victim".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "interrupted-ha-victim", + b"victim", + )) .unwrap(); - let renamed = service - .rename( - InodeId::root(), - &old_name, - InodeId::root(), - new_name.clone(), - ) + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - assert_eq!(renamed.attr.inode, created.attr.inode); + service.disable_sync_metadata_log().unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); drop(service); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - assert!(reopened - .lookup_plus(InodeId::root(), &old_name) + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!(matches!( + reopened.recover_object_gc_claim(), + Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) + )); + let claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) .unwrap() - .is_none()); + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert!(objects.head(&object).unwrap().is_some()); assert_eq!( - reopened.lookup_plus(InodeId::root(), &new_name).unwrap(), - Some(renamed) + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 ); - assert_eq!(reopened.read_dir_plus(InodeId::root()).unwrap().len(), 1); } #[test] -fn open_existing_does_not_reuse_removed_inode() { - let dir = tempfile::tempdir().unwrap(); +fn enabling_failover_durability_fences_a_crash_left_deleting_claim_before_recovery() { + let metadata = HoltMetadataStore::open_memory().unwrap(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let first_name = DentryName::new(b"first".to_vec()).unwrap(); - let second_name = DentryName::new(b"second".to_vec()).unwrap(); - let first = service - .create_file(InodeId::root(), first_name.clone(), 0o644, 1000, 1000) + let name = DentryName::new(b"pre-marker-interrupted".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "pre-marker-interrupted", + b"victim", + )) .unwrap(); - service.remove_file(InodeId::root(), &first_name).unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); drop(service); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let second = reopened - .create_file(InodeId::root(), second_name.clone(), 0o644, 1000, 1000) + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + reopened + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - assert!(second.attr.inode > first.attr.inode); - assert!(second.attr.generation > first.attr.generation); - assert!(reopened - .lookup_plus(InodeId::root(), &first_name) + reopened.disable_sync_metadata_log().unwrap(); + let deleting = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) .unwrap() - .is_none()); + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &deleting.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + + assert!(matches!( + reopened.recover_object_gc_claim(), + Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) + )); + let still_deleting = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &still_deleting.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert!(objects.head(&object).unwrap().is_some()); assert_eq!( - reopened.lookup_plus(InodeId::root(), &second_name).unwrap(), - Some(second) + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 ); } #[test] -fn pending_object_gc_survives_service_reopen() { - let dir = tempfile::tempdir().unwrap(); +fn failover_recovery_does_not_reopen_after_external_delete_completed() { + let metadata = HoltMetadataStore::open_memory().unwrap(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let name = DentryName::new(b"deleted-before-crash".to_vec()).unwrap(); let published = service - .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) + .publish_artifact(artifact_request( + name.clone(), + "deleted-before-crash", + b"victim", + )) .unwrap(); - let body = published.body.clone().unwrap(); + let body = published.body.as_ref().unwrap(); let object = block_key(published.attr.inode, body.generation, 0, 0); service.remove_file(InodeId::root(), &name).unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = leave_object_gc_deleting(&service, &gc_row); + assert!(objects.delete(&object).unwrap()); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + service.disable_sync_metadata_log().unwrap(); drop(service); - let reopened = - NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects.clone(), 0).unwrap(); - let cleanup = reopened.cleanup_pending_objects(100).unwrap(); - assert_eq!(cleanup.deleted, 1); - assert_eq!(cleanup.records_removed, 1); + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!(matches!( + reopened.recover_object_gc_claim(), + Err(MetadError::ObjectGcRecoveryRequiresIntervention { .. }) + )); assert!(objects.head(&object).unwrap().is_none()); + let claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); } #[test] -fn snapshot_pin_survives_service_reopen() { - let dir = tempfile::tempdir().unwrap(); +fn reopen_missing_claim_is_initialized_before_enabling_failover_durability() { + let metadata = HoltMetadataStore::open_memory().unwrap(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + delete_object_gc_claim(&service); drop(service); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); + assert!(metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + reopened + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata-log", + "shard-0", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + let claim = metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); assert_eq!( - reopened.snapshot_pin(snapshot.snapshot_id).unwrap(), - Some(snapshot) + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Open ); - assert_eq!(reopened.metadata_store_stats().active_snapshot_pin_total, 1); + assert!(metadata + .get( + RecordFamily::System, + &failover_durability_required_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); } #[test] -fn failed_publish_returns_staged_objects_for_cleanup_and_does_not_reuse_identity() { - let dir = tempfile::tempdir().unwrap(); +fn reopen_missing_claim_is_initialized_by_cleanup_recovery() { + let metadata = HoltMetadataStore::open_memory().unwrap(); let objects = MemoryObjectStore::new(); - let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); service.bootstrap_root(0o755, 1000, 1000).unwrap(); - let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); - let first = service - .publish_artifact(artifact_request(name.clone(), "first", b"first")) + let name = DentryName::new(b"legacy-gc-victim".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request( + name.clone(), + "legacy-gc-victim", + b"victim", + )) .unwrap(); - let err = service - .publish_artifact(artifact_request(name.clone(), "duplicate", b"duplicate")) - .unwrap_err(); - let staged = match err { - MetadError::PublishArtifactFailed { source, staged } => { - assert!(matches!( - *source, - MetadError::Metadata(MetadataError::PredicateFailed) - )); - staged - } - err => panic!("unexpected publish error: {err:?}"), - }; - assert_eq!(staged.len(), 1); - for object in staged.objects() { - assert!(objects.head(&object.key).unwrap().is_some()); - } - assert_eq!( - service.lookup_plus(InodeId::root(), &name).unwrap(), - Some(first.clone()) - ); - - let cleanup = service.cleanup_staged_objects(&staged).unwrap(); - assert_eq!(cleanup.attempted, staged.len()); - assert_eq!(cleanup.deleted, staged.len()); - assert_eq!(cleanup.missing, 0); - for object in staged.objects() { - assert!(objects.head(&object.key).unwrap().is_none()); - } + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + delete_object_gc_claim(&service); drop(service); - let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); - let next_name = DentryName::new(b"next.bin".to_vec()).unwrap(); - let next = reopened - .publish_artifact(artifact_request(next_name, "next", b"next")) + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + let outcome = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + let claim = metadata + .get( + RecordFamily::System, + &object_gc_claim_key(reopened.mount), + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() .unwrap(); - - assert!(next.attr.inode.get() > first.attr.inode.get() + 1); - assert!(next.attr.generation > first.attr.generation + 1); + assert_eq!( + decode_object_gc_claim(reopened.mount, &claim.0).unwrap(), + ObjectGcClaim::Open + ); } -fn dname(raw: &[u8]) -> DentryName { - DentryName::new(raw.to_vec()).unwrap() -} +#[test] +fn read_lease_grace_blocks_recent_object_gc_records() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) + .unwrap(); + let new_body = replaced.entry.body.clone().unwrap(); + let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); -fn block_count_for(objects: &MemoryObjectStore, inode: InodeId, generation: u64) -> usize { - // Count the published blocks the base file owns under its (inode, generation). - let mut count = 0; - let mut block = 0; - while objects - .head(&block_key(inode, generation, 0, block)) - .unwrap() - .is_some() - { - count += 1; - block += 1; - } - count + let blocked = service + .cleanup_pending_objects_with_grace(100, std::time::Duration::from_secs(3_600)) + .unwrap(); + assert_eq!(blocked.scanned, 1); + assert_eq!(blocked.blocked_by_snapshots, 0); + assert_eq!(blocked.blocked_by_read_leases, 1); + assert_eq!(blocked.attempted, 0); + assert_eq!(blocked.records_removed, 0); + assert!(objects.head(&old_object).unwrap().is_some()); + assert!(objects.head(&new_object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert!(objects.head(&new_object).unwrap().is_some()); } #[test] -fn clone_subtree_shares_base_blocks_diverges_on_write_and_keeps_gc_safe() { +fn replace_artifact_cleanup_deletes_only_old_generation() { let (service, objects) = service_with_objects(); - // 1. Base namespace: /base with files a ("AAA..") and b ("BBB.."). - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let a = publish_path_artifact(&service, "/base/a", "base/a", &vec![b'A'; 4096]); - let b = publish_path_artifact(&service, "/base/b", "base/b", &vec![b'B'; 4096]); - let a_gen = a.body.as_ref().unwrap().generation; - let b_gen = b.body.as_ref().unwrap().generation; - let a_block = block_key(a.attr.inode, a_gen, 0, 0); - let b_block = block_key(b.attr.inode, b_gen, 0, 0); - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); - let objects_after_base = objects.object_count(); - - // 2. Writable O(1)-ish fork of /base. - let fork = service.clone_subtree_path("/base").unwrap(); - assert_ne!(fork.root, base.attr.inode); - - // 3. Sharing: the fork sees the base content, with NO duplicate blocks. - let fork_a = service - .lookup_plus(fork.root, &dname(b"a")) - .unwrap() + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) .unwrap(); - let fork_b = service - .lookup_plus(fork.root, &dname(b"b")) - .unwrap() + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) .unwrap(); - assert_ne!( - fork_a.attr.inode, a.attr.inode, - "fork must use a fresh inode" - ); - // Shared files keep the source's content generation (the CoW sharing signal). - assert_eq!(fork_a.attr.generation, a_gen); - assert_eq!(fork_b.attr.generation, b_gen); - assert_eq!(fork_b.body.as_ref().unwrap().generation, b_gen); + let new_body = replaced.entry.body.clone().unwrap(); + let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); + assert!(objects.head(&old_object).unwrap().is_some()); + assert!(objects.head(&new_object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert!(objects.head(&new_object).unwrap().is_some()); assert_eq!( - service.read_artifact(fork.root, &dname(b"a")).unwrap(), - vec![b'A'; 4096] + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" ); +} + +#[test] +fn snapshot_after_replace_does_not_block_old_object_cleanup() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) + .unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + assert_eq!( - service.read_artifact(fork.root, &dname(b"b")).unwrap(), - vec![b'B'; 4096] + service + .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") + .unwrap(), + b"new-body" ); - // Zero-copy: clone added metadata only, not object blocks. + assert!(objects.head(&old_object).unwrap().is_some()); + + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.scanned, 1); + assert_eq!(cleanup.blocked_by_snapshots, 0); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); assert_eq!( - objects.object_count(), - objects_after_base, - "clone must share base blocks, not copy them" + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" ); - // The fork's a/b manifests reference the SAME object keys as the base. assert_eq!( - service - .read_file_plan(fork_a.attr.inode, fork_a.attr.generation, 0, 4096) - .unwrap() - .blocks[0] - .object_key, - a_block.as_str() + replaced.entry.body.unwrap().generation, + snapshot.read_version ); +} - // 4. Divergence: rewrite a in the fork and add a new file c. - service - .replace_artifact(PublishArtifact { - parent: fork.root, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:zzz".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fork/a".to_owned(), - bytes: vec![b'Z'; 4096], - mode: 0o644, - uid: 1000, - gid: 1000, - }) +#[test] +fn snapshot_preserves_old_artifact_and_blocks_object_gc_until_retired() { + let (service, objects) = service_with_objects(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) .unwrap(); - service - .publish_artifact(PublishArtifact { - parent: fork.root, - name: dname(b"c"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:ccc".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "fork/c".to_owned(), - bytes: vec![b'C'; 4096], - mode: 0o644, - uid: 1000, - gid: 1000, - }) + let old_body = first.body.clone().unwrap(); + let old_object = block_key(first.attr.inode, old_body.generation, 0, 0); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + + let replaced = service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) .unwrap(); + let new_body = replaced.entry.body.clone().unwrap(); + let new_object = block_key(replaced.entry.attr.inode, new_body.generation, 0, 0); - // 4a. Fork now sees a="ZZZ..", b="BBB..", c present. assert_eq!( - service.read_artifact(fork.root, &dname(b"a")).unwrap(), - vec![b'Z'; 4096] + service + .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") + .unwrap(), + b"old" ); assert_eq!( - service.read_artifact(fork.root, &dname(b"b")).unwrap(), - vec![b'B'; 4096] + service + .get_attr_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name)) + .unwrap(), + Some(first.attr.clone()) ); assert_eq!( - service.read_artifact(fork.root, &dname(b"c")).unwrap(), - vec![b'C'; 4096] + service + .read_file_at_snapshot("/", snapshot.snapshot_id, std::slice::from_ref(&name), 0, 3,) + .unwrap(), + b"old" ); - // 4b. Base is unchanged: a="AAA..", no c. assert_eq!( service - .read_artifact(base.attr.inode, &dname(b"a")) + .read_dir_plus_at_snapshot("/", snapshot.snapshot_id, &[]) .unwrap(), - vec![b'A'; 4096] + vec![first.clone()] ); - assert!(service - .lookup_plus(base.attr.inode, &dname(b"c")) - .unwrap() - .is_none()); - - // 6. Diff reports exactly { modified: a, added: c }; b (shared) is not reported. - let mut diff = service.diff_subtrees(base.attr.inode, fork.root).unwrap(); - diff.sort_by(|left, right| left.path.cmp(&right.path)); - let summary: Vec<(&str, SubtreeDeltaKind)> = - diff.iter().map(|d| (d.path.as_str(), d.kind)).collect(); assert_eq!( - summary, - vec![ - ("/a", SubtreeDeltaKind::Modified), - ("/c", SubtreeDeltaKind::Added), - ] + service.read_artifact(InodeId::root(), &name).unwrap(), + b"new-body" ); - // The enriched diff carries the changed file's content digest. - assert!(diff - .iter() - .find(|d| d.path == "/a") - .unwrap() - .digest - .is_some()); + let blocked = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(blocked.scanned, 1); + assert_eq!(blocked.blocked_by_snapshots, 1); + assert_eq!(blocked.attempted, 0); + assert!(objects.head(&old_object).unwrap().is_some()); + assert!(objects.head(&new_object).unwrap().is_some()); - // 5. GC safety: reclaim must NOT touch base blocks the fork's divergent write - // abandoned but the base still references; they are owned by the base inode and - // protected by the fork's retained snapshot pin. - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert_eq!(reclaim.deleted, 0, "no base block may be reclaimed yet"); - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); - assert_eq!( - service - .read_artifact(base.attr.inode, &dname(b"a")) - .unwrap(), - vec![b'A'; 4096] - ); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + assert!(!service.retire_snapshot(snapshot.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&old_object).unwrap().is_none()); + assert!(objects.head(&new_object).unwrap().is_some()); +} - // Drop the fork: remove its files and retire its snapshot pin. The fork-only - // blocks (the divergent a' and the new c) then become reclaimable, while the - // base's blocks remain because the base still references them. - let fork_a_diverged = service - .lookup_plus(fork.root, &dname(b"a")) - .unwrap() +#[test] +fn snapshot_path_reads_are_rooted_at_snapshot_subtree_and_support_ranges() { + let service = service(); + let scope = service + .create_dir_path("/scope", 0o755, 1000, 1000) .unwrap(); - let fork_c = service - .lookup_plus(fork.root, &dname(b"c")) + let nested = service + .create_dir_path("/scope/nested", 0o755, 1000, 1000) + .unwrap(); + let outside = service + .create_dir_path("/outside", 0o755, 1000, 1000) + .unwrap(); + let name = DentryName::new(b"model.bin".to_vec()).unwrap(); + let inside_old = service + .publish_artifact(PublishArtifact { + parent: nested.attr.inode, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:inside-old".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "inside-old".to_owned(), + bytes: b"inside-old".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + service + .publish_artifact(PublishArtifact { + parent: outside.attr.inode, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:outside".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "outside".to_owned(), + bytes: b"outside".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/scope").unwrap(); + service + .replace_artifact(PublishArtifact { + parent: nested.attr.inode, + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:inside-new".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "inside-new".to_owned(), + bytes: b"inside-new".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + + let root = service + .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/") .unwrap() .unwrap(); - let fork_a_block = block_key( - fork_a_diverged.attr.inode, - fork_a_diverged.body.as_ref().unwrap().generation, - 0, - 0, - ); - let fork_c_block = block_key( - fork_c.attr.inode, - fork_c.body.as_ref().unwrap().generation, - 0, - 0, - ); - service.remove_file(fork.root, &dname(b"a")).unwrap(); - service.remove_file(fork.root, &dname(b"b")).unwrap(); - service.remove_file(fork.root, &dname(b"c")).unwrap(); - assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert!(reclaim.deleted >= 2, "fork-only blocks must be reclaimable"); - assert!(objects.head(&fork_a_block).unwrap().is_none()); - assert!(objects.head(&fork_c_block).unwrap().is_none()); - // Base remains fully intact and readable. - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); + assert_eq!(root.attr.inode, scope.attr.inode); assert_eq!( service - .read_artifact(base.attr.inode, &dname(b"a")) + .read_dir_plus_path_at_snapshot("/scope", snapshot.snapshot_id, "/") .unwrap(), - vec![b'A'; 4096] + vec![nested.clone()] ); + let file = service + .stat_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin") + .unwrap() + .unwrap(); + assert_eq!(file.attr.generation, inside_old.attr.generation); + assert_eq!(file.body, inside_old.body); assert_eq!( service - .read_artifact(base.attr.inode, &dname(b"b")) + .read_file_path_at_snapshot("/scope", snapshot.snapshot_id, "/nested/model.bin", 7, 3,) .unwrap(), - vec![b'B'; 4096] + b"old" ); - assert_eq!(block_count_for(&objects, a.attr.inode, a_gen), 1); + assert!(matches!( + service.read_file_path_at_snapshot( + "/scope", + snapshot.snapshot_id, + "/outside/model.bin", + 0, + 7, + ), + Err(MetadError::NotFound) + )); } #[test] -fn clone_subtree_copies_nested_dirs_and_diff_reports_removed() { +fn snapshot_path_list_pages_include_entries_deleted_after_snapshot() { let service = service(); - service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + for name in [ + b"a.txt".as_slice(), + b"b.txt".as_slice(), + b"c.txt".as_slice(), + ] { + service + .create_file( + InodeId::root(), + DentryName::new(name.to_vec()).unwrap(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + } + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); service - .create_dir_path("/base/sub", 0o755, 1000, 1000) + .rename( + InodeId::root(), + &DentryName::new(b"b.txt".to_vec()).unwrap(), + InodeId::root(), + DentryName::new(b"z.txt".to_vec()).unwrap(), + ) + .unwrap(); + service + .remove_file( + InodeId::root(), + &DentryName::new(b"c.txt".to_vec()).unwrap(), + ) .unwrap(); - publish_path_artifact(&service, "/base/sub/deep", "base/deep", b"deep-bytes"); - publish_path_artifact(&service, "/base/top", "base/top", b"top-bytes"); - let fork = service.clone_subtree_path("/base").unwrap(); - // Nested structure is reproduced under fresh inodes. - let sub = service - .lookup_plus(fork.root, &dname(b"sub")) - .unwrap() + let first = service + .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", None, 2) .unwrap(); - assert_eq!(sub.attr.file_type, FileType::Directory); assert_eq!( - service - .read_artifact(sub.attr.inode, &dname(b"deep")) - .unwrap(), - b"deep-bytes" + first + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(), + vec![b"a.txt".as_slice(), b"b.txt".as_slice()] ); + let cursor = first.next_cursor.unwrap(); + assert_eq!(cursor.as_bytes(), b"b.txt"); - // Identical subtree => no deltas. - let base = service.resolve_directory_path("/base").unwrap(); - assert!(service.diff_subtrees(base, fork.root).unwrap().is_empty()); - - // Remove a nested file in the fork => Removed delta at the nested path, - // direction base -> fork. - service - .remove_file(sub.attr.inode, &dname(b"deep")) + let second = service + .read_dir_plus_path_at_snapshot_page("/", snapshot.snapshot_id, "/", Some(&cursor), 2) .unwrap(); - let removed = service.diff_subtrees(base, fork.root).unwrap(); - assert_eq!(removed.len(), 1); - assert_eq!(removed[0].path, "/sub/deep"); - assert_eq!(removed[0].kind, SubtreeDeltaKind::Removed); - assert_eq!(removed[0].size_delta, -(b"deep-bytes".len() as i64)); - // Reversed direction reports it as Added, with the net size flipped. - let added = service.diff_subtrees(fork.root, base).unwrap(); - assert_eq!(added.len(), 1); - assert_eq!(added[0].path, "/sub/deep"); - assert_eq!(added[0].kind, SubtreeDeltaKind::Added); - assert_eq!(added[0].size_delta, b"deep-bytes".len() as i64); + assert_eq!(second.entries.len(), 1); + assert_eq!(second.entries[0].dentry.name.as_bytes(), b"c.txt"); + assert!(second.next_cursor.is_none()); } #[test] -fn clone_subtree_path_rejects_non_directory() { +fn history_cleanup_keeps_snapshot_reads_until_snapshot_retired() { let service = service(); - publish_path_artifact(&service, "/file.bin", "file", b"bytes"); - assert!(matches!( - service.clone_subtree_path("/file.bin"), - Err(MetadError::NotDirectory) - )); -} - -fn read_artifact_at_path( - service: &NoKvFs, - path: &str, -) -> Vec { - let (parent, name) = service.resolve_parent_path(path).unwrap(); - service.read_artifact(parent, &name).unwrap() -} - -#[test] -fn rollback_subtree_restores_snapshot_shares_blocks_and_reclaims_delta() { - let (service, objects) = service_with_objects(); - // 1. Build /ws with files a="A1", b="B1", sub/c="C1" (real object data). - let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); service - .create_dir_path("/ws/sub", 0o755, 1000, 1000) + .publish_artifact(artifact_request(name.clone(), "checkpoint/old", b"old")) + .unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + service + .replace_artifact(artifact_request( + name.clone(), + "checkpoint/new", + b"new-body", + )) .unwrap(); - let a = publish_path_artifact(&service, "/ws/a", "ws/a", &vec![b'1'; 4096]); - let b = publish_path_artifact(&service, "/ws/b", "ws/b", &vec![b'2'; 4096]); - let c = publish_path_artifact(&service, "/ws/sub/c", "ws/sub/c", &vec![b'3'; 4096]); - let a_gen = a.body.as_ref().unwrap().generation; - let b_gen = b.body.as_ref().unwrap().generation; - let c_gen = c.body.as_ref().unwrap().generation; - let a1_block = block_key(a.attr.inode, a_gen, 0, 0); - let b1_block = block_key(b.attr.inode, b_gen, 0, 0); - let c1_block = block_key(c.attr.inode, c_gen, 0, 0); - assert!(objects.head(&a1_block).unwrap().is_some()); - assert!(objects.head(&b1_block).unwrap().is_some()); - assert!(objects.head(&c1_block).unwrap().is_some()); - // 2. Snapshot /ws. - let snap = service.snapshot_subtree_path("/ws").unwrap(); + let retained = service.cleanup_history(100).unwrap(); + assert!(retained.retained_by_snapshots > 0); + assert_eq!( + service + .read_artifact_path_at_snapshot("/", snapshot.snapshot_id, "/checkpoint.bin") + .unwrap(), + b"old" + ); - // 3. Diverge /ws: rewrite a->"A2", add d="D1", delete b. + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + let pruned = service.cleanup_history(100).unwrap(); + assert!(pruned.removed > 0); + assert_eq!( + service + .metadata + .get( + RecordFamily::Dentry, + &dentry_key(service.mount, InodeId::root(), &name), + Version::new(snapshot.read_version).unwrap(), + ReadPurpose::Snapshot, + ) + .unwrap(), + None + ); +} + +#[test] +fn remove_empty_dir_rejects_non_empty_directory() { + let service = service(); + let dir = DentryName::new(b"runs".to_vec()).unwrap(); + let child = DentryName::new(b"1".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + .unwrap(); service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), + .create_dir(created.attr.inode, child, 0o755, 1000, 1000) + .unwrap(); + + let err = service.remove_empty_dir(InodeId::root(), &dir).unwrap_err(); + assert!(matches!(err, MetadError::DirectoryNotEmpty)); + assert!(service + .lookup_plus(InodeId::root(), &dir) + .unwrap() + .is_some()); +} + +#[test] +fn remove_empty_dir_deletes_empty_directory() { + let service = service(); + let dir = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + .unwrap(); + + let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); + assert_eq!(removed, created); + assert!(service + .lookup_plus(InodeId::root(), &dir) + .unwrap() + .is_none()); + assert!(service.get_attr(created.attr.inode).unwrap().is_none()); +} + +#[test] +fn remove_empty_dir_allows_directory_after_last_child_removed() { + let service = service(); + let dir = DentryName::new(b"runs".to_vec()).unwrap(); + let child = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), dir.clone(), 0o755, 1000, 1000) + .unwrap(); + service + .publish_artifact(PublishArtifact { + parent: created.attr.inode, + name: child.clone(), producer: "unit-test".to_owned(), - digest_uri: "sha256:a2".to_owned(), + digest_uri: "sha256:test".to_owned(), content_type: "application/octet-stream".to_owned(), - manifest_id: "ws/a2".to_owned(), - bytes: vec![b'4'; 4096], + manifest_id: "runs/checkpoint.bin".to_owned(), + bytes: b"payload".to_vec(), mode: 0o644, uid: 1000, gid: 1000, }) .unwrap(); - let d = publish_path_artifact(&service, "/ws/d", "ws/d", &vec![b'5'; 4096]); - service.remove_file(ws.attr.inode, &dname(b"b")).unwrap(); - // Capture the delta's private blocks so we can assert their fate. - let a_diverged = service - .lookup_plus(ws.attr.inode, &dname(b"a")) - .unwrap() - .unwrap(); - let a2_block = block_key( - a_diverged.attr.inode, - a_diverged.body.as_ref().unwrap().generation, - 0, - 0, - ); - let d1_block = block_key(d.attr.inode, d.body.as_ref().unwrap().generation, 0, 0); - assert!(objects.head(&a2_block).unwrap().is_some()); - assert!(objects.head(&d1_block).unwrap().is_some()); - // Pre-rollback /ws is the diverged state. - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'4'; 4096]); + + service.remove_file(created.attr.inode, &child).unwrap(); + let removed = service.remove_empty_dir(InodeId::root(), &dir).unwrap(); + + assert_eq!(removed, created); assert!(service - .lookup_plus(ws.attr.inode, &dname(b"b")) + .lookup_plus(InodeId::root(), &dir) .unwrap() .is_none()); +} - // 4. Roll /ws back to the snapshot. - service - .rollback_subtree_path("/ws", snap.snapshot_id) +#[test] +fn rename_moves_dentry_without_changing_inode() { + let service = service(); + let old_name = DentryName::new(b"old".to_vec()).unwrap(); + let new_name = DentryName::new(b"new".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) .unwrap(); - // 5. /ws now exactly matches the snapshot: a="A1", b="B1" (restored), sub/c="C1", - // and d is gone. The target keeps its inode identity. - assert_eq!( - service.resolve_directory_path("/ws").unwrap(), - ws.attr.inode, - "rollback keeps the target root's identity" - ); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); - assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'3'; 4096] - ); - assert!( - service - .lookup_plus(ws.attr.inode, &dname(b"d")) - .unwrap() - .is_none(), - "the delta-only file d must be gone after rollback" - ); - - // 6. The rolled-back /ws is identical to a fresh clone of the snapshot: an empty - // diff in both directions. - let reference = service - .clone_subtree_path_into("/ws", "/reference") + let renamed = service + .rename( + InodeId::root(), + &old_name, + InodeId::root(), + new_name.clone(), + ) .unwrap(); + assert_eq!(renamed.attr.inode, created.attr.inode); assert!(service - .diff_subtrees(ws.attr.inode, reference.root) - .unwrap() - .is_empty()); - assert!(service - .diff_subtrees(reference.root, ws.attr.inode) + .lookup_plus(InodeId::root(), &old_name) .unwrap() - .is_empty()); - - // 7. GC: the discarded delta's private blocks (A2, D1) are reclaimable, while the - // restored shared blocks (A1, B1, C1) survive and stay readable. The reference - // clone shares the snapshot's blocks, so retire its pin too before reclaiming. - assert!(service.retire_snapshot(snap.snapshot_id).unwrap()); - assert!(service.retire_snapshot(reference.snapshot_id).unwrap()); - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert!( - reclaim.deleted >= 2, - "delta-only blocks must be reclaimable" - ); - assert!( - objects.head(&a2_block).unwrap().is_none(), - "A2 must be reclaimed" - ); - assert!( - objects.head(&d1_block).unwrap().is_none(), - "D1 must be reclaimed" - ); - assert!( - objects.head(&a1_block).unwrap().is_some(), - "A1 must survive" - ); - assert!( - objects.head(&b1_block).unwrap().is_some(), - "B1 must survive" - ); - assert!( - objects.head(&c1_block).unwrap().is_some(), - "C1 must survive" - ); - // Restored content is still readable from the shared blocks after reclaim. - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + .is_none()); assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'3'; 4096] + service.lookup_plus(InodeId::root(), &new_name).unwrap(), + Some(renamed) ); } #[test] -fn rollback_subtree_rejects_foreign_or_missing_snapshot() { +fn rename_replace_returns_replaced_file_body() { let service = service(); - service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); - service - .create_dir_path("/other", 0o755, 1000, 1000) + let source_name = DentryName::new(b"stage".to_vec()).unwrap(); + let final_name = DentryName::new(b"final".to_vec()).unwrap(); + let source = service + .publish_artifact(artifact_request(source_name.clone(), "stage", b"new")) + .unwrap(); + let old = service + .publish_artifact(artifact_request(final_name.clone(), "final-old", b"old")) .unwrap(); - let other_root = service.resolve_directory_path("/other").unwrap(); - let snap = service.snapshot_subtree_path("/other").unwrap(); - // A snapshot of /other cannot roll back /ws. - assert!(matches!( - service.rollback_subtree_path("/ws", snap.snapshot_id), - Err(MetadError::InvalidPath(_)) - )); - // An unknown snapshot id is not found. - assert!(matches!( - service.rollback_subtree_path("/ws", snap.snapshot_id + 9_999), - Err(MetadError::NotFound) - )); - // The rejected target is untouched and the legitimate one still works. + let result = service + .rename_replace( + InodeId::root(), + &source_name, + InodeId::root(), + final_name.clone(), + ) + .unwrap(); + assert_eq!(result.entry.attr.inode, source.attr.inode); + assert_eq!(result.replaced, Some(old.clone())); assert!(service - .rollback_subtree(other_root, snap.snapshot_id) - .is_ok()); + .lookup_plus(InodeId::root(), &source_name) + .unwrap() + .is_none()); + assert_eq!( + service.lookup_plus(InodeId::root(), &final_name).unwrap(), + Some(result.entry) + ); + assert!(service.get_attr(old.attr.inode).unwrap().is_none()); +} + +#[test] +fn watch_replay_returns_typed_events_after_cursor() { + let service = service(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); + let name = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) + .unwrap(); + + let events = service.replay_watch(InodeId::root(), cursor, 100).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].event.kind, WatchEventKind::Create); + assert_eq!(events[0].event.parent, Some(InodeId::root())); + assert_eq!(events[0].event.name, Some(name.clone())); + assert_eq!(events[0].event.inode, created.attr.inode); + + let next_name = DentryName::new(b"checkpoint.bin".to_vec()).unwrap(); + service + .publish_artifact(artifact_request( + next_name.clone(), + "checkpoint.bin", + b"body", + )) + .unwrap(); + let resumed = service + .replay_watch(InodeId::root(), events[0].cursor, 100) + .unwrap(); + assert_eq!(resumed.len(), 1); + assert_eq!(resumed[0].event.kind, WatchEventKind::PublishArtifact); + assert_eq!(resumed[0].event.name, Some(next_name)); +} + +#[test] +fn watch_replay_resumes_from_cursor_without_scanning_old_events() { + let service = service(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); + for name in ["a", "b", "c"] { + service + .create_dir( + InodeId::root(), + DentryName::new(name.as_bytes().to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + } + + let before_first = service.metadata_store_stats(); + let first = service.replay_watch(InodeId::root(), cursor, 1).unwrap(); + let after_first = service.metadata_store_stats(); + assert_eq!(first.len(), 1); + assert_eq!( + first[0].event.name.as_ref().map(DentryName::as_bytes), + Some(b"a".as_slice()) + ); + assert_eq!( + after_first.scan_key_visited_total - before_first.scan_key_visited_total, + 1 + ); + assert_eq!( + after_first.scan_key_returned_total - before_first.scan_key_returned_total, + 1 + ); + + let before_second = service.metadata_store_stats(); + let second = service + .replay_watch(InodeId::root(), first[0].cursor, 1) + .unwrap(); + let after_second = service.metadata_store_stats(); + assert_eq!(second.len(), 1); + assert_eq!( + second[0].event.name.as_ref().map(DentryName::as_bytes), + Some(b"b".as_slice()) + ); + assert_eq!( + after_second.scan_key_visited_total - before_second.scan_key_visited_total, + 1 + ); + assert_eq!( + after_second.scan_key_returned_total - before_second.scan_key_returned_total, + 1 + ); +} + +#[test] +fn rename_replay_notifies_old_and_new_parent_scopes() { + let service = service(); + let old_parent_name = DentryName::new(b"old-parent".to_vec()).unwrap(); + let new_parent_name = DentryName::new(b"new-parent".to_vec()).unwrap(); + let old_parent = service + .create_dir(InodeId::root(), old_parent_name, 0o755, 1000, 1000) + .unwrap(); + let new_parent = service + .create_dir(InodeId::root(), new_parent_name, 0o755, 1000, 1000) + .unwrap(); + let file_name = DentryName::new(b"artifact".to_vec()).unwrap(); + let source = service + .create_file(old_parent.attr.inode, file_name.clone(), 0o644, 1000, 1000) + .unwrap(); + let old_cursor = service.watch_subtree(old_parent.attr.inode).unwrap(); + let new_cursor = service.watch_subtree(new_parent.attr.inode).unwrap(); + + service + .rename( + old_parent.attr.inode, + &file_name, + new_parent.attr.inode, + file_name.clone(), + ) + .unwrap(); + + let old_events = service + .replay_watch(old_parent.attr.inode, old_cursor, 100) + .unwrap(); + assert_eq!(old_events.len(), 1); + assert_eq!(old_events[0].event.kind, WatchEventKind::Remove); + assert_eq!(old_events[0].event.inode, source.attr.inode); + + let new_events = service + .replay_watch(new_parent.attr.inode, new_cursor, 100) + .unwrap(); + assert_eq!(new_events.len(), 1); + assert_eq!(new_events[0].event.kind, WatchEventKind::Rename); + assert_eq!(new_events[0].event.name, Some(file_name)); + assert_eq!(new_events[0].event.inode, source.attr.inode); +} + +#[test] +fn watch_replay_survives_service_reopen() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); + let name = DentryName::new(b"runs".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), name.clone(), 0o755, 1000, 1000) + .unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let events = reopened.replay_watch(InodeId::root(), cursor, 100).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].event.kind, WatchEventKind::Create); + assert_eq!(events[0].event.name, Some(name)); + assert_eq!(events[0].event.inode, created.attr.inode); +} + +#[test] +fn open_existing_recovers_inode_and_version_allocators() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let first = service + .create_dir( + InodeId::root(), + DentryName::new(b"first".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let second = reopened + .create_dir( + InodeId::root(), + DentryName::new(b"second".to_vec()).unwrap(), + 0o755, + 1000, + 1000, + ) + .unwrap(); + assert!(second.attr.inode > first.attr.inode); + assert!(second.attr.generation > first.attr.generation); +} + +#[test] +fn refresh_allocator_state_advances_live_read_version_after_external_commit() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let original = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + original.bootstrap_root(0o755, 1000, 1000).unwrap(); + + let external = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let external_file = external + .create_file_path("/external.bin", 0o644, 1000, 1000) + .unwrap(); + assert!(original.stat_path("/external.bin").unwrap().is_none()); + + original.refresh_allocator_state().unwrap(); + let visible = original + .stat_path("/external.bin") + .unwrap() + .expect("external commit should be visible after refresh"); + assert_eq!(visible.attr, external_file.attr); + let local_file = original + .create_file_path("/after-refresh.bin", 0o644, 1000, 1000) + .unwrap(); + assert!(local_file.attr.inode > external_file.attr.inode); + assert!(local_file.attr.generation > external_file.attr.generation); +} + +#[test] +fn open_existing_recovers_after_dentry_only_rename() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let old_name = DentryName::new(b"old".to_vec()).unwrap(); + let new_name = DentryName::new(b"new".to_vec()).unwrap(); + let created = service + .create_dir(InodeId::root(), old_name.clone(), 0o755, 1000, 1000) + .unwrap(); + let renamed = service + .rename( + InodeId::root(), + &old_name, + InodeId::root(), + new_name.clone(), + ) + .unwrap(); + assert_eq!(renamed.attr.inode, created.attr.inode); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + assert!(reopened + .lookup_plus(InodeId::root(), &old_name) + .unwrap() + .is_none()); + assert_eq!( + reopened.lookup_plus(InodeId::root(), &new_name).unwrap(), + Some(renamed) + ); + assert_eq!(reopened.read_dir_plus(InodeId::root()).unwrap().len(), 1); +} + +#[test] +fn open_existing_does_not_reuse_removed_inode() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let first_name = DentryName::new(b"first".to_vec()).unwrap(); + let second_name = DentryName::new(b"second".to_vec()).unwrap(); + let first = service + .create_file(InodeId::root(), first_name.clone(), 0o644, 1000, 1000) + .unwrap(); + service.remove_file(InodeId::root(), &first_name).unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let second = reopened + .create_file(InodeId::root(), second_name.clone(), 0o644, 1000, 1000) + .unwrap(); + assert!(second.attr.inode > first.attr.inode); + assert!(second.attr.generation > first.attr.generation); + assert!(reopened + .lookup_plus(InodeId::root(), &first_name) + .unwrap() + .is_none()); + assert_eq!( + reopened.lookup_plus(InodeId::root(), &second_name).unwrap(), + Some(second) + ); +} + +#[test] +fn pending_object_gc_survives_service_reopen() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "artifact.bin", b"old")) + .unwrap(); + let body = published.body.clone().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file(InodeId::root(), &name).unwrap(); + drop(service); + + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects.clone(), 0).unwrap(); + let cleanup = reopened.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + assert_eq!(cleanup.records_removed, 1); + assert!(objects.head(&object).unwrap().is_none()); +} + +#[test] +fn snapshot_pin_survives_service_reopen() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let snapshot = service.snapshot_subtree(InodeId::root()).unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + assert_eq!( + reopened.snapshot_pin(snapshot.snapshot_id).unwrap(), + Some(snapshot) + ); + assert_eq!(reopened.metadata_store_stats().active_snapshot_pin_total, 1); +} + +#[test] +fn failed_publish_returns_staged_objects_for_cleanup_and_does_not_reuse_identity() { + let dir = tempfile::tempdir().unwrap(); + let objects = MemoryObjectStore::new(); + let metadata = HoltMetadataStore::open_file(dir.path().join("meta")).unwrap(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = DentryName::new(b"artifact.bin".to_vec()).unwrap(); + let first = service + .publish_artifact(artifact_request(name.clone(), "first", b"first")) + .unwrap(); + let err = service + .publish_artifact(artifact_request(name.clone(), "duplicate", b"duplicate")) + .unwrap_err(); + let staged = match err { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::Metadata(MetadataError::PredicateFailed) + )); + staged + } + err => panic!("unexpected publish error: {err:?}"), + }; + assert_eq!(staged.len(), 1); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_some()); + } + assert_eq!( + service.lookup_plus(InodeId::root(), &name).unwrap(), + Some(first.clone()) + ); + + let cleanup = service.cleanup_staged_objects(&staged).unwrap(); + assert_eq!(cleanup.attempted, staged.len()); + assert_eq!(cleanup.deleted, staged.len()); + assert_eq!(cleanup.missing, 0); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_none()); + } + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let next_name = DentryName::new(b"next.bin".to_vec()).unwrap(); + let next = reopened + .publish_artifact(artifact_request(next_name, "next", b"next")) + .unwrap(); + + assert!(next.attr.inode.get() > first.attr.inode.get() + 1); + assert!(next.attr.generation > first.attr.generation + 1); +} + +fn dname(raw: &[u8]) -> DentryName { + DentryName::new(raw.to_vec()).unwrap() +} + +fn block_count_for(objects: &MemoryObjectStore, inode: InodeId, generation: u64) -> usize { + // Count the published blocks the base file owns under its (inode, generation). + let mut count = 0; + let mut block = 0; + while objects + .head(&block_key(inode, generation, 0, block)) + .unwrap() + .is_some() + { + count += 1; + block += 1; + } + count +} + +#[test] +fn clone_subtree_shares_base_blocks_diverges_on_write_and_keeps_gc_safe() { + let (service, objects) = service_with_objects(); + // 1. Base namespace: /base with files a ("AAA..") and b ("BBB.."). + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let a = publish_path_artifact(&service, "/base/a", "base/a", &vec![b'A'; 4096]); + let b = publish_path_artifact(&service, "/base/b", "base/b", &vec![b'B'; 4096]); + let a_gen = a.body.as_ref().unwrap().generation; + let b_gen = b.body.as_ref().unwrap().generation; + let a_block = block_key(a.attr.inode, a_gen, 0, 0); + let b_block = block_key(b.attr.inode, b_gen, 0, 0); + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + let objects_after_base = objects.object_count(); + + // 2. Writable O(1)-ish fork of /base. + let fork = service.clone_subtree_path("/base").unwrap(); + assert_ne!(fork.root, base.attr.inode); + + // 3. Sharing: the fork sees the base content, with NO duplicate blocks. + let fork_a = service + .lookup_plus(fork.root, &dname(b"a")) + .unwrap() + .unwrap(); + let fork_b = service + .lookup_plus(fork.root, &dname(b"b")) + .unwrap() + .unwrap(); + assert_ne!( + fork_a.attr.inode, a.attr.inode, + "fork must use a fresh inode" + ); + // Shared files keep the source's content generation (the CoW sharing signal). + assert_eq!(fork_a.attr.generation, a_gen); + assert_eq!(fork_b.attr.generation, b_gen); + assert_eq!(fork_b.body.as_ref().unwrap().generation, b_gen); + assert_eq!( + service.read_artifact(fork.root, &dname(b"a")).unwrap(), + vec![b'A'; 4096] + ); + assert_eq!( + service.read_artifact(fork.root, &dname(b"b")).unwrap(), + vec![b'B'; 4096] + ); + // Zero-copy: clone added metadata only, not object blocks. + assert_eq!( + objects.object_count(), + objects_after_base, + "clone must share base blocks, not copy them" + ); + // The fork's a/b manifests reference the SAME object keys as the base. + assert_eq!( + service + .read_file_plan(fork_a.attr.inode, fork_a.attr.generation, 0, 4096) + .unwrap() + .blocks[0] + .object_key, + a_block.as_str() + ); + + // 4. Divergence: rewrite a in the fork and add a new file c. + service + .replace_artifact(PublishArtifact { + parent: fork.root, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:zzz".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork/a".to_owned(), + bytes: vec![b'Z'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + service + .publish_artifact(PublishArtifact { + parent: fork.root, + name: dname(b"c"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:ccc".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork/c".to_owned(), + bytes: vec![b'C'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + + // 4a. Fork now sees a="ZZZ..", b="BBB..", c present. + assert_eq!( + service.read_artifact(fork.root, &dname(b"a")).unwrap(), + vec![b'Z'; 4096] + ); + assert_eq!( + service.read_artifact(fork.root, &dname(b"b")).unwrap(), + vec![b'B'; 4096] + ); + assert_eq!( + service.read_artifact(fork.root, &dname(b"c")).unwrap(), + vec![b'C'; 4096] + ); + // 4b. Base is unchanged: a="AAA..", no c. + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"a")) + .unwrap(), + vec![b'A'; 4096] + ); + assert!(service + .lookup_plus(base.attr.inode, &dname(b"c")) + .unwrap() + .is_none()); + + // 6. Diff reports exactly { modified: a, added: c }; b (shared) is not reported. + let mut diff = service.diff_subtrees(base.attr.inode, fork.root).unwrap(); + diff.sort_by(|left, right| left.path.cmp(&right.path)); + let summary: Vec<(&str, SubtreeDeltaKind)> = + diff.iter().map(|d| (d.path.as_str(), d.kind)).collect(); + assert_eq!( + summary, + vec![ + ("/a", SubtreeDeltaKind::Modified), + ("/c", SubtreeDeltaKind::Added), + ] + ); + // The enriched diff carries the changed file's content digest. + assert!(diff + .iter() + .find(|d| d.path == "/a") + .unwrap() + .digest + .is_some()); + + // 5. GC safety: reclaim must NOT touch base blocks the fork's divergent write + // abandoned but the base still references; they are owned by the base inode and + // protected by the fork's retained snapshot pin. + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(reclaim.deleted, 0, "no base block may be reclaimed yet"); + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"a")) + .unwrap(), + vec![b'A'; 4096] + ); + + // Drop the fork: remove its files and retire its snapshot pin. The fork-only + // blocks (the divergent a' and the new c) then become reclaimable, while the + // base's blocks remain because the base still references them. + let fork_a_diverged = service + .lookup_plus(fork.root, &dname(b"a")) + .unwrap() + .unwrap(); + let fork_c = service + .lookup_plus(fork.root, &dname(b"c")) + .unwrap() + .unwrap(); + let fork_a_block = block_key( + fork_a_diverged.attr.inode, + fork_a_diverged.body.as_ref().unwrap().generation, + 0, + 0, + ); + let fork_c_block = block_key( + fork_c.attr.inode, + fork_c.body.as_ref().unwrap().generation, + 0, + 0, + ); + service.remove_file(fork.root, &dname(b"a")).unwrap(); + service.remove_file(fork.root, &dname(b"b")).unwrap(); + service.remove_file(fork.root, &dname(b"c")).unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert!(reclaim.deleted >= 2, "fork-only blocks must be reclaimable"); + assert!(objects.head(&fork_a_block).unwrap().is_none()); + assert!(objects.head(&fork_c_block).unwrap().is_none()); + // Base remains fully intact and readable. + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"a")) + .unwrap(), + vec![b'A'; 4096] + ); + assert_eq!( + service + .read_artifact(base.attr.inode, &dname(b"b")) + .unwrap(), + vec![b'B'; 4096] + ); + assert_eq!(block_count_for(&objects, a.attr.inode, a_gen), 1); +} + +#[test] +fn clone_subtree_copies_nested_dirs_and_diff_reports_removed() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/base/sub", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact(&service, "/base/sub/deep", "base/deep", b"deep-bytes"); + publish_path_artifact(&service, "/base/top", "base/top", b"top-bytes"); + + let fork = service.clone_subtree_path("/base").unwrap(); + // Nested structure is reproduced under fresh inodes. + let sub = service + .lookup_plus(fork.root, &dname(b"sub")) + .unwrap() + .unwrap(); + assert_eq!(sub.attr.file_type, FileType::Directory); + assert_eq!( + service + .read_artifact(sub.attr.inode, &dname(b"deep")) + .unwrap(), + b"deep-bytes" + ); + + // Identical subtree => no deltas. + let base = service.resolve_directory_path("/base").unwrap(); + assert!(service.diff_subtrees(base, fork.root).unwrap().is_empty()); + + // Remove a nested file in the fork => Removed delta at the nested path, + // direction base -> fork. + service + .remove_file(sub.attr.inode, &dname(b"deep")) + .unwrap(); + let removed = service.diff_subtrees(base, fork.root).unwrap(); + assert_eq!(removed.len(), 1); + assert_eq!(removed[0].path, "/sub/deep"); + assert_eq!(removed[0].kind, SubtreeDeltaKind::Removed); + assert_eq!(removed[0].size_delta, -(b"deep-bytes".len() as i64)); + // Reversed direction reports it as Added, with the net size flipped. + let added = service.diff_subtrees(fork.root, base).unwrap(); + assert_eq!(added.len(), 1); + assert_eq!(added[0].path, "/sub/deep"); + assert_eq!(added[0].kind, SubtreeDeltaKind::Added); + assert_eq!(added[0].size_delta, b"deep-bytes".len() as i64); +} + +#[test] +fn clone_subtree_path_rejects_non_directory() { + let service = service(); + publish_path_artifact(&service, "/file.bin", "file", b"bytes"); + assert!(matches!( + service.clone_subtree_path("/file.bin"), + Err(MetadError::NotDirectory) + )); +} + +#[test] +fn clone_link_rejects_an_expired_reaped_pin_before_exposing_the_fork() { + let store = SnapshotCommitBarrierStore::new(CommandKind::CreateDir, 1, 2) + .matching_request_prefix(b"clone-subtree-link"); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(PublishArtifact { + parent: base.attr.inode, + name: dname(b"data.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:clone-link-race".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "clone-link-race".to_owned(), + bytes: b"payload".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + + let clone_service = Arc::clone(&service); + let clone = std::thread::spawn(move || clone_service.clone_subtree_path_into("/base", "/fork")); + store.wait_until_blocked(); + assert!(service.object_gc_gate.try_lock().is_err()); + + let pins = store + .scan(ScanRequest { + family: RecordFamily::Snapshot, + prefix: snapshot_pin_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap(); + assert_eq!(pins.len(), 1); + let pin = decode_snapshot_pin(&pins[0].value.0).unwrap(); + + service + .remove_file(base.attr.inode, &dname(b"data.bin")) + .unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + + let cleanup_started = Arc::new(Barrier::new(2)); + let cleanup_service = Arc::clone(&service); + let cleanup_thread_started = Arc::clone(&cleanup_started); + let cleanup = std::thread::spawn(move || { + cleanup_thread_started.wait(); + cleanup_service.cleanup_pending_objects(100) + }); + cleanup_started.wait(); + store.release_blocked(); + + assert!(matches!( + clone.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + let cleanup = cleanup.join().unwrap().unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&object).unwrap().is_none()); + assert!(service.lookup_path("/fork").unwrap().is_none()); +} + +#[test] +fn fork_binding_survives_pin_reaping_and_a_hardlink_escaping_the_fork_root() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-retention/source", + b"shared bytes", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + service + .link(fork_file.attr.inode, InodeId::root(), dname(b"escaped.bin")) + .unwrap(); + service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + service.remove_empty_dir_path("/fork").unwrap(); + + // The source can stop naming its object and the construction pin can expire; + // neither event retires the durable binding. The escaped hardlink remains a + // current borrowed reference even though the fork root itself is gone. + service.remove_file_path("/base/data.bin").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + assert!(service.snapshot_pin(fork.snapshot_id).unwrap().is_none()); + assert_eq!( + service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: crate::layout::fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(cleanup.blocked_by_snapshots >= 1); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(), + b"shared bytes" + ); + + let error = service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap_err(); + assert!( + matches!( + error, + MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + } if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_file.attr.inode + ), + "unexpected retirement error: {error:?}" + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(), + b"shared bytes" + ); + + // Retirement becomes valid after every borrowed fork reference, including + // links outside the original fork root, has been removed. + service + .remove_file(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(); + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_none()); +} + +#[test] +fn detached_fork_binding_is_a_legal_link_and_rename_root() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "detached-namespace/source", + b"shared bytes", + ); + let fork = service.clone_subtree_path("/base").unwrap(); + assert!( + !service.materialization_orphan_slow_path_enabled(), + "a durable detached ForkBinding must restore the healthy fast path" + ); + let reachability_scans = service.namespace_reachability_scan_count(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + + let renamed = service + .rename( + fork.root, + &dname(b"data.bin"), + fork.root, + dname(b"renamed.bin"), + ) + .unwrap(); + assert_eq!(renamed.attr.inode, fork_file.attr.inode); + let escaped = service + .link(fork_file.attr.inode, InodeId::root(), dname(b"escaped.bin")) + .unwrap(); + assert_eq!(escaped.attr.inode, fork_file.attr.inode); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"escaped.bin")) + .unwrap(), + b"shared bytes" + ); + assert_eq!( + service.namespace_reachability_scan_count(), + reachability_scans, + "healthy detached-root rename/link must not scan namespace reachability" + ); +} + +#[test] +fn reopen_recovers_an_unbound_materialization_into_the_slow_path() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "reopen-orphan/source", + b"borrowed", + ); + let pin = service.snapshot_subtree(base.attr.inode).unwrap(); + let orphan_root = { + let _object_gc_gate = service.object_gc_gate.lock().unwrap(); + service + .materialize_subtree_at(base.attr.inode, Version::new(pin.read_version).unwrap()) + .unwrap() + }; + let orphan = service + .lookup_plus(orphan_root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + assert!(service.materialization_orphan_slow_path_enabled()); + + let reopened = + NoKvFs::open_existing(service.mount, metadata, objects, service.shard_index()).unwrap(); + assert!( + reopened.materialization_orphan_slow_path_enabled(), + "reopen must recover an unbound detached tree before serving" + ); + assert_eq!(reopened.namespace_reachability_scan_count(), 1); + assert!(matches!( + reopened.link(orphan.attr.inode, InodeId::root(), dname(b"reopened-link")), + Err(MetadError::NotFound) + )); + assert!(matches!( + reopened.rename( + orphan_root, + &dname(b"data.bin"), + InodeId::root(), + dname(b"reopened-rename") + ), + Err(MetadError::NotFound) + )); + assert!(reopened + .lookup_plus(InodeId::root(), &dname(b"reopened-link")) + .unwrap() + .is_none()); + assert!(reopened + .lookup_plus(InodeId::root(), &dname(b"reopened-rename")) + .unwrap() + .is_none()); +} + +#[test] +fn open_existing_allows_empty_namespace_until_bootstrap_then_uses_fast_path() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + + assert!( + service.materialization_orphan_slow_path_enabled(), + "an empty namespace remains fail-closed until its root is bootstrapped" + ); + assert_eq!(service.namespace_reachability_scan_count(), 0); + + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + assert!( + !service.materialization_orphan_slow_path_enabled(), + "root bootstrap must prove the pristine namespace healthy" + ); + assert_eq!(service.namespace_reachability_scan_count(), 1); + + let source = service + .create_file(InodeId::root(), dname(b"source.bin"), 0o644, 1000, 1000) + .unwrap(); + let scans_after_bootstrap = service.namespace_reachability_scan_count(); + service + .link(source.attr.inode, InodeId::root(), dname(b"linked.bin")) + .unwrap(); + assert_eq!( + service.namespace_reachability_scan_count(), + scans_after_bootstrap, + "healthy link after bootstrap must not run namespace reachability" + ); +} + +#[test] +fn open_existing_rejects_missing_root_with_unbound_materialization_records() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let writer = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + let orphan = InodeId::new(InodeId::ROOT_RAW + 1).unwrap(); + let version = writer.next_version().unwrap(); + let attr = directory_attr(orphan, 0o755, 1000, 1000, version.get()); + writer + .commit_metadata(MetadataCommand { + request_id: request_id(b"test-rootless-orphan", writer.mount, orphan, version), + kind: CommandKind::CreateDir, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Inode, + primary_key: inode_key(writer.mount, orphan), + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::Inode, + key: inode_key(writer.mount, orphan), + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&attr))), + }], + watch: Vec::new(), + }) + .unwrap(); + + let reopened = NoKvFs::open_existing(writer.mount, metadata, objects, writer.shard_index()); + assert!(matches!( + reopened, + Err(MetadError::Codec(message)) + if message == "mount root is missing while namespace records still exist" + )); +} + +#[test] +fn fork_binding_retains_object_backed_symlink_after_source_deletion() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = service + .create_symlink( + base.attr.inode, + dname(b"latest"), + b"runs/42/checkpoint.bin".to_vec(), + 0o777, + 1000, + 1000, + ) + .unwrap(); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let fork_symlink = service + .lookup_plus(fork.root, &dname(b"latest")) + .unwrap() + .unwrap(); + + service.remove_file_path("/base/latest").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service.read_symlink(fork_symlink.attr.inode).unwrap(), + b"runs/42/checkpoint.bin" + ); + + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_symlink.attr.inode + )); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service.read_symlink(fork_symlink.attr.inode).unwrap(), + b"runs/42/checkpoint.bin" + ); + + service.remove_file(fork.root, &dname(b"latest")).unwrap(); + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + assert!(objects.head(&source_block).unwrap().is_none()); +} + +#[test] +fn fork_binding_can_retire_after_the_fork_rewrites_onto_self_owned_blocks() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-rewrite/source", + b"borrowed bytes", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + + let rewritten = service + .replace_artifact(PublishArtifact { + parent: fork.root, + name: dname(b"data.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:fork-rewrite".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork-rewrite/self-owned".to_owned(), + bytes: b"self-owned bytes".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let rewritten_body = rewritten.entry.body.as_ref().unwrap(); + let rewritten_block = block_key(rewritten.entry.attr.inode, rewritten_body.generation, 0, 0); + service + .create_file( + InodeId::root(), + dname(b"unrelated-empty"), + 0o644, + 1000, + 1000, + ) + .unwrap(); + service.remove_file_path("/base/data.bin").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + assert!(objects.head(&source_block).unwrap().is_none()); + assert!(objects.head(&rewritten_block).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/fork/data.bin"), + b"self-owned bytes" + ); +} + +#[test] +fn unrelated_cross_shard_graft_does_not_block_safe_fork_retirement() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let foreign = InodeId::compose(1, 42).unwrap(); + service + .create_graft( + InodeId::root(), + dname(b"foreign"), + foreign, + 0o755, + 1000, + 1000, + ) + .unwrap(); + + assert!(service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert!(!service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap()); + assert_eq!( + service + .lookup_plus(InodeId::root(), &dname(b"foreign")) + .unwrap() + .unwrap() + .attr + .inode, + foreign + ); +} + +#[test] +fn fork_binding_cannot_retire_when_append_still_inherits_borrowed_blocks() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-append/source", + b"borrowed base", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + + let prepared = service + .prepare_artifact_replace(fork.root, dname(b"data.bin")) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "fork-append/delta", + &[PublishArtifactRange { + offset: source_body.size, + bytes: b" + self-owned delta".to_vec(), + }], + 0, + ) + .unwrap(); + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: fork.root, + name: dname(b"data.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:fork-append".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "fork-append/delta".to_owned(), + size: source_body.size + b" + self-owned delta".len() as u64, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + + service.remove_file_path("/base/data.bin").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/fork/data.bin"), + b"borrowed base + self-owned delta" + ); + + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_file.attr.inode + )); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/fork/data.bin"), + b"borrowed base + self-owned delta" + ); +} + +#[test] +fn fork_binding_retirement_fails_closed_on_corrupt_dentry_projection_body() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "fork-corruption/source", + b"must remain retained", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path_into("/base", "/fork").unwrap(); + + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + + let dentry = dentry_key(service.mount, fork.root, &dname(b"data.bin")); + let current = service.read_version().unwrap(); + let row = service + .metadata + .get_versioned( + RecordFamily::Dentry, + &dentry, + current, + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let mut projection = decode_dentry_projection(&row.value.0).unwrap(); + projection + .body + .as_mut() + .unwrap() + .manifest_id + .push_str("/forged"); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"inject-corrupt-dentry-projection", + service.mount, + projection.attr.inode, + version, + ), + kind: CommandKind::UpdateAttr, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::VersionEquals(row.version), + }], + mutations: vec![Mutation { + family: RecordFamily::Dentry, + key: dentry, + op: MutationOp::Put, + value: Some(Value(encode_dentry_projection(&projection))), + }], + watch: Vec::new(), + }) + .unwrap(); + service.remove_file_path("/base/data.bin").unwrap(); + + let error = service + .retire_snapshot_path("/base", fork.snapshot_id) + .unwrap_err(); + assert!( + matches!(&error, MetadError::Codec(message) if message.contains("body descriptor")), + "unexpected retirement error: {error:?}" + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&source_block).unwrap().is_some()); +} + +#[test] +fn detached_fork_binding_protects_after_source_deletion_and_pin_reaping() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact( + &service, + "/base/data.bin", + "detached-retention/source", + b"detached bytes", + ); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let fork = service.clone_subtree_path("/base").unwrap(); + + service.remove_file_path("/base/data.bin").unwrap(); + service.remove_empty_dir_path("/base").unwrap(); + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!( + service + .read_artifact(fork.root, &dname(b"data.bin")) + .unwrap(), + b"detached bytes" + ); + + // A deleted source path cannot accidentally release the retention root. + // The unbound service primitive remains able to retire it after the detached + // fork has dropped its last borrowed reference. + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::NotFound) + )); + service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_none()); +} + +#[test] +fn fork_binding_can_be_retired_through_the_sources_new_path_after_rename() { + let (service, _objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "renamed-retention/source", + b"renamed bytes", + ); + let fork = service.clone_subtree_path("/base").unwrap(); + service + .rename_path("/base", "/renamed") + .expect("source directory rename"); + service.remove_file(fork.root, &dname(b"data.bin")).unwrap(); + + let pin = service.snapshot_pin(fork.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + assert!(matches!( + service.retire_snapshot_path("/base", fork.snapshot_id), + Err(MetadError::NotFound) + )); + assert!(service + .retire_snapshot_path("/renamed", fork.snapshot_id) + .unwrap()); + assert!(!service.retire_snapshot(fork.snapshot_id).unwrap()); +} + +fn read_artifact_at_path( + service: &NoKvFs, + path: &str, +) -> Vec { + let (parent, name) = service.resolve_parent_path(path).unwrap(); + service.read_artifact(parent, &name).unwrap() +} + +#[test] +fn failed_rollback_orphan_does_not_block_unrelated_fork_retirement() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenameReplace, 1, 2) + .matching_request_prefix(b"rollback-subtree-swap"); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .publish_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a1".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-orphan/a1".to_owned(), + bytes: b"A1".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let ws_inode = ws.attr.inode; + let snapshot_id = snapshot.snapshot_id; + + let rollback_service = Arc::clone(&service); + let rollback = + std::thread::spawn(move || rollback_service.rollback_subtree(ws_inode, snapshot_id)); + store.wait_until_blocked(); + assert!( + service.materialization_orphan_slow_path_enabled(), + "materialization must enter slow mode before its detached tree can commit" + ); + + // Change a swap-guarded dentry after rollback materialized its detached tree. + // The graft must fail, leaving that tree unreachable and without a binding. + // The rollback commit is deliberately paused while the service-local + // commit-to-log ordering gate is held. Model the conflicting write from a + // separately recovered owner so it can reach the shared store and invalidate + // the already-planned swap without deadlocking on that process-local gate. + let concurrent = + NoKvFs::open_existing(service.mount, store.clone(), objects, service.shard_index()) + .unwrap(); + concurrent + .replace_artifact(PublishArtifact { + parent: ws_inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-orphan/a2".to_owned(), + bytes: b"A2".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + store.release_blocked(); + assert!(matches!( + rollback.join().unwrap(), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + service.refresh_allocator_state().unwrap(); + assert_eq!( + service.read_artifact(ws_inode, &dname(b"a")).unwrap(), + b"A2" + ); + assert!(service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .iter() + .all(|binding| binding.binding.snapshot_id != snapshot_id)); + + // A real detached clone remains a retention root through its binding. Once + // its borrower is removed, the failed rollback's unbound orphan must not keep + // this otherwise-unrelated binding alive forever. + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + service + .publish_artifact(PublishArtifact { + parent: base.attr.inode, + name: dname(b"data"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:base".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-orphan/base".to_owned(), + bytes: b"base".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let fork = service.clone_subtree_path("/base").unwrap(); + let fork_file = service + .lookup_plus(fork.root, &dname(b"data")) + .unwrap() + .unwrap(); + assert!(matches!( + service.retire_snapshot(fork.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == fork.snapshot_id + && fork_root == fork.root + && borrower == fork_file.attr.inode + )); + service.remove_file(fork.root, &dname(b"data")).unwrap(); + assert!(service.retire_snapshot(fork.snapshot_id).unwrap()); +} + +#[test] +fn failed_rollback_orphan_cannot_race_retirement_or_be_resurrected() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RetireSnapshot, 1, 2) + .rejecting(CommandKind::RenameReplace); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = service + .publish_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:orphan-a1".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-resurrection/a1".to_owned(), + bytes: b"A1".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let original_body = original.body.as_ref().unwrap(); + let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:orphan-a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-resurrection/a2".to_owned(), + bytes: b"A2".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + + // Reject only the atomic graft. All prior materialization commits remain, + // leaving an inode + dentry tree with neither a mount path nor ForkBinding. + assert!(matches!( + service.rollback_subtree(ws.attr.inode, snapshot.snapshot_id), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + assert!( + service.materialization_orphan_slow_path_enabled(), + "failed materialization must leave inode exposure fail-closed" + ); + let (orphan_row, orphan) = service + .metadata + .scan(ScanRequest { + family: RecordFamily::Dentry, + prefix: dentry_mount_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .into_iter() + .find_map(|row| { + let projection = decode_dentry_projection(&row.value.0).unwrap(); + projection + .body + .as_ref() + .is_some_and(|body| body.manifest_id == "rollback-resurrection/a1") + .then_some((row, projection)) + }) + .expect("failed rollback leaves its materialized child"); + let orphan_parent = orphan.dentry.parent; + let orphan_inode = orphan.attr.inode; + let orphan_name = orphan.dentry.name.clone(); + assert_ne!(orphan_parent, ws.attr.inode); + assert!(service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .iter() + .all(|binding| binding.binding.snapshot_id != snapshot.snapshot_id)); + + // Hold an unrelated retirement after its mount-wide proof but before its + // binding CAS. A racing hardlink must wait on object_gc_gate; after the CAS + // removes the last legal detached root, it must re-prove and reject the + // unbound rollback orphan instead of exposing it under the mount root. + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + service.as_ref(), + "/base/data", + "rollback-resurrection/base", + b"base", + ); + let fork = service.clone_subtree_path("/base").unwrap(); + service.remove_file(fork.root, &dname(b"data")).unwrap(); + + let retire_service = Arc::clone(&service); + let retire = std::thread::spawn(move || retire_service.retire_snapshot(fork.snapshot_id)); + store.wait_until_blocked(); + assert!(service.object_gc_gate.try_lock().is_err()); + + let (link_tx, link_rx) = std::sync::mpsc::channel(); + let link_service = Arc::clone(&service); + let racing_link = std::thread::spawn(move || { + let result = + link_service.link(orphan_inode, InodeId::root(), dname(b"racing-resurrection")); + link_tx.send(result).unwrap(); + }); + let early_link = link_rx.recv_timeout(Duration::from_millis(100)); + store.release_blocked(); + assert!(retire.join().unwrap().unwrap()); + let link_result = match early_link { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => link_rx.recv().unwrap(), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + panic!("racing hardlink thread disconnected") + } + }; + racing_link.join().unwrap(); + assert!( + matches!(link_result, Err(MetadError::NotFound)), + "unbound orphan hardlink result: {link_result:?}" + ); + assert!(service + .lookup_plus(InodeId::root(), &dname(b"racing-resurrection")) + .unwrap() + .is_none()); + + // Release the failed rollback's construction snapshot as well, allowing A1 + // to be reclaimed. The orphan metadata is now a dangling borrower; neither + // inode-addressed operation may make it reachable again. + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert!(cleanup.deleted >= 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); + + let orphan_inode_key = inode_key(service.mount, orphan_inode); + let before_inode = service + .metadata + .get_versioned( + RecordFamily::Inode, + &orphan_inode_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert!(matches!( + service.link( + orphan_inode, + InodeId::root(), + dname(b"hardlink-resurrection") + ), + Err(MetadError::NotFound) + )); + assert!(matches!( + service.rename( + orphan_parent, + &orphan_name, + InodeId::root(), + dname(b"rename-resurrection") + ), + Err(MetadError::NotFound) + )); + + let live = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + assert!(matches!( + service.link( + live.attr.inode, + orphan_parent, + dname(b"orphan-destination-link") + ), + Err(MetadError::NotFound) + )); + assert!(matches!( + service.rename( + ws.attr.inode, + &dname(b"a"), + orphan_parent, + dname(b"orphan-destination-rename") + ), + Err(MetadError::NotFound) + )); + + let after_dentry = service + .metadata + .get_versioned( + RecordFamily::Dentry, + &orphan_row.key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let after_inode = service + .metadata + .get_versioned( + RecordFamily::Inode, + &orphan_inode_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + assert_eq!(after_dentry.version, orphan_row.version); + assert_eq!(after_dentry.value, orphan_row.value); + assert_eq!(after_inode.version, before_inode.version); + assert_eq!(after_inode.value, before_inode.value); + assert!(service + .lookup_plus(InodeId::root(), &dname(b"hardlink-resurrection")) + .unwrap() + .is_none()); + assert!(service + .lookup_plus(InodeId::root(), &dname(b"rename-resurrection")) + .unwrap() + .is_none()); + assert!(service + .lookup_plus(orphan_parent, &dname(b"orphan-destination-link")) + .unwrap() + .is_none()); + assert!(service + .lookup_plus(orphan_parent, &dname(b"orphan-destination-rename")) + .unwrap() + .is_none()); + assert!(objects.head(&original_block).unwrap().is_none()); + assert_eq!( + service.read_artifact(ws.attr.inode, &dname(b"a")).unwrap(), + b"A2" + ); +} + +#[test] +fn rollback_binding_survives_pin_reaping_without_an_auxiliary_clone() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = publish_path_artifact(&service, "/ws/a", "rollback-hold/a1", b"A1"); + let original_body = original.body.as_ref().unwrap(); + let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); + let snap = service.snapshot_subtree_path("/ws").unwrap(); + let diverged = service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-hold/a2".to_owned(), + bytes: b"A2".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap() + .entry; + let diverged_block = block_key( + diverged.attr.inode, + diverged.body.as_ref().unwrap().generation, + 0, + 0, + ); + + service + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap(); + let restored = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + assert_ne!(restored.attr.inode, original.attr.inode); + let binding = service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .into_iter() + .find(|binding| binding.binding.snapshot_id == snap.snapshot_id) + .expect("rollback installs durable retention"); + assert_eq!(binding.binding.source_root, ws.attr.inode); + assert_ne!(binding.binding.fork_root, ws.attr.inode); + assert!(service + .get_attr(binding.binding.fork_root) + .unwrap() + .is_none()); + + let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(cleanup.blocked_by_snapshots >= 1); + assert!(objects.head(&original_block).unwrap().is_some()); + assert!(objects.head(&diverged_block).unwrap().is_some()); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1"); + assert!(matches!( + service.retire_snapshot_path("/ws", snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) if snapshot_id == snap.snapshot_id + && fork_root == binding.binding.fork_root + && borrower == restored.attr.inode + )); + + let current = service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a3".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-hold/a3".to_owned(), + bytes: b"A3".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap() + .entry; + let current_block = block_key( + current.attr.inode, + current.body.as_ref().unwrap().generation, + 0, + 0, + ); + assert!(service + .retire_snapshot_path("/ws", snap.snapshot_id) + .unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert!(cleanup.deleted >= 2, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); + assert!(objects.head(&diverged_block).unwrap().is_none()); + assert!(objects.head(¤t_block).unwrap().is_some()); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A3"); +} + +#[test] +fn rollback_binding_protects_an_owner_gc_row_enqueued_after_the_swap() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let source = publish_path_artifact(&service, "/ws/a", "rollback-late-owner", b"shared"); + let source_body = source.body.as_ref().unwrap(); + let source_block = block_key(source.attr.inode, source_body.generation, 0, 0); + let snap = service.snapshot_subtree_path("/ws").unwrap(); + service + .link(source.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + assert_eq!(service.cleanup_pending_objects(100).unwrap().attempted, 0); + + service + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap(); + let restored = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + + service + .remove_file(InodeId::root(), &dname(b"outside")) + .unwrap(); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 0, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&source_block).unwrap().is_some()); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"shared"); + assert!(matches!( + service.retire_snapshot_path("/ws", snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + borrower, + .. + }) if snapshot_id == snap.snapshot_id && borrower == restored.attr.inode + )); + + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:self-owned".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-late-owner/self-owned".to_owned(), + bytes: b"self-owned".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + assert!(service + .retire_snapshot_path("/ws", snap.snapshot_id) + .unwrap()); + assert!(service.cleanup_pending_objects(100).unwrap().deleted >= 1); + assert!(objects.head(&source_block).unwrap().is_none()); +} + +#[test] +fn rollback_enqueues_a_missing_gc_row_for_a_restored_delta_base() { + let (service, objects) = service_with_objects(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = publish_path_artifact(&service, "/ws/a", "rollback-delta/a1", b"A1"); + let original_body = original.body.as_ref().unwrap(); + let original_generation = original_body.generation; + let original_block = block_key(original.attr.inode, original_generation, 0, 0); + let snap = service.snapshot_subtree_path("/ws").unwrap(); + + let prepared = service + .prepare_artifact_replace(ws.attr.inode, dname(b"a")) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "rollback-delta/a2", + &[PublishArtifactRange { + offset: original_body.size, + bytes: b"-delta".to_vec(), + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + let chunks = written.chunk_manifests(); + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-delta/a2".to_owned(), + size: original_body.size + 6, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + let appended = service.lookup_path("/ws/a").unwrap().unwrap(); + assert_eq!( + appended.body.as_ref().unwrap().base_generation, + original_generation + ); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1-delta"); + + service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + let gc_records = || { + service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_object_gc_record(&row.value.0).unwrap()) + .collect::>() + }; + assert!( + gc_records() + .iter() + .all(|record| record.object_key != original_block.as_str()), + "deleting a delta top alone does not discover its historical base object" + ); + + service + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap(); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"A1"); + assert!( + gc_records() + .iter() + .any(|record| record.object_key == original_block.as_str()), + "rollback must proactively make every restored owner object reclaimable" + ); + + let pin = service.snapshot_pin(snap.snapshot_id).unwrap().unwrap(); + service.set_clock_override_ms(pin.lease_expires_unix_ms); + assert_eq!( + service.reclaim_expired_snapshot_pins(100).unwrap().reaped, + 1 + ); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 0); + assert!(objects.head(&original_block).unwrap().is_some()); + + service.remove_file(ws.attr.inode, &dname(b"a")).unwrap(); + assert!(service + .retire_snapshot_path("/ws", snap.snapshot_id) + .unwrap()); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert!(cleanup.deleted >= 2, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); +} + +#[test] +fn rollback_on_a_clone_root_keeps_both_retention_bindings() { + let service = service(); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/base/a", "layered/base", b"base"); + let clone = service.clone_subtree_path_into("/base", "/fork").unwrap(); + let snap = service.snapshot_subtree_path("/fork").unwrap(); + service + .replace_artifact(PublishArtifact { + parent: clone.root, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:diverged".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "layered/diverged".to_owned(), + bytes: b"diverged".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + service + .rollback_subtree_path("/fork", snap.snapshot_id) + .unwrap(); + + let bindings = service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap(); + let clone_binding = bindings + .iter() + .find(|binding| binding.binding.snapshot_id == clone.snapshot_id) + .unwrap(); + let rollback_binding = bindings + .iter() + .find(|binding| binding.binding.snapshot_id == snap.snapshot_id) + .unwrap(); + assert_eq!(clone_binding.binding.fork_root, clone.root); + assert_eq!(rollback_binding.binding.source_root, clone.root); + assert_ne!(rollback_binding.binding.fork_root, clone.root); + assert_ne!(clone_binding.key, rollback_binding.key); + assert!(matches!( + service.retire_snapshot(clone.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + .. + }) if snapshot_id == clone.snapshot_id + )); + assert!(matches!( + service.retire_snapshot(snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + .. + }) if snapshot_id == snap.snapshot_id + )); + assert_eq!(read_artifact_at_path(&service, "/fork/a"), b"base"); +} + +#[test] +fn rollback_rejects_hardlinks_in_snapshot_or_current_tree() { + let snapshot_linked = service(); + snapshot_linked + .create_dir_path("/ws", 0o755, 1000, 1000) + .unwrap(); + let file = publish_path_artifact(&snapshot_linked, "/ws/a", "hardlink-snapshot", b"body"); + snapshot_linked + .link(file.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + let snap = snapshot_linked.snapshot_subtree_path("/ws").unwrap(); + let error = snapshot_linked + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap_err(); + assert!(matches!( + &error, + MetadError::InvalidPath(message) if message.contains("hardlink-free") + )); + assert_eq!( + snapshot_linked + .read_artifact(InodeId::root(), &dname(b"outside")) + .unwrap(), + b"body" + ); + + let current_linked = service(); + let ws = current_linked + .create_dir_path("/ws", 0o755, 1000, 1000) + .unwrap(); + let file = publish_path_artifact(¤t_linked, "/ws/a", "hardlink-current", b"body"); + let snap = current_linked.snapshot_subtree_path("/ws").unwrap(); + current_linked + .link(file.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + let error = current_linked + .rollback_subtree(ws.attr.inode, snap.snapshot_id) + .unwrap_err(); + assert!(matches!( + &error, + MetadError::InvalidPath(message) if message.contains("hardlink-free") + )); + assert_eq!(read_artifact_at_path(¤t_linked, "/ws/a"), b"body"); + assert_eq!( + current_linked + .read_artifact(InodeId::root(), &dname(b"outside")) + .unwrap(), + b"body" + ); +} + +#[test] +fn rollback_subtree_restores_snapshot_shares_blocks_and_reclaims_delta() { + let (service, objects) = service_with_objects(); + // 1. Build /ws with files a="A1", b="B1", sub/c="C1" (real object data). + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/ws/sub", 0o755, 1000, 1000) + .unwrap(); + let a = publish_path_artifact(&service, "/ws/a", "ws/a", &vec![b'1'; 4096]); + let b = publish_path_artifact(&service, "/ws/b", "ws/b", &vec![b'2'; 4096]); + let c = publish_path_artifact(&service, "/ws/sub/c", "ws/sub/c", &vec![b'3'; 4096]); + let a_gen = a.body.as_ref().unwrap().generation; + let b_gen = b.body.as_ref().unwrap().generation; + let c_gen = c.body.as_ref().unwrap().generation; + let a1_block = block_key(a.attr.inode, a_gen, 0, 0); + let b1_block = block_key(b.attr.inode, b_gen, 0, 0); + let c1_block = block_key(c.attr.inode, c_gen, 0, 0); + assert!(objects.head(&a1_block).unwrap().is_some()); + assert!(objects.head(&b1_block).unwrap().is_some()); + assert!(objects.head(&c1_block).unwrap().is_some()); + + // 2. Snapshot /ws. + let snap = service.snapshot_subtree_path("/ws").unwrap(); + + // 3. Diverge /ws: rewrite a->"A2", add d="D1", delete b. + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "ws/a2".to_owned(), + bytes: vec![b'4'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let d = publish_path_artifact(&service, "/ws/d", "ws/d", &vec![b'5'; 4096]); + service.remove_file(ws.attr.inode, &dname(b"b")).unwrap(); + // Capture the delta's private blocks so we can assert their fate. + let a_diverged = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + let a2_block = block_key( + a_diverged.attr.inode, + a_diverged.body.as_ref().unwrap().generation, + 0, + 0, + ); + let d1_block = block_key(d.attr.inode, d.body.as_ref().unwrap().generation, 0, 0); + assert!(objects.head(&a2_block).unwrap().is_some()); + assert!(objects.head(&d1_block).unwrap().is_some()); + // Pre-rollback /ws is the diverged state. + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'4'; 4096]); + assert!(service + .lookup_plus(ws.attr.inode, &dname(b"b")) + .unwrap() + .is_none()); + + // 4. Roll /ws back to the snapshot. + service + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap(); + + // 5. /ws now exactly matches the snapshot: a="A1", b="B1" (restored), sub/c="C1", + // and d is gone. The target keeps its inode identity. + assert_eq!( + service.resolve_directory_path("/ws").unwrap(), + ws.attr.inode, + "rollback keeps the target root's identity" + ); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + assert_eq!( + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'3'; 4096] + ); + assert!( + service + .lookup_plus(ws.attr.inode, &dname(b"d")) + .unwrap() + .is_none(), + "the delta-only file d must be gone after rollback" + ); + + // 6. The rolled-back /ws is identical to a fresh clone of the snapshot: an empty + // diff in both directions. + let reference = service + .clone_subtree_path_into("/ws", "/reference") + .unwrap(); + assert!(service + .diff_subtrees(ws.attr.inode, reference.root) + .unwrap() + .is_empty()); + assert!(service + .diff_subtrees(reference.root, ws.attr.inode) + .unwrap() + .is_empty()); + + // Remove the reference fork itself. Its binding still cannot be retired: + // rollback propagated the same borrowed keys onto fresh /ws inodes, which + // the mount-global retirement proof must discover even though they were + // never descendants of `reference.root`. + let reference_sub = service + .lookup_plus(reference.root, &dname(b"sub")) + .unwrap() + .unwrap(); + service + .remove_file(reference_sub.attr.inode, &dname(b"c")) + .unwrap(); + service + .remove_empty_dir(reference.root, &dname(b"sub")) + .unwrap(); + service.remove_file(reference.root, &dname(b"a")).unwrap(); + service.remove_file(reference.root, &dname(b"b")).unwrap(); + service.remove_empty_dir_path("/reference").unwrap(); + + // 7. Both durable bindings fail closed while /ws still borrows the restored + // blocks. The rollback binding is intentionally conservative: its old + // mount-wide floor also delays the discarded delta until the borrowers + // are rewritten. + assert!(matches!( + service.retire_snapshot(snap.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + .. + }) if snapshot_id == snap.snapshot_id + )); + assert!(matches!( + service.retire_snapshot(reference.snapshot_id), + Err(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + .. + }) if snapshot_id == reference.snapshot_id && fork_root == reference.root + )); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(reclaim.deleted, 0, "cleanup outcome: {reclaim:?}"); + assert!(objects.head(&a2_block).unwrap().is_some()); + assert!(objects.head(&d1_block).unwrap().is_some()); + assert!( + objects.head(&a1_block).unwrap().is_some(), + "A1 must survive" + ); + assert!( + objects.head(&b1_block).unwrap().is_some(), + "B1 must survive" + ); + assert!( + objects.head(&c1_block).unwrap().is_some(), + "C1 must survive" + ); + // Restored content is still readable from the shared blocks after reclaim. + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + assert_eq!( + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'3'; 4096] + ); + + // Once every rollback borrower owns a fresh generation, each binding can be + // retired independently and both the old snapshot blocks and discarded delta + // become reclaimable. + let restored_sub = service + .lookup_plus(ws.attr.inode, &dname(b"sub")) + .unwrap() + .unwrap(); + let rewrite = |parent: InodeId, name: &[u8], manifest: &str, byte: u8| { + service + .replace_artifact(PublishArtifact { + parent, + name: dname(name), + producer: "unit-test".to_owned(), + digest_uri: format!("sha256:{manifest}"), + content_type: "application/octet-stream".to_owned(), + manifest_id: manifest.to_owned(), + bytes: vec![byte; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap() + .entry + }; + let a3 = rewrite(ws.attr.inode, b"a", "ws/a3", b'6'); + let b3 = rewrite(ws.attr.inode, b"b", "ws/b3", b'7'); + let c3 = rewrite(restored_sub.attr.inode, b"c", "ws/c3", b'8'); + let a3_block = block_key(a3.attr.inode, a3.body.as_ref().unwrap().generation, 0, 0); + let b3_block = block_key(b3.attr.inode, b3.body.as_ref().unwrap().generation, 0, 0); + let c3_block = block_key(c3.attr.inode, c3.body.as_ref().unwrap().generation, 0, 0); + assert!(service.retire_snapshot(snap.snapshot_id).unwrap()); + assert!(service.retire_snapshot(reference.snapshot_id).unwrap()); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert!(reclaim.deleted >= 5, "cleanup outcome: {reclaim:?}"); + for old in [&a1_block, &b1_block, &c1_block, &a2_block, &d1_block] { + assert!(objects.head(old).unwrap().is_none(), "old block {old:?}"); + } + for current in [&a3_block, &b3_block, &c3_block] { + assert!( + objects.head(current).unwrap().is_some(), + "current block {current:?}" + ); + } + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'6'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'7'; 4096]); + assert_eq!( + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'8'; 4096] + ); +} + +#[test] +fn rollback_subtree_rejects_an_expired_snapshot_before_materializing() { + let service = service(); + service.set_clock_override_ms(1_000); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/ws/a", "rollback-expired", b"original"); + let pin = service + .snapshot_subtree_with_lease(ws.attr.inode, 500) + .unwrap(); + + service.set_clock_override_ms(1_500); + assert!(matches!( + service.rollback_subtree(ws.attr.inode, pin.snapshot_id), + Err(MetadError::SnapshotLeaseExpired { + snapshot_id, + lease_expires_unix_ms: 1_500, + now_ms: 1_500, + }) if snapshot_id == pin.snapshot_id + )); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), b"original"); +} + +#[test] +fn rollback_holds_the_gc_gate_until_restored_references_are_live() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenameReplace, 1, 2); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let ws_inode = ws.attr.inode; + let original = service + .publish_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:original".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-race/original".to_owned(), + bytes: b"original".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let original_body = original.body.as_ref().unwrap(); + let original_object = block_key(original.attr.inode, original_body.generation, 0, 0); + let pin = service + .snapshot_subtree_with_lease(ws.attr.inode, 500) + .unwrap(); + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:delta".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "rollback-race/delta".to_owned(), + bytes: b"delta".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + + let rollback_service = Arc::clone(&service); + let rollback = + std::thread::spawn(move || rollback_service.rollback_subtree(ws_inode, pin.snapshot_id)); + store.wait_until_blocked(); + assert!(service.object_gc_gate.try_lock().is_err()); + + let deadline_ms = start_ms + 500; + service.set_clock_override_ms(deadline_ms); + let cleanup_started = Arc::new(Barrier::new(2)); + let cleanup_service = Arc::clone(&service); + let cleanup_thread_started = Arc::clone(&cleanup_started); + let cleanup = std::thread::spawn(move || { + cleanup_thread_started.wait(); + cleanup_service.cleanup_pending_objects(100) + }); + cleanup_started.wait(); + store.release_blocked(); + + rollback.join().unwrap().unwrap(); + cleanup.join().unwrap().unwrap(); + assert!(objects.head(&original_object).unwrap().is_some()); + assert_eq!( + service.read_artifact(ws_inode, &dname(b"a")).unwrap(), + b"original" + ); +} + +#[test] +fn rollback_subtree_rejects_foreign_or_missing_snapshot() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/other", 0o755, 1000, 1000) + .unwrap(); + let other_root = service.resolve_directory_path("/other").unwrap(); + let snap = service.snapshot_subtree_path("/other").unwrap(); + + // A snapshot of /other cannot roll back /ws. + assert!(matches!( + service.rollback_subtree_path("/ws", snap.snapshot_id), + Err(MetadError::InvalidPath(_)) + )); + // An unknown snapshot id is not found. + assert!(matches!( + service.rollback_subtree_path("/ws", snap.snapshot_id + 9_999), + Err(MetadError::NotFound) + )); + // The rejected target is untouched and the legitimate one still works. + assert!(service + .rollback_subtree(other_root, snap.snapshot_id) + .is_ok()); } #[test] fn metadata_backup_then_restore_into_fresh_store_recovers_namespace() { let (service, objects) = service_with_objects(); - // Build a small namespace; file bodies land in the shared object store. + // Build a small namespace; file bodies land in the shared object store. + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + service.create_dir_path("/data", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha-body"); + publish_path_artifact(&service, "/data/b.bin", "m-b", b"bravo-body-2"); + let want_runs = service.lookup_path("/runs/a.bin").unwrap(); + let want_data = service.lookup_path("/data/b.bin").unwrap(); + assert!(want_runs.is_some()); + + let config = MetadataArchiveConfig::new("meta/checkpoints", 3); + let backup = service.backup_metadata(&config).unwrap(); + assert!(backup.image_bytes > 0); + assert!(backup.checkpoint_key.starts_with("meta/checkpoints/ckpt/")); + + // Simulate total loss of the metadata node: a brand-new empty Holt store, + // pointed at the SAME object store (the clone shares the backing map). + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + // The fresh node has no namespace at all (not even a root) until restore. + let outcome = recovered.restore_metadata(&config).unwrap(); + assert_eq!( + outcome.as_ref().map(|o| o.checkpoint_key.as_str()), + Some(backup.checkpoint_key.as_str()) + ); + + // Namespace entries (dentry + attr + body descriptor) are identical after + // restore, and a subsequent create allocates a fresh, non-colliding inode. + assert_eq!(recovered.lookup_path("/runs/a.bin").unwrap(), want_runs); + assert_eq!(recovered.lookup_path("/data/b.bin").unwrap(), want_data); + let fresh = publish_path_artifact(&recovered, "/runs/c.bin", "m-c", b"charlie"); + assert_ne!(fresh.attr.inode, want_runs.unwrap().attr.inode); +} + +#[test] +fn restore_metadata_without_archive_returns_none() { + let (service, _objects) = service_with_objects(); + let config = MetadataArchiveConfig::new("meta/empty", 3); + assert!(service.restore_metadata(&config).unwrap().is_none()); +} + +#[test] +fn restore_rejects_a_pre_fence_archive_before_installing_its_image() { + let (service, objects) = service_with_objects(); + service + .create_dir_path("/unsafe", 0o755, 1000, 1000) + .unwrap(); + let config = MetadataArchiveConfig::new("meta/legacy", 3); + let backup = service.backup_metadata(&config).unwrap(); + + // Downgrade only CURRENT to the legacy format. The referenced image is + // actually valid and fenced, so an empty target proves rejection happened + // from the pointer proof before install_checkpoint_image could mutate it. + let current_key = ObjectKey::new("meta/legacy/CURRENT").unwrap(); + let current = String::from_utf8(objects.get(¤t_key, None).unwrap()).unwrap(); + let legacy = current + .lines() + .enumerate() + .filter_map(|(index, line)| { + if line.starts_with("object_gc_failover_fenced\t") { + None + } else if index == 0 { + Some("nokv-metadata-archive\t1".to_owned()) + } else { + Some(line.to_owned()) + } + }) + .collect::>() + .join("\n") + + "\n"; + objects.put(¤t_key, legacy.into_bytes()).unwrap(); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + assert!(matches!( + recovered.restore_metadata(&config), + Err(MetadError::MetadataArchiveMissingObjectGcFence { checkpoint_key }) + if checkpoint_key == backup.checkpoint_key + )); + assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); +} + +#[test] +fn metadata_backup_retains_only_keep_last_checkpoints() { + let (service, objects) = service_with_objects(); + let config = MetadataArchiveConfig::new("meta/ck", 2); + let b1 = service.backup_metadata(&config).unwrap(); + let _b2 = service.backup_metadata(&config).unwrap(); + let b3 = service.backup_metadata(&config).unwrap(); + // keep_last=2: the third backup prunes exactly the first checkpoint object. + assert_eq!(b3.pruned, 1); + let pruned = ObjectKey::new(b1.checkpoint_key.clone()).unwrap(); + assert!(objects.head(&pruned).unwrap().is_none()); + let live = ObjectKey::new(b3.checkpoint_key.clone()).unwrap(); + assert!(objects.head(&live).unwrap().is_some()); + // Restore (into a fresh store) always selects the latest checkpoint. + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + let restored = recovered.restore_metadata(&config).unwrap().unwrap(); + assert_eq!(restored.checkpoint_key, b3.checkpoint_key); +} + +fn log_test_command(request_id: &[u8], commit_version: u64) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::CreateFile, + read_version: Version::new(commit_version - 1).unwrap(), + commit_version: Version::new(commit_version).unwrap(), + primary_family: RecordFamily::Dentry, + primary_key: b"log-primary".to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::Dentry, + key: b"log-primary".to_vec(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Dentry, + key: b"log-primary".to_vec(), + op: MutationOp::Put, + value: Some(Value(b"log-value".to_vec())), + }], + watch: Vec::new(), + } +} + +fn log_test_entry( + request_id: &[u8], + lsn: u64, + commit_version: u64, + prev_digest: [u8; 32], +) -> MetadataLogEntry { + MetadataLogEntry::seal( + "mount-1:/runs", + 1, + lsn, + log_test_command(request_id, commit_version), + CommitResult { + commit_version: Version::new(commit_version).unwrap(), + applied_mutations: 1, + watch_events: 0, + }, + prev_digest, + ) + .unwrap() +} + +fn snapshot_segment_keys(snapshot: &MetadataLogSyncSnapshot) -> Vec { + snapshot + .segments + .iter() + .map(|segment| segment.segment_key.clone()) + .collect() +} + +fn log_replay_command( + request_id: &[u8], + key: &[u8], + value: &[u8], + read_version: u64, + commit_version: u64, +) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::RegisterNamespaceIndex, + read_version: Version::new(read_version).unwrap(), + commit_version: Version::new(commit_version).unwrap(), + primary_family: RecordFamily::System, + primary_key: key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.to_vec(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.to_vec(), + op: MutationOp::Put, + value: Some(Value(value.to_vec())), + }], + watch: Vec::new(), + } +} + +fn ordered_log_put_command( + request_id: &[u8], + key: &[u8], + value: &[u8], + read_version: u64, + commit_version: u64, +) -> MetadataCommand { + MetadataCommand { + request_id: request_id.to_vec(), + kind: CommandKind::RegisterNamespaceIndex, + read_version: Version::new(read_version).unwrap(), + commit_version: Version::new(commit_version).unwrap(), + primary_family: RecordFamily::System, + primary_key: key.to_vec(), + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.to_vec(), + op: MutationOp::Put, + value: Some(Value(value.to_vec())), + }], + watch: Vec::new(), + } +} + +#[test] +fn metadata_log_segment_archive_round_trips_through_object_store() { + let (service, objects) = service_with_objects(); + let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); + let second = log_test_entry(b"req-log-2", 2, 12, first.digest); + let segment = MetadataLogSegment::seal(vec![first, second]).unwrap(); + let config = MetadataLogArchiveConfig::new("meta/shared-log"); + + let archived = service + .archive_metadata_log_segment(&config, &segment) + .unwrap(); + + assert!(archived.segment_key.starts_with("meta/shared-log/log/")); + assert!(archived + .segment_key + .contains("/00000000000000000001-00000000000000000002-")); + assert!(archived.segment_key.ends_with(".segment")); + assert_eq!(archived.first_lsn, 1); + assert_eq!(archived.last_lsn, 2); + assert_eq!( + archived.encoded_bytes, + segment.encode().unwrap().len() as u64 + ); + assert!(objects + .head(&ObjectKey::new(archived.segment_key.clone()).unwrap()) + .unwrap() + .is_some()); + + let loaded = service + .load_metadata_log_segment(&archived.segment_key) + .unwrap(); + assert_eq!(loaded, segment); +} + +#[test] +fn metadata_log_archive_config_rejects_keys_outside_the_exact_shard_prefix() { + let config = MetadataLogArchiveConfig::new("meta/shared-log/mount_1__"); + let valid = concat!( + "meta/shared-log/mount_1__/log/", + "00000000000000000001-00000000000000000001-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" + ); + config.validate_segment_key(valid).unwrap(); + + for invalid in [ + concat!( + "meta/shared-log/mount_2__/log/", + "00000000000000000001-00000000000000000001-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" + ), + concat!( + "meta/shared-log/mount_1__/log/nested/", + "00000000000000000001-00000000000000000001-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.segment" + ), + "meta/shared-log/mount_1__/log/", + ] { + assert!(matches!( + config.validate_segment_key(invalid), + Err(MetadError::Codec(_)) + )); + } +} + +#[test] +fn metadata_log_segment_load_rejects_corrupted_object() { + let (service, objects) = service_with_objects(); + let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); + let segment = MetadataLogSegment::seal(vec![first]).unwrap(); + let config = MetadataLogArchiveConfig::new("meta/shared-log"); + let archived = service + .archive_metadata_log_segment(&config, &segment) + .unwrap(); + let key = ObjectKey::new(archived.segment_key.clone()).unwrap(); + objects.put(&key, b"corrupted-segment".to_vec()).unwrap(); + + assert!(matches!( + service.load_metadata_log_segment(&archived.segment_key), + Err(MetadError::Codec(_)) + )); +} + +#[test] +fn restore_metadata_with_archived_log_segments_replays_after_checkpoint() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-replay", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + + let key = b"log-replay-system-key".to_vec(); + let value = b"after-checkpoint".to_vec(); + let commit_version = checkpoint.commit_version + 1; + let command = log_replay_command( + b"req-log-replay", + &key, + &value, + checkpoint.commit_version, + commit_version, + ); + let result = service.commit_metadata(command.clone()).unwrap(); + let entry = + MetadataLogEntry::seal("mount-1:/", 1, 1, command, result, METADATA_LOG_ZERO_DIGEST) + .unwrap(); + let segment = MetadataLogSegment::seal(vec![entry.clone()]).unwrap(); + let log_config = MetadataLogArchiveConfig::new("meta/shared-log-replay"); + let archived = service + .archive_metadata_log_segment(&log_config, &segment) + .unwrap(); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + std::slice::from_ref(&archived.segment_key), + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); + + assert_eq!(outcome.checkpoint.checkpoint_key, checkpoint.checkpoint_key); + assert_eq!(outcome.replayed_entries, 1); + assert_eq!(outcome.durable_lsn, 1); + assert_eq!(outcome.last_digest, entry.digest); + assert!(recovered.read_version().unwrap().get() >= commit_version); + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + &key, + Version::new(commit_version).unwrap(), + ReadPurpose::UserStrong + ) + .unwrap(), + Some(Value(value)) + ); +} + +#[test] +fn log_replay_recovers_partial_materialization_as_an_unbound_orphan() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + publish_path_artifact( + &service, + "/base/data.bin", + "replay-orphan/source", + b"borrowed", + ); + let pin = service.snapshot_subtree(base.attr.inode).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/replay-orphan-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + // The checkpoint is healthy. Only the subsequent detached materialization + // commits enter the retained log tail; no ForkBinding commit follows them. + let checkpoint_config = MetadataArchiveConfig::new("meta/replay-orphan-checkpoint", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + assert!(!service.materialization_orphan_slow_path_enabled()); + let orphan_root = { + let _object_gc_gate = service.object_gc_gate.lock().unwrap(); + service + .materialize_subtree_at(base.attr.inode, Version::new(pin.read_version).unwrap()) + .unwrap() + }; + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert!(snapshot.durable_lsn > checkpoint.log_lsn); + assert!(!snapshot.segments.is_empty()); + let segments = snapshot + .segments + .iter() + .map(|pointer| service.load_metadata_log_segment(&pointer.segment_key)) + .collect::, _>>() + .unwrap(); + assert!(segments + .iter() + .flat_map(|segment| &segment.entries) + .flat_map(|entry| &entry.command.mutations) + .all(|mutation| mutation.family != RecordFamily::ForkBinding)); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + let outcome = recovered + .restore_metadata_with_log_segments( + &checkpoint_config, + "mount-1:/", + &segments, + checkpoint.log_lsn, + checkpoint.log_digest, + ) + .unwrap() + .unwrap(); + assert_eq!(outcome.replayed_entries, 2); + assert!( + recovered.materialization_orphan_slow_path_enabled(), + "partial materialization replay must remain fail-closed" + ); + let orphan = recovered + .lookup_plus(orphan_root, &dname(b"data.bin")) + .unwrap() + .unwrap(); + assert!(matches!( + recovered.link(orphan.attr.inode, InodeId::root(), dname(b"replayed-link")), + Err(MetadError::NotFound) + )); + assert!(matches!( + recovered.rename( + orphan_root, + &dname(b"data.bin"), + InodeId::root(), + dname(b"replayed-rename") + ), + Err(MetadError::NotFound) + )); +} + +#[test] +fn restore_metadata_with_log_segments_rejects_chain_gap_before_restore() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-gap", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + + let command = log_replay_command( + b"req-log-gap", + b"log-gap-system-key", + b"value", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + let result = service.commit_metadata(command.clone()).unwrap(); + let entry = + MetadataLogEntry::seal("mount-1:/", 1, 2, command, result, METADATA_LOG_ZERO_DIGEST) + .unwrap(); + let segment = MetadataLogSegment::seal(vec![entry]).unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + + let err = recovered + .restore_metadata_with_log_segments( + &checkpoint_config, + "mount-1:/", + &[segment], + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap_err(); + + assert!(matches!(err, MetadError::Codec(message) if message.contains("lsn gap"))); +} + +#[test] +fn sync_metadata_log_archives_commit_before_recovery_ack() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-log", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let key = b"sync-log-system-key".to_vec(); + let value = b"sync-after-checkpoint".to_vec(); + let commit_version = checkpoint.commit_version + 1; + let command = log_replay_command( + b"req-sync-log", + &key, + &value, + checkpoint.commit_version, + commit_version, + ); + let result = service.commit_metadata(command).unwrap(); + assert_eq!(result.commit_version.get(), commit_version); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 1); + assert_eq!(snapshot.segments.len(), 1); + let segment_pointer = snapshot.segments.last().unwrap(); + assert!(segment_pointer + .segment_key + .starts_with("meta/sync-log/log/")); + + let loaded = service + .load_metadata_log_segment(&segment_pointer.segment_key) + .unwrap(); + assert_eq!(loaded.first_lsn, 1); + assert_eq!(loaded.last_lsn, 1); + assert_eq!(loaded.last_digest, snapshot.last_digest); + + let segment_keys = snapshot_segment_keys(&snapshot); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); + + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + &key, + Version::new(commit_version).unwrap(), + ReadPurpose::UserStrong + ) + .unwrap(), + Some(Value(value)) + ); +} + +#[test] +fn log_replay_does_not_fold_a_foreign_graft_inode_into_the_local_allocator() { + let (service, objects) = service_with_objects(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-graft-log-replay", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/graft-log-replay", + "mount-1:/", + 1, + checkpoint.log_lsn, + checkpoint.log_digest, + )) + .unwrap(); + + let foreign_inode = InodeId::compose(1, 42).unwrap(); + service + .create_graft( + InodeId::root(), + dname(b"dataset"), + foreign_inode, + 0o755, + 1000, + 1000, + ) + .unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.segments.len(), 1); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ) + .with_shard_index(0); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + checkpoint.log_lsn, + checkpoint.log_digest, + ) + .unwrap() + .unwrap(); + + let graft = recovered + .lookup_plus(InodeId::root(), &dname(b"dataset")) + .unwrap(); + assert_eq!(graft.unwrap().dentry.child, foreign_inode); + let local = recovered + .create_file( + InodeId::root(), + dname(b"local-after-replay"), + 0o644, + 1000, + 1000, + ) + .unwrap(); + assert_eq!( + local.attr.inode.shard_index(), + 0, + "a foreign graft target replayed from the log must not poison the local allocator" + ); +} + +#[test] +fn concurrent_metadata_apply_and_log_archive_preserve_one_recovery_order() { + const FIRST_REQUEST: &[u8] = b"req-ordered-log-first"; + const SECOND_REQUEST: &[u8] = b"req-ordered-log-second"; + let store = PostCommitBarrierStore::new(FIRST_REQUEST); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-ordered-sync-log", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/ordered-sync-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let key = b"ordered-sync-log-key".to_vec(); + let first = ordered_log_put_command( + FIRST_REQUEST, + &key, + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + let second = ordered_log_put_command( + SECOND_REQUEST, + &key, + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + + let first_service = Arc::clone(&service); + let first_commit = std::thread::spawn(move || first_service.commit_metadata(first)); + store.wait_until_applied(); + + let (second_tx, second_rx) = std::sync::mpsc::sync_channel(1); + let second_service = Arc::clone(&service); + let second_commit = std::thread::spawn(move || { + let result = second_service.commit_metadata(second); + second_tx.send(result).unwrap(); + }); + let early = second_rx.recv_timeout(Duration::from_millis(100)); + store.release_after_apply(); + first_commit.join().unwrap().unwrap(); + assert!(matches!( + early, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + )); + second_rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap(); + second_commit.join().unwrap(); + + assert_eq!( + service + .metadata + .get( + RecordFamily::System, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + assert_eq!(snapshot.segments.len(), 2); + let first_segment = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .unwrap(); + let second_segment = service + .load_metadata_log_segment(&snapshot.segments[1].segment_key) + .unwrap(); + assert_eq!(first_segment.entries[0].command.request_id, FIRST_REQUEST); + assert_eq!(second_segment.entries[0].command.request_id, SECOND_REQUEST); + assert_eq!(first_segment.last_digest, second_segment.prev_digest); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); + assert_eq!(outcome.replayed_entries, 2); + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + &key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); +} + +#[test] +fn metadata_commits_remain_concurrent_while_sync_log_is_disabled() { + const FIRST_REQUEST: &[u8] = b"req-no-sync-first"; + let store = PostCommitBarrierStore::new(FIRST_REQUEST); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let base = service.read_version().unwrap().get(); + let key = b"no-sync-concurrent-key"; + let first = ordered_log_put_command(FIRST_REQUEST, key, b"first", base, base + 1); + let second = ordered_log_put_command(b"req-no-sync-second", key, b"second", base + 1, base + 2); + + let first_service = Arc::clone(&service); + let first_commit = std::thread::spawn(move || first_service.commit_metadata(first)); + store.wait_until_applied(); + + let (second_tx, second_rx) = std::sync::mpsc::sync_channel(1); + let second_service = Arc::clone(&service); + let second_commit = std::thread::spawn(move || { + second_tx + .send(second_service.commit_metadata(second)) + .unwrap(); + }); + let second_result = second_rx + .recv_timeout(Duration::from_secs(5)) + .expect("a disabled sync log must not serialize unrelated metadata commits"); + store.release_after_apply(); + + second_result.unwrap(); + second_commit.join().unwrap(); + first_commit.join().unwrap().unwrap(); + assert_eq!( + service + .metadata + .get( + RecordFamily::System, + key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); +} + +#[test] +fn sync_metadata_log_snapshot_keeps_durable_tail_after_checkpoint_prune() { + let (service, _objects) = service_with_objects(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-prune", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + service + .create_dir_path("/before-checkpoint", 0o755, 1000, 1000) + .unwrap(); + let first = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(first.segments.len(), 1); + assert_eq!(first.segments[0].last_lsn, first.durable_lsn); + assert_eq!(first.segments[0].last_digest, first.last_digest); + + service.prune_sync_metadata_log_segments(first.durable_lsn); + let after_first_prune = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(after_first_prune.durable_lsn, first.durable_lsn); + assert_eq!(after_first_prune.last_digest, first.last_digest); + assert!(after_first_prune.segments.is_empty()); + + service + .create_dir_path("/after-checkpoint", 0o755, 1000, 1000) + .unwrap(); + let second = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(second.segments.len(), 1); + assert_eq!(second.segments[0].first_lsn, first.durable_lsn + 1); + assert_eq!(second.segments[0].last_lsn, second.durable_lsn); + assert_eq!(second.segments[0].last_digest, second.last_digest); + let continued = service + .load_metadata_log_segment(&second.segments[0].segment_key) + .unwrap(); + assert_eq!(continued.prev_digest, first.last_digest); + + service.prune_sync_metadata_log_segments(second.durable_lsn); + let after_second_prune = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(after_second_prune.durable_lsn, second.durable_lsn); + assert_eq!(after_second_prune.last_digest, second.last_digest); + assert!(after_second_prune.segments.is_empty()); +} + +#[test] +fn sync_metadata_log_prune_never_deletes_pointer_outside_archive_prefix() { + let (service, objects) = service_with_objects(); + let unrelated = ObjectKey::new("unrelated/keep-me").unwrap(); + objects.put(&unrelated, b"keep".to_vec()).unwrap(); + service + .enable_sync_metadata_log( + MetadataLogSyncConfig::new("meta/sync-log-delete-scope", "mount-1:/", 1, 1, [7; 32]) + .with_segments(vec![MetadataLogSegmentPointer { + segment_key: unrelated.as_str().to_owned(), + first_lsn: 1, + last_lsn: 1, + last_digest: [7; 32], + }]), + ) + .unwrap(); + + let outcome = service.prune_sync_metadata_log_segments(1); + + assert_eq!(outcome.pointers_pruned, 1); + assert_eq!(outcome.objects_deleted, 0); + assert_eq!(outcome.objects_missing, 0); + assert_eq!(outcome.delete_failures, 1); + assert!(objects.head(&unrelated).unwrap().is_some()); + assert!(service + .sync_metadata_log_snapshot() + .unwrap() + .segments + .is_empty()); +} + +#[test] +fn sync_metadata_log_rejects_duplicate_enable_without_replacing_tail() { + let (service, _objects) = service_with_objects(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-original", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + service + .create_dir_path("/before-duplicate-enable", 0o755, 1000, 1000) + .unwrap(); + let before = service.sync_metadata_log_snapshot().unwrap(); + + let error = service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-replacement", + "mount-1:/", + 2, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap_err(); + assert!(matches!( + error, + MetadError::Codec(message) if message.contains("already enabled") + )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap(), before); + + service + .create_dir_path("/after-duplicate-enable", 0o755, 1000, 1000) + .unwrap(); + let after = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(after.epoch, 1); + assert_eq!(after.durable_lsn, before.durable_lsn + 1); + assert!(after + .segments + .last() + .unwrap() + .segment_key + .starts_with("meta/sync-log-original/log/")); +} + +#[test] +fn sync_metadata_log_preflights_lsn_capacity_before_metadata_apply() { + let (service, _objects) = service_with_objects(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-exhausted", + "mount-1:/", + 1, + u64::MAX - 1, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let error = service + .create_dir_path("/must-not-apply", 0o755, 1000, 1000) + .unwrap_err(); + assert!(matches!( + error, + MetadError::SyncLogArchiveFailed { + committed: false, + message, + } if message.contains("LSN is exhausted before commit") + )); + assert!(service.lookup_path("/must-not-apply").unwrap().is_none()); + assert_eq!( + service.sync_metadata_log_snapshot().unwrap().durable_lsn, + u64::MAX - 1 + ); +} + +#[test] +fn restore_metadata_with_sync_log_advances_allocator_after_replay() { + let (service, objects) = service_with_objects(); service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - service.create_dir_path("/data", 0o755, 1000, 1000).unwrap(); - publish_path_artifact(&service, "/runs/a.bin", "m-a", b"alpha-body"); - publish_path_artifact(&service, "/data/b.bin", "m-b", b"bravo-body-2"); - let want_runs = service.lookup_path("/runs/a.bin").unwrap(); - let want_data = service.lookup_path("/data/b.bin").unwrap(); - assert!(want_runs.is_some()); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-allocator", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-allocator-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); - let config = MetadataArchiveConfig::new("meta/checkpoints", 3); - let backup = service.backup_metadata(&config).unwrap(); - assert!(backup.image_bytes > 0); - assert!(backup.checkpoint_key.starts_with("meta/checkpoints/ckpt/")); + let post_checkpoint = service + .create_dir_path("/runs/post-checkpoint", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + let segment_keys = snapshot_segment_keys(&snapshot); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); + + assert_eq!( + recovered + .lookup_path("/runs/post-checkpoint") + .unwrap() + .unwrap() + .attr + .inode, + post_checkpoint.attr.inode + ); + + let after_failover = recovered + .create_dir_path("/after-failover", 0o755, 1000, 1000) + .unwrap(); + assert!( + after_failover.attr.inode.get() > post_checkpoint.attr.inode.get(), + "failover replay must advance allocator past replayed namespace state" + ); + assert_eq!( + recovered + .lookup_path("/runs/post-checkpoint") + .unwrap() + .unwrap() + .attr + .inode, + post_checkpoint.attr.inode + ); +} + +#[test] +fn prepare_only_allocator_reservation_replays_before_failover_reuse() { + let (service, objects) = service_with_objects(); + // Initialize the durable object-GC claim before the checkpoint so the + // prepare-only workload below changes only allocator reservations. + service + .prepare_artifact_create(InodeId::root(), dname(b"warmup.bin")) + .unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-prepare-allocator", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/prepare-allocator-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let mut max_prepared_inode = 0; + let mut max_prepared_generation = 0; + for _ in 0..(ALLOCATOR_INODE_RESERVATION + 4) { + let prepared = service + .prepare_artifact_create(InodeId::root(), dname(b"never-published.bin")) + .unwrap(); + max_prepared_inode = max_prepared_inode.max(prepared.inode.get()); + max_prepared_generation = max_prepared_generation.max(prepared.generation); + } + + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert!(snapshot.durable_lsn > 0); + let segments = snapshot + .segments + .iter() + .map(|segment| { + service + .load_metadata_log_segment(&segment.segment_key) + .unwrap() + }) + .collect::>(); + assert!(segments + .iter() + .flat_map(|segment| &segment.entries) + .any(|entry| { entry.command.kind == CommandKind::ReserveAllocator })); - // Simulate total loss of the metadata node: a brand-new empty Holt store, - // pointed at the SAME object store (the clone shares the backing map). let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); + + let after_failover = recovered + .prepare_artifact_create(InodeId::root(), dname(b"after-failover.bin")) + .unwrap(); + assert!( + after_failover.inode.get() > max_prepared_inode, + "recovery must not reuse an inode from a prepare-only reservation" + ); + assert!( + after_failover.generation > max_prepared_generation, + "recovery must not reuse a generation from a prepare-only reservation" + ); +} + +#[test] +fn failed_allocator_archive_keeps_local_watermark_slow_until_pending_flush() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let service = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), objects.clone(), ); - // The fresh node has no namespace at all (not even a root) until restore. - let outcome = recovered.restore_metadata(&config).unwrap(); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + // Initialize the object-GC claim before enabling the log so the only logged + // command in this workload is the allocator reservation at the boundary. + service + .prepare_artifact_create(InodeId::root(), dname(b"claim-warmup.bin")) + .unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/allocator-pending-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + while service.clock.load(Ordering::Relaxed).saturating_add(1) + <= service.reserved_version.load(Ordering::Relaxed) + && service.next_inode.load(Ordering::Relaxed).saturating_add(1) + <= service.reserved_next_inode.load(Ordering::Relaxed) + { + service + .prepare_artifact_create(InodeId::root(), dname(b"boundary.bin")) + .unwrap(); + } + let reserved_version_before = service.reserved_version.load(Ordering::Relaxed); + let reserved_inode_before = service.reserved_next_inode.load(Ordering::Relaxed); + objects.fail_puts_containing("meta/allocator-pending-log/log/"); + + assert!(matches!( + service.prepare_artifact_create(InodeId::root(), dname(b"first-fails.bin")), + Err(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); assert_eq!( - outcome.as_ref().map(|o| o.checkpoint_key.as_str()), - Some(backup.checkpoint_key.as_str()) + service.reserved_version.load(Ordering::Relaxed), + reserved_version_before, + "a locally applied but unarchived reservation must not open the fast path" ); + assert_eq!( + service.reserved_next_inode.load(Ordering::Relaxed), + reserved_inode_before, + "inode reservation watermark must stay behind until pending archive flush" + ); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); - // Namespace entries (dentry + attr + body descriptor) are identical after - // restore, and a subsequent create allocates a fresh, non-colliding inode. - assert_eq!(recovered.lookup_path("/runs/a.bin").unwrap(), want_runs); - assert_eq!(recovered.lookup_path("/data/b.bin").unwrap(), want_data); - let fresh = publish_path_artifact(&recovered, "/runs/c.bin", "m-c", b"charlie"); - assert_ne!(fresh.attr.inode, want_runs.unwrap().attr.inode); -} - -#[test] -fn restore_metadata_without_archive_returns_none() { - let (service, _objects) = service_with_objects(); - let config = MetadataArchiveConfig::new("meta/empty", 3); - assert!(service.restore_metadata(&config).unwrap().is_none()); + objects.clear_faults(); + service + .prepare_artifact_create(InodeId::root(), dname(b"second-flushes.bin")) + .unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!( + snapshot.durable_lsn, 2, + "the retry must archive the prior applied reservation before its own reservation" + ); + let kinds = snapshot + .segments + .iter() + .map(|segment| { + service + .load_metadata_log_segment(&segment.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.kind) + .collect::>(); + assert_eq!( + kinds, + vec![CommandKind::ReserveAllocator, CommandKind::ReserveAllocator] + ); } #[test] -fn metadata_backup_retains_only_keep_last_checkpoints() { +fn sync_metadata_log_archives_independent_batch_as_one_segment() { let (service, objects) = service_with_objects(); - let config = MetadataArchiveConfig::new("meta/ck", 2); - let b1 = service.backup_metadata(&config).unwrap(); - let _b2 = service.backup_metadata(&config).unwrap(); - let b3 = service.backup_metadata(&config).unwrap(); - // keep_last=2: the third backup prunes exactly the first checkpoint object. - assert_eq!(b3.pruned, 1); - let pruned = ObjectKey::new(b1.checkpoint_key.clone()).unwrap(); - assert!(objects.head(&pruned).unwrap().is_none()); - let live = ObjectKey::new(b3.checkpoint_key.clone()).unwrap(); - assert!(objects.head(&live).unwrap().is_some()); - // Restore (into a fresh store) always selects the latest checkpoint. + service.create_dir_path("/left", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/right", 0o755, 1000, 1000) + .unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-batch-log", 2); + service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-batch-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let results = service.create_file_batches_in_dir_path(vec![ + CreateInDirPathBatch { + parent_path: "/left".to_owned(), + names: vec![DentryName::new(b"a.bin".to_vec()).unwrap()], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + CreateInDirPathBatch { + parent_path: "/right".to_owned(), + names: vec![DentryName::new(b"b.bin".to_vec()).unwrap()], + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ]); + + assert_eq!(results.len(), 2); + for result in &results { + assert_eq!(result.as_ref().unwrap().len(), 1); + } + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + assert_eq!(snapshot.segments.len(), 1); + let segment = service + .load_metadata_log_segment(&snapshot.segments.last().unwrap().segment_key) + .unwrap(); + assert_eq!(segment.first_lsn, 1); + assert_eq!(segment.last_lsn, 2); + assert_eq!(segment.entries.len(), 2); + + let segment_keys = snapshot_segment_keys(&snapshot); let recovered = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), objects.clone(), ); - let restored = recovered.restore_metadata(&config).unwrap().unwrap(); - assert_eq!(restored.checkpoint_key, b3.checkpoint_key); -} + let outcome = recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &segment_keys, + 0, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap() + .unwrap(); -fn log_test_command(request_id: &[u8], commit_version: u64) -> MetadataCommand { - MetadataCommand { - request_id: request_id.to_vec(), - kind: CommandKind::CreateFile, - read_version: Version::new(commit_version - 1).unwrap(), - commit_version: Version::new(commit_version).unwrap(), - primary_family: RecordFamily::Dentry, - primary_key: b"log-primary".to_vec(), - predicates: vec![PredicateRef { - family: RecordFamily::Dentry, - key: b"log-primary".to_vec(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::Dentry, - key: b"log-primary".to_vec(), - op: MutationOp::Put, - value: Some(Value(b"log-value".to_vec())), - }], - watch: Vec::new(), - } + assert_eq!(outcome.replayed_entries, 2); + assert!(recovered.lookup_path("/left/a.bin").unwrap().is_some()); + assert!(recovered.lookup_path("/right/b.bin").unwrap().is_some()); } -fn log_test_entry( - request_id: &[u8], - lsn: u64, - commit_version: u64, - prev_digest: [u8; 32], -) -> MetadataLogEntry { - MetadataLogEntry::seal( - "mount-1:/runs", - 1, - lsn, - log_test_command(request_id, commit_version), - CommitResult { - commit_version: Version::new(commit_version).unwrap(), - applied_mutations: 1, - watch_events: 0, - }, - prev_digest, - ) - .unwrap() -} +#[test] +fn post_commit_backend_readback_preserves_single_commit_log_order() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-post-commit-readback", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/post-commit-readback-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); -fn snapshot_segment_keys(snapshot: &MetadataLogSyncSnapshot) -> Vec { - snapshot + store.arm(); + let first = ordered_log_put_command( + b"req-post-commit-first", + b"post-commit-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + service.commit_metadata(first).unwrap(); + let second = ordered_log_put_command( + b"req-post-commit-second", + b"post-commit-second", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + service.commit_metadata(second).unwrap(); + + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + let request_ids = snapshot .segments .iter() - .map(|segment| segment.segment_key.clone()) - .collect() -} + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.request_id) + .collect::>(); + assert_eq!( + request_ids, + vec![ + b"req-post-commit-first".to_vec(), + b"req-post-commit-second".to_vec() + ] + ); -fn log_replay_command( - request_id: &[u8], - key: &[u8], - value: &[u8], - read_version: u64, - commit_version: u64, -) -> MetadataCommand { - MetadataCommand { - request_id: request_id.to_vec(), - kind: CommandKind::RegisterNamespaceIndex, - read_version: Version::new(read_version).unwrap(), - commit_version: Version::new(commit_version).unwrap(), - primary_family: RecordFamily::System, - primary_key: key.to_vec(), - predicates: vec![PredicateRef { - family: RecordFamily::System, - key: key.to_vec(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::System, - key: key.to_vec(), - op: MutationOp::Put, - value: Some(Value(value.to_vec())), - }], - watch: Vec::new(), - } + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + recovered + .restore_metadata_with_archived_log_segments( + &checkpoint_config, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + checkpoint.log_lsn, + checkpoint.log_digest, + ) + .unwrap() + .unwrap(); + assert_eq!( + recovered + .metadata + .get( + RecordFamily::System, + b"post-commit-second", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())) + ); } #[test] -fn metadata_log_segment_archive_round_trips_through_object_store() { - let (service, objects) = service_with_objects(); - let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); - let second = log_test_entry(b"req-log-2", 2, 12, first.digest); - let segment = MetadataLogSegment::seal(vec![first, second]).unwrap(); - let config = MetadataLogArchiveConfig::new("meta/shared-log"); - - let archived = service - .archive_metadata_log_segment(&config, &segment) +fn readback_error_blocks_later_apply_and_both_checkpoint_publication_paths() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-readback-block", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/readback-block-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - assert!(archived.segment_key.starts_with("meta/shared-log/log/")); - assert!(archived - .segment_key - .contains("/00000000000000000001-00000000000000000002-")); - assert!(archived.segment_key.ends_with(".segment")); - assert_eq!(archived.first_lsn, 1); - assert_eq!(archived.last_lsn, 2); + store.arm(); + store.fail_next_readbacks(4); + let first = ordered_log_put_command( + b"req-readback-block-first", + b"readback-block-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + assert!(matches!( + service.commit_metadata(first), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); + + let blocked = ordered_log_put_command( + b"req-readback-block-second", + b"readback-block-second", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + assert!(matches!( + service.commit_metadata(blocked), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); assert_eq!( - archived.encoded_bytes, - segment.encode().unwrap().len() as u64 + service + .metadata + .get( + RecordFamily::System, + b"readback-block-second", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + None ); - assert!(objects - .head(&ObjectKey::new(archived.segment_key.clone()).unwrap()) - .unwrap() - .is_some()); - let loaded = service - .load_metadata_log_segment(&archived.segment_key) - .unwrap(); - assert_eq!(loaded, segment); + assert!(matches!( + service.backup_metadata(&checkpoint_config), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); + let controlled = MetadataArchiveConfig::new("meta/controlled-readback-block", 2); + assert!(matches!( + service.prepare_immutable_metadata_backup(&controlled), + Err(MetadError::Codec(message)) if message.contains("readback failed") + )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + + store.clear_readback_failures(); + let third = ordered_log_put_command( + b"req-readback-block-third", + b"readback-block-third", + b"third", + checkpoint.commit_version + 1, + checkpoint.commit_version + 3, + ); + service.commit_metadata(third).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + let request_ids = snapshot + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.request_id) + .collect::>(); + assert_eq!( + request_ids, + vec![ + b"req-readback-block-first".to_vec(), + b"req-readback-block-third".to_vec() + ] + ); } #[test] -fn metadata_log_segment_load_rejects_corrupted_object() { - let (service, objects) = service_with_objects(); - let first = log_test_entry(b"req-log-1", 1, 11, METADATA_LOG_ZERO_DIGEST); - let segment = MetadataLogSegment::seal(vec![first]).unwrap(); - let config = MetadataLogArchiveConfig::new("meta/shared-log"); - let archived = service - .archive_metadata_log_segment(&config, &segment) +fn mismatched_readback_result_blocks_later_apply_until_exact_result_is_visible() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + MemoryObjectStore::new(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-readback-mismatch", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/readback-mismatch-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) .unwrap(); - let key = ObjectKey::new(archived.segment_key.clone()).unwrap(); - objects.put(&key, b"corrupted-segment".to_vec()).unwrap(); + store.arm(); + store.mismatch_next_readbacks(2); + let first = ordered_log_put_command( + b"req-readback-mismatch-first", + b"readback-mismatch-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); assert!(matches!( - service.load_metadata_log_segment(&archived.segment_key), - Err(MetadError::Codec(_)) + service.commit_metadata(first), + Err(MetadError::Codec(message)) if message.contains("result mismatch") )); + let blocked = ordered_log_put_command( + b"req-readback-mismatch-blocked", + b"readback-mismatch-blocked", + b"blocked", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + assert!(matches!( + service.commit_metadata(blocked), + Err(MetadError::Codec(message)) if message.contains("result mismatch") + )); + assert!(service + .metadata + .get( + RecordFamily::System, + b"readback-mismatch-blocked", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + + store.clear_readback_mismatches(); + let trigger = ordered_log_put_command( + b"req-readback-mismatch-trigger", + b"readback-mismatch-trigger", + b"trigger", + checkpoint.commit_version + 1, + checkpoint.commit_version + 3, + ); + service.commit_metadata(trigger).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); } #[test] -fn restore_metadata_with_archived_log_segments_replays_after_checkpoint() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-replay", 2); +fn batch_post_commit_backend_readback_archives_full_batch_in_input_order() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-batch-readback", 2); let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/batch-readback-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); - let key = b"log-replay-system-key".to_vec(); - let value = b"after-checkpoint".to_vec(); - let commit_version = checkpoint.commit_version + 1; - let command = log_replay_command( - b"req-log-replay", - &key, - &value, - checkpoint.commit_version, - commit_version, - ); - let result = service.commit_metadata(command.clone()).unwrap(); - let entry = - MetadataLogEntry::seal("mount-1:/", 1, 1, command, result, METADATA_LOG_ZERO_DIGEST) - .unwrap(); - let segment = MetadataLogSegment::seal(vec![entry.clone()]).unwrap(); - let log_config = MetadataLogArchiveConfig::new("meta/shared-log-replay"); - let archived = service - .archive_metadata_log_segment(&log_config, &segment) + let commands = vec![ + ordered_log_put_command( + b"req-batch-readback-first", + b"batch-readback-first", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ), + ordered_log_put_command( + b"req-batch-readback-second", + b"batch-readback-second", + b"second", + checkpoint.commit_version, + checkpoint.commit_version + 2, + ), + ]; + store.fail_next_batch_results(vec![0, 1]); + let results = service.commit_independent_metadata_batch(&commands); + assert!(results.iter().all(Result::is_ok)); + + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + assert_eq!(snapshot.segments.len(), 1); + let segment = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) .unwrap(); + assert_eq!( + segment + .entries + .iter() + .map(|entry| entry.command.request_id.as_slice()) + .collect::>(), + vec![ + b"req-batch-readback-first".as_slice(), + b"req-batch-readback-second".as_slice() + ] + ); let recovered = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + objects, ); let outcome = recovered .restore_metadata_with_archived_log_segments( &checkpoint_config, - std::slice::from_ref(&archived.segment_key), - 0, - METADATA_LOG_ZERO_DIGEST, + "mount-1:/", + &snapshot_segment_keys(&snapshot), + checkpoint.log_lsn, + checkpoint.log_digest, ) .unwrap() .unwrap(); - - assert_eq!(outcome.checkpoint.checkpoint_key, checkpoint.checkpoint_key); - assert_eq!(outcome.replayed_entries, 1); - assert_eq!(outcome.durable_lsn, 1); - assert_eq!(outcome.last_digest, entry.digest); - assert!(recovered.read_version().unwrap().get() >= commit_version); + assert_eq!(outcome.replayed_entries, 2); assert_eq!( recovered .metadata .get( RecordFamily::System, - &key, - Version::new(commit_version).unwrap(), - ReadPurpose::UserStrong + b"batch-readback-second", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, ) .unwrap(), - Some(Value(value)) + Some(Value(b"second".to_vec())) + ); +} + +#[test] +fn unresolved_early_batch_subgroup_freezes_later_success_and_mixed_failure() { + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-subgroup-readback", 2); + let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/subgroup-readback-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let first = ordered_log_put_command( + b"req-subgroup-first", + b"subgroup-shared-key", + b"first", + checkpoint.commit_version, + checkpoint.commit_version + 1, + ); + // Same key forces Holt to flush the first atomic subgroup before planning + // and applying this later successful subgroup. + let second = ordered_log_put_command( + b"req-subgroup-second", + b"subgroup-shared-key", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + let mut rejected = ordered_log_put_command( + b"req-subgroup-rejected", + b"subgroup-rejected-key", + b"rejected", + checkpoint.commit_version, + checkpoint.commit_version + 3, + ); + rejected.predicates.push(PredicateRef { + family: RecordFamily::System, + key: b"missing-subgroup-predicate-key".to_vec(), + predicate: Predicate::Exists, + }); + store.fail_next_batch_results(vec![0]); + store.fail_next_readbacks(1); + let results = + service.commit_independent_metadata_batch(&[first.clone(), second.clone(), rejected]); + assert!(results + .iter() + .all(|result| matches!(result, Err(MetadError::Codec(message)) if message.contains("readback failed")))); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + assert_eq!( + service + .metadata + .get( + RecordFamily::System, + b"subgroup-shared-key", + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap(), + Some(Value(b"second".to_vec())), + "the later subgroup applied but must not be acknowledged or archived early" + ); + + store.clear_readback_failures(); + let trigger = ordered_log_put_command( + b"req-subgroup-trigger", + b"subgroup-trigger-key", + b"trigger", + checkpoint.commit_version + 2, + checkpoint.commit_version + 4, + ); + service.commit_metadata(trigger).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 3); + assert_eq!(snapshot.segments.len(), 2); + let resolved_group = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) + .unwrap(); + assert_eq!( + resolved_group + .entries + .iter() + .map(|entry| entry.command.request_id.as_slice()) + .collect::>(), + vec![first.request_id.as_slice(), second.request_id.as_slice()], + "the true committed subset must archive once in original execution order" ); } -#[test] -fn restore_metadata_with_log_segments_rejects_chain_gap_before_restore() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-log-gap", 2); - let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); +use nokv_object::{ObjectInfo, ObjectRange}; +use std::sync::atomic::AtomicUsize; + +/// An [`ObjectStore`] wrapper that injects PUT failures to simulate a crash at a +/// chosen point (e.g. after the checkpoint object is written but before the +/// `CURRENT` pointer is swapped). Reads and deletes always pass through. +#[derive(Clone)] +struct FaultObjectStore { + inner: MemoryObjectStore, + fail_put_substring: Arc>>, + injected_put_failures: Arc, + get_keys: Arc>>, +} + +impl FaultObjectStore { + fn new(inner: MemoryObjectStore) -> Self { + Self { + inner, + fail_put_substring: Arc::new(Mutex::new(None)), + injected_put_failures: Arc::new(AtomicUsize::new(0)), + get_keys: Arc::new(Mutex::new(Vec::new())), + } + } + + fn fail_puts_containing(&self, substring: &str) { + *self.fail_put_substring.lock().unwrap() = Some(substring.to_owned()); + } + + fn clear_faults(&self) { + *self.fail_put_substring.lock().unwrap() = None; + } + + fn injected_put_failures(&self) -> usize { + self.injected_put_failures.load(Ordering::Relaxed) + } + + fn clear_get_keys(&self) { + self.get_keys.lock().unwrap().clear(); + } + + fn got_key(&self, key: &str) -> bool { + self.get_keys.lock().unwrap().iter().any(|got| got == key) + } +} + +impl ObjectStore for FaultObjectStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + if let Some(substring) = self.fail_put_substring.lock().unwrap().clone() { + if key.as_str().contains(&substring) { + self.injected_put_failures.fetch_add(1, Ordering::Relaxed); + return Err(ObjectError::Backend("injected put fault".to_owned())); + } + } + self.inner.put(key, bytes) + } - let command = log_replay_command( - b"req-log-gap", - b"log-gap-system-key", - b"value", - checkpoint.commit_version, - checkpoint.commit_version + 1, - ); - let result = service.commit_metadata(command.clone()).unwrap(); - let entry = - MetadataLogEntry::seal("mount-1:/", 1, 2, command, result, METADATA_LOG_ZERO_DIGEST) - .unwrap(); - let segment = MetadataLogSegment::seal(vec![entry]).unwrap(); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects.clone(), - ); + fn get(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { + self.get_keys.lock().unwrap().push(key.as_str().to_owned()); + self.inner.get(key, range) + } - let err = recovered - .restore_metadata_with_log_segments( - &checkpoint_config, - &[segment], - 0, - METADATA_LOG_ZERO_DIGEST, - ) - .unwrap_err(); + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } - assert!(matches!(err, MetadError::Codec(message) if message.contains("lsn gap"))); + fn delete(&self, key: &ObjectKey) -> Result { + self.inner.delete(key) + } } #[test] -fn sync_metadata_log_archives_commit_before_recovery_ack() { - let (service, objects) = service_with_objects(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-log", 2); +fn post_commit_backend_with_archive_failure_retains_exact_pending_segment() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let store = PostCommitErrorStore::new_disarmed(CommandKind::RegisterNamespaceIndex); + let service = NoKvFs::new(MountId::new(1).unwrap(), store.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-backend-archive-failure", 2); let checkpoint = service.backup_metadata(&checkpoint_config).unwrap(); service .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-log", + "meta/backend-archive-failure-log", "mount-1:/", 1, 0, @@ -5386,260 +10879,488 @@ fn sync_metadata_log_archives_commit_before_recovery_ack() { )) .unwrap(); - let key = b"sync-log-system-key".to_vec(); - let value = b"sync-after-checkpoint".to_vec(); - let commit_version = checkpoint.commit_version + 1; - let command = log_replay_command( - b"req-sync-log", - &key, - &value, + objects.fail_puts_containing("meta/backend-archive-failure-log/log/"); + store.arm(); + let first = ordered_log_put_command( + b"req-backend-archive-first", + b"backend-archive-first", + b"first", checkpoint.commit_version, - commit_version, + checkpoint.commit_version + 1, ); - let result = service.commit_metadata(command).unwrap(); - assert_eq!(result.commit_version.get(), commit_version); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 1); - assert_eq!(snapshot.segments.len(), 1); - let segment_pointer = snapshot.segments.last().unwrap(); - assert!(segment_pointer - .segment_key - .starts_with("meta/sync-log/log/")); + assert!(matches!( + service.commit_metadata(first), + Err(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); - let loaded = service - .load_metadata_log_segment(&segment_pointer.segment_key) - .unwrap(); - assert_eq!(loaded.first_lsn, 1); - assert_eq!(loaded.last_lsn, 1); - assert_eq!(loaded.last_digest, snapshot.last_digest); + objects.clear_faults(); + let second = ordered_log_put_command( + b"req-backend-archive-second", + b"backend-archive-second", + b"second", + checkpoint.commit_version + 1, + checkpoint.commit_version + 2, + ); + service.commit_metadata(second).unwrap(); + let snapshot = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(snapshot.durable_lsn, 2); + let request_ids = snapshot + .segments + .iter() + .map(|pointer| { + service + .load_metadata_log_segment(&pointer.segment_key) + .unwrap() + }) + .flat_map(|segment| segment.entries.into_iter()) + .map(|entry| entry.command.request_id) + .collect::>(); + assert_eq!( + request_ids, + vec![ + b"req-backend-archive-first".to_vec(), + b"req-backend-archive-second".to_vec() + ] + ); +} - let segment_keys = snapshot_segment_keys(&snapshot); - let recovered = NoKvFs::new( +#[test] +fn publish_remains_readable_when_sync_log_ack_fails_after_metadata_commit() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let service = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), objects.clone(), ); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - &segment_keys, + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "meta/sync-log-ack", + "mount-1:/", + 1, 0, METADATA_LOG_ZERO_DIGEST, - ) - .unwrap() + )) .unwrap(); + objects.fail_puts_containing("meta/sync-log-ack/log/"); + + let name = dname(b"committed.bin"); + let payload = b"metadata committed before the archive acknowledgement"; + let error = service + .publish_artifact(artifact_request(name.clone(), "committed", payload)) + .unwrap_err(); + let staged = match error { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } + )); + staged + } + other => panic!("unexpected publish error: {other:?}"), + }; + let committed = service + .lookup_plus(InodeId::root(), &name) + .unwrap() + .expect("the metadata transaction committed"); assert_eq!( - recovered - .metadata - .get( - RecordFamily::System, - &key, - Version::new(commit_version).unwrap(), - ReadPurpose::UserStrong - ) - .unwrap(), - Some(Value(value)) + service.read_artifact(InodeId::root(), &name).unwrap(), + payload + ); + assert_eq!(staged.len(), 1); + assert_eq!( + staged.objects()[0].key, + block_key( + committed.attr.inode, + committed.body.as_ref().unwrap().generation, + 0, + 0, + ) ); + assert!(backing.head(&staged.objects()[0].key).unwrap().is_some()); } #[test] -fn restore_metadata_with_sync_log_advances_allocator_after_replay() { - let (service, objects) = service_with_objects(); - service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-allocator", 2); - service.backup_metadata(&checkpoint_config).unwrap(); +fn checkpoint_keeps_referenced_blocks_after_committed_sync_log_failure() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-allocator-log", + "meta/checkpoint-sync-log-ack", "mount-1:/", 1, 0, METADATA_LOG_ZERO_DIGEST, )) .unwrap(); + objects.fail_puts_containing("meta/checkpoint-sync-log-ack/log/"); - let post_checkpoint = service - .create_dir_path("/runs/post-checkpoint", 0o755, 1000, 1000) - .unwrap(); - let snapshot = service.sync_metadata_log_snapshot().unwrap(); - let segment_keys = snapshot_segment_keys(&snapshot); + let payload = b"checkpoint shard remains reachable after log ACK failure".to_vec(); + let error = service + .publish_checkpoint( + InodeId::root(), + vec![CheckpointShard { + name: dname(b"rank-0.ckpt"), + bytes: payload.clone(), + }], + 1000, + 1000, + ) + .unwrap_err(); + assert!(matches!( + error, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } + )); - let recovered = NoKvFs::new( - MountId::new(1).unwrap(), - HoltMetadataStore::open_memory().unwrap(), - objects, + let committed = service + .lookup_path("/rank-0.ckpt") + .unwrap() + .expect("checkpoint metadata was atomically committed"); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"rank-0.ckpt")) + .unwrap(), + payload ); - recovered - .restore_metadata_with_archived_log_segments( - &checkpoint_config, - &segment_keys, - 0, - METADATA_LOG_ZERO_DIGEST, + let body = committed.body.expect("checkpoint has a staged object body"); + let object = block_key(committed.attr.inode, body.generation, 0, 0); + assert!(backing.head(&object).unwrap().is_some()); +} + +#[test] +fn checkpoint_keeps_blocks_after_uncertain_backend_ack_failure() { + let metadata = PostCommitErrorStore::new(CommandKind::PublishArtifact); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata, objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let payload = b"checkpoint visible despite a lost backend acknowledgement".to_vec(); + + let error = service + .publish_checkpoint( + InodeId::root(), + vec![CheckpointShard { + name: dname(b"uncertain-rank.ckpt"), + bytes: payload.clone(), + }], + 1000, + 1000, ) - .unwrap() - .unwrap(); + .unwrap_err(); + assert!(matches!( + error, + MetadError::Metadata(MetadataError::Backend(message)) + if message.contains("journal acknowledgement") + )); + let committed = service + .lookup_path("/uncertain-rank.ckpt") + .unwrap() + .expect("the injected backend applied before losing its ACK"); assert_eq!( - recovered - .lookup_path("/runs/post-checkpoint") - .unwrap() - .unwrap() - .attr - .inode, - post_checkpoint.attr.inode + service + .read_artifact(InodeId::root(), &dname(b"uncertain-rank.ckpt")) + .unwrap(), + payload ); + let body = committed.body.expect("checkpoint has an object body"); + assert!(objects + .head(&block_key(committed.attr.inode, body.generation, 0, 0,)) + .unwrap() + .is_some()); +} - let after_failover = recovered - .create_dir_path("/after-failover", 0o755, 1000, 1000) - .unwrap(); - assert!( - after_failover.attr.inode.get() > post_checkpoint.attr.inode.get(), - "failover replay must advance allocator past replayed namespace state" - ); - assert_eq!( - recovered - .lookup_path("/runs/post-checkpoint") - .unwrap() - .unwrap() - .attr - .inode, - post_checkpoint.attr.inode +#[test] +fn pending_sync_log_blocks_single_and_batch_apply_until_prior_commit_is_archived() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), ); -} - -#[test] -fn sync_metadata_log_archives_independent_batch_as_one_segment() { - let (service, objects) = service_with_objects(); - service.create_dir_path("/left", 0o755, 1000, 1000).unwrap(); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); service - .create_dir_path("/right", 0o755, 1000, 1000) + .create_dir_path("/batch", 0o755, 1000, 1000) .unwrap(); - let checkpoint_config = MetadataArchiveConfig::new("meta/ck-sync-batch-log", 2); + let checkpoint_config = MetadataArchiveConfig::new("meta/ck-pending-sync-log", 2); service.backup_metadata(&checkpoint_config).unwrap(); service .enable_sync_metadata_log(MetadataLogSyncConfig::new( - "meta/sync-batch-log", + "meta/pending-sync-log", "mount-1:/", 1, 0, METADATA_LOG_ZERO_DIGEST, )) .unwrap(); + objects.fail_puts_containing("meta/pending-sync-log/log/"); - let results = service.create_file_batches_in_dir_path(vec![ - CreateInDirPathBatch { - parent_path: "/left".to_owned(), - names: vec![DentryName::new(b"a.bin".to_vec()).unwrap()], - mode: 0o644, - uid: 1000, - gid: 1000, - }, - CreateInDirPathBatch { - parent_path: "/right".to_owned(), - names: vec![DentryName::new(b"b.bin".to_vec()).unwrap()], - mode: 0o644, - uid: 1000, - gid: 1000, - }, - ]); + let first_error = service + .create_dir_path("/first", 0o755, 1000, 1000) + .unwrap_err(); + assert!(matches!( + first_error, + MetadError::SyncLogArchiveFailed { + committed: true, + .. + } + )); + assert!(service.lookup_path("/first").unwrap().is_some()); + + let blocked_single = service + .create_dir_path("/second", 0o755, 1000, 1000) + .unwrap_err(); + assert!(matches!( + blocked_single, + MetadError::SyncLogArchiveFailed { + committed: false, + .. + } + )); + assert!(service.lookup_path("/second").unwrap().is_none()); + + let blocked = service.create_file_batches_in_dir_path(vec![CreateInDirPathBatch { + parent_path: "/batch".to_owned(), + names: vec![dname(b"third.bin")], + mode: 0o644, + uid: 1000, + gid: 1000, + }]); + assert!(matches!( + &blocked[0], + Err(MetadError::SyncLogArchiveFailed { + committed: false, + .. + }) + )); + assert!(service.lookup_path("/batch/third.bin").unwrap().is_none()); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + + objects.clear_faults(); + service + .create_dir_path("/second", 0o755, 1000, 1000) + .unwrap(); + let retried = service.create_file_batches_in_dir_path(vec![CreateInDirPathBatch { + parent_path: "/batch".to_owned(), + names: vec![dname(b"third.bin")], + mode: 0o644, + uid: 1000, + gid: 1000, + }]); + assert_eq!(retried[0].as_ref().unwrap().len(), 1); - assert_eq!(results.len(), 2); - for result in &results { - assert_eq!(result.as_ref().unwrap().len(), 1); - } let snapshot = service.sync_metadata_log_snapshot().unwrap(); - assert_eq!(snapshot.durable_lsn, 2); - assert_eq!(snapshot.segments.len(), 1); - let segment = service - .load_metadata_log_segment(&snapshot.segments.last().unwrap().segment_key) + assert_eq!(snapshot.durable_lsn, 3); + assert_eq!(snapshot.segments.len(), 3); + let first_segment = service + .load_metadata_log_segment(&snapshot.segments[0].segment_key) .unwrap(); - assert_eq!(segment.first_lsn, 1); - assert_eq!(segment.last_lsn, 2); - assert_eq!(segment.entries.len(), 2); + let second_segment = service + .load_metadata_log_segment(&snapshot.segments[1].segment_key) + .unwrap(); + let third_segment = service + .load_metadata_log_segment(&snapshot.segments[2].segment_key) + .unwrap(); + assert_eq!((first_segment.first_lsn, first_segment.last_lsn), (1, 1)); + assert_eq!((second_segment.first_lsn, second_segment.last_lsn), (2, 2)); + assert_eq!(first_segment.last_digest, second_segment.prev_digest); + assert_eq!((third_segment.first_lsn, third_segment.last_lsn), (3, 3)); + assert_eq!(second_segment.last_digest, third_segment.prev_digest); - let segment_keys = snapshot_segment_keys(&snapshot); let recovered = NoKvFs::new( MountId::new(1).unwrap(), HoltMetadataStore::open_memory().unwrap(), - objects.clone(), + objects, ); let outcome = recovered .restore_metadata_with_archived_log_segments( &checkpoint_config, - &segment_keys, + "mount-1:/", + &snapshot_segment_keys(&snapshot), 0, METADATA_LOG_ZERO_DIGEST, ) .unwrap() .unwrap(); - - assert_eq!(outcome.replayed_entries, 2); - assert!(recovered.lookup_path("/left/a.bin").unwrap().is_some()); - assert!(recovered.lookup_path("/right/b.bin").unwrap().is_some()); + assert_eq!(outcome.replayed_entries, 3); + assert!(recovered.lookup_path("/first").unwrap().is_some()); + assert!(recovered.lookup_path("/second").unwrap().is_some()); + assert!(recovered.lookup_path("/batch/third.bin").unwrap().is_some()); } -use nokv_object::{ObjectInfo, ObjectRange}; -use std::sync::atomic::AtomicUsize; +#[test] +fn immutable_checkpoint_restores_only_its_exact_control_identity() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + source + .create_dir_path("/controlled", 0o755, 1000, 1000) + .unwrap(); + let config = MetadataArchiveConfig::new("meta/exact", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest.clone(), + }; + assert!(backing + .head(&ObjectKey::new("meta/exact/CURRENT").unwrap()) + .unwrap() + .is_none()); -/// An [`ObjectStore`] wrapper that injects PUT failures to simulate a crash at a -/// chosen point (e.g. after the checkpoint object is written but before the -/// `CURRENT` pointer is swapped). Reads and deletes always pass through. -#[derive(Clone)] -struct FaultObjectStore { - inner: MemoryObjectStore, - fail_put_substring: Arc>>, - injected_put_failures: Arc, + // Drift standalone CURRENT to a different image. Controlled restore must + // still install only the exact control identity above. + source.create_dir_path("/drift", 0o755, 1000, 1000).unwrap(); + source.backup_metadata(&config).unwrap(); + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + let restored = recovered + .restore_metadata_checkpoint(&config, &identity) + .unwrap(); + assert_eq!(restored.checkpoint_key, prepared.checkpoint_key); + assert!(recovered.lookup_path("/controlled").unwrap().is_some()); + assert!(recovered.lookup_path("/drift").unwrap().is_none()); } -impl FaultObjectStore { - fn new(inner: MemoryObjectStore) -> Self { - Self { - inner, - fail_put_substring: Arc::new(Mutex::new(None)), - injected_put_failures: Arc::new(AtomicUsize::new(0)), - } - } - - fn fail_puts_containing(&self, substring: &str) { - *self.fail_put_substring.lock().unwrap() = Some(substring.to_owned()); - } - - fn clear_faults(&self) { - *self.fail_put_substring.lock().unwrap() = None; - } +#[test] +fn exact_checkpoint_rejects_missing_proof_before_fetching_image() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing.clone()); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + let config = MetadataArchiveConfig::new("meta/proof", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest, + }; + let proof_key = ObjectKey::new(format!("{}.proof", identity.checkpoint_key)).unwrap(); + assert!(backing.delete(&proof_key).unwrap()); + objects.clear_get_keys(); - fn injected_put_failures(&self) -> usize { - self.injected_put_failures.load(Ordering::Relaxed) - } + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + assert!(matches!( + recovered.restore_metadata_checkpoint(&config, &identity), + Err(MetadError::MetadataArchiveMissingObjectGcFence { checkpoint_key }) + if checkpoint_key == identity.checkpoint_key + )); + assert!(!objects.got_key(&identity.checkpoint_key)); + assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); } -impl ObjectStore for FaultObjectStore { - fn put( - &self, - key: &ObjectKey, - bytes: impl Into, - ) -> Result { - if let Some(substring) = self.fail_put_substring.lock().unwrap().clone() { - if key.as_str().contains(&substring) { - self.injected_put_failures.fetch_add(1, Ordering::Relaxed); - return Err(ObjectError::Backend("injected put fault".to_owned())); - } - } - self.inner.put(key, bytes) - } +#[test] +fn exact_checkpoint_rejects_cross_prefix_or_same_key_digest_mismatch() { + let backing = MemoryObjectStore::new(); + let objects = FaultObjectStore::new(backing); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + let config = MetadataArchiveConfig::new("meta/owned", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest, + }; + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + assert!(matches!( + recovered.restore_metadata_checkpoint( + &MetadataArchiveConfig::new("meta/other", 4), + &identity, + ), + Err(MetadError::Codec(message)) if message.contains("does not match archive prefix") + )); - fn get(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { - self.inner.get(key, range) - } + let mismatched = MetadataCheckpointIdentity { + checkpoint_key: identity.checkpoint_key.clone(), + image_bytes: identity.image_bytes, + image_digest: format!("sha256:{}", "0".repeat(64)), + }; + objects.clear_get_keys(); + assert!(matches!( + recovered.restore_metadata_checkpoint(&config, &mismatched), + Err(MetadError::Codec(message)) if message.contains("does not match archive prefix") + )); + assert!(!objects.got_key(&identity.checkpoint_key)); +} - fn head(&self, key: &ObjectKey) -> Result, ObjectError> { - self.inner.head(key) - } +#[test] +fn exact_checkpoint_rejects_tampered_same_key_image_before_install() { + let backing = MemoryObjectStore::new(); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + backing.clone(), + ); + source.bootstrap_root(0o755, 1000, 1000).unwrap(); + source + .create_dir_path("/must-not-install", 0o755, 1000, 1000) + .unwrap(); + let config = MetadataArchiveConfig::new("meta/tamper", 4); + let prepared = source.prepare_immutable_metadata_backup(&config).unwrap(); + let identity = MetadataCheckpointIdentity { + checkpoint_key: prepared.checkpoint_key.clone(), + image_bytes: prepared.image_bytes, + image_digest: prepared.image_digest, + }; + let image_key = ObjectKey::new(identity.checkpoint_key.clone()).unwrap(); + let mut tampered = backing.get(&image_key, None).unwrap(); + tampered[0] ^= 0xff; + assert_eq!(tampered.len() as u64, identity.image_bytes); + backing.put(&image_key, tampered).unwrap(); - fn delete(&self, key: &ObjectKey) -> Result { - self.inner.delete(key) - } + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + backing, + ); + assert!(matches!( + recovered.restore_metadata_checkpoint(&config, &identity), + Err(MetadError::Codec(message)) if message.contains("image digest mismatch") + )); + assert!(recovered.get_attr(InodeId::root()).unwrap().is_none()); } #[test] @@ -5848,6 +11569,63 @@ fn renew_snapshot_rejects_expiry_at_the_deadline() { ); } +#[test] +fn renew_snapshot_cannot_revive_a_pin_after_gc_crosses_its_expiry() { + let store = SnapshotCommitBarrierStore::new(CommandKind::RenewSnapshot, 1, 2) + .rejecting(CommandKind::RetireSnapshot); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + store.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let start_ms = service.now_ms(); + service.set_clock_override_ms(start_ms); + let name = DentryName::new(b"renew-gc-race.bin".to_vec()).unwrap(); + let published = service + .publish_artifact(artifact_request(name.clone(), "renew-gc-race", b"payload")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + let pin = service + .snapshot_subtree_with_lease(InodeId::root(), 500) + .unwrap(); + let snapshot_id = pin.snapshot_id; + service.remove_file(InodeId::root(), &name).unwrap(); + + let renew_service = Arc::clone(&service); + let renew = std::thread::spawn(move || renew_service.renew_snapshot(snapshot_id, 10_000)); + store.wait_until_blocked(); + + let deadline_ms = start_ms + 500; + service.set_clock_override_ms(deadline_ms); + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.snapshot_reap.conflicted, 1); + assert_eq!(cleanup.deleted, 1, "cleanup outcome: {cleanup:?}"); + assert!(objects.head(&object).unwrap().is_none()); + store.release_blocked(); + + assert!(matches!( + renew.join().unwrap(), + Err(MetadError::SnapshotLeaseExpired { + snapshot_id, + lease_expires_unix_ms, + now_ms, + }) if snapshot_id == pin.snapshot_id + && lease_expires_unix_ms == deadline_ms + && now_ms == deadline_ms + )); + assert_eq!( + service + .snapshot_pin(pin.snapshot_id) + .unwrap() + .unwrap() + .lease_expires_unix_ms, + deadline_ms + ); +} + #[test] fn renew_snapshot_is_extend_only_and_never_shortens_protection() { let (service, _objects) = service_with_objects(); @@ -6943,6 +12721,13 @@ impl MetadataStore for FixedRecoveryStore { unreachable!("recovery is read-only") } + fn committed_request_result( + &self, + _request_id: &[u8], + ) -> Result, MetadataError> { + unreachable!("recovery is read-only") + } + fn prune_history( &self, _request: HistoryPruneRequest, @@ -7193,6 +12978,40 @@ fn fallback_recovery_survives_command_dedupe_rows_on_real_store() { assert_eq!(minted.attr.inode.shard_index(), 0); } +#[test] +fn fallback_allocator_recovery_folds_fork_binding_commit_version() { + let (service, store) = shard_service(0); + service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let fork = service.clone_subtree_path("/base").unwrap(); + let binding = service + .versioned_fork_bindings_at(service.read_version().unwrap(), ReadPurpose::UserStrong) + .unwrap() + .into_iter() + .find(|binding| binding.binding.fork_root == fork.root) + .expect("detached clone publishes a durable fork binding"); + assert_eq!(binding.binding.created_version, binding.version.get()); + + // The binding commit only leaves a ForkBinding row (the dedupe value has a + // special encoding and is intentionally excluded from fallback scans). + // Removing the allocator record simulates recovery when that durable fast + // path is unavailable. + drop_allocator_record(&service); + let recovered = recover_allocator_state(&store, service.mount_id(), 0).unwrap(); + assert!( + recovered.last_commit_version >= binding.binding.created_version, + "fallback clock {} must cover binding commit {}", + recovered.last_commit_version, + binding.binding.created_version + ); + + let reopened = + NoKvFs::open_existing(service.mount_id(), store, MemoryObjectStore::new(), 0).unwrap(); + assert!( + reopened.next_version().unwrap().get() > binding.binding.created_version, + "reopened allocator must not reuse a visible ForkBinding version" + ); +} + /// Install `/dataset` as a cross-shard graft on shard 0 pointing at a real /// subtree dir owned by shard 1, returning both shards and the graft name. The /// child subtree dir already holds a file so any blind emptiness check on the diff --git a/crates/nokv-protocol/src/lib.rs b/crates/nokv-protocol/src/lib.rs index ce060495c..08452b7b6 100644 --- a/crates/nokv-protocol/src/lib.rs +++ b/crates/nokv-protocol/src/lib.rs @@ -370,6 +370,9 @@ pub enum MetadataRpcRequest { path: String, replace: bool, }, + RefreshPreparedArtifactObjectGcEpoch { + prepared: WirePreparedArtifact, + }, PublishPreparedArtifact { prepared: WirePreparedArtifact, body: Box, @@ -469,6 +472,10 @@ pub enum WireMetadataError { /// The target dentry is a cross-shard graft point; remove/rename of it must /// go through the graft lifecycle. The client maps this to `EBUSY`. GraftPoint, + StalePreparedArtifactObjectGcEpoch { + expected: u64, + current: u64, + }, SnapshotLeaseExpired { snapshot_id: u64, lease_expires_unix_ms: u64, @@ -483,6 +490,11 @@ pub enum WireMetadataError { SnapshotBindingChanged { root_path: String, }, + ForkRetentionActive { + snapshot_id: u64, + fork_root: u64, + borrower: u64, + }, SnapshotRenewContended { snapshot_id: u64, attempts: usize, @@ -1129,6 +1141,7 @@ pub struct WirePreparedArtifact { pub replace: bool, pub dentry_version: Option, pub old_generation: Option, + pub object_gc_claim_version: u64, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -1719,6 +1732,9 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { MetadataRpcRequest::PublishPreparedArtifact { prepared, .. } => { RoutingKey::Inode(prepared.parent) } + MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared } => { + RoutingKey::Inode(prepared.parent) + } MetadataRpcRequest::PublishPreparedArtifactStagedSession { prepared, .. } => { RoutingKey::Inode(prepared.parent) } @@ -1734,6 +1750,53 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { mod tests { use super::*; + #[test] + fn prepared_artifact_object_gc_epoch_round_trips() { + let prepared = WirePreparedArtifact { + mount: 1, + parent: 2, + name: "artifact.bin".to_owned(), + path: Some("/workspace/artifact.bin".to_owned()), + inode: 42, + generation: 7, + mtime_ms: 8, + ctime_ms: 9, + replace: true, + dentry_version: Some(6), + old_generation: Some(5), + object_gc_claim_version: 12, + }; + let mut encoded = Vec::new(); + prepared + .serialize(&mut rmp_serde::Serializer::new(&mut encoded).with_struct_map()) + .unwrap(); + let decoded: WirePreparedArtifact = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded, prepared); + + let request = MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { + prepared: prepared.clone(), + }; + assert_eq!( + decode_request(&encode_request(&request).unwrap()).unwrap(), + request + ); + + let error = WireMetadataError::StalePreparedArtifactObjectGcEpoch { + expected: 12, + current: 14, + }; + let envelope = MetadataRpcEnvelope { + ok: false, + result: None, + error: Some("stale prepared artifact".to_owned()), + error_kind: Some(error.clone()), + }; + assert_eq!( + decode_envelope(&encode_envelope(&envelope).unwrap()).unwrap(), + envelope + ); + } + #[test] fn binary_codec_round_trips_metadata_request() { let request = MetadataRpcRequest::GrepPaths { diff --git a/crates/nokv-server/src/control.rs b/crates/nokv-server/src/control.rs index fb499540f..52036e781 100644 --- a/crates/nokv-server/src/control.rs +++ b/crates/nokv-server/src/control.rs @@ -1,8 +1,10 @@ -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::time::Duration; use nokv_control::{ - CheckpointRef, ControlStore, LogRef, NodeId, ShardId, ShardLease, ShardRecord, ShardState, + CheckpointRef, ControlError, ControlStore, LogRef, NodeId, ShardId, ShardLease, ShardRecord, + ShardState, }; use nokv_meta::{MetadataStore, NoKvFs}; use nokv_object::ObjectStore; @@ -69,6 +71,9 @@ pub struct ServerShardOwnerState { pub(crate) struct ServerShardOwner { store: Arc, lease: ShardLease, + recovery_publish_gate: Arc>, + published_recovery_state: Arc>>, + recovery_dirty: Arc, /// Lease TTL (ms) for the wall-clock self-fence; `None` when auto-renewal is /// disabled (manual/test owners and single-node dev, which keep the fence /// off and rely on the epoch fence alone). @@ -165,7 +170,8 @@ impl ServerShardOwner { .map(|renewal| renewal.lease_ttl.as_millis() as u64) .filter(|ms| *ms > 0); let basis_ms = service.now_ms(); - let lease = match options.acquisition { + let acquisition = options.acquisition; + let lease = match acquisition { ServerShardAcquisition::Fresh => { store.acquire_unassigned(options.shard_id, options.node_id)? } @@ -173,17 +179,130 @@ impl ServerShardOwner { store.acquire_after_failure(options.shard_id, options.node_id, previous_epoch)? } }; - service.install_owner_epoch(lease.epoch)?; + // This check is deliberately after the atomic acquire. A pre-read has a + // TOCTOU window with another owner generation; the returned lease plus + // an exact record read-back is the authoritative proof. Epoch > 1 may + // still be a controlled Fresh resurrection only when no generation has + // ever reached Serving and no recovery identity has ever been published. + if matches!(acquisition, ServerShardAcquisition::Fresh) && lease.epoch != 1 { + let record = match store.get_shard(&lease.shard_id) { + Ok(record) => record, + Err(err) => { + let _ = store.release(&lease); + return Err(err.into()); + } + }; + let exact_lease = record.owner.as_ref() == Some(&lease.owner) + && record.epoch == lease.epoch + && record.lease_id == lease.lease_id + && record.state == ShardState::Recovering; + let never_served_empty = !record.ever_served + && record.checkpoint.is_none() + && record.log.is_none() + && record.durable_lsn == 0; + if !exact_lease || !never_served_empty { + let err = nokv_control::ControlError::FreshAcquireRequiresFailover { + shard_id: lease.shard_id.clone(), + epoch: lease.epoch, + }; + let _ = store.release(&lease); + return Err(err.into()); + } + } + if let Err(err) = service.install_owner_epoch(lease.epoch) { + // The control lease already exists, but no owner object has been + // constructed yet to release it. Do not strand a Recovering owner + // when local epoch installation rejects startup. + let _ = store.release(&lease); + return Err(err.into()); + } if let Some(ttl) = lease_ttl_ms { service.set_lease_deadline(basis_ms.saturating_add(ttl)); } Ok(Self { store, lease, + recovery_publish_gate: Arc::new(RwLock::new(())), + published_recovery_state: Arc::new(RwLock::new(None)), + recovery_dirty: Arc::new(AtomicBool::new(false)), lease_ttl_ms, }) } + pub(crate) fn lock_recovery_visibility(&self) -> RwLockReadGuard<'_, ()> { + self.recovery_publish_gate + .read() + .unwrap_or_else(|err| err.into_inner()) + } + + pub(crate) fn lock_recovery_publication(&self) -> RwLockWriteGuard<'_, ()> { + self.recovery_publish_gate + .write() + .unwrap_or_else(|err| err.into_inner()) + } + + pub(crate) fn published_recovery_state(&self) -> Option { + self.published_recovery_state + .read() + .unwrap_or_else(|err| err.into_inner()) + .clone() + } + + pub(crate) fn mark_recovery_dirty(&self) { + self.recovery_dirty.store(true, Ordering::Release); + } + + pub(crate) fn mark_recovery_clean(&self) { + self.recovery_dirty.store(false, Ordering::Release); + } + + pub(crate) fn recovery_is_dirty(&self) -> bool { + self.recovery_dirty.load(Ordering::Acquire) + } + + pub(crate) fn verify_read_lease(&self, service: &NoKvFs) -> Result<(), ServerError> + where + M: MetadataStore, + O: ObjectStore, + { + if self.lease_ttl_ms.is_some() { + service.verify_owner_lease().map_err(ServerError::Metadata) + } else { + // Explicit no-background-renew mode has no local expiry deadline. + // Validate the stable control session on every read instead of + // allowing a replaced owner to serve stale data indefinitely. + match self.renew(service) { + Ok(_) => service.verify_owner_lease().map_err(ServerError::Metadata), + Err(err) + if matches!( + &err, + ServerError::Control( + ControlError::NotOwner { .. } | ControlError::StaleLease { .. } + ) + ) => + { + let endpoint = self + .store + .get_shard(&self.lease.shard_id) + .ok() + .and_then(|record| record.endpoint); + Err(ServerError::NotOwner { + shard_id: self.lease.shard_id.as_str().to_owned(), + endpoint, + }) + } + Err(err) => Err(err), + } + } + } + + fn cache_published_recovery_state(&self, state: ServerShardOwnerState) { + *self + .published_recovery_state + .write() + .unwrap_or_else(|err| err.into_inner()) = Some(state); + } + pub(crate) fn mark_serving( &self, service: &NoKvFs, @@ -206,15 +325,63 @@ impl ServerShardOwner { M: MetadataStore, O: ObjectStore, { - let basis_ms = service.now_ms(); - let record = self + let _publication = self.lock_recovery_publication(); + self.mark_serving_with_recovery_refs_locked(service, checkpoint, log, durable_lsn) + } + + /// Publish while the caller already holds `recovery_publish_gate`. This is + /// used when the protected operation begins before the control CAS (for + /// example, snapshotting the sync-log tail or preparing a checkpoint). + pub(crate) fn mark_serving_with_recovery_refs_locked( + &self, + service: &NoKvFs, + checkpoint: Option, + log: Option, + durable_lsn: u64, + ) -> Result + where + M: MetadataStore, + O: ObjectStore, + { + // Local deadline/epoch fencing is checked immediately before the + // control-plane lease CAS. An owner that self-fenced must not revive + // itself by publishing a recovery reference. + service.verify_owner_lease()?; + let cached = self.published_recovery_state(); + let checkpoint_for_readback = checkpoint.clone(); + let log_for_readback = log.clone(); + let record = match self .store - .mark_serving(&self.lease, checkpoint, log, durable_lsn)?; + .mark_serving(&self.lease, checkpoint, log, durable_lsn) + { + Ok(record) => record, + Err(mark_err) => match self.store.get_shard(&self.lease.shard_id) { + Ok(record) + if recovery_publication_matches( + &self.lease, + &record, + cached.as_ref(), + checkpoint_for_readback.as_ref(), + log_for_readback.as_ref(), + durable_lsn, + ) => + { + record + } + Ok(record) => { + service.observe_required_owner_epoch(record.epoch)?; + return Err(mark_err.into()); + } + Err(_) => return Err(mark_err.into()), + }, + }; service.install_owner_epoch(record.epoch)?; - if let Some(ttl) = self.lease_ttl_ms { - service.set_lease_deadline(basis_ms.saturating_add(ttl)); - } - Ok(owner_state(&self.lease, &record)) + // `mark_serving` is a control-record CAS, not a lease keepalive. Never + // extend the local deadline here: only a successful `renew` round trip + // proves that the control plane extended the real lease. + let state = owner_state(&self.lease, &record); + self.cache_published_recovery_state(state.clone()); + Ok(state) } pub(crate) fn renew( @@ -234,7 +401,9 @@ impl ServerShardOwner { if let Some(ttl) = self.lease_ttl_ms { service.set_lease_deadline(basis_ms.saturating_add(ttl)); } - Ok(owner_state(&self.lease, &record)) + let state = owner_state(&self.lease, &record); + self.cache_published_recovery_state(state.clone()); + Ok(state) } Err(err) => { // Best-effort: observe a bumped epoch if the control plane is @@ -262,6 +431,48 @@ impl ServerShardOwner { } } +fn recovery_publication_matches( + lease: &ShardLease, + record: &ShardRecord, + cached: Option<&ServerShardOwnerState>, + checkpoint: Option<&CheckpointRef>, + log: Option<&LogRef>, + durable_lsn: u64, +) -> bool { + if record.owner.as_ref() != Some(&lease.owner) + || record.epoch != lease.epoch + || record.lease_id != lease.lease_id + || record.state != ShardState::Serving + || record.durable_lsn != durable_lsn + { + return false; + } + + let expected_checkpoint = checkpoint + .cloned() + .or_else(|| cached.and_then(|state| state.checkpoint.clone())); + let Some(expected_checkpoint) = expected_checkpoint else { + // An ACK-lost no-reference publication cannot be proven without an + // exact previously-published recovery identity. + return false; + }; + if record.checkpoint.as_ref() != Some(&expected_checkpoint) { + return false; + } + + let mut expected_log = log + .cloned() + .or_else(|| cached.and_then(|state| state.log.clone())); + if expected_log + .as_ref() + .is_some_and(|expected_log| expected_log.durable_lsn <= expected_checkpoint.lsn) + || (checkpoint.is_some() && expected_checkpoint.lsn == durable_lsn && log.is_none()) + { + expected_log = None; + } + record.log == expected_log +} + fn owner_state(lease: &ShardLease, record: &ShardRecord) -> ServerShardOwnerState { ServerShardOwnerState { shard_id: lease.shard_id.clone(), diff --git a/crates/nokv-server/src/http.rs b/crates/nokv-server/src/http.rs index 0d50b43e0..5af627e97 100644 --- a/crates/nokv-server/src/http.rs +++ b/crates/nokv-server/src/http.rs @@ -17,9 +17,9 @@ struct HttpRequest { #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct HttpResponse { - status: &'static str, - content_type: &'static str, - body: String, + pub(crate) status: &'static str, + pub(crate) content_type: &'static str, + pub(crate) body: String, keep_alive: bool, } @@ -143,7 +143,12 @@ pub(crate) fn handle_parts(server: &Server, line: &str, _body: &[u8]) -> HttpRes let (path, query) = split_target(target); match (method, path) { ("GET", "/healthz") => HttpResponse::text("200 OK", "ok\n"), - ("GET", "/readyz") => HttpResponse::text("200 OK", "ready\n"), + ("GET", "/readyz") => match server.check_readiness() { + Ok(()) => HttpResponse::text("200 OK", "ready\n"), + Err(err) => { + HttpResponse::text("503 Service Unavailable", format!("not ready: {err}\n")) + } + }, ("GET", "/stats") => HttpResponse::json("200 OK", server.stats_json()), ("GET", "/gc") | ("POST", "/gc") => { let limit = match gc_limit(query) { diff --git a/crates/nokv-server/src/rpc/batch.rs b/crates/nokv-server/src/rpc/batch.rs index 1b3a3295e..78e3eaac9 100644 --- a/crates/nokv-server/src/rpc/batch.rs +++ b/crates/nokv-server/src/rpc/batch.rs @@ -12,6 +12,46 @@ pub(super) fn execute_batch( // route to the same shard. Resolve that single slot up front and reject // cross-shard batches (the client must split them per shard). let slot = resolve_batch_slot(server, &requests)?; + execute_resolved_batch(server, slot, requests) +} + +/// Opportunistically coalesce independently framed requests. A routing or +/// nested-batch mismatch is reported as `None` before execution so transport can +/// fall back to individual frames. Any error after resolution is returned and +/// must not trigger re-execution because some batch entries may have committed. +pub(super) fn try_execute_coalesced_batch( + server: &Server, + requests: Vec, +) -> Result>, ServerError> { + let slot = match resolve_batch_slot(server, &requests) { + Ok(slot) => slot, + Err(_) => return Ok(None), + }; + match execute_resolved_batch(server, slot, requests)? { + MetadataRpcResult::Batch { results } => Ok(Some(results)), + _ => unreachable!("resolved batch executor always returns a batch result"), + } +} + +fn execute_resolved_batch( + server: &Server, + slot: &ShardSlot, + requests: Vec, +) -> Result { + let may_commit = requests.iter().any(super::commits_metadata_view); + // Resolve selected one shard before taking the visibility gate. The whole + // mixed/coalesced batch then holds that gate once; subrequests execute raw + // to avoid lock re-entry, and the post barrier runs on both Ok and Err. + server.execute_with_shard_visibility(slot, may_commit, || { + execute_batch_on_slot(server, slot, requests) + }) +} + +fn execute_batch_on_slot( + server: &Server, + slot: &ShardSlot, + requests: Vec, +) -> Result { let mut results = Vec::with_capacity(requests.len()); let mut iter = requests.into_iter().peekable(); while let Some(request) = iter.next() { @@ -114,7 +154,7 @@ fn resolve_batch_slot<'a>( } fn execute_envelope(server: &Server, request: MetadataRpcRequest) -> MetadataRpcEnvelope { - match super::execute(server, request) { + match super::execute_unfenced(server, request) { Ok(result) => ok_envelope(result), Err(err) => err_envelope(err), } diff --git a/crates/nokv-server/src/rpc/mod.rs b/crates/nokv-server/src/rpc/mod.rs index 468cbbba6..9deadcfc1 100644 --- a/crates/nokv-server/src/rpc/mod.rs +++ b/crates/nokv-server/src/rpc/mod.rs @@ -40,35 +40,15 @@ use wire::{ fn handle_binary_rpc(server: &Server, body: &[u8]) -> Result, ServerError> { let envelope = match decode_request(body) { - Ok(request) => { - // Resolve which shard owns this request BEFORE consuming it, so a - // committing op publishes the LogRef of the shard that handled it - // (not the default shard). Batches publish per-shard inside execute, - // so they are excluded here. - let publish_index = if !matches!(request, MetadataRpcRequest::Batch { .. }) - && commits_metadata_view(&request) - { - server.route(&request).ok().map(|slot| slot.shard_index()) - } else { - None - }; - match execute(server, request) { - Ok(result) => match if let Some(shard_index) = publish_index { - server.publish_latest_metadata_log_ref_for_index(shard_index) - } else { - Ok(None) - } { - Ok(_) => MetadataRpcEnvelope { - ok: true, - result: Some(result), - error: None, - error_kind: None, - }, - Err(err) => err_envelope(err), - }, - Err(err) => err_envelope(err), - } - } + Ok(request) => match execute_visible(server, request) { + Ok(result) => MetadataRpcEnvelope { + ok: true, + result: Some(result), + error: None, + error_kind: None, + }, + Err(err) => err_envelope(err), + }, Err(err) => MetadataRpcEnvelope { ok: false, result: None, @@ -86,7 +66,23 @@ fn handle_binary_rpc(server: &Server, body: &[u8]) -> Result, ServerErro }) } -fn execute(server: &Server, request: MetadataRpcRequest) -> Result { +fn execute_visible( + server: &Server, + request: MetadataRpcRequest, +) -> Result { + if matches!(request, MetadataRpcRequest::Batch { .. }) { + return execute_unfenced(server, request); + } + let slot = server.route(&request)?; + let commits_metadata = commits_metadata_view(&request); + server + .execute_with_shard_visibility(slot, commits_metadata, || execute_unfenced(server, request)) +} + +fn execute_unfenced( + server: &Server, + request: MetadataRpcRequest, +) -> Result { // A batch re-routes each sub-request to its own shard, so resolve it there. if let MetadataRpcRequest::Batch { requests } = request { return execute_batch(server, requests); @@ -931,6 +927,19 @@ fn execute(server: &Server, request: MetadataRpcRequest) -> Result { + if prepared.mount != slot.service().mount_id().get() { + return Err(ServerError::Metadata(MetadError::Codec( + "prepared artifact mount does not match server mount".to_owned(), + ))); + } + let prepared = slot + .service() + .refresh_prepared_artifact_object_gc_epoch(prepared_artifact(prepared)?)?; + Ok(MetadataRpcResult::PreparedArtifact { + prepared: wire_prepared_artifact(slot.service().mount_id(), &prepared), + }) + } MetadataRpcRequest::PublishPreparedArtifact { prepared, body, @@ -1107,6 +1116,7 @@ fn refreshes_metadata_view(request: &MetadataRpcRequest) -> bool { | MetadataRpcRequest::RenewSnapshot { .. } | MetadataRpcRequest::PrepareArtifact { .. } | MetadataRpcRequest::PrepareArtifactPath { .. } + | MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { .. } | MetadataRpcRequest::PublishPreparedArtifact { .. } | MetadataRpcRequest::PublishPreparedArtifactStagedSession { .. } => false, } diff --git a/crates/nokv-server/src/rpc/tests.rs b/crates/nokv-server/src/rpc/tests.rs index b54e31121..5e049a218 100644 --- a/crates/nokv-server/src/rpc/tests.rs +++ b/crates/nokv-server/src/rpc/tests.rs @@ -1,11 +1,10 @@ use super::*; -use crate::server::tests::{test_options, test_server}; -use crate::{ServerShardOwnerOptions, ServerSharedLogOptions}; -use nokv_control::{ControlStore, InMemoryControlStore, ShardId}; -use nokv_object::{ - ConfiguredObjectStore, HotFillMode, LocalObjectStoreOptions, MemoryObjectStore, - ObjectStoreConfig, S3ObjectStoreOptions, TieredObjectStoreOptions, TieredPutPolicy, +use crate::server::tests::{ + test_options, test_server, BlockingServingControl, FailingCheckpointPublishControl, }; +use crate::{ServerShardOwnerOptions, ServerShardOwnerRenewalOptions, ServerSharedLogOptions}; +use nokv_control::{ControlStore, InMemoryControlStore, ShardId}; +use nokv_object::{ConfiguredObjectStore, MemoryObjectStore, S3ObjectStoreOptions}; use nokv_protocol::{ decode_envelope, encode_request, WireBlockDescriptor, WireBodyDescriptor, WireChunkManifest, WireDentryWithAttr, WireMetadataError, WireOpenPathReadPlanRequest, WireSliceManifest, @@ -15,8 +14,9 @@ use std::io::{Read, Write}; use std::net::TcpListener; use std::net::TcpStream; use std::path::Path; -use std::sync::Arc; +use std::sync::{mpsc, Arc}; use std::thread; +use std::time::Duration; use tempfile::tempdir; fn request_envelope(server: &Server, request: MetadataRpcRequest) -> MetadataRpcEnvelope { @@ -55,19 +55,6 @@ fn fake_s3_options() -> S3ObjectStoreOptions { } } -fn shared_tiered_object_config(root: &Path) -> ObjectStoreConfig { - ObjectStoreConfig::tiered_local_with_options( - LocalObjectStoreOptions::new(root.join("hot")), - fake_s3_options(), - TieredObjectStoreOptions { - put_policy: TieredPutPolicy::HotThenBackgroundCold, - populate_hot_on_get: true, - hot_fill_mode: HotFillMode::Inline, - pending_cold_put_root: Some(root.join("pending-cold")), - }, - ) -} - #[test] fn rpc_creates_and_lists_directory() { let server = test_server(); @@ -96,14 +83,17 @@ fn rpc_creates_and_lists_directory() { fn rpc_write_publishes_sync_shared_log_before_ack() { let dir = tempdir().unwrap(); let mut options = test_options(dir.path()); - options.object = shared_tiered_object_config(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/rpc-sync-checkpoint".to_owned()); let control = Arc::new(InMemoryControlStore::new()); - let server = Server::open_with_control( + let server = Server::open_with_objects( options, - control.clone(), - vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") - .with_renewal(None) - .with_shared_log(Some(ServerSharedLogOptions::new("meta/rpc-sync-log")))], + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/rpc-sync-log")))], + )), ) .unwrap(); @@ -133,6 +123,446 @@ fn rpc_write_publishes_sync_shared_log_before_ack() { assert_eq!(log.digest.len(), 64); } +#[test] +fn rpc_prepare_with_an_empty_log_tail_preserves_the_checkpoint_publication() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/rpc-empty-tail-checkpoint".to_owned()); + let control = Arc::new(InMemoryControlStore::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/rpc-empty-tail-log")))], + )), + ) + .unwrap(); + + let initial_log = server.service().sync_metadata_log_snapshot().unwrap(); + assert!( + initial_log.segments.is_empty(), + "startup checkpoint must cover and prune the complete live log chain" + ); + let before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + let checkpoint = before + .checkpoint + .clone() + .expect("controlled startup must publish a checkpoint"); + assert_eq!(before.durable_lsn, initial_log.durable_lsn); + assert_eq!(checkpoint.lsn, initial_log.durable_lsn); + assert!(before.log.is_none()); + + let prepared = request_envelope( + &server, + MetadataRpcRequest::PrepareArtifact { + parent: 1, + name: "prepared-only.bin".to_owned(), + replace: false, + }, + ); + assert!(prepared.ok, "unexpected prepare RPC error: {prepared:?}"); + + let after_prepare = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(after_prepare.durable_lsn, before.durable_lsn); + assert_eq!(after_prepare.checkpoint, Some(checkpoint.clone())); + assert!( + after_prepare.log.is_none(), + "an empty live tail must not publish an invalid empty LogRef" + ); + + let committed = request_envelope( + &server, + MetadataRpcRequest::CreateDirPath { + path: "/after-prepare".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + ); + assert!( + committed.ok, + "unexpected committing RPC error: {committed:?}" + ); + let after_commit = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + let log = after_commit + .log + .expect("a real commit above the checkpoint must publish a LogRef"); + assert_eq!(log.segments[0].first_lsn, checkpoint.lsn + 1); + assert_eq!(log.durable_lsn, after_commit.durable_lsn); +} + +#[test] +fn rpc_publish_control_failure_does_not_hide_committed_metadata_or_local_log() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/rpc-phase-checkpoint".to_owned()); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/rpc-phase-log")))], + )), + ) + .unwrap(); + + let prepared = match request_envelope( + &server, + MetadataRpcRequest::PrepareArtifact { + parent: 1, + name: "phase.bin".to_owned(), + replace: false, + }, + ) + .result + .unwrap() + { + MetadataRpcResult::PreparedArtifact { prepared } => prepared, + other => panic!("unexpected prepare result: {other:?}"), + }; + let control_before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + let local_before = server + .service() + .sync_metadata_log_snapshot() + .unwrap() + .durable_lsn; + assert_eq!(control_before.durable_lsn, local_before); + + control.fail_next_marks(2); + let envelope = request_envelope( + &server, + MetadataRpcRequest::PublishPreparedArtifact { + body: Box::new(WireBodyDescriptor { + producer: "unit-test".to_owned(), + digest_uri: "sha256:empty".to_owned(), + size: 0, + content_type: "application/octet-stream".to_owned(), + manifest_id: "phase.bin".to_owned(), + generation: prepared.generation, + base_generation: 0, + chunk_size: 64 * 1024 * 1024, + block_size: 4 * 1024 * 1024, + }), + chunks: Vec::new(), + prepared, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ); + assert!(!envelope.ok, "control publication must gate the RPC ACK"); + assert!(matches!( + envelope.error_kind, + Some(WireMetadataError::Metadata { message }) + if message.contains("injected checkpoint publish failure") + )); + + let committed = server + .service() + .lookup_path("/phase.bin") + .unwrap() + .expect("metadata apply must remain visible after the phase-3 control error"); + assert!( + committed.body.is_some(), + "the committed manifest must remain visible" + ); + let local_after = server.service().sync_metadata_log_snapshot().unwrap(); + assert!(local_after.durable_lsn > local_before); + let control_after = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!( + control_after.durable_lsn, control_before.durable_lsn, + "failed control publication must not advance the authoritative tail" + ); + assert_eq!(control_after.log, control_before.log); +} + +#[test] +fn controlled_read_waits_for_inflight_write_control_publication() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/read-visibility-checkpoint".to_owned()); + let control = Arc::new(BlockingServingControl::new()); + let server = Arc::new( + Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new( + "meta/read-visibility-log", + )))], + )), + ) + .unwrap(), + ); + + control.arm_mark(); + let writer_server = Arc::clone(&server); + let writer = thread::spawn(move || { + request_envelope( + &writer_server, + MetadataRpcRequest::CreateDirPath { + path: "/published-before-read".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + ) + }); + control.wait_until_mark_reached(); + + let (read_tx, read_rx) = mpsc::channel(); + let reader_server = Arc::clone(&server); + let reader = thread::spawn(move || { + let result = request_envelope( + &reader_server, + MetadataRpcRequest::StatPath { + path: "/published-before-read".to_owned(), + }, + ); + read_tx.send(result).unwrap(); + }); + assert!( + read_rx.recv_timeout(Duration::from_millis(100)).is_err(), + "strong read must remain blocked while the write owns publication" + ); + + control.release_mark(); + let write = writer.join().unwrap(); + assert!( + write.ok, + "write should ACK after control publication: {write:?}" + ); + let read = read_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert!(read.ok, "read should run after publication: {read:?}"); + reader.join().unwrap(); +} + +#[test] +fn controlled_read_rejects_local_only_write_while_control_publication_fails() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/read-fail-checkpoint".to_owned()); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/read-fail-log")))], + )), + ) + .unwrap(); + let before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + + // Each repair gets two mark attempts (initial + authoritative retry). + // Keep write, read, readyz, and stats persistently failed. + control.fail_next_marks(8); + let write = request_envelope( + &server, + MetadataRpcRequest::CreateDirPath { + path: "/local-only".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + ); + assert!(!write.ok); + assert!(server + .service() + .lookup_path("/local-only") + .unwrap() + .is_some()); + assert_eq!( + control.get_shard(&ShardId::new("mount-1:/")).unwrap(), + before + ); + + let read = request_envelope( + &server, + MetadataRpcRequest::StatPath { + path: "/local-only".to_owned(), + }, + ); + assert!( + !read.ok, + "dirty local state must never escape in a read ACK" + ); + assert!(read.result.is_none()); + assert_eq!( + control.get_shard(&ShardId::new("mount-1:/")).unwrap(), + before + ); + + let not_ready = crate::http::handle_parts(&server, "GET /readyz HTTP/1.1", &[]); + assert_eq!(not_ready.status, "503 Service Unavailable"); + assert!(server.stats_json().contains("\"ready\":false")); + + control.fail_next_marks(0); + let ready = crate::http::handle_parts(&server, "GET /readyz HTTP/1.1", &[]); + assert_eq!(ready.status, "200 OK"); + assert!(server.stats_json().contains("\"ready\":true")); + let repaired = request_envelope( + &server, + MetadataRpcRequest::StatPath { + path: "/local-only".to_owned(), + }, + ); + assert!(repaired.ok, "readiness repair must make the write visible"); +} + +#[test] +fn transient_control_publication_failure_is_repaired_before_write_ack() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/transient-mark-checkpoint".to_owned()); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/transient-mark-log")))], + )), + ) + .unwrap(); + let before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + + control.fail_next_marks(1); + let write = request_envelope( + &server, + MetadataRpcRequest::CreateDirPath { + path: "/transient".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + ); + assert!( + write.ok, + "one transient mark failure should repair: {write:?}" + ); + assert!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .durable_lsn + > before.durable_lsn + ); +} + +#[test] +fn controlled_clean_read_fails_at_local_lease_deadline() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/read-deadline-checkpoint".to_owned()); + let control = Arc::new(InMemoryControlStore::new()); + let renewal = ServerShardOwnerRenewalOptions { + interval: Duration::from_secs(3600), + run_immediately: false, + lease_ttl: Duration::from_secs(10), + }; + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control, + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(Some(renewal)) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/read-deadline-log")))], + )), + ) + .unwrap(); + let deadline = server.service().lease_deadline_ms(); + assert!(deadline > 0); + server.service().set_clock_override_ms(deadline); + + let read = request_envelope(&server, MetadataRpcRequest::GetAttr { inode: 1 }); + assert!(!read.ok); + assert!(read + .error + .as_deref() + .is_some_and(|message| message.contains("lease expired"))); +} + +#[test] +fn no_renew_controlled_read_requires_live_control_session() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/read-session-checkpoint".to_owned()); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/read-session-log")))], + )), + ) + .unwrap(); + control.fail_next_renews(1); + + let read = request_envelope(&server, MetadataRpcRequest::GetAttr { inode: 1 }); + assert!(!read.ok); + assert!(read + .error + .as_deref() + .is_some_and(|message| message.contains("injected owner renew failure"))); +} + +#[test] +fn controlled_read_rejects_an_observed_successor_epoch() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/read-epoch-checkpoint".to_owned()); + let control = Arc::new(InMemoryControlStore::new()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/read-epoch-log")))], + )), + ) + .unwrap(); + let epoch = control.get_shard(&ShardId::new("mount-1:/")).unwrap().epoch; + server + .service() + .observe_required_owner_epoch(epoch + 1) + .unwrap(); + assert_eq!(server.service().required_owner_epoch(), epoch + 1); + assert!(matches!( + server.service().verify_owner_lease(), + Err(nokv_meta::MetadError::StaleOwnerEpoch { .. }) + )); + + let read = request_envelope(&server, MetadataRpcRequest::GetAttr { inode: 1 }); + assert!(!read.ok); + assert!(matches!( + read.error_kind, + Some(WireMetadataError::StaleOwnerEpoch { + owner_epoch, + required_epoch, + }) if owner_epoch == epoch && required_epoch == epoch + 1 + )); +} + #[test] fn controlled_failover_restores_checkpoint_and_replays_shared_log() { let dir = tempdir().unwrap(); @@ -171,15 +601,34 @@ fn controlled_failover_restores_checkpoint_and_replays_shared_log() { .checkpoint .expect("manual backup should publish checkpoint ref"); - expect_dentry(request_envelope( + let partial_batch = request_envelope( &first, - MetadataRpcRequest::CreateDirPath { - path: "/after".to_owned(), - mode: 0o755, - uid: 1000, - gid: 1000, + MetadataRpcRequest::Batch { + requests: vec![ + MetadataRpcRequest::CreateDirPath { + path: "/after".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + MetadataRpcRequest::CreateDirPath { + path: "/before".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }, + ], }, - )); + ); + assert!( + partial_batch.ok, + "unexpected batch error: {partial_batch:?}" + ); + let MetadataRpcResult::Batch { results } = partial_batch.result.unwrap() else { + panic!("unexpected partial batch result") + }; + assert!(results[0].ok, "the first batch entry must commit"); + assert!(!results[1].ok, "the duplicate batch entry must fail"); let after_record = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); assert!(after_record.durable_lsn > checkpoint_record.lsn); assert!(after_record.log.is_some()); @@ -203,8 +652,36 @@ fn controlled_failover_restores_checkpoint_and_replays_shared_log() { assert_eq!(state.epoch, 2); assert_eq!(state.state, nokv_control::ShardState::Serving); assert_eq!(state.durable_lsn, after_record.durable_lsn); - assert_eq!(state.checkpoint, after_record.checkpoint); - assert_eq!(state.log, after_record.log); + let refreshed_checkpoint = state + .checkpoint + .as_ref() + .expect("failover startup must publish a fresh fenced checkpoint"); + assert_ne!( + refreshed_checkpoint.object_key, checkpoint_record.object_key, + "serving must not retain the historical checkpoint ref" + ); + assert_eq!(refreshed_checkpoint.lsn, after_record.durable_lsn); + assert!(refreshed_checkpoint + .object_key + .starts_with("meta/failover-ck/mount_1__/controlled/sha256/")); + assert!( + state.log.is_none(), + "the fresh checkpoint covers and clears the replayed log chain" + ); + + // Startup checkpoint publication pruned every shared-log segment it now + // covers. A second checkpoint with no intervening write must still publish + // the durable tail from the log allocator, not regress to LSN zero merely + // because the live segment list is empty. + second.run_manual_backup().unwrap(); + let no_write_state = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + let no_write_checkpoint = no_write_state + .checkpoint + .as_ref() + .expect("no-write backup should retain a checkpoint ref"); + assert_eq!(no_write_state.durable_lsn, after_record.durable_lsn); + assert_eq!(no_write_checkpoint.lsn, after_record.durable_lsn); + assert_eq!(no_write_checkpoint.digest, refreshed_checkpoint.digest); expect_dentry(request_envelope( &second, @@ -806,7 +1283,10 @@ fn rpc_lists_indexed_path_pages_without_plain_namespace_entries() { logical_offset: 0, len: 2, blocks: vec![WireBlockDescriptor { - object_key: format!("blocks/1/{}/{}", prepared.inode, prepared.generation), + object_key: format!( + "blocks/1/{}/{}/0/0", + prepared.inode, prepared.generation + ), logical_offset: 0, object_offset: 0, len: 2, @@ -1469,7 +1949,7 @@ fn rpc_prepares_and_publishes_artifact_manifest() { logical_offset: 0, len: 4, blocks: vec![WireBlockDescriptor { - object_key: format!("blocks/1/{}/{}", prepared.inode, prepared.generation), + object_key: format!("blocks/1/{}/{}/0/0", prepared.inode, prepared.generation), logical_offset: 0, object_offset: 0, len: 4, @@ -1491,6 +1971,73 @@ fn rpc_prepares_and_publishes_artifact_manifest() { assert!(result.1.is_none()); } +#[test] +fn rpc_refreshes_stale_prepared_object_gc_epoch_and_preserves_typed_error() { + let server = test_server(); + let prepared = match request_envelope( + &server, + MetadataRpcRequest::PrepareArtifact { + parent: 1, + name: "refresh.bin".to_owned(), + replace: false, + }, + ) + .result + .unwrap() + { + MetadataRpcResult::PreparedArtifact { prepared } => prepared, + other => panic!("unexpected prepare result: {other:?}"), + }; + assert_ne!(prepared.object_gc_claim_version, 0); + + let mut stale = prepared.clone(); + stale.object_gc_claim_version = prepared.object_gc_claim_version.saturating_add(1000); + let stale_publish = request_envelope( + &server, + MetadataRpcRequest::PublishPreparedArtifact { + prepared: stale.clone(), + body: Box::new(WireBodyDescriptor { + producer: "unit-test".to_owned(), + digest_uri: "sha256:empty".to_owned(), + size: 0, + content_type: "application/octet-stream".to_owned(), + manifest_id: "refresh.bin".to_owned(), + generation: stale.generation, + base_generation: 0, + chunk_size: 64 * 1024 * 1024, + block_size: 4 * 1024 * 1024, + }), + chunks: Vec::new(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ); + assert!(!stale_publish.ok); + assert_eq!( + stale_publish.error_kind, + Some(WireMetadataError::StalePreparedArtifactObjectGcEpoch { + expected: stale.object_gc_claim_version, + current: prepared.object_gc_claim_version, + }) + ); + + let refreshed = request_envelope( + &server, + MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared: stale }, + ); + assert!(refreshed.ok, "refresh failed: {refreshed:?}"); + let refreshed = match refreshed.result.unwrap() { + MetadataRpcResult::PreparedArtifact { prepared } => prepared, + other => panic!("unexpected refresh result: {other:?}"), + }; + assert!(refreshed.generation > prepared.generation); + assert_eq!( + refreshed.object_gc_claim_version, + prepared.object_gc_claim_version + ); +} + #[test] fn rpc_staged_session_publish_preserves_old_prefix_on_shrink() { let server = test_server(); @@ -1508,7 +2055,7 @@ fn rpc_staged_session_publish_preserves_old_prefix_on_shrink() { MetadataRpcResult::PreparedArtifact { prepared } => prepared, other => panic!("unexpected prepare result: {other:?}"), }; - let old_object_key = format!("blocks/1/{}/{}", prepared.inode, prepared.generation); + let old_object_key = format!("blocks/1/{}/{}/0/0", prepared.inode, prepared.generation); let envelope = request_envelope( &server, MetadataRpcRequest::PublishPreparedArtifact { @@ -1643,7 +2190,12 @@ fn publish_synthetic_file(server: &Server, path: &str, manifest_id: &str, replac logical_offset: 0, len: 4, blocks: vec![nokv_types::BlockDescriptor { - object_key: format!("blocks/{manifest_id}"), + object_key: format!( + "blocks/{}/{}/{}/0/0", + server.service().mount_id().get(), + prepared.inode.get(), + prepared.generation + ), logical_offset: 0, object_offset: 0, len: 4, @@ -1712,7 +2264,7 @@ fn rpc_clone_subtree_links_navigable_fork_and_diff_tracks_divergence() { other => panic!("unexpected clone result: {other:?}"), }; assert!(fork_root > base.attr.inode, "fork gets a fresh root inode"); - assert!(snapshot_id > 0, "clone retains a snapshot pin"); + assert!(snapshot_id > 0, "clone retains a durable source binding"); // The fork is a real, navigable directory at /fork: listing it through the // path RPC surfaces the base's entries under fresh inodes. @@ -2030,13 +2582,22 @@ mod multi_shard { DATASET_INDEX, ) .unwrap(); - let server = Server::open_with_control( - test_options(dir.path()), - control.clone(), - vec![ - ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None), - ServerShardOwnerOptions::fresh("mount-1:/dataset", "node-a").with_renewal(None), - ], + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("meta/multi-shard-checkpoint".to_owned()); + let server = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ + ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/multi-shard-log"))), + ServerShardOwnerOptions::fresh("mount-1:/dataset", "node-a") + .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new("meta/multi-shard-log"))), + ], + )), ) .unwrap(); (server, control, dir) @@ -2107,7 +2668,7 @@ mod multi_shard { ); // Reconcile heals it locally. - server.reconcile_local_grafts(); + server.reconcile_local_grafts().unwrap(); let after = expect_dentry(request_envelope( &server, @@ -2123,7 +2684,7 @@ mod multi_shard { ); // Idempotent: a second reconcile is a no-op (the dentry already exists). - server.reconcile_local_grafts(); + server.reconcile_local_grafts().unwrap(); let still = expect_dentry(request_envelope( &server, MetadataRpcRequest::LookupPlus { @@ -2747,14 +3308,24 @@ mod fleet_e2e { make_owner: impl FnOnce(SocketAddr) -> ServerShardOwnerOptions + Send + 'static, ) -> RunningServer { let meta_root = meta_root.to_path_buf(); - let prefix = checkpoint_prefix.map(ToOwned::to_owned); + let prefix = Some( + checkpoint_prefix + .unwrap_or("meta/fleet-control-checkpoint") + .to_owned(), + ); RunningServer::spawn(move |addr| { let mut options = fleet_options(&meta_root); options.metadata_checkpoint_archive_prefix = prefix; + let owner = make_owner(addr); + let owner = if owner.shared_log.is_some() { + owner + } else { + owner.with_shared_log(Some(ServerSharedLogOptions::new("meta/fleet-log"))) + }; Server::open_with_objects( options, objects, - Some((control as Arc, vec![make_owner(addr)])), + Some((control as Arc, vec![owner])), ) .unwrap() }) @@ -2930,9 +3501,9 @@ mod fleet_e2e { /// hitting the still-cached B returns a typed handoff error. /// - The client's next `/dataset` request hits B, gets `StaleOwnerEpoch`, /// refreshes the shard map from control, re-resolves to B', and succeeds — - /// all inside one client call, transparently. With no checkpoint archive - /// configured, B' starts shard 1 fresh (the documented "fresh if nothing - /// was checkpointed" path); the stronger restore variant is the next test. + /// all inside one client call, transparently. B' restores an exact + /// checkpoint because failover never treats an empty local disk as proof + /// of an empty namespace. #[test] fn fleet_transparently_reresolves_after_owner_handoff() { let dir_a = tempdir().unwrap(); @@ -2941,6 +3512,7 @@ mod fleet_e2e { let object = memory_object_store(); let control = Arc::new(InMemoryControlStore::new()); register_two_shards(&control); + let checkpoint_prefix = "meta/fleet-reresolve"; let server_a = spawn_member( dir_a.path(), @@ -2953,7 +3525,7 @@ mod fleet_e2e { dir_b.path(), object.clone(), Arc::clone(&control), - None, + Some(checkpoint_prefix), |addr| { ServerShardOwnerOptions::fresh("mount-1:/dataset", addr.to_string()) .with_renewal(None) @@ -2975,6 +3547,7 @@ mod fleet_e2e { // Seed shard 1 through B and confirm the client routes there. let pre = metadata.mkdir("/dataset", MODE_DIR, 1000, 1000).unwrap(); assert_eq!(pre.attr.inode.shard_index(), DATASET_INDEX); + server_b.server().run_manual_backup().unwrap(); // Bring up B' as a failover owner of shard 1 on a brand-new port. This // bumps the shard epoch (1 -> 2) and rewrites the control endpoint to B'. @@ -2982,10 +3555,13 @@ mod fleet_e2e { dir_b_prime.path(), object.clone(), Arc::clone(&control), - None, + Some(checkpoint_prefix), move |addr| { ServerShardOwnerOptions::failover("mount-1:/dataset", addr.to_string(), b_epoch) .with_renewal(None) + .with_shared_log(Some(ServerSharedLogOptions::new( + "meta/fleet-reresolve-log", + ))) }, ); assert_ne!( @@ -3017,16 +3593,8 @@ mod fleet_e2e { // The first client call after the handoff hits cached B -> StaleOwnerEpoch // -> refresh from control -> re-resolve to B' -> served there. The - // re-resolve is invisible to the caller; it just sees a successful op. - // (B' is a fresh shard with no checkpoint, so its /dataset directory must - // be recreated before files land under it — that recreate IS the request - // that drives the transparent re-resolve.) - let post_dir = metadata.mkdir("/dataset", MODE_DIR, 1000, 1000).unwrap(); - assert_eq!( - post_dir.attr.inode.shard_index(), - DATASET_INDEX, - "the re-resolved /dataset directory is still a shard-1 inode (served by B')" - ); + // re-resolve is invisible to the caller; it just sees a successful op in + // the checkpoint-restored `/dataset` directory. let post = metadata .create_file("/dataset/after-handoff.bin", MODE_FILE, 1000, 1000) .unwrap(); diff --git a/crates/nokv-server/src/rpc/transport.rs b/crates/nokv-server/src/rpc/transport.rs index 64d243020..a6649efde 100644 --- a/crates/nokv-server/src/rpc/transport.rs +++ b/crates/nokv-server/src/rpc/transport.rs @@ -9,7 +9,7 @@ use std::time::Duration; #[cfg(test)] use nokv_protocol::{decode_envelope, encode_request, MetadataRpcRequest}; -use nokv_protocol::{decode_request, encode_envelope, MetadataRpcEnvelope, MetadataRpcResult}; +use nokv_protocol::{decode_request, encode_envelope, MetadataRpcEnvelope}; use super::wire::wire_server_error; use crate::server::{Server, ServerError}; @@ -244,17 +244,22 @@ fn handle_binary_rpc_frames( } } - let Ok(MetadataRpcResult::Batch { results }) = super::execute_batch(server, requests) else { - return frames - .into_iter() - .map(|frame| handle_binary_rpc_frame(server, frame)) - .collect(); + let results = match super::batch::try_execute_coalesced_batch(server, requests) { + Ok(Some(results)) => results, + Ok(None) => { + return frames + .into_iter() + .map(|frame| handle_binary_rpc_frame(server, frame)) + .collect(); + } + Err(err) => return encode_batch_error_frames(frames, &err), }; if results.len() != frames.len() { - return frames - .into_iter() - .map(|frame| handle_binary_rpc_frame(server, frame)) - .collect(); + let err = ServerError::Io(io::Error::new( + io::ErrorKind::InvalidData, + "coalesced metadata batch returned an unexpected result count", + )); + return encode_batch_error_frames(frames, &err); } frames @@ -275,6 +280,34 @@ fn handle_binary_rpc_frames( .collect() } +fn encode_batch_error_frames( + frames: Vec, + err: &ServerError, +) -> Vec<(u64, u32, Result, ServerError>)> { + match encode_server_error(err) { + Ok(encoded) => frames + .into_iter() + .map(|frame| (frame.request_id, frame.flags, Ok(encoded.clone()))) + .collect(), + Err(encode_err) => { + let message = encode_err.to_string(); + frames + .into_iter() + .map(|frame| { + ( + frame.request_id, + frame.flags, + Err(ServerError::Io(io::Error::new( + io::ErrorKind::InvalidData, + message.clone(), + ))), + ) + }) + .collect() + } + } +} + fn handle_binary_rpc_frame( server: &Server, frame: RpcFrame, diff --git a/crates/nokv-server/src/rpc/wire.rs b/crates/nokv-server/src/rpc/wire.rs index e4d053a2e..153b5c2eb 100644 --- a/crates/nokv-server/src/rpc/wire.rs +++ b/crates/nokv-server/src/rpc/wire.rs @@ -134,6 +134,12 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { dest_shard: *dest_shard, }, MetadError::GraftPoint => WireMetadataError::GraftPoint, + MetadError::StalePreparedArtifactObjectGcEpoch { expected, current } => { + WireMetadataError::StalePreparedArtifactObjectGcEpoch { + expected: *expected, + current: *current, + } + } MetadError::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, @@ -159,6 +165,15 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { root_path: root_path.clone(), } } + MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + } => WireMetadataError::ForkRetentionActive { + snapshot_id: *snapshot_id, + fork_root: fork_root.get(), + borrower: borrower.get(), + }, MetadError::SnapshotRenewContended { snapshot_id, attempts, @@ -240,6 +255,7 @@ pub(super) fn wire_prepared_artifact( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, } } @@ -258,6 +274,7 @@ pub(super) fn prepared_artifact( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, }) } diff --git a/crates/nokv-server/src/server.rs b/crates/nokv-server/src/server.rs index 0a6b7a4a4..c53674aee 100644 --- a/crates/nokv-server/src/server.rs +++ b/crates/nokv-server/src/server.rs @@ -12,9 +12,10 @@ use nokv_control::{CheckpointRef, ControlError, ControlStore, LogRef, LogSegment use nokv_meta::holtstore::HoltMetadataStore; use nokv_meta::{ HistoryGcWorker, HistoryGcWorkerState, MetadError, MetadataArchiveConfig, - MetadataBackupOptions, MetadataBackupOutcome, MetadataBackupWorker, MetadataCheckpointStore, - MetadataLogSegmentPointer, MetadataLogSyncConfig, NoKvFs, ObjectGcWorker, ObjectGcWorkerState, - METADATA_LOG_ZERO_DIGEST, + MetadataBackupOptions, MetadataBackupOutcome, MetadataBackupWorker, MetadataBackupWorkerState, + MetadataCheckpointIdentity, MetadataCheckpointStore, MetadataLogArchiveConfig, + MetadataLogPublicationState, MetadataLogSegment, MetadataLogSegmentPointer, + MetadataLogSyncConfig, NoKvFs, ObjectGcWorker, ObjectGcWorkerState, METADATA_LOG_ZERO_DIGEST, }; use nokv_object::{ConfiguredObjectStore, ObjectError}; use nokv_protocol::{request_routing_key, MetadataRpcRequest, RoutingKey}; @@ -38,6 +39,20 @@ const DEFAULT_ARCHIVE_KEEP_LAST: usize = 8; type OpenedControlStore = (Arc, Vec); +enum ServerMetadataBackupWorker { + Standalone(MetadataBackupWorker), + Controlled(ControlledMetadataBackupWorker), +} + +impl ServerMetadataBackupWorker { + fn state(&self) -> MetadataBackupWorkerState { + match self { + Self::Standalone(worker) => worker.state(), + Self::Controlled(worker) => worker.state(), + } + } +} + /// One metadata shard hosted by this node: its routing identity, its own Holt /// engine + service, and the per-shard background workers and control-plane /// owner lease. A single-node dev server holds exactly one (the default shard). @@ -52,7 +67,7 @@ pub(crate) struct ShardSlot { renewal: Option, object_gc: ObjectGcWorker, history_gc: HistoryGcWorker, - metadata_backup: Option, + metadata_backup: Option, metadata_archive: Option, } @@ -66,6 +81,23 @@ impl ShardSlot { } } +impl Drop for ShardSlot { + fn drop(&mut self) { + // A slot can be dropped before `Server` is fully constructed when a + // later shard fails startup. Keep lease renewal active while every + // state-mutating worker drains; otherwise a successor could acquire the + // shard while an old GC iteration is still committing or deleting. + // Release only after renewal itself has stopped. + self.metadata_backup.take(); + self.object_gc.stop(); + self.history_gc.stop(); + self.renewal.take(); + if let Some(owner) = self.owner.as_ref() { + let _ = owner.release(); + } + } +} + pub struct Server { shards: BTreeMap, shard_map: ShardMap, @@ -79,20 +111,6 @@ pub struct Server { _test_meta_dir: Option, } -impl Drop for Server { - fn drop(&mut self) { - // Stop renewing BEFORE releasing so a renewal worker cannot re-assert - // ownership after the release, then relinquish each lease so a standby - // can take over immediately instead of waiting out the full lease TTL. - for slot in self.shards.values_mut() { - slot.renewal.take(); - if let Some(owner) = slot.owner.as_ref() { - let _ = owner.release(); - } - } - } -} - #[derive(Debug)] pub enum ServerError { Io(io::Error), @@ -118,6 +136,14 @@ pub fn run(options: ServerOptions) -> Result<(), ServerError> { /// local store, without serving. Run this on a replacement node with an empty /// `--meta-path` before starting the server. Returns a JSON report. pub fn restore(options: ServerOptions) -> Result { + let objects = options.object.open()?; + restore_with_objects(options, objects) +} + +fn restore_with_objects( + options: ServerOptions, + objects: ConfiguredObjectStore, +) -> Result { let Some(prefix) = options.metadata_checkpoint_archive_prefix.clone() else { return Err(ServerError::Metadata(MetadError::InvalidPath( "metadata checkpoint archive is not configured \ @@ -125,7 +151,6 @@ pub fn restore(options: ServerOptions) -> Result { .to_owned(), ))); }; - let objects = options.object.open()?; let metadata_state_path = default_metadata_state_path(&options.meta_path); let store = HoltMetadataStore::open_file(&metadata_state_path).map_err(MetadError::from)?; let metadata = ServerMetadataStore::direct(store); @@ -198,12 +223,28 @@ impl Server { .map(|prefix| { MetadataArchiveConfig::new(prefix.clone(), DEFAULT_ARCHIVE_KEEP_LAST) }); + // An archive makes this state recoverable on another server. + // Persist the failover fence before resolving a crash-left + // deletion claim or admitting workers and writers. + if metadata_archive.is_some() { + service.require_failover_durability()?; + } + service.recover_object_gc_claim()?; + if let Some(archive) = metadata_archive.as_ref() { + // Publish a fresh CURRENT before serving. Historical + // checkpoints may reference objects deleted before the + // failover fence was installed. + service.backup_metadata(archive)?; + } let object_gc = ObjectGcWorker::spawn(Arc::clone(&service), options.object_gc); let history_gc = HistoryGcWorker::spawn(Arc::clone(&service), options.history_gc); let metadata_backup = metadata_archive.as_ref().map(|archive| { let mut backup = MetadataBackupOptions::new(archive.clone()); backup.run_immediately = false; - MetadataBackupWorker::spawn(Arc::clone(&service), backup) + ServerMetadataBackupWorker::Standalone(MetadataBackupWorker::spawn( + Arc::clone(&service), + backup, + )) }); shards.insert( shard_id, @@ -262,33 +303,39 @@ impl Server { /// `list_shards` and, for every subtree shard with a durable /// `subtree_root_inode` whose parent prefix it owns LOCALLY, idempotently /// re-creates the graft dentry against its own local service (no RPC — the - /// write lands on this very shard). Best-effort: a reconcile failure is logged - /// and does not block serving (the CLI `reconcile-grafts` can retry). - pub(crate) fn reconcile_local_grafts(&self) { + /// write lands on this very shard). Reconciliation is fail-closed: the + /// server cannot accept reads until every local mutation is represented by + /// the exact control-published recovery tail. + pub(crate) fn reconcile_local_grafts(&self) -> Result<(), ServerError> { let Some(control) = &self.control else { - return; - }; - let records = match control.list_shards() { - Ok(records) => records, - Err(err) => { - eprintln!("nokv-server: graft reconcile skipped (list_shards failed): {err}"); - return; - } + return Ok(()); }; + let records = control.list_shards()?; + // Reconciliation ownership is determined from the complete control + // topology, not merely from the shards hosted by this process. Without + // the global map, a nested graft whose parent belongs to a remote + // subtree shard incorrectly falls back to the local default shard. + let global_shard_map = control_shard_map_for_mount(self.mount, &records)?; for record in records { + let parsed = ShardPrefix::parse(record.shard_id.as_str()).map_err(|err| { + ServerError::Metadata(MetadError::Codec(format!( + "control shard {} has invalid identity: {err}", + record.shard_id + ))) + })?; + if parsed.mount != self.mount { + continue; + } if record.shard_index == DEFAULT_SHARD_INDEX { continue; } let Some(subtree_root_raw) = record.subtree_root_inode else { continue; }; - let Ok(child_inode) = InodeId::new(subtree_root_raw) else { - continue; - }; // Which shard owns the PARENT prefix? If it is not one this server // hosts, skip — that parent shard reconciles its own grafts. let (parent_prefix, basename) = split_graft_prefix(&record.prefix); - let parent_index = self.shard_map.route(self.mount, &parent_prefix); + let parent_index = global_shard_map.route(self.mount, &parent_prefix); let Some(parent_slot) = self .shards .values() @@ -296,49 +343,96 @@ impl Server { else { continue; }; - let Ok(name) = DentryName::new(basename.into_bytes()) else { - continue; - }; - // Resolve the parent inode on the parent shard's own namespace. The - // root-level case (parent == "/") is the global root; a nested parent - // is resolved via the parent shard's local path lookup. - let parent_inode = if parent_prefix == "/" { - InodeId::root() - } else { - match parent_slot.service().lookup_path(&parent_prefix) { - Ok(Some(entry)) => entry.attr.inode, - _ => continue, + let child_inode = InodeId::new(subtree_root_raw).map_err(|err| { + ServerError::Metadata(MetadError::Codec(format!( + "hosted graft {} has invalid subtree root inode: {err}", + record.prefix + ))) + })?; + if child_inode.local() == 0 || child_inode.shard_index() != record.shard_index { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "hosted graft {} subtree root inode {} does not belong to shard {}", + record.prefix, subtree_root_raw, record.shard_index + )))); + } + let name = DentryName::new(basename.into_bytes()).map_err(|err| { + ServerError::Metadata(MetadError::Codec(format!( + "hosted graft {} has invalid basename: {err}", + record.prefix + ))) + })?; + let reconcile = self.execute_with_shard_visibility(parent_slot, true, || { + // Resolve the parent while holding the same write gate as the + // mutation and its control publication. + let parent_inode = if parent_prefix == "/" { + InodeId::root() + } else { + parent_slot + .service() + .lookup_path(&parent_prefix)? + .ok_or(MetadError::NotFound)? + .attr + .inode + }; + let create = parent_slot + .service() + .create_graft( + parent_inode, + name.clone(), + child_inode, + DEFAULT_ROOT_MODE, + 0, + 0, + ); + match create { + Ok(entry) => Ok(entry), + Err(MetadError::Metadata(err)) if is_predicate_failed(&err) => { + // PredicateFailed only proves that a name already + // exists. It is idempotent success exclusively when the + // existing projection is the exact registered graft; + // accepting any other file/dir would silently bind the + // control topology to unrelated namespace state. + match parent_slot.service().lookup_plus(parent_inode, &name)? { + Some(existing) + if existing.attr.file_type + == nokv_types::FileType::Directory + && existing.attr.inode == child_inode => + { + Ok(existing) + } + Some(existing) => Err(ServerError::Metadata( + MetadError::InvalidPath(format!( + "graft {} conflicts with existing {:?} inode {}", + record.prefix, + existing.attr.file_type, + existing.attr.inode.get() + )), + )), + None => Err(ServerError::Metadata(MetadError::InvalidPath(format!( + "graft {} create conflicted but no exact existing target was visible", + record.prefix + )))), + } + } + Err(err) => Err(ServerError::Metadata(err)), } - }; - match parent_slot.service().create_graft( - parent_inode, - name, - child_inode, - DEFAULT_ROOT_MODE, - 0, - 0, - ) { + }); + match reconcile { Ok(_) => { eprintln!( "nokv-server: reconciled missing graft for prefix {} -> inode {}", record.prefix, subtree_root_raw ); } - // Already present (the common case): nothing to heal. - Err(MetadError::Metadata(err)) if is_predicate_failed(&err) => {} - Err(err) => { - eprintln!( - "nokv-server: graft reconcile for prefix {} failed: {err:?}", - record.prefix - ); - } + Err(err) => return Err(err), } } + Ok(()) } pub fn serve(self, listener: TcpListener) -> Result<(), ServerError> { // Heal any graft whose parent we own before accepting traffic. - self.reconcile_local_grafts(); + self.reconcile_local_grafts()?; let server = Arc::new(self); let workers = ConnectionWorkerPool::new( Arc::clone(&server), @@ -443,24 +537,6 @@ impl Server { .transpose() } - fn publish_slot_checkpoint_ref( - slot: &ShardSlot, - checkpoint: CheckpointRef, - ) -> Result, ServerError> { - let durable_lsn = checkpoint.lsn; - slot.owner - .as_ref() - .map(|owner| { - owner.mark_serving_with_recovery_refs( - slot.service(), - Some(checkpoint), - None, - durable_lsn, - ) - }) - .transpose() - } - /// Publish a log ref for the default/sole shard. Retained for callers (and /// tests) that target the primary shard directly. pub fn publish_shard_owner_log_ref( @@ -470,76 +546,117 @@ impl Server { Self::publish_slot_log_ref(self.default_slot(), log) } - pub fn publish_shard_owner_checkpoint_ref( - &self, - checkpoint: CheckpointRef, - ) -> Result, ServerError> { - Self::publish_slot_checkpoint_ref(self.default_slot(), checkpoint) - } - - fn publish_checkpoint_for_backup( + fn publish_slot_latest_log_ref( slot: &ShardSlot, - outcome: &MetadataBackupOutcome, ) -> Result, ServerError> { - if slot.owner.is_none() { + let Some(owner) = slot.owner.as_ref() else { return Ok(None); - } - // Use the (lsn, digest) captured atomically with the image inside - // backup_metadata, NOT a fresh snapshot read — a concurrent commit - // between the export and a later read could otherwise stamp the - // CheckpointRef with an LSN ahead of the image and drop that write on - // restore. - let state = Self::publish_slot_checkpoint_ref( - slot, - CheckpointRef { - object_key: outcome.checkpoint_key.clone(), - lsn: outcome.log_lsn, - image_bytes: outcome.image_bytes, - digest: hex_digest(&outcome.log_digest), - }, - )?; - // Segments fully covered by this checkpoint are redundant now; drop them - // from the live chain so the chain and published LogRef stay bounded. - slot.service() - .prune_sync_metadata_log_segments(outcome.log_lsn); - Ok(state) + }; + Self::publish_owner_latest_log_ref(slot.service(), owner) } - /// Publish the latest sync-log ref of the shard that handled a committing - /// request, identified by its index. - pub(crate) fn publish_latest_metadata_log_ref_for_index( + pub(crate) fn execute_with_shard_visibility( &self, - shard_index: u16, - ) -> Result, ServerError> { - let Some(slot) = self.slot_by_index(shard_index) else { - return Ok(None); + slot: &ShardSlot, + commits_metadata: bool, + execute: impl FnOnce() -> Result, + ) -> Result { + let Some(owner) = slot.owner.as_ref() else { + return execute(); }; - Self::publish_slot_latest_log_ref(slot) + + if commits_metadata { + let _visibility = owner.lock_recovery_publication(); + // Set before raw execution so a partial commit, returned error, or + // panic cannot release the gate looking clean. + owner.mark_recovery_dirty(); + let result = execute(); + // A clone/rollback or any other multi-stage mutator may have + // committed even when its final business result is an error. The + // publication barrier therefore runs on both result paths and its + // failure takes precedence, keeping later reads fail-closed. + Self::publish_owner_latest_log_ref_locked(slot.service(), owner)?; + return result; + } + + let visibility = owner.lock_recovery_visibility(); + owner.verify_read_lease(slot.service())?; + if metadata_log_publication_matches_cached_control(slot.service(), owner) { + return execute(); + } + drop(visibility); + + // Dirty readers serialize behind one repairer. Hold the write side + // through the read itself so another writer cannot create a new local + // tail between repair and observation. + let _visibility = owner.lock_recovery_publication(); + Self::publish_owner_latest_log_ref_locked(slot.service(), owner)?; + execute() } - fn publish_slot_latest_log_ref( - slot: &ShardSlot, - ) -> Result, ServerError> { - let Some(snapshot) = slot.service().sync_metadata_log_snapshot() else { - return Ok(None); + fn publish_owner_latest_log_ref( + service: &NoKvFs, + owner: &ServerShardOwner, + ) -> Result, ServerError> + where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, + { + // Snapshot selection and the control CAS are one owner-local + // publication critical section. Otherwise an older request can pause + // after taking its snapshot, let a newer request publish, and then + // overwrite the control LogRef with the older chain. + let _publication = owner.lock_recovery_publication(); + Self::publish_owner_latest_log_ref_locked(service, owner) + } + + fn publish_owner_latest_log_ref_locked( + service: &NoKvFs, + owner: &ServerShardOwner, + ) -> Result, ServerError> + where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, + { + service.verify_owner_lease()?; + service.flush_sync_metadata_log_for_publication()?; + let Some(mut publication) = service.sync_metadata_log_publication_state() else { + return Err(ServerError::Metadata(MetadError::Codec( + "control-owned shard has no synchronous metadata log publication state".to_owned(), + ))); }; - Self::publish_slot_log_ref( - slot, - LogRef { - segments: snapshot - .segments - .iter() - .map(|segment| LogSegmentRef { - segment_key: segment.segment_key.clone(), - first_lsn: segment.first_lsn, - last_lsn: segment.last_lsn, - digest: hex_digest(&segment.last_digest), - }) - .collect(), - durable_lsn: snapshot.durable_lsn, - digest: hex_digest(&snapshot.last_digest), - }, - ) + + if let Some(cached) = owner.published_recovery_state() { + prune_control_covered_log_segments(service, &publication, &cached)?; + publication = service + .sync_metadata_log_publication_state() + .expect("enabled metadata log remains enabled under publication gate"); + if metadata_log_publication_matches_control(&publication, &cached) { + owner.mark_recovery_clean(); + return Ok(Some(cached)); + } + } + + match publish_metadata_log_snapshot_locked(service, owner, &publication) { + Ok(state) => complete_metadata_log_publication(service, owner, state).map(Some), + Err(_) => { + // A checkpoint/log CAS may have committed and lost its ACK, or + // another publication by this same owner may have advanced the + // authoritative checkpoint while local covered pointers remain. + // Dirty repair alone may perform this remote validation. + let authoritative = owner.renew(service)?; + prune_control_covered_log_segments(service, &publication, &authoritative)?; + let repaired = service + .sync_metadata_log_publication_state() + .expect("enabled metadata log remains enabled under publication gate"); + if metadata_log_publication_matches_control(&repaired, &authoritative) { + owner.mark_recovery_clean(); + return Ok(Some(authoritative)); + } + let state = publish_metadata_log_snapshot_locked(service, owner, &repaired)?; + complete_metadata_log_publication(service, owner, state).map(Some) + } + } } /// Publish the latest sync-log ref of the default/sole shard. Retained for @@ -562,7 +679,20 @@ impl Server { &self.framed_rpc_workers } + /// Readiness is the same safety claim as a controlled strong read: every + /// hosted owner lease is live and the exact local recovery tail is already + /// represented in control. A dirty slot may repair under its publication + /// write gate; publication failure keeps the server unready. Standalone + /// slots pass through unchanged. + pub(crate) fn check_readiness(&self) -> Result<(), ServerError> { + for slot in self.shards.values() { + self.execute_with_shard_visibility(slot, false, || Ok(()))?; + } + Ok(()) + } + pub fn stats_json(&self) -> String { + let ready = self.check_readiness().is_ok(); let slot = self.default_slot(); let service = slot.service(); let objects = service.object_stats(); @@ -570,8 +700,14 @@ impl Server { let metadata_service = service.metadata_service_stats(); let object_gc = slot.object_gc.state(); let history_gc = slot.history_gc.state(); + // Stats are an operational safety surface. If the persisted marker + // cannot be read or decoded, report the failover requirement as active + // rather than presenting an unsafe deployment as unconstrained. + let failover_durability_required = + service.failover_durability_is_required().unwrap_or(true); format!( - "{{\"ready\":true,\"block_cache_enabled\":{},\"object_puts\":{},\"object_put_bytes\":{},\"object_gets\":{},\"object_get_bytes\":{},\"coalesced_gets\":{},\"coalesced_get_bytes\":{},\"cache_hits\":{},\"cache_hit_bytes\":{},\"prefetch_enqueued\":{},\"prefetch_dropped\":{},\"prefetch_completed\":{},\"prefetch_failed\":{},\"prefetch_object_gets\":{},\"prefetch_object_get_bytes\":{},\"prefetch_cache_hits\":{},\"prefetch_cache_hit_bytes\":{},\"read_plan_cache_hits\":{},\"read_plan_cache_misses\":{},\"object_writeback_enqueued\":{},\"object_writeback_inline\":{},\"object_writeback_completed\":{},\"object_writeback_failed\":{},\"object_writeback_staged_bytes\":{},\"object_writeback_uploaded_bytes\":{},\"object_writeback_queue_wait_ns\":{},\"object_writeback_queue_max_wait_ns\":{},\"object_writeback_upload_ns\":{},\"object_writeback_upload_max_ns\":{},\"object_writeback_collect_ns\":{},\"object_writeback_digest_ns\":{},\"object_writeback_store_put_ns\":{},\"object_writeback_cache_put_ns\":{},\"manifest_chunks\":{},\"manifest_blocks\":{},\"metadata_store\":{},\"metadata_service\":{},\"shard_owner\":{},\"object_gc\":{},\"history_gc\":{},\"metadata_backup\":{}}}\n", + "{{\"ready\":{},\"block_cache_enabled\":{},\"object_puts\":{},\"object_put_bytes\":{},\"object_gets\":{},\"object_get_bytes\":{},\"coalesced_gets\":{},\"coalesced_get_bytes\":{},\"cache_hits\":{},\"cache_hit_bytes\":{},\"prefetch_enqueued\":{},\"prefetch_dropped\":{},\"prefetch_completed\":{},\"prefetch_failed\":{},\"prefetch_object_gets\":{},\"prefetch_object_get_bytes\":{},\"prefetch_cache_hits\":{},\"prefetch_cache_hit_bytes\":{},\"read_plan_cache_hits\":{},\"read_plan_cache_misses\":{},\"object_writeback_enqueued\":{},\"object_writeback_inline\":{},\"object_writeback_completed\":{},\"object_writeback_failed\":{},\"object_writeback_staged_bytes\":{},\"object_writeback_uploaded_bytes\":{},\"object_writeback_queue_wait_ns\":{},\"object_writeback_queue_max_wait_ns\":{},\"object_writeback_upload_ns\":{},\"object_writeback_upload_max_ns\":{},\"object_writeback_collect_ns\":{},\"object_writeback_digest_ns\":{},\"object_writeback_store_put_ns\":{},\"object_writeback_cache_put_ns\":{},\"manifest_chunks\":{},\"manifest_blocks\":{},\"metadata_store\":{},\"metadata_service\":{},\"shard_owner\":{},\"object_gc\":{},\"history_gc\":{},\"metadata_backup\":{}}}\n", + ready, service.block_cache_enabled(), objects.object_puts, objects.object_put_bytes, @@ -610,7 +746,7 @@ impl Server { metadata_store_json(&metadata), metadata_service_json(&metadata_service), self.shard_owner_json(), - object_gc_json(&object_gc), + object_gc_json(&object_gc, failover_durability_required), history_gc_json(&history_gc), self.metadata_backup_json(), ) @@ -627,6 +763,7 @@ impl Server { object.scanned += object_outcome.scanned; object.blocked_by_snapshots += object_outcome.blocked_by_snapshots; object.blocked_by_read_leases += object_outcome.blocked_by_read_leases; + object.blocked_by_failover_durability += object_outcome.blocked_by_failover_durability; object.attempted += object_outcome.attempted; object.deleted += object_outcome.deleted; object.missing += object_outcome.missing; @@ -641,11 +778,12 @@ impl Server { history.retained_by_snapshots += history_outcome.retained_by_snapshots; } Ok(format!( - r#"{{"object_gc":{{"scanned":{},"blocked_by_snapshots":{},"blocked_by_read_leases":{},"attempted":{},"deleted":{},"missing":{},"records_removed":{},"snapshot_reap":{{"scanned":{},"expired_candidates":{},"reaped":{},"conflicted":{}}}}},"history_gc":{{"scanned":{},"removed":{},"retained_by_snapshots":{}}}}} + r#"{{"object_gc":{{"scanned":{},"blocked_by_snapshots":{},"blocked_by_read_leases":{},"blocked_by_failover_durability":{},"attempted":{},"deleted":{},"missing":{},"records_removed":{},"snapshot_reap":{{"scanned":{},"expired_candidates":{},"reaped":{},"conflicted":{}}}}},"history_gc":{{"scanned":{},"removed":{},"retained_by_snapshots":{}}}}} "#, object.scanned, object.blocked_by_snapshots, object.blocked_by_read_leases, + object.blocked_by_failover_durability, object.attempted, object.deleted, object.missing, @@ -682,13 +820,21 @@ impl Server { .to_owned(), ))); }; - let outcome = slot.service().backup_metadata(archive)?; - let _checkpoint_state = Self::publish_checkpoint_for_backup(slot, &outcome)?; + let outcome = match slot.owner.as_ref() { + Some(owner) => run_controlled_metadata_backup_once(slot.service(), owner, archive)?, + None => slot.service().backup_metadata(archive)?, + }; let key = format!("\"{}\"", escape_json_string(&outcome.checkpoint_key)); Ok(format!( - r#"{{"checkpoint_key":{key},"image_bytes":{},"commit_version":{},"pruned":{}}} + r#"{{"checkpoint_key":{key},"image_bytes":{},"commit_version":{},"pruned":{},"log_segments_pruned":{},"log_segment_objects_deleted":{},"log_segment_objects_missing":{},"log_segment_delete_failures":{}}} "#, - outcome.image_bytes, outcome.commit_version, outcome.pruned, + outcome.image_bytes, + outcome.commit_version, + outcome.pruned, + outcome.log_segments_pruned, + outcome.log_segment_objects_deleted, + outcome.log_segment_objects_missing, + outcome.log_segment_delete_failures, )) } @@ -771,6 +917,163 @@ impl Server { } } +fn metadata_log_ref(snapshot: &nokv_meta::MetadataLogSyncSnapshot) -> Option { + (!snapshot.segments.is_empty()).then(|| LogRef { + segments: snapshot + .segments + .iter() + .map(|segment| LogSegmentRef { + segment_key: segment.segment_key.clone(), + first_lsn: segment.first_lsn, + last_lsn: segment.last_lsn, + digest: hex_digest(&segment.last_digest), + }) + .collect(), + durable_lsn: snapshot.durable_lsn, + digest: hex_digest(&snapshot.last_digest), + }) +} + +fn metadata_log_publication_matches_control( + publication: &MetadataLogPublicationState, + control: &ServerShardOwnerState, +) -> bool { + if publication.has_pending_segment + || publication.has_unresolved_commit_group + || control.state != nokv_control::ShardState::Serving + || control.durable_lsn != publication.snapshot.durable_lsn + { + return false; + } + match metadata_log_ref(&publication.snapshot) { + Some(log) => control.log.as_ref() == Some(&log), + None => { + control.log.is_none() + && control.checkpoint.as_ref().is_some_and(|checkpoint| { + checkpoint.lsn == publication.snapshot.durable_lsn + && checkpoint.digest == hex_digest(&publication.snapshot.last_digest) + }) + } + } +} + +fn metadata_log_publication_matches_cached_control( + service: &NoKvFs, + owner: &ServerShardOwner, +) -> bool +where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, +{ + if owner.recovery_is_dirty() { + return false; + } + let Some(publication) = service.sync_metadata_log_publication_state() else { + return false; + }; + owner + .published_recovery_state() + .as_ref() + .is_some_and(|control| metadata_log_publication_matches_control(&publication, control)) +} + +fn publish_metadata_log_snapshot_locked( + service: &NoKvFs, + owner: &ServerShardOwner, + publication: &MetadataLogPublicationState, +) -> Result +where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, +{ + if publication.has_pending_segment || publication.has_unresolved_commit_group { + return Err(ServerError::Metadata(MetadError::SyncLogArchiveFailed { + committed: true, + message: "metadata log publication state remains unresolved after flush".to_owned(), + })); + } + let snapshot = &publication.snapshot; + match metadata_log_ref(snapshot) { + Some(log) => owner.mark_serving_with_recovery_refs_locked( + service, + None, + Some(log), + snapshot.durable_lsn, + ), + None => { + owner.mark_serving_with_recovery_refs_locked(service, None, None, snapshot.durable_lsn) + } + } +} + +fn prune_control_covered_log_segments( + service: &NoKvFs, + publication: &MetadataLogPublicationState, + control: &ServerShardOwnerState, +) -> Result<(), ServerError> +where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, +{ + let Some(checkpoint) = control.checkpoint.as_ref() else { + return Ok(()); + }; + let Some(last_covered) = publication + .snapshot + .segments + .iter() + .take_while(|segment| segment.last_lsn <= checkpoint.lsn) + .last() + else { + return Ok(()); + }; + let expected_digest = parse_hex_digest(&checkpoint.digest)?; + if last_covered.last_lsn != checkpoint.lsn || last_covered.last_digest != expected_digest { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "authoritative checkpoint tail {}:{} does not match the local covered log boundary", + checkpoint.lsn, checkpoint.digest + )))); + } + let outcome = service.prune_sync_metadata_log_segments(checkpoint.lsn); + if outcome.delete_failures > 0 { + eprintln!( + "nokv-server: {} control-covered metadata log segment object(s) leaked during publication repair", + outcome.delete_failures + ); + } + Ok(()) +} + +fn complete_metadata_log_publication( + service: &NoKvFs, + owner: &ServerShardOwner, + control: ServerShardOwnerState, +) -> Result +where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, +{ + let publication = service + .sync_metadata_log_publication_state() + .ok_or_else(|| { + ServerError::Metadata(MetadError::Codec( + "synchronous metadata log disappeared during publication".to_owned(), + )) + })?; + prune_control_covered_log_segments(service, &publication, &control)?; + let final_publication = service + .sync_metadata_log_publication_state() + .expect("enabled metadata log remains enabled under publication gate"); + if !metadata_log_publication_matches_control(&final_publication, &control) { + return Err(ServerError::Metadata(MetadError::Codec( + "authoritative recovery refs do not exactly cover the local metadata log tail" + .to_owned(), + ))); + } + owner.mark_recovery_clean(); + Ok(control) +} + fn slot_owner_state(slot: &ShardSlot) -> Result, ServerError> { slot.owner.as_ref().map(ServerShardOwner::state).transpose() } @@ -811,6 +1114,54 @@ fn split_graft_prefix(prefix: &str) -> (String, String) { } } +/// Build the authoritative longest-prefix routing table for one mount from +/// control-plane records. Every record for the mount must carry an identity, +/// prefix, and default/non-default index that agree exactly; startup must not +/// guess around malformed topology. +fn control_shard_map_for_mount( + mount: MountId, + records: &[nokv_control::ShardRecord], +) -> Result { + let mut routes = Vec::new(); + for record in records { + let parsed = ShardPrefix::parse(record.shard_id.as_str()).map_err(|err| { + ServerError::Metadata(MetadError::Codec(format!( + "control shard {} has invalid identity: {err}", + record.shard_id + ))) + })?; + if parsed.mount != mount { + continue; + } + if parsed.path != record.prefix { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "control shard {} prefix {:?} does not match its identity prefix {:?}", + record.shard_id, record.prefix, parsed.path + )))); + } + if record.shard_index == DEFAULT_SHARD_INDEX { + if parsed.path != "/" { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "control shard {} uses the default shard index for non-root prefix {:?}", + record.shard_id, record.prefix + )))); + } + continue; + } + if parsed.path == "/" { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "control shard {} uses non-default index {} for the root prefix", + record.shard_id, record.shard_index + )))); + } + routes.push(ShardRoute { + shard_index: record.shard_index, + prefix: parsed, + }); + } + Ok(ShardMap::from_routes(routes)) +} + /// Whether a metadata backend error is a predicate failure — the idempotent /// "graft dentry already exists" signal during reconcile. fn is_predicate_failed(err: &nokv_meta::MetadataError) -> bool { @@ -832,10 +1183,95 @@ fn shard_archive_prefix(prefix: &str, sanitized_shard_id: &str) -> String { format!("{}/{}", prefix.trim_end_matches('/'), sanitized_shard_id) } +fn checkpoint_ref_for_backup(outcome: &MetadataBackupOutcome) -> CheckpointRef { + CheckpointRef { + object_key: outcome.checkpoint_key.clone(), + lsn: outcome.log_lsn, + image_bytes: outcome.image_bytes, + image_digest: outcome.image_digest.clone(), + digest: hex_digest(&outcome.log_digest), + } +} + +fn checkpoint_identity(checkpoint: &CheckpointRef) -> MetadataCheckpointIdentity { + MetadataCheckpointIdentity { + checkpoint_key: checkpoint.object_key.clone(), + image_bytes: checkpoint.image_bytes, + image_digest: checkpoint.image_digest.clone(), + } +} + +/// Publish one controlled checkpoint without touching standalone `CURRENT`. +/// The old control ref remains authoritative through every upload failure and +/// is pruned only after the new exact identity wins the owner-lease CAS. +fn run_controlled_metadata_backup_once( + service: &NoKvFs, + owner: &ServerShardOwner, + archive: &MetadataArchiveConfig, +) -> Result +where + M: nokv_meta::MetadataStore + MetadataCheckpointStore, + O: nokv_object::ObjectStore, +{ + let _publication = owner.lock_recovery_publication(); + service.verify_owner_lease()?; + let prior = owner.renew(service)?; + let mut outcome = service.prepare_immutable_metadata_backup(archive)?; + let published = owner.mark_serving_with_recovery_refs_locked( + service, + Some(checkpoint_ref_for_backup(&outcome)), + None, + outcome.log_lsn, + )?; + let log_prune = service.prune_sync_metadata_log_segments(outcome.log_lsn); + outcome.log_segments_pruned = log_prune.pointers_pruned; + outcome.log_segment_objects_deleted = log_prune.objects_deleted; + outcome.log_segment_objects_missing = log_prune.objects_missing; + outcome.log_segment_delete_failures = log_prune.delete_failures; + if log_prune.delete_failures > 0 { + // The control CAS already made the checkpoint authoritative. Cleanup + // failure is observable but cannot turn publication into an error: a + // caller retry would publish another checkpoint and obscure success. + eprintln!( + "nokv-server: {} covered metadata log segment object(s) leaked after checkpoint publication", + log_prune.delete_failures + ); + } + let final_publication = service + .sync_metadata_log_publication_state() + .ok_or_else(|| { + ServerError::Metadata(MetadError::Codec( + "control-owned shard lost synchronous metadata log state during checkpoint publication" + .to_owned(), + )) + })?; + if !metadata_log_publication_matches_control(&final_publication, &published) { + return Err(ServerError::Metadata(MetadError::Codec( + "published checkpoint does not exactly cover the local metadata log tail".to_owned(), + ))); + } + owner.mark_recovery_clean(); + + if let Some(previous) = prior.checkpoint.as_ref() { + if previous.object_key != outcome.checkpoint_key && !previous.image_digest.is_empty() { + match service.prune_immutable_metadata_backup(archive, &checkpoint_identity(previous)) { + Ok(pruned) => outcome.pruned = pruned, + Err(err) => { + // The control CAS already made the new image authoritative; + // pruning failure is a safe leak and must not invite a retry + // that could obscure the successful publication. + eprintln!("nokv-server: controlled metadata checkpoint prune skipped: {err}"); + } + } + } + } + Ok(outcome) +} + /// Build one [`ShardSlot`] from its control-plane owner options: open the shard's /// own Holt engine, derive its index/prefix from the (registered) control record, -/// acquire the owner lease, restore on failover, enable the shared log, mark -/// serving, and spawn the per-shard background workers. +/// acquire the owner lease, restore on failover, install durability, recover a +/// crash-left object-GC claim, mark serving, and spawn the background workers. fn open_shard_slot( options: &ServerOptions, objects: &ConfiguredObjectStore, @@ -844,6 +1280,21 @@ fn open_shard_slot( ) -> Result<(ShardId, ShardSlot), ServerError> { let shard_id = shard_owner.shard_id.clone(); let sanitized = sanitize_shard_id(shard_id.as_str()); + // Every control-owned generation can eventually be replaced by another + // server. Require the shared log before even registering/reading control + // state, creating a local directory, or touching object storage; otherwise + // a Fresh owner could ACK writes above its checkpoint that a later owner + // has no configured way to recover. + let shared_log_options = shard_owner.shared_log.clone().ok_or_else(|| { + ControlError::InvalidOptions(format!( + "control-owned shard {} requires synchronous shared_log configuration", + shard_id.as_str() + )) + })?; + let shared_log_archive = MetadataLogArchiveConfig::new(shard_archive_prefix( + &shared_log_options.archive_prefix, + &sanitized, + )); // When this owner declares its own shard index (a multi-process fleet, where // no separate registration step has seeded identity), register the shard's @@ -859,6 +1310,52 @@ fn open_shard_slot( // ensure_shard returns (creating if absent) the record so we read its index // and prefix; for the default shard the derived prefix is "/" and index 0. let record = store.ensure_shard(shard_id.clone())?; + let is_failover = matches!( + shard_owner.acquisition, + ServerShardAcquisition::Failover { .. } + ); + // Derive and validate the exact recovery namespace before creating a local + // Holt directory, acquiring/bumping the epoch, or touching object storage. + let metadata_archive = options + .metadata_checkpoint_archive_prefix + .as_ref() + .map(|prefix| { + MetadataArchiveConfig::new( + shard_archive_prefix(prefix, &sanitized), + DEFAULT_ARCHIVE_KEEP_LAST, + ) + }); + if metadata_archive.is_none() { + let message = if is_failover { + "shard failover recovery requires a metadata checkpoint archive" + } else { + "control-owned shard requires a metadata checkpoint archive" + }; + return Err(ServerError::Metadata(MetadError::InvalidPath( + message.to_owned(), + ))); + } + if is_failover { + let checkpoint = record.checkpoint.as_ref().ok_or_else(|| { + ControlError::InvalidOptions(format!( + "shard {} has no durable checkpoint identity for failover", + shard_id.as_str() + )) + })?; + metadata_archive + .as_ref() + .expect("archive presence checked above") + .validate_controlled_checkpoint_identity(&checkpoint_identity(checkpoint))?; + // The image identity validation above covers object key, proof-key + // derivation, size, and image digest. Validate the logical tail digest + // here as well so restore cannot discover it only after acquisition. + parse_hex_digest(&checkpoint.digest)?; + if let Some(log) = record.log.as_ref() { + for segment in &log.segments { + shared_log_archive.validate_segment_key(&segment.segment_key)?; + } + } + } let shard_index = record.shard_index; let prefix = ShardPrefix::parse(&format!("mount-{}:{}", options.mount.get(), record.prefix)) .unwrap_or_else(|_| ShardPrefix::new(options.mount, record.prefix.clone())); @@ -874,10 +1371,6 @@ fn open_shard_slot( let holt = HoltMetadataStore::open_file(&metadata_state_path).map_err(MetadError::from)?; let metadata = ServerMetadataStore::direct(holt); - let is_failover = matches!( - shard_owner.acquisition, - ServerShardAcquisition::Failover { .. } - ); let service = if is_failover { Arc::new( NoKvFs::new(options.mount, metadata, objects.clone()).with_shard_index(shard_index), @@ -892,49 +1385,55 @@ fn open_shard_slot( }; let renewal_options = shard_owner.renewal; - let shared_log_options = shard_owner.shared_log.clone(); - // Each shard's checkpoint archive is isolated under its own prefix. - let metadata_archive = options - .metadata_checkpoint_archive_prefix - .as_ref() - .map(|prefix| { - MetadataArchiveConfig::new( - shard_archive_prefix(prefix, &sanitized), - DEFAULT_ARCHIVE_KEEP_LAST, - ) - }); - let owner = ServerShardOwner::acquire(store, shard_owner, service.as_ref())?; + // Renewal starts immediately after acquisition and remains the same worker + // throughout restore, checkpoint export/upload, and publication. A large + // checkpoint must not consume the whole lease before the shard can serve. + let mut renewal = renewal_options.map(|renewal| { + ServerShardOwnerRenewalWorker::spawn(Arc::clone(&service), owner.clone(), renewal) + }); - let restored_from_control = if is_failover { - match metadata_archive.as_ref() { - Some(archive) => restore_shard_owner_recovery_refs(service.as_ref(), &owner, archive)?, - None => { - let state = owner.state()?; - if state.checkpoint.is_some() { - return Err(ServerError::Metadata(MetadError::InvalidPath( - "metadata checkpoint archive is required for shard failover restore" - .to_owned(), - ))); + let startup = (|| -> Result<(), ServerError> { + let restored_from_control = if is_failover { + match metadata_archive.as_ref() { + Some(archive) => { + if !restore_shard_owner_recovery_refs( + service.as_ref(), + &owner, + archive, + &shared_log_archive, + )? { + return Err(ServerError::Metadata(MetadError::NotFound)); + } + true } - false + None => unreachable!("failover without a checkpoint archive is rejected above"), } + } else { + false + }; + if !restored_from_control { + service.bootstrap_root(DEFAULT_ROOT_MODE, options.uid, options.gid)?; + } + + // Every control-owned shard can be recovered by a different server. + // Install this policy before claim recovery, readiness, or GC workers. + service.require_failover_durability()?; + service.recover_object_gc_claim()?; + if is_failover { + // Recovery proved the durable claim is Open. Rewrite it at a fresh + // version before enabling writers so every prepared upload minted + // by the failed owner is fenced from publishing after takeover. + service.rotate_object_gc_claim_for_failover()?; } - } else { - false - }; - if !restored_from_control { - service.bootstrap_root(DEFAULT_ROOT_MODE, options.uid, options.gid)?; - } - if let Some(shared_log) = shared_log_options { let state = owner.state()?; let last_digest = control_recovery_digest(&state)?; let inherited_segments = inherited_log_segments(&state)?; // Isolate each shard's shared-log archive under its own prefix. service.enable_sync_metadata_log( MetadataLogSyncConfig::new( - shard_archive_prefix(&shared_log.archive_prefix, &sanitized), + shared_log_archive.prefix.clone(), state.shard_id.as_str(), state.epoch, state.durable_lsn, @@ -942,19 +1441,35 @@ fn open_shard_slot( ) .with_segments(inherited_segments), )?; + // A fresh archive supersedes any historical CURRENT that could retain + // object references from before the deletion fence. The immutable image + // is prepared first; only the owner-lease CAS makes it authoritative. + if let Some(archive) = metadata_archive.as_ref() { + run_controlled_metadata_backup_once(service.as_ref(), &owner, archive)?; + } else { + owner.mark_serving(service.as_ref())?; + } + Ok(()) + })(); + if let Err(err) = startup { + // Stop before release so an in-flight renew cannot reassert a lease that + // startup has abandoned. Stale release is best-effort after failover. + renewal.take(); + let _ = owner.release(); + return Err(err); } - owner.mark_serving(service.as_ref())?; - let renewal = renewal_options.map(|renewal| { - ServerShardOwnerRenewalWorker::spawn(Arc::clone(&service), owner.clone(), renewal) - }); let object_gc = ObjectGcWorker::spawn(Arc::clone(&service), options.object_gc); let history_gc = HistoryGcWorker::spawn(Arc::clone(&service), options.history_gc); let metadata_backup = metadata_archive.as_ref().map(|archive| { let mut backup = MetadataBackupOptions::new(archive.clone()); // Back up on the interval, not on every boot (avoids startup stalls). backup.run_immediately = false; - MetadataBackupWorker::spawn(Arc::clone(&service), backup) + ServerMetadataBackupWorker::Controlled(ControlledMetadataBackupWorker::spawn( + Arc::clone(&service), + owner.clone(), + backup, + )) }); Ok(( @@ -991,32 +1506,31 @@ fn open_etcd_control_store( .into()) } -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct ServerShardOwnerRenewalWorkerState { - iterations: u64, - last_error: Option, -} - -struct ServerShardOwnerRenewalWorker { +struct ControlledMetadataBackupWorker { stop: Arc<(Mutex, Condvar)>, - state: Arc>, + state: Arc>, handle: Option>, } -impl ServerShardOwnerRenewalWorker { +impl ControlledMetadataBackupWorker { fn spawn( service: Arc>, owner: ServerShardOwner, - options: ServerShardOwnerRenewalOptions, + options: MetadataBackupOptions, ) -> Self { let stop = Arc::new((Mutex::new(false), Condvar::new())); - let state = Arc::new(Mutex::new(ServerShardOwnerRenewalWorkerState::default())); + let state = Arc::new(Mutex::new(MetadataBackupWorkerState::default())); let worker_stop = Arc::clone(&stop); let worker_state = Arc::clone(&state); let interval = options.interval.max(Duration::from_millis(1)); let handle = thread::spawn(move || { if options.run_immediately { - run_shard_owner_renewal_once(&service, &owner, &worker_state); + run_controlled_metadata_backup_worker_once( + &service, + &owner, + &options.config, + &worker_state, + ); } loop { let (lock, cvar) = &*worker_stop; @@ -1035,7 +1549,12 @@ impl ServerShardOwnerRenewalWorker { break; } drop(stopped); - run_shard_owner_renewal_once(&service, &owner, &worker_state); + run_controlled_metadata_backup_worker_once( + &service, + &owner, + &options.config, + &worker_state, + ); } }); Self { @@ -1045,7 +1564,7 @@ impl ServerShardOwnerRenewalWorker { } } - fn state(&self) -> ServerShardOwnerRenewalWorkerState { + fn state(&self) -> MetadataBackupWorkerState { self.state .lock() .map(|state| state.clone()) @@ -1064,24 +1583,129 @@ impl ServerShardOwnerRenewalWorker { } } -impl Drop for ServerShardOwnerRenewalWorker { +impl Drop for ControlledMetadataBackupWorker { fn drop(&mut self) { self.stop(); } } -fn run_shard_owner_renewal_once( - service: &Arc>, +fn run_controlled_metadata_backup_worker_once( + service: &NoKvFs, owner: &ServerShardOwner, - state: &Arc>, + config: &MetadataArchiveConfig, + state: &Arc>, ) { - let result = owner.renew(service.as_ref()); - let mut state = match state.lock() { - Ok(state) => state, - Err(err) => err.into_inner(), - }; - state.iterations = state.iterations.saturating_add(1); - state.last_error = result.err().map(|err| err.to_string()); + let outcome = run_controlled_metadata_backup_once(service, owner, config); + if let Ok(mut state) = state.lock() { + state.iterations += 1; + match outcome { + Ok(outcome) => { + state.last_outcome = Some(outcome); + state.last_error = None; + } + Err(err) => state.last_error = Some(err.to_string()), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct ServerShardOwnerRenewalWorkerState { + iterations: u64, + last_error: Option, +} + +struct ServerShardOwnerRenewalWorker { + stop: Arc<(Mutex, Condvar)>, + state: Arc>, + handle: Option>, +} + +impl ServerShardOwnerRenewalWorker { + fn spawn( + service: Arc>, + owner: ServerShardOwner, + options: ServerShardOwnerRenewalOptions, + ) -> Self + where + M: nokv_meta::MetadataStore + Send + Sync + 'static, + O: nokv_object::ObjectStore + Send + Sync + 'static, + { + let stop = Arc::new((Mutex::new(false), Condvar::new())); + let state = Arc::new(Mutex::new(ServerShardOwnerRenewalWorkerState::default())); + let worker_stop = Arc::clone(&stop); + let worker_state = Arc::clone(&state); + let interval = options.interval.max(Duration::from_millis(1)); + let handle = thread::spawn(move || { + if options.run_immediately { + run_shard_owner_renewal_once(&service, &owner, &worker_state); + } + loop { + let (lock, cvar) = &*worker_stop; + let stopped = match lock.lock() { + Ok(stopped) => stopped, + Err(_) => break, + }; + if *stopped { + break; + } + let (stopped, _) = match cvar.wait_timeout(stopped, interval) { + Ok(waited) => waited, + Err(_) => break, + }; + if *stopped { + break; + } + drop(stopped); + run_shard_owner_renewal_once(&service, &owner, &worker_state); + } + }); + Self { + stop, + state, + handle: Some(handle), + } + } + + fn state(&self) -> ServerShardOwnerRenewalWorkerState { + self.state + .lock() + .map(|state| state.clone()) + .unwrap_or_else(|err| err.into_inner().clone()) + } + + fn stop(&mut self) { + let (lock, cvar) = &*self.stop; + if let Ok(mut stopped) = lock.lock() { + *stopped = true; + cvar.notify_all(); + } + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +impl Drop for ServerShardOwnerRenewalWorker { + fn drop(&mut self) { + self.stop(); + } +} + +fn run_shard_owner_renewal_once( + service: &Arc>, + owner: &ServerShardOwner, + state: &Arc>, +) where + M: nokv_meta::MetadataStore, + O: nokv_object::ObjectStore, +{ + let result = owner.renew(service.as_ref()); + let mut state = match state.lock() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.iterations = state.iterations.saturating_add(1); + state.last_error = result.err().map(|err| err.to_string()); } struct ConnectionWorkerPool { @@ -1192,6 +1816,7 @@ fn restore_shard_owner_recovery_refs( service: &NoKvFs, owner: &ServerShardOwner, archive: &MetadataArchiveConfig, + log_archive: &MetadataLogArchiveConfig, ) -> Result { let state = owner.state()?; let Some(checkpoint) = state.checkpoint.clone() else { @@ -1201,31 +1826,38 @@ fn restore_shard_owner_recovery_refs( // Replay every archived segment whose tail is above the checkpoint LSN, in // order. A single-pointer LogRef would drop all but the newest segment and // silently lose acknowledged metadata on any multi-segment failover. - let segment_keys: Vec = state + let segment_refs: Vec = state .log .as_ref() .map(|log| { log.segments .iter() .filter(|segment| segment.last_lsn > checkpoint.lsn) - .map(|segment| segment.segment_key.clone()) + .cloned() .collect::>() }) .unwrap_or_default(); - if state.durable_lsn > checkpoint.lsn && segment_keys.is_empty() { + if state.durable_lsn > checkpoint.lsn && segment_refs.is_empty() { return Err(ServerError::Metadata(MetadError::Codec( "control record is missing log segments for checkpoint replay".to_owned(), ))); } - let Some(outcome) = service.restore_metadata_with_archived_log_segments( + let mut segments = Vec::with_capacity(segment_refs.len()); + for segment_ref in &segment_refs { + log_archive.validate_segment_key(&segment_ref.segment_key)?; + let segment = service.load_metadata_log_segment(&segment_ref.segment_key)?; + validate_control_log_segment_identity(segment_ref, &segment)?; + segments.push(segment); + } + let identity = checkpoint_identity(&checkpoint); + let outcome = service.restore_metadata_checkpoint_with_log_segments( archive, - &segment_keys, + state.shard_id.as_str(), + &identity, + &segments, checkpoint.lsn, checkpoint_digest, - )? - else { - return Err(ServerError::Metadata(MetadError::NotFound)); - }; + )?; if outcome.checkpoint.checkpoint_key != checkpoint.object_key { return Err(ServerError::Metadata(MetadError::Codec(format!( "restored checkpoint {} does not match control checkpoint {}", @@ -1238,14 +1870,61 @@ fn restore_shard_owner_recovery_refs( outcome.durable_lsn, state.durable_lsn )))); } + let expected_digest = control_recovery_digest(&state)?; + if outcome.last_digest != expected_digest { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "restored metadata log digest {} does not match control digest {}", + hex_digest(&outcome.last_digest), + hex_digest(&expected_digest) + )))); + } Ok(true) } -fn object_gc_json(state: &ObjectGcWorkerState) -> String { +fn validate_control_log_segment_identity( + reference: &LogSegmentRef, + segment: &MetadataLogSegment, +) -> Result<(), ServerError> { + let expected_digest = parse_hex_digest(&reference.digest)?; + if segment.first_lsn != reference.first_lsn + || segment.last_lsn != reference.last_lsn + || segment.last_digest != expected_digest + { + return Err(ServerError::Metadata(MetadError::Codec(format!( + "metadata log segment {} does not match control identity {}..{}:{}", + reference.segment_key, reference.first_lsn, reference.last_lsn, reference.digest, + )))); + } + Ok(()) +} + +fn object_gc_json(state: &ObjectGcWorkerState, failover_durability_required: bool) -> String { + let outcome = state.last_outcome.map_or_else( + || "null".to_owned(), + |outcome| { + format!( + "{{\"scanned\":{},\"blocked_by_snapshots\":{},\"blocked_by_read_leases\":{},\"blocked_by_failover_durability\":{},\"attempted\":{},\"deleted\":{},\"missing\":{},\"records_removed\":{},\"snapshot_reap\":{{\"scanned\":{},\"expired_candidates\":{},\"reaped\":{},\"conflicted\":{}}}}}", + outcome.scanned, + outcome.blocked_by_snapshots, + outcome.blocked_by_read_leases, + outcome.blocked_by_failover_durability, + outcome.attempted, + outcome.deleted, + outcome.missing, + outcome.records_removed, + outcome.snapshot_reap.scanned, + outcome.snapshot_reap.expired_candidates, + outcome.snapshot_reap.reaped, + outcome.snapshot_reap.conflicted, + ) + }, + ); let reap = state.snapshot_reap; format!( - "{{\"iterations\":{},\"last_error\":{},\"snapshot_reap\":{{\"scanned\":{},\"expired_candidates\":{},\"reaped\":{},\"conflicted\":{}}}}}", + "{{\"failover_durability_required\":{},\"iterations\":{},\"last_outcome\":{},\"last_error\":{},\"snapshot_reap\":{{\"scanned\":{},\"expired_candidates\":{},\"reaped\":{},\"conflicted\":{}}}}}", + failover_durability_required, state.iterations, + outcome, json_string_or_null(state.last_error.as_deref()), reap.scanned, reap.expired_candidates, @@ -1445,16 +2124,27 @@ impl Error for ServerError {} pub(crate) mod tests { use super::*; use std::path::Path; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering}; use std::time::{Duration, Instant}; #[cfg(feature = "etcd")] use std::time::{SystemTime, UNIX_EPOCH}; #[cfg(feature = "etcd")] use std::{env, process}; - use nokv_control::InMemoryControlStore; - use nokv_meta::{HistoryGcOptions, ObjectGcOptions}; - use nokv_object::{ObjectStoreConfig, S3ObjectStoreOptions}; - use nokv_types::MountId; + use nokv_control::{InMemoryControlStore, NodeId, ShardLease, ShardRecord, ShardState}; + use nokv_meta::layout::{ + decode_snapshot_pin, failover_durability_required_key, gc_object_key, object_gc_claim_key, + snapshot_pin_prefix, + }; + use nokv_meta::{ + CommandKind, CommitResult, HistoryGcOptions, MetadataCommand, MetadataLogEntry, + MetadataStore, Mutation, MutationOp, ObjectGcOptions, Predicate, PredicateRef, ReadPurpose, + ScanRequest, Value, Version, + }; + use nokv_object::{ + MemoryObjectStore, ObjectKey, ObjectStore, ObjectStoreConfig, S3ObjectStoreOptions, + }; + use nokv_types::{MountId, RecordFamily}; use tempfile::tempdir; pub(crate) fn test_options(root: &Path) -> ServerOptions { @@ -1498,159 +2188,2449 @@ pub(crate) mod tests { server } - fn fast_renewal_options(run_immediately: bool) -> ServerShardOwnerRenewalOptions { - ServerShardOwnerRenewalOptions { - interval: Duration::from_millis(10), - run_immediately, - ..ServerShardOwnerRenewalOptions::default() - } + fn open_memory_controlled( + root: &Path, + control: Arc, + owners: Vec, + ) -> Result + where + C: ControlStore + 'static, + { + let mut options = test_options(root); + options.metadata_checkpoint_archive_prefix = Some("metadata/control-test".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let control: Arc = control; + let owners = owners + .into_iter() + .map(|owner| { + if owner.shared_log.is_some() { + owner + } else { + owner.with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/control-test-log", + ))) + } + }) + .collect(); + Server::open_with_objects(options, objects, Some((control, owners))) } - fn wait_until(mut condition: impl FnMut() -> bool) { - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline { - if condition() { - return; + pub(crate) struct FailingCheckpointPublishControl { + inner: InMemoryControlStore, + fail_marks: AtomicUsize, + fail_renews: AtomicUsize, + mark_calls: AtomicUsize, + fail_on_call: AtomicUsize, + } + + impl FailingCheckpointPublishControl { + pub(crate) fn new() -> Self { + Self { + inner: InMemoryControlStore::new(), + fail_marks: AtomicUsize::new(0), + fail_renews: AtomicUsize::new(0), + mark_calls: AtomicUsize::new(0), + fail_on_call: AtomicUsize::new(0), } - thread::sleep(Duration::from_millis(5)); } - panic!("condition was not met before deadline"); - } - #[test] - fn manual_gc_reports_empty_outcomes() { - let server = test_server(); - assert!(server.stats_json().contains("\"ready\":true")); - assert!(server - .stats_json() - .contains("\"shard_owner\":{\"enabled\":false}")); - let body = server.run_manual_gc(128).unwrap(); - assert!(body.contains("\"object_gc\"")); - assert!(body.contains("\"history_gc\"")); - } + pub(crate) fn fail_next_marks(&self, count: usize) { + self.fail_marks.store(count, AtomicOrdering::SeqCst); + } - #[test] - fn object_gc_health_keeps_cumulative_reaper_conflicts() { - let state = ObjectGcWorkerState { - iterations: 9, - last_outcome: Some(nokv_meta::PendingObjectCleanupOutcome::default()), - snapshot_reap: nokv_meta::SnapshotReapOutcome { - scanned: 12, - expired_candidates: 3, - reaped: 2, - conflicted: 1, - }, - last_error: None, - }; + pub(crate) fn fail_next_renews(&self, count: usize) { + self.fail_renews.store(count, AtomicOrdering::SeqCst); + } - let json = object_gc_json(&state); - assert!(json.contains("\"iterations\":9")); - assert!(json.contains("\"conflicted\":1")); + fn fail_mark_on_call(&self, call: usize) { + self.mark_calls.store(0, AtomicOrdering::SeqCst); + self.fail_on_call.store(call, AtomicOrdering::SeqCst); + } } - #[test] - fn controlled_open_acquires_shard_and_installs_owner_epoch() { - let dir = tempdir().unwrap(); - let control = Arc::new(InMemoryControlStore::new()); - let server = Server::open_with_control( - test_options(dir.path()), - control, - vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], - ) - .unwrap(); + impl ControlStore for FailingCheckpointPublishControl { + fn ensure_shard(&self, shard_id: ShardId) -> Result { + self.inner.ensure_shard(shard_id) + } - let state = server.shard_owner_state().unwrap().unwrap(); - assert_eq!(state.shard_id.as_str(), "mount-1:/"); - assert_eq!(state.node_id.as_str(), "node-a"); - assert_eq!(state.epoch, 1); - assert_eq!(state.lease_id, 1); - assert_eq!(state.state, nokv_control::ShardState::Serving); - assert_eq!(server.service().allocator_epoch(), 1); - assert_eq!(server.service().required_owner_epoch(), 1); - assert!(server - .stats_json() - .contains("\"shard_owner\":{\"enabled\":true")); - assert!(server - .stats_json() - .contains("\"renewal\":{\"enabled\":true")); - } + fn register_shard( + &self, + shard_id: ShardId, + prefix: String, + shard_index: u16, + ) -> Result { + self.inner.register_shard(shard_id, prefix, shard_index) + } - /// An owner that declares `shard_index` registers its shard identity (prefix + - /// index) on open even when nothing pre-registered it — the path a multi-process - /// `nokv serve --shard-index N` fleet relies on. The control record then carries - /// the declared index, and the server routes the shard's subtree to that slot. - #[test] - fn controlled_open_with_shard_index_registers_identity() { - use nokv_protocol::MetadataRpcRequest; - let dir = tempdir().unwrap(); - let control = Arc::new(InMemoryControlStore::new()); - // No register_shard call: the owner's declared index is the only source of - // the shard's identity. - let server = Server::open_with_control( - test_options(dir.path()), - control.clone(), - vec![ServerShardOwnerOptions::fresh("mount-1:/dataset", "node-a") - .with_renewal(None) - .with_shard_index(Some(1))], - ) - .unwrap(); + fn set_subtree_root_inode( + &self, + shard_id: &ShardId, + subtree_root_inode: Option, + ) -> Result { + self.inner + .set_subtree_root_inode(shard_id, subtree_root_inode) + } - // The control record now carries the declared index and the derived prefix. - let record = control - .get_shard(&nokv_control::ShardId::new("mount-1:/dataset")) - .unwrap(); - assert_eq!(record.shard_index, 1, "open registered the declared index"); - assert_eq!( - record.prefix, "/dataset", - "prefix derived from the shard id" - ); + fn list_shards(&self) -> Result, ControlError> { + self.inner.list_shards() + } - // The server routes a /dataset path to the index-1 slot it just registered. - let slot = server - .route(&MetadataRpcRequest::CreateDirPath { - path: "/dataset/run".to_owned(), - mode: 0o755, - uid: 1000, - gid: 1000, - }) - .unwrap(); - assert_eq!(slot.shard_index(), 1); - } + fn get_shard(&self, shard_id: &ShardId) -> Result { + self.inner.get_shard(shard_id) + } - #[test] - fn controlled_failover_installs_bumped_epoch() { - let first_dir = tempdir().unwrap(); - let control = Arc::new(InMemoryControlStore::new()); - let first = Server::open_with_control( - test_options(first_dir.path()), - control.clone(), - vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], - ) - .unwrap(); - assert_eq!(first.shard_owner_state().unwrap().unwrap().epoch, 1); + fn acquire_unassigned( + &self, + shard_id: ShardId, + owner: NodeId, + ) -> Result { + self.inner.acquire_unassigned(shard_id, owner) + } - let second_dir = tempdir().unwrap(); - let second = Server::open_with_control( - test_options(second_dir.path()), - control, - vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1)], - ) - .unwrap(); + fn acquire_after_failure( + &self, + shard_id: ShardId, + owner: NodeId, + previous_epoch: u64, + ) -> Result { + self.inner + .acquire_after_failure(shard_id, owner, previous_epoch) + } - let state = second.shard_owner_state().unwrap().unwrap(); - assert_eq!(state.node_id.as_str(), "node-b"); - assert_eq!(state.epoch, 2); - assert_eq!(state.lease_id, 2); - assert_eq!(state.state, nokv_control::ShardState::Serving); - assert_eq!(second.service().allocator_epoch(), 2); - assert_eq!(second.service().required_owner_epoch(), 2); + fn renew(&self, lease: &ShardLease) -> Result { + if self + .fail_renews + .fetch_update( + AtomicOrdering::SeqCst, + AtomicOrdering::SeqCst, + |remaining| remaining.checked_sub(1), + ) + .is_ok() + { + return Err(ControlError::Backend( + "injected owner renew failure".to_owned(), + )); + } + self.inner.renew(lease) + } + + fn mark_serving( + &self, + lease: &ShardLease, + checkpoint: Option, + log: Option, + durable_lsn: u64, + ) -> Result { + let call = self.mark_calls.fetch_add(1, AtomicOrdering::SeqCst) + 1; + if self.fail_on_call.load(AtomicOrdering::SeqCst) == call { + return Err(ControlError::Backend( + "injected checkpoint publish failure".to_owned(), + )); + } + if self + .fail_marks + .fetch_update( + AtomicOrdering::SeqCst, + AtomicOrdering::SeqCst, + |remaining| remaining.checked_sub(1), + ) + .is_ok() + { + return Err(ControlError::Backend( + "injected checkpoint publish failure".to_owned(), + )); + } + self.inner.mark_serving(lease, checkpoint, log, durable_lsn) + } + + fn release(&self, lease: &ShardLease) -> Result { + self.inner.release(lease) + } } - #[cfg(feature = "etcd")] - #[test] - fn configured_etcd_control_store_expires_session_and_allows_failover() { - let endpoints = match env::var("NOKV_ETCD_ENDPOINTS") { - Ok(raw) if !raw.trim().is_empty() => raw + #[derive(Default)] + struct BlockingGateState { + armed: bool, + reached: bool, + released: bool, + } + + #[derive(Clone)] + struct BlockingPutStore { + inner: MemoryObjectStore, + gate: Arc<(Mutex, Condvar)>, + fail_put_substring: Arc>>, + fail_delete_substring: Arc>>, + } + + impl BlockingPutStore { + fn new() -> Self { + Self { + inner: MemoryObjectStore::new(), + gate: Arc::new((Mutex::new(BlockingGateState::default()), Condvar::new())), + fail_put_substring: Arc::new(Mutex::new(None)), + fail_delete_substring: Arc::new(Mutex::new(None)), + } + } + + fn fail_puts_containing(&self, substring: &str) { + *self.fail_put_substring.lock().unwrap() = Some(substring.to_owned()); + } + + fn clear_put_failures(&self) { + *self.fail_put_substring.lock().unwrap() = None; + } + + fn fail_deletes_containing(&self, substring: &str) { + *self.fail_delete_substring.lock().unwrap() = Some(substring.to_owned()); + } + + fn arm(&self) { + let (lock, _) = &*self.gate; + *lock.lock().unwrap() = BlockingGateState { + armed: true, + reached: false, + released: false, + }; + } + + fn wait_until_reached(&self) { + wait_for_blocking_gate(&self.gate, "checkpoint PUT"); + } + + fn has_reached(&self) -> bool { + let (lock, _) = &*self.gate; + lock.lock().unwrap().reached + } + + fn release(&self) { + release_blocking_gate(&self.gate); + } + } + + impl ObjectStore for BlockingPutStore { + fn put( + &self, + key: &ObjectKey, + bytes: impl Into, + ) -> Result { + let bytes = bytes.into(); + if self + .fail_put_substring + .lock() + .unwrap() + .as_ref() + .is_some_and(|substring| key.as_str().contains(substring)) + { + return Err(ObjectError::Backend( + "injected object PUT failure".to_owned(), + )); + } + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + if state.armed { + state.armed = false; + state.reached = true; + changed.notify_all(); + while !state.released { + state = changed.wait(state).unwrap(); + } + } + drop(state); + self.inner.put(key, bytes) + } + + fn get( + &self, + key: &ObjectKey, + range: Option, + ) -> Result, ObjectError> { + self.inner.get(key, range) + } + + fn head(&self, key: &ObjectKey) -> Result, ObjectError> { + self.inner.head(key) + } + + fn delete(&self, key: &ObjectKey) -> Result { + if self + .fail_delete_substring + .lock() + .unwrap() + .as_ref() + .is_some_and(|substring| key.as_str().contains(substring)) + { + return Err(ObjectError::Backend( + "injected object DELETE failure".to_owned(), + )); + } + self.inner.delete(key) + } + } + + fn controlled_sync_log_backup_fixture( + control: Arc, + objects: BlockingPutStore, + prefix: &str, + ) -> ( + Arc>, + ServerShardOwner, + MetadataArchiveConfig, + MetadataBackupOutcome, + ) { + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + )); + service + .bootstrap_root(DEFAULT_ROOT_MODE, 1000, 1000) + .unwrap(); + let owner = ServerShardOwner::acquire( + control, + ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None), + service.as_ref(), + ) + .unwrap(); + let archive = MetadataArchiveConfig::new(format!("{prefix}/checkpoint"), 2); + let owner_state = owner.state().unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + format!("{prefix}/shared-log"), + owner_state.shard_id.as_str(), + owner_state.epoch, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + let initial = + run_controlled_metadata_backup_once(service.as_ref(), &owner, &archive).unwrap(); + (service, owner, archive, initial) + } + + fn local_snapshot_pins( + service: &NoKvFs, + ) -> Vec { + service + .metadata_store() + .scan(ScanRequest { + family: RecordFamily::Snapshot, + prefix: snapshot_pin_prefix(MountId::new(1).unwrap()), + start_after: None, + version: Version::new(u64::MAX).unwrap(), + limit: usize::MAX, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|item| decode_snapshot_pin(&item.value.0).unwrap()) + .collect() + } + + fn local_log_segment_keys(snapshot: &nokv_meta::MetadataLogSyncSnapshot) -> Vec { + snapshot + .segments + .iter() + .map(|segment| segment.segment_key.clone()) + .collect() + } + + pub(crate) struct BlockingServingControl { + inner: InMemoryControlStore, + block_next_mark: AtomicBool, + gate: Arc<(Mutex, Condvar)>, + } + + impl BlockingServingControl { + pub(crate) fn new() -> Self { + Self { + inner: InMemoryControlStore::new(), + block_next_mark: AtomicBool::new(false), + gate: Arc::new((Mutex::new(BlockingGateState::default()), Condvar::new())), + } + } + + pub(crate) fn arm_mark(&self) { + self.block_next_mark.store(true, AtomicOrdering::SeqCst); + let (lock, _) = &*self.gate; + *lock.lock().unwrap() = BlockingGateState { + armed: true, + reached: false, + released: false, + }; + } + + pub(crate) fn wait_until_mark_reached(&self) { + wait_for_blocking_gate(&self.gate, "mark-serving CAS"); + } + + fn mark_reached(&self) -> bool { + let (lock, _) = &*self.gate; + lock.lock().unwrap().reached + } + + pub(crate) fn release_mark(&self) { + release_blocking_gate(&self.gate); + } + } + + fn wait_for_blocking_gate(gate: &Arc<(Mutex, Condvar)>, operation: &str) { + let (lock, changed) = &**gate; + let mut state = lock.lock().unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while !state.reached { + let now = Instant::now(); + assert!(now < deadline, "timed out waiting for {operation}"); + let (next, timeout) = changed.wait_timeout(state, deadline - now).unwrap(); + state = next; + assert!( + !timeout.timed_out() || state.reached, + "timed out waiting for {operation}" + ); + } + } + + fn release_blocking_gate(gate: &Arc<(Mutex, Condvar)>) { + let (lock, changed) = &**gate; + let mut state = lock.lock().unwrap(); + state.released = true; + changed.notify_all(); + } + + impl ControlStore for BlockingServingControl { + fn ensure_shard(&self, shard_id: ShardId) -> Result { + self.inner.ensure_shard(shard_id) + } + + fn register_shard( + &self, + shard_id: ShardId, + prefix: String, + shard_index: u16, + ) -> Result { + self.inner.register_shard(shard_id, prefix, shard_index) + } + + fn set_subtree_root_inode( + &self, + shard_id: &ShardId, + subtree_root_inode: Option, + ) -> Result { + self.inner + .set_subtree_root_inode(shard_id, subtree_root_inode) + } + + fn list_shards(&self) -> Result, ControlError> { + self.inner.list_shards() + } + + fn get_shard(&self, shard_id: &ShardId) -> Result { + self.inner.get_shard(shard_id) + } + + fn acquire_unassigned( + &self, + shard_id: ShardId, + owner: NodeId, + ) -> Result { + self.inner.acquire_unassigned(shard_id, owner) + } + + fn acquire_after_failure( + &self, + shard_id: ShardId, + owner: NodeId, + previous_epoch: u64, + ) -> Result { + self.inner + .acquire_after_failure(shard_id, owner, previous_epoch) + } + + fn renew(&self, lease: &ShardLease) -> Result { + self.inner.renew(lease) + } + + fn mark_serving( + &self, + lease: &ShardLease, + checkpoint: Option, + log: Option, + durable_lsn: u64, + ) -> Result { + if self.block_next_mark.swap(false, AtomicOrdering::SeqCst) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + state.reached = true; + changed.notify_all(); + while !state.released { + state = changed.wait(state).unwrap(); + } + } + self.inner.mark_serving(lease, checkpoint, log, durable_lsn) + } + + fn release(&self, lease: &ShardLease) -> Result { + self.inner.release(lease) + } + } + + /// Seed the exact durable state left by a crash after object-GC claimed a + /// row but before it reopened the claim. The malformed row makes startup + /// ordering observable: ordinary recovery quarantines it, while failover + /// durability must preserve it. + struct SeededObjectGcClaim { + metadata_path: PathBuf, + gc_key: Vec, + claim_key: Vec, + encoded_claim: Vec, + operation_token: u64, + } + + fn seed_crash_left_object_gc_claim( + options: &ServerOptions, + metadata_dir: &Path, + objects: ConfiguredObjectStore, + ) -> SeededObjectGcClaim { + std::fs::create_dir_all(metadata_dir).unwrap(); + let metadata_path = default_metadata_state_path(metadata_dir); + let holt = HoltMetadataStore::open_file(&metadata_path).unwrap(); + let service = NoKvFs::new( + options.mount, + ServerMetadataStore::direct(holt.clone()), + objects, + ); + service + .bootstrap_root(DEFAULT_ROOT_MODE, options.uid, options.gid) + .unwrap(); + + let claim_key = object_gc_claim_key(options.mount); + let claim = holt + .get_versioned( + RecordFamily::System, + &claim_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + // Bootstrap reserves a large version window. Stay within that window + // while placing the synthetic crash record after every bootstrap row. + let commit_version = Version::new(claim.version.get() + 100).unwrap(); + let operation_token = claim.version.get(); + let gc_key = gc_object_key( + options.mount, + commit_version.get(), + InodeId::root(), + 1, + 0, + 0, + ); + let mut deleting_claim = Vec::with_capacity(29 + gc_key.len()); + deleting_claim.push(2); + deleting_claim.extend_from_slice(&1_u64.to_be_bytes()); + deleting_claim.extend_from_slice(&operation_token.to_be_bytes()); + deleting_claim.extend_from_slice(&commit_version.get().to_be_bytes()); + deleting_claim.extend_from_slice(&(gc_key.len() as u32).to_be_bytes()); + deleting_claim.extend_from_slice(&gc_key); + holt.commit_metadata(MetadataCommand { + request_id: b"seed-crash-left-object-gc-claim".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: Version::new(commit_version.get() - 1).unwrap(), + commit_version, + primary_family: RecordFamily::System, + primary_key: claim_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: gc_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::Gc, + key: gc_key.clone(), + op: MutationOp::Put, + value: Some(Value(vec![0xff])), + }, + Mutation { + family: RecordFamily::System, + key: claim_key.clone(), + op: MutationOp::Put, + value: Some(Value(deleting_claim.clone())), + }, + ], + watch: Vec::new(), + }) + .unwrap(); + drop(service); + drop(holt); + SeededObjectGcClaim { + metadata_path, + gc_key, + claim_key, + encoded_claim: deleting_claim, + operation_token, + } + } + + fn assert_startup_requires_object_gc_intervention( + result: Result, + operation_token: u64, + ) { + match result { + Err(ServerError::Metadata(MetadError::ObjectGcRecoveryRequiresIntervention { + owner_epoch, + operation_token: actual_token, + })) => { + assert_eq!(owner_epoch, 1); + assert_eq!(actual_token, operation_token); + } + Err(err) => panic!("unexpected startup error: {err}"), + Ok(_) => panic!("server must not become ready with an uncertain object deletion"), + } + } + + fn assert_failover_gate_preserves_crash_claim( + options: &ServerOptions, + seeded: &SeededObjectGcClaim, + expect_marker: bool, + ) { + let holt = HoltMetadataStore::open_file(&seeded.metadata_path).unwrap(); + let latest = Version::new(u64::MAX).unwrap(); + assert_eq!( + holt.get( + RecordFamily::Gc, + &seeded.gc_key, + latest, + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + vec![0xff], + "failover policy must preserve the claimed GC row" + ); + assert_eq!( + holt.get( + RecordFamily::System, + &seeded.claim_key, + latest, + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + seeded.encoded_claim, + "failover policy must keep the deletion claim closed" + ); + assert_eq!( + holt.get( + RecordFamily::System, + &failover_durability_required_key(options.mount), + latest, + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some(), + expect_marker, + "deployment preflight must not install the marker before shared-log validation" + ); + } + + fn fast_renewal_options(run_immediately: bool) -> ServerShardOwnerRenewalOptions { + ServerShardOwnerRenewalOptions { + interval: Duration::from_millis(10), + run_immediately, + ..ServerShardOwnerRenewalOptions::default() + } + } + + fn wait_until(mut condition: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + if condition() { + return; + } + thread::sleep(Duration::from_millis(5)); + } + panic!("condition was not met before deadline"); + } + + #[test] + fn controlled_checkpoint_cas_failure_does_not_delete_covered_log_segment() { + let objects = BlockingPutStore::new(); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let (service, owner, archive, initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects.clone(), + "metadata/log-prune-cas-failure", + ); + service + .create_dir_path("/covered", 0o755, 1000, 1000) + .unwrap(); + let before = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(before.segments.len(), 1); + let covered_key = ObjectKey::new(before.segments[0].segment_key.clone()).unwrap(); + let prior = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!( + prior.checkpoint.as_ref().unwrap().object_key, + initial.checkpoint_key + ); + + control.fail_next_marks(1); + assert!(matches!( + run_controlled_metadata_backup_once(service.as_ref(), &owner, &archive), + Err(ServerError::Control(ControlError::Backend(message))) + if message.contains("injected checkpoint publish failure") + )); + + assert_eq!(service.sync_metadata_log_snapshot().unwrap(), before); + assert!(objects.head(&covered_key).unwrap().is_some()); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint, + prior.checkpoint + ); + owner.release().unwrap(); + } + + #[test] + fn controlled_checkpoint_deletes_only_segments_at_or_before_captured_lsn() { + let objects = BlockingPutStore::new(); + let control = Arc::new(InMemoryControlStore::new()); + let (service, owner, archive, _initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects.clone(), + "metadata/log-prune-boundary", + ); + service + .create_dir_path("/covered", 0o755, 1000, 1000) + .unwrap(); + let covered = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(covered.segments.len(), 1); + let covered_pointer = covered.segments[0].clone(); + let covered_key = ObjectKey::new(covered_pointer.segment_key.clone()).unwrap(); + + objects.arm(); + let backup_service = Arc::clone(&service); + let backup_owner = owner.clone(); + let backup_archive = archive.clone(); + let backup = thread::spawn(move || { + run_controlled_metadata_backup_once( + backup_service.as_ref(), + &backup_owner, + &backup_archive, + ) + }); + objects.wait_until_reached(); + + // Checkpoint capture is complete and its image PUT is blocked. This + // later commit must remain as the replay tail above the captured LSN. + service.create_dir_path("/tail", 0o755, 1000, 1000).unwrap(); + let before_publish = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(before_publish.segments.len(), 2); + let tail_pointer = before_publish.segments[1].clone(); + let tail_key = ObjectKey::new(tail_pointer.segment_key.clone()).unwrap(); + assert!(tail_pointer.first_lsn > covered_pointer.last_lsn); + + objects.release(); + let err = backup.join().unwrap().unwrap_err(); + assert!( + err.to_string() + .contains("published checkpoint does not exactly cover"), + "unexpected stale-capture result: {err}" + ); + assert!(objects.head(&covered_key).unwrap().is_none()); + assert!(objects.head(&tail_key).unwrap().is_some()); + assert_eq!( + service.sync_metadata_log_snapshot().unwrap().segments, + vec![tail_pointer.clone()] + ); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap() + .lsn, + covered_pointer.last_lsn + ); + let published = Server::publish_owner_latest_log_ref(service.as_ref(), &owner) + .unwrap() + .unwrap(); + assert_eq!( + published.log, + metadata_log_ref(&service.sync_metadata_log_snapshot().unwrap()) + ); + owner.release().unwrap(); + } + + #[test] + fn controlled_checkpoint_delete_failure_keeps_publication_success_observable() { + let objects = BlockingPutStore::new(); + let control = Arc::new(InMemoryControlStore::new()); + let (service, owner, archive, initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects.clone(), + "metadata/log-prune-delete-failure", + ); + service + .create_dir_path("/covered", 0o755, 1000, 1000) + .unwrap(); + let covered = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(covered.segments.len(), 1); + let covered_key = ObjectKey::new(covered.segments[0].segment_key.clone()).unwrap(); + objects.fail_deletes_containing(covered_key.as_str()); + + let outcome = + run_controlled_metadata_backup_once(service.as_ref(), &owner, &archive).unwrap(); + + assert_ne!(outcome.checkpoint_key, initial.checkpoint_key); + assert_eq!(outcome.log_segments_pruned, 1); + assert_eq!(outcome.log_segment_objects_deleted, 0); + assert_eq!(outcome.log_segment_objects_missing, 0); + assert_eq!(outcome.log_segment_delete_failures, 1); + assert!(objects.head(&covered_key).unwrap().is_some()); + assert!(service + .sync_metadata_log_snapshot() + .unwrap() + .segments + .is_empty()); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap() + .object_key, + outcome.checkpoint_key + ); + owner.release().unwrap(); + } + + #[test] + fn snapshot_create_archive_failure_stays_unpublished_until_exact_pending_flush() { + let objects = BlockingPutStore::new(); + let control = Arc::new(InMemoryControlStore::new()); + let (service, owner, archive, initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects.clone(), + "metadata/snapshot-create-pending", + ); + service + .create_dir_path("/scope", 0o755, 1000, 1000) + .unwrap(); + Server::publish_owner_latest_log_ref(service.as_ref(), &owner).unwrap(); + let before = service.sync_metadata_log_snapshot().unwrap(); + let control_before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(control_before.durable_lsn, before.durable_lsn); + + objects.fail_puts_containing("metadata/snapshot-create-pending/shared-log/log/"); + owner.mark_recovery_dirty(); + assert!(matches!( + service.snapshot_subtree_path_with_lease("/scope", 10_000), + Err(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + assert!(matches!( + Server::publish_owner_latest_log_ref(service.as_ref(), &owner), + Err(ServerError::Metadata(MetadError::SyncLogArchiveFailed { + committed: true, + .. + })) + )); + let pins = local_snapshot_pins(service.as_ref()); + assert_eq!(pins.len(), 1, "the failed ACK still applied one exact pin"); + assert_eq!(service.sync_metadata_log_snapshot().unwrap(), before); + assert_eq!( + control.get_shard(&ShardId::new("mount-1:/")).unwrap(), + control_before + ); + + // Retrying while the exact committed segment is still unavailable must + // fail before applying a second snapshot pin. + assert!(matches!( + service.snapshot_subtree_path_with_lease("/scope", 10_000), + Err(MetadError::SyncLogArchiveFailed { + committed: false, + .. + }) + )); + assert_eq!(local_snapshot_pins(service.as_ref()), pins); + + objects.clear_put_failures(); + service + .create_dir_path("/flush-trigger", 0o755, 1000, 1000) + .unwrap(); + let published = Server::publish_owner_latest_log_ref(service.as_ref(), &owner) + .unwrap() + .unwrap(); + assert!(!owner.recovery_is_dirty()); + let tail = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(published.durable_lsn, tail.durable_lsn); + assert!(tail.durable_lsn >= before.durable_lsn + 2); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + recovered + .restore_metadata_checkpoint_with_archived_log_segments( + &archive, + "mount-1:/", + &MetadataCheckpointIdentity { + checkpoint_key: initial.checkpoint_key, + image_bytes: initial.image_bytes, + image_digest: initial.image_digest, + }, + &local_log_segment_keys(&tail), + initial.log_lsn, + initial.log_digest, + ) + .unwrap(); + assert_eq!( + recovered + .snapshot_pin_path("/scope", pins[0].snapshot_id) + .unwrap(), + Some(pins[0].clone()) + ); + owner.release().unwrap(); + } + + #[test] + fn snapshot_renew_exact_retry_cannot_ack_before_pending_tail_publication() { + let objects = BlockingPutStore::new(); + let control = Arc::new(InMemoryControlStore::new()); + let (service, owner, archive, initial) = controlled_sync_log_backup_fixture( + Arc::clone(&control) as Arc, + objects.clone(), + "metadata/snapshot-renew-pending", + ); + service.set_clock_override_ms(1_000); + service + .create_dir_path("/scope", 0o755, 1000, 1000) + .unwrap(); + let pin = service + .snapshot_subtree_path_with_lease("/scope", 10_000) + .unwrap(); + Server::publish_owner_latest_log_ref(service.as_ref(), &owner).unwrap(); + let before = service.sync_metadata_log_snapshot().unwrap(); + let control_before = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + + objects.fail_puts_containing("metadata/snapshot-renew-pending/shared-log/log/"); + owner.mark_recovery_dirty(); + assert!(matches!( + service.renew_snapshot_path("/scope", pin.snapshot_id, 20_000), + Err(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) + )); + let renewed = service + .snapshot_pin_path("/scope", pin.snapshot_id) + .unwrap() + .unwrap(); + assert_eq!(renewed.lease_expires_unix_ms, 21_000); + assert_eq!(service.sync_metadata_log_snapshot().unwrap(), before); + assert_eq!( + control.get_shard(&ShardId::new("mount-1:/")).unwrap(), + control_before + ); + + // Exact retry observes the locally-applied expiry and returns a no-op; + // the RPC post barrier must still reject it while the pending segment + // cannot be archived and control remains at the old tail. + assert!(matches!( + service + .renew_snapshot_path("/scope", pin.snapshot_id, 20_000) + .unwrap(), + nokv_meta::SnapshotRenewOutcome::Renewed { + extended: false, + .. + } + )); + assert!(matches!( + Server::publish_owner_latest_log_ref(service.as_ref(), &owner), + Err(ServerError::Metadata(MetadError::SyncLogArchiveFailed { + committed: true, + .. + })) + )); + assert_eq!( + control.get_shard(&ShardId::new("mount-1:/")).unwrap(), + control_before + ); + + objects.clear_put_failures(); + assert!(matches!( + service + .renew_snapshot_path("/scope", pin.snapshot_id, 20_000) + .unwrap(), + nokv_meta::SnapshotRenewOutcome::Renewed { + extended: false, + .. + } + )); + let published = Server::publish_owner_latest_log_ref(service.as_ref(), &owner) + .unwrap() + .unwrap(); + let tail = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(published.durable_lsn, before.durable_lsn + 1); + assert_eq!(tail.durable_lsn, published.durable_lsn); + assert!(!owner.recovery_is_dirty()); + + let recovered = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + recovered + .restore_metadata_checkpoint_with_archived_log_segments( + &archive, + "mount-1:/", + &MetadataCheckpointIdentity { + checkpoint_key: initial.checkpoint_key, + image_bytes: initial.image_bytes, + image_digest: initial.image_digest, + }, + &local_log_segment_keys(&tail), + initial.log_lsn, + initial.log_digest, + ) + .unwrap(); + assert_eq!( + recovered + .snapshot_pin_path("/scope", pin.snapshot_id) + .unwrap() + .unwrap() + .lease_expires_unix_ms, + renewed.lease_expires_unix_ms + ); + owner.release().unwrap(); + } + + #[test] + fn controlled_startup_renewal_survives_checkpoint_put_longer_than_ttl() { + let objects = BlockingPutStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + )); + service + .bootstrap_root(DEFAULT_ROOT_MODE, 1000, 1000) + .unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let renewal_options = ServerShardOwnerRenewalOptions { + interval: Duration::from_millis(5), + run_immediately: true, + lease_ttl: Duration::from_millis(30), + }; + let owner = ServerShardOwner::acquire( + Arc::clone(&control) as Arc, + ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(Some(renewal_options)), + service.as_ref(), + ) + .unwrap(); + let state = owner.state().unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata/slow-startup-log", + state.shard_id.as_str(), + state.epoch, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + let mut renewal = ServerShardOwnerRenewalWorker::spawn( + Arc::clone(&service), + owner.clone(), + renewal_options, + ); + let archive = MetadataArchiveConfig::new("metadata/slow-startup", 2); + objects.arm(); + let backup_service = Arc::clone(&service); + let backup_owner = owner.clone(); + let backup = thread::spawn(move || { + run_controlled_metadata_backup_once(backup_service.as_ref(), &backup_owner, &archive) + }); + objects.wait_until_reached(); + let recovering = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(recovering.state, ShardState::Recovering); + assert!(recovering.checkpoint.is_none()); + let blocked_at = Instant::now(); + wait_until(|| { + blocked_at.elapsed() >= Duration::from_millis(60) && renewal.state().iterations >= 3 + }); + objects.release(); + + let outcome = backup.join().unwrap().unwrap(); + assert!(outcome.checkpoint_key.contains("/controlled/sha256/")); + assert!(renewal.state().last_error.is_none()); + let state = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(state.state, ShardState::Serving); + assert_eq!(state.checkpoint.unwrap().object_key, outcome.checkpoint_key); + renewal.stop(); + owner.release().unwrap(); + } + + #[test] + fn concurrent_controlled_backups_publish_in_capture_order_and_prune_after_cas() { + let objects = BlockingPutStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + )); + service + .bootstrap_root(DEFAULT_ROOT_MODE, 1000, 1000) + .unwrap(); + let control = Arc::new(BlockingServingControl::new()); + let owner = ServerShardOwner::acquire( + Arc::clone(&control) as Arc, + ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None), + service.as_ref(), + ) + .unwrap(); + let archive = MetadataArchiveConfig::new("metadata/ordered-checkpoint", 2); + let state = owner.state().unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata/ordered-checkpoint-log", + state.shard_id.as_str(), + state.epoch, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + + let _initial = + run_controlled_metadata_backup_once(service.as_ref(), &owner, &archive).unwrap(); + service + .create_dir_path("/captured-by-a", 0o755, 1000, 1000) + .unwrap(); + + control.arm_mark(); + let a_service = Arc::clone(&service); + let a_owner = owner.clone(); + let a_archive = archive.clone(); + let backup_a = thread::spawn(move || { + run_controlled_metadata_backup_once(a_service.as_ref(), &a_owner, &a_archive) + }); + control.wait_until_mark_reached(); + + // A has captured and uploaded its image but has not published it. The + // later mutation must belong only to B's checkpoint. + service + .create_dir_path("/captured-by-b", 0o755, 1000, 1000) + .unwrap(); + objects.arm(); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let b_service = Arc::clone(&service); + let b_owner = owner.clone(); + let b_archive = archive.clone(); + let backup_b = thread::spawn(move || { + started_tx.send(()).unwrap(); + run_controlled_metadata_backup_once(b_service.as_ref(), &b_owner, &b_archive) + }); + started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + thread::sleep(Duration::from_millis(100)); + assert!( + !objects.has_reached(), + "backup B must not capture/upload while backup A owns the publication gate" + ); + + control.release_mark(); + objects.wait_until_reached(); + let err = backup_a.join().unwrap().unwrap_err(); + assert!( + err.to_string() + .contains("published checkpoint does not exactly cover"), + "unexpected stale-capture result: {err}" + ); + let after_a = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + let checkpoint_a = after_a.checkpoint.as_ref().unwrap().clone(); + assert!(objects + .head(&ObjectKey::new(checkpoint_a.object_key.clone()).unwrap()) + .unwrap() + .is_some()); + assert!(objects + .head(&ObjectKey::new(format!("{}.proof", checkpoint_a.object_key)).unwrap()) + .unwrap() + .is_some()); + + // B's image PUT is still blocked, so A remains both authoritative and + // physically present. Only B's successful control CAS may prune A. + objects.release(); + let outcome_b = backup_b.join().unwrap().unwrap(); + let final_record = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + let final_checkpoint = final_record.checkpoint.unwrap(); + assert_eq!(final_checkpoint.object_key, outcome_b.checkpoint_key); + assert_ne!(checkpoint_a.object_key, outcome_b.checkpoint_key); + assert!(objects + .head(&ObjectKey::new(checkpoint_a.object_key.clone()).unwrap()) + .unwrap() + .is_none()); + assert!(objects + .head(&ObjectKey::new(format!("{}.proof", checkpoint_a.object_key)).unwrap()) + .unwrap() + .is_none()); + assert!(objects + .head(&ObjectKey::new(outcome_b.checkpoint_key.clone()).unwrap()) + .unwrap() + .is_some()); + + let restored = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects, + ); + restored + .restore_metadata_checkpoint(&archive, &checkpoint_identity(&final_checkpoint)) + .unwrap(); + assert!(restored.lookup_path("/captured-by-a").unwrap().is_some()); + assert!(restored.lookup_path("/captured-by-b").unwrap().is_some()); + owner.release().unwrap(); + } + + #[test] + fn controlled_startup_failure_releases_current_and_prior_shard_owners() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/partial-open".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let control = Arc::new(FailingCheckpointPublishControl::new()); + control.fail_mark_on_call(2); + + let result = Server::open_with_objects( + options, + objects, + Some(( + control.clone(), + vec![ + ServerShardOwnerOptions::fresh("mount-1:/", "node-root") + .with_renewal(None) + .with_shard_index(Some(0)) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/partial-open-log", + ))), + ServerShardOwnerOptions::fresh("mount-1:/dataset", "node-dataset") + .with_renewal(None) + .with_shard_index(Some(1)) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/partial-open-log", + ))), + ], + )), + ); + assert!(result.is_err()); + + for shard_id in ["mount-1:/", "mount-1:/dataset"] { + let record = control.get_shard(&ShardId::new(shard_id)).unwrap(); + assert!(record.owner.is_none(), "{shard_id} owner must be released"); + assert_eq!(record.state, ShardState::Unassigned); + } + + // Neither a failed current slot nor an already-open prior slot may + // strand ownership and block an immediate retry. + let lease = control + .acquire_unassigned(ShardId::new("mount-1:/"), NodeId::new("node-retry")) + .unwrap(); + assert_eq!( + control.get_shard(&lease.shard_id).unwrap().state, + ShardState::Recovering + ); + control.release(&lease).unwrap(); + } + + #[test] + fn controlled_startup_lost_lease_never_marks_serving() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/lost-startup".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let control = Arc::new(BlockingServingControl::new()); + control.arm_mark(); + let open_control = Arc::clone(&control); + let open = thread::spawn(move || { + Server::open_with_objects( + options, + objects, + Some(( + open_control as Arc, + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(Some(ServerShardOwnerRenewalOptions { + interval: Duration::from_millis(10), + run_immediately: true, + lease_ttl: Duration::from_secs(5), + })) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/lost-startup-log", + )))], + )), + ) + }); + let deadline = Instant::now() + Duration::from_secs(2); + while !control.mark_reached() && !open.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + if open.is_finished() { + match open.join().unwrap() { + Err(err) => panic!("startup failed before mark-serving CAS: {err}"), + Ok(_) => panic!("startup unexpectedly completed before mark-serving CAS"), + } + } + control.wait_until_mark_reached(); + control + .acquire_after_failure(ShardId::new("mount-1:/"), NodeId::new("node-b"), 1) + .unwrap(); + control.release_mark(); + + assert!(open.join().unwrap().is_err()); + let state = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(state.owner, Some(NodeId::new("node-b"))); + assert_eq!(state.epoch, 2); + assert_eq!(state.state, ShardState::Recovering); + assert!(state.checkpoint.is_none()); + } + + #[test] + fn offline_restore_preserves_archived_failover_marker() { + let dir = tempdir().unwrap(); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let source_dir = dir.path().join("source"); + std::fs::create_dir_all(&source_dir).unwrap(); + let source_store = HoltMetadataStore::open_file(source_dir.join("metadata.holt")).unwrap(); + let source = NoKvFs::new( + MountId::new(1).unwrap(), + ServerMetadataStore::direct(source_store), + objects.clone(), + ); + source + .bootstrap_root(DEFAULT_ROOT_MODE, 1000, 1000) + .unwrap(); + let archive = MetadataArchiveConfig::new("metadata/offline-restore", 2); + let backup = source.backup_metadata(&archive).unwrap(); + drop(source); + + let target_root = dir.path().join("target"); + let mut options = test_options(&target_root); + options.metadata_checkpoint_archive_prefix = Some(archive.prefix.clone()); + let report = restore_with_objects(options.clone(), objects).unwrap(); + assert!(report.contains(&format!("\"commit_version\":{}", backup.commit_version))); + + let restored = + HoltMetadataStore::open_file(default_metadata_state_path(&options.meta_path)).unwrap(); + let marker = restored + .get_versioned( + RecordFamily::System, + &failover_durability_required_key(options.mount), + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .expect("offline restore must persist the failover marker"); + assert_eq!(marker.value.0, vec![1]); + } + + #[test] + fn single_node_startup_recovers_object_gc_claim_before_ready() { + let dir = tempdir().unwrap(); + let options = test_options(dir.path()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let seeded = seed_crash_left_object_gc_claim(&options, &options.meta_path, objects.clone()); + + let server = Server::open_with_objects(options, objects, None).unwrap(); + assert!(server.stats_json().contains("\"ready\":true")); + drop(server); + + let holt = HoltMetadataStore::open_file(&seeded.metadata_path).unwrap(); + let latest = Version::new(u64::MAX).unwrap(); + assert!(holt + .get_versioned( + RecordFamily::Gc, + &seeded.gc_key, + latest, + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert_eq!( + holt.get( + RecordFamily::System, + &seeded.claim_key, + latest, + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap() + .0, + vec![1] + ); + } + + #[test] + fn single_node_archive_marker_blocks_uncertain_object_gc_recovery() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/single-node".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let seeded = seed_crash_left_object_gc_claim(&options, &options.meta_path, objects.clone()); + + let result = Server::open_with_objects(options.clone(), objects, None); + assert_startup_requires_object_gc_intervention(result, seeded.operation_token); + assert_failover_gate_preserves_crash_claim(&options, &seeded, true); + } + + #[test] + fn shared_log_marker_is_installed_before_object_gc_claim_recovery() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/startup-order-ck".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let shard_id = "mount-1:/"; + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard_id)); + let seeded = seed_crash_left_object_gc_claim(&options, &shard_meta_dir, objects.clone()); + let control: Arc = Arc::new(InMemoryControlStore::new()); + let owner = ServerShardOwnerOptions::fresh(shard_id, "node-a").with_shared_log(Some( + crate::ServerSharedLogOptions::new("metadata/startup-order"), + )); + + let result = + Server::open_with_objects(options.clone(), objects, Some((control, vec![owner]))); + assert_startup_requires_object_gc_intervention(result, seeded.operation_token); + assert_failover_gate_preserves_crash_claim(&options, &seeded, true); + } + + #[test] + fn controlled_marker_without_shared_log_blocks_uncertain_object_gc_recovery() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/controlled-marker".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let shard_id = "mount-1:/"; + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard_id)); + let seeded = seed_crash_left_object_gc_claim(&options, &shard_meta_dir, objects.clone()); + let control: Arc = Arc::new(InMemoryControlStore::new()); + let owner = ServerShardOwnerOptions::fresh(shard_id, "node-a"); + + let result = + Server::open_with_objects(options.clone(), objects, Some((control, vec![owner]))); + assert!(matches!( + result, + Err(ServerError::Control(ControlError::InvalidOptions(message))) + if message.contains("requires synchronous shared_log") + )); + assert_failover_gate_preserves_crash_claim(&options, &seeded, false); + } + + #[test] + fn manual_gc_reports_empty_outcomes() { + let server = test_server(); + assert!(server.stats_json().contains("\"ready\":true")); + assert!(server + .stats_json() + .contains("\"shard_owner\":{\"enabled\":false}")); + let body = server.run_manual_gc(128).unwrap(); + assert!(body.contains("\"object_gc\"")); + assert!(body.contains("\"history_gc\"")); + } + + #[test] + fn controlled_stats_report_persisted_failover_fence_with_empty_gc_queue() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let server = open_memory_controlled( + dir.path(), + control, + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], + ) + .unwrap(); + + let cleanup = server.service().cleanup_pending_objects(128).unwrap(); + assert_eq!(cleanup.scanned, 0); + assert!(server + .stats_json() + .contains("\"object_gc\":{\"failover_durability_required\":true")); + } + + #[test] + fn object_gc_health_keeps_cumulative_reaper_conflicts() { + let last_outcome = nokv_meta::PendingObjectCleanupOutcome { + blocked_by_failover_durability: 4, + ..Default::default() + }; + let state = ObjectGcWorkerState { + iterations: 9, + last_outcome: Some(last_outcome), + snapshot_reap: nokv_meta::SnapshotReapOutcome { + scanned: 12, + expired_candidates: 3, + reaped: 2, + conflicted: 1, + }, + last_error: None, + }; + + let json = object_gc_json(&state, true); + assert!(json.contains("\"failover_durability_required\":true")); + assert!(json.contains("\"iterations\":9")); + assert!(json.contains("\"blocked_by_failover_durability\":4")); + assert!(json.contains("\"conflicted\":1")); + } + + #[test] + fn controlled_open_acquires_shard_and_installs_owner_epoch() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let server = open_memory_controlled( + dir.path(), + control, + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], + ) + .unwrap(); + + let state = server.shard_owner_state().unwrap().unwrap(); + assert_eq!(state.shard_id.as_str(), "mount-1:/"); + assert_eq!(state.node_id.as_str(), "node-a"); + assert_eq!(state.epoch, 1); + assert_eq!(state.lease_id, 1); + assert_eq!(state.state, nokv_control::ShardState::Serving); + assert_eq!(server.service().allocator_epoch(), 1); + assert_eq!(server.service().required_owner_epoch(), 1); + assert!(server + .stats_json() + .contains("\"shard_owner\":{\"enabled\":true")); + assert!(server + .stats_json() + .contains("\"renewal\":{\"enabled\":true")); + } + + #[test] + fn controlled_archive_is_refreshed_and_published_before_serving() { + let dir = tempdir().unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/controlled".to_owned()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let control = Arc::new(InMemoryControlStore::new()); + + let server = Server::open_with_objects( + options, + objects, + Some(( + control.clone(), + vec![ + ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_shared_log(Some( + crate::ServerSharedLogOptions::new("metadata/controlled-log"), + )), + ], + )), + ) + .unwrap(); + + let state = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(state.state, nokv_control::ShardState::Serving); + let checkpoint = state + .checkpoint + .expect("serving transition must publish the startup checkpoint"); + assert!(checkpoint + .object_key + .starts_with("metadata/controlled/mount_1__/controlled/sha256/")); + assert!(checkpoint.image_bytes > 0); + assert!(checkpoint.image_digest.starts_with("sha256:")); + assert_eq!(checkpoint.lsn, state.durable_lsn); + assert!(server.stats_json().contains("\"ready\":true")); + } + + #[test] + fn controlled_periodic_backup_is_exactly_restorable_after_failover() { + let dir = tempdir().unwrap(); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let control = Arc::new(InMemoryControlStore::new()); + let mut first_options = test_options(&dir.path().join("first")); + first_options.metadata_checkpoint_archive_prefix = Some("metadata/periodic".to_owned()); + let first = Server::open_with_objects( + first_options, + objects.clone(), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new("metadata/log")))], + )), + ) + .unwrap(); + first + .service() + .create_dir_path("/periodic", 0o755, 1000, 1000) + .unwrap(); + + let slot = first.default_slot(); + let mut options = MetadataBackupOptions::new(slot.metadata_archive.clone().unwrap()); + options.interval = Duration::from_secs(3600); + options.run_immediately = true; + let mut worker = ControlledMetadataBackupWorker::spawn( + Arc::clone(&slot.service), + slot.owner.clone().unwrap(), + options, + ); + wait_until(|| worker.state().iterations >= 1); + assert!(worker.state().last_error.is_none()); + worker.stop(); + let periodic_ref = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap(); + drop(first); + + let mut second_options = test_options(&dir.path().join("second")); + second_options.metadata_checkpoint_archive_prefix = Some("metadata/periodic".to_owned()); + let second = Server::open_with_objects( + second_options, + objects.clone(), + Some(( + control, + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/periodic-log", + )))], + )), + ) + .unwrap(); + assert!(second.service().lookup_path("/periodic").unwrap().is_some()); + let successor_ref = second + .shard_owner_state() + .unwrap() + .unwrap() + .checkpoint + .unwrap(); + let digest = successor_ref.image_digest.strip_prefix("sha256:").unwrap(); + assert_eq!( + successor_ref.object_key, + format!("metadata/periodic/mount_1__/controlled/sha256/{digest}.image") + ); + assert!(objects + .head(&ObjectKey::new(format!("{}.proof", successor_ref.object_key)).unwrap()) + .unwrap() + .is_some()); + assert_ne!( + successor_ref.object_key, periodic_ref.object_key, + "failover rotates the object-GC claim version before its startup checkpoint" + ); + assert!(objects + .head(&ObjectKey::new(periodic_ref.object_key.clone()).unwrap()) + .unwrap() + .is_none()); + assert!(objects + .head(&ObjectKey::new(format!("{}.proof", periodic_ref.object_key)).unwrap()) + .unwrap() + .is_none()); + } + + #[test] + fn controlled_startup_upload_window_keeps_previous_ref_recoverable() { + let dir = tempdir().unwrap(); + let memory = Arc::new(MemoryObjectStore::new()); + let objects = ConfiguredObjectStore::Memory(Arc::clone(&memory)); + let control = Arc::new(InMemoryControlStore::new()); + let mut first_options = test_options(&dir.path().join("first")); + first_options.metadata_checkpoint_archive_prefix = Some("metadata/window".to_owned()); + let first = Server::open_with_objects( + first_options, + objects.clone(), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/window-log", + )))], + )), + ) + .unwrap(); + first + .service() + .create_dir_path("/authoritative", 0o755, 1000, 1000) + .unwrap(); + first.run_manual_backup().unwrap(); + let authoritative = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap(); + first + .service() + .create_dir_path("/orphan", 0o755, 1000, 1000) + .unwrap(); + let archive = first.default_slot().metadata_archive.as_ref().unwrap(); + let orphan = first + .service() + .prepare_immutable_metadata_backup(archive) + .unwrap(); + assert_ne!(orphan.checkpoint_key, authoritative.object_key); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .as_ref(), + Some(&authoritative) + ); + assert!(memory + .head(&ObjectKey::new(authoritative.object_key.clone()).unwrap()) + .unwrap() + .is_some()); + assert!(memory + .head(&ObjectKey::new(orphan.checkpoint_key).unwrap()) + .unwrap() + .is_some()); + assert!(memory + .head(&ObjectKey::new("metadata/window/mount_1__/CURRENT").unwrap()) + .unwrap() + .is_none()); + drop(first); + + let mut second_options = test_options(&dir.path().join("second")); + second_options.metadata_checkpoint_archive_prefix = Some("metadata/window".to_owned()); + let second = Server::open_with_objects( + second_options, + objects, + Some(( + control, + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/window-log", + )))], + )), + ) + .unwrap(); + assert!(second + .service() + .lookup_path("/authoritative") + .unwrap() + .is_some()); + assert!(second.service().lookup_path("/orphan").unwrap().is_none()); + } + + #[test] + fn expired_or_stale_owner_cannot_publish_or_prune_controlled_archive() { + let dir = tempdir().unwrap(); + let memory = Arc::new(MemoryObjectStore::new()); + let objects = ConfiguredObjectStore::Memory(Arc::clone(&memory)); + let control = Arc::new(InMemoryControlStore::new()); + let mut first_options = test_options(&dir.path().join("first")); + first_options.metadata_checkpoint_archive_prefix = Some("metadata/fenced-owner".to_owned()); + let first = Server::open_with_objects( + first_options, + objects.clone(), + Some(( + control.clone(), + vec![ + ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_shared_log(Some( + crate::ServerSharedLogOptions::new("metadata/fenced-owner-log"), + )), + ], + )), + ) + .unwrap(); + let old_service = Arc::clone(&first.default_slot().service); + let old_owner = first.default_slot().owner.clone().unwrap(); + let archive = first.default_slot().metadata_archive.clone().unwrap(); + let first_ref = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap(); + + let deadline = old_service.lease_deadline_ms(); + assert!(deadline > 0); + old_service.set_clock_override_ms(deadline); + assert!(matches!( + run_controlled_metadata_backup_once(&old_service, &old_owner, &archive), + Err(ServerError::Metadata(MetadError::LeaseExpired { .. })) + )); + old_service.set_clock_override_ms(0); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .as_ref(), + Some(&first_ref) + ); + drop(first); + + let mut second_options = test_options(&dir.path().join("second")); + second_options.metadata_checkpoint_archive_prefix = + Some("metadata/fenced-owner".to_owned()); + let second = Server::open_with_objects( + second_options, + objects, + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/fenced-owner-log", + )))], + )), + ) + .unwrap(); + let successor_ref = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap(); + assert!(run_controlled_metadata_backup_once(&old_service, &old_owner, &archive).is_err()); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .as_ref(), + Some(&successor_ref) + ); + assert!(memory + .head(&ObjectKey::new(successor_ref.object_key.clone()).unwrap()) + .unwrap() + .is_some()); + assert!(memory + .head(&ObjectKey::new(format!("{}.proof", successor_ref.object_key)).unwrap()) + .unwrap() + .is_some()); + assert!(memory + .head(&ObjectKey::new("metadata/fenced-owner/mount_1__/CURRENT").unwrap()) + .unwrap() + .is_none()); + drop(second); + } + + #[test] + fn repeated_control_publish_failures_preserve_previous_checkpoint() { + let dir = tempdir().unwrap(); + let memory = Arc::new(MemoryObjectStore::new()); + let objects = ConfiguredObjectStore::Memory(Arc::clone(&memory)); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/publish-failure".to_owned()); + let server = Server::open_with_objects( + options, + objects, + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/publish-failure-log", + )))], + )), + ) + .unwrap(); + let previous = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap(); + server + .service() + .create_dir_path("/new-state", 0o755, 1000, 1000) + .unwrap(); + control.fail_next_marks(2); + assert!(server.run_manual_backup().is_err()); + assert!(server.run_manual_backup().is_err()); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .as_ref(), + Some(&previous) + ); + assert!(memory + .head(&ObjectKey::new(previous.object_key.clone()).unwrap()) + .unwrap() + .is_some()); + assert!(memory + .head(&ObjectKey::new(format!("{}.proof", previous.object_key)).unwrap()) + .unwrap() + .is_some()); + assert!(memory + .head(&ObjectKey::new("metadata/publish-failure/mount_1__/CURRENT").unwrap()) + .unwrap() + .is_none()); + + let report = server.run_manual_backup().unwrap(); + assert!(report.contains("\"checkpoint_key\"")); + let published = control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .checkpoint + .unwrap(); + assert_ne!(published.object_key, previous.object_key); + assert!(memory + .head(&ObjectKey::new(previous.object_key).unwrap()) + .unwrap() + .is_none()); + } + + /// An owner that declares `shard_index` registers its shard identity (prefix + + /// index) on open even when nothing pre-registered it — the path a multi-process + /// `nokv serve --shard-index N` fleet relies on. The control record then carries + /// the declared index, and the server routes the shard's subtree to that slot. + #[test] + fn controlled_open_with_shard_index_registers_identity() { + use nokv_protocol::MetadataRpcRequest; + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + // No register_shard call: the owner's declared index is the only source of + // the shard's identity. + let server = open_memory_controlled( + dir.path(), + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/dataset", "node-a") + .with_renewal(None) + .with_shard_index(Some(1))], + ) + .unwrap(); + + // The control record now carries the declared index and the derived prefix. + let record = control + .get_shard(&nokv_control::ShardId::new("mount-1:/dataset")) + .unwrap(); + assert_eq!(record.shard_index, 1, "open registered the declared index"); + assert_eq!( + record.prefix, "/dataset", + "prefix derived from the shard id" + ); + + // The server routes a /dataset path to the index-1 slot it just registered. + let slot = server + .route(&MetadataRpcRequest::CreateDirPath { + path: "/dataset/run".to_owned(), + mode: 0o755, + uid: 1000, + gid: 1000, + }) + .unwrap(); + assert_eq!(slot.shard_index(), 1); + } + + #[test] + fn graft_reconcile_does_not_fall_back_to_default_for_a_remote_nested_parent() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + nokv_control::register_shard( + control.as_ref(), + ShardId::new("mount-1:/"), + "/", + DEFAULT_SHARD_INDEX, + ) + .unwrap(); + nokv_control::register_shard( + control.as_ref(), + ShardId::new("mount-1:/dataset"), + "/dataset", + 1, + ) + .unwrap(); + nokv_control::register_shard( + control.as_ref(), + ShardId::new("mount-1:/dataset/images"), + "/dataset/images", + 2, + ) + .unwrap(); + + // This process hosts only the default shard. Leave a local /dataset + // directory in place so the former local-only routing bug would have + // inserted the nested "images" graft into it. + let server = open_memory_controlled( + dir.path(), + Arc::clone(&control), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-root").with_renewal(None)], + ) + .unwrap(); + server + .service() + .create_dir_path("/dataset", 0o755, 1000, 1000) + .unwrap(); + server.publish_latest_metadata_log_ref().unwrap(); + + let child = InodeId::compose(2, 7).unwrap(); + control + .set_subtree_root_inode(&ShardId::new("mount-1:/dataset/images"), Some(child.get())) + .unwrap(); + assert!(server + .service() + .lookup_path("/dataset/images") + .unwrap() + .is_none()); + + server.reconcile_local_grafts().unwrap(); + + assert!(server + .service() + .lookup_path("/dataset/images") + .unwrap() + .is_none()); + } + + fn assert_failover_without_checkpoint_archive_is_rejected( + control: Arc, + with_shared_log: bool, + ) { + let dir = tempdir().unwrap(); + let mut owner = + ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1).with_renewal(None); + if with_shared_log { + owner = owner.with_shared_log(Some(crate::ServerSharedLogOptions::new("metadata/log"))); + } + let result = Server::open_with_objects( + test_options(dir.path()), + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some((control, vec![owner])), + ); + match result { + Err(ServerError::Metadata(MetadError::InvalidPath(message))) => assert!( + message.contains("failover recovery requires a metadata checkpoint archive"), + "unexpected strict-restore error: {message}" + ), + Err(err) => panic!("unexpected failover error: {err}"), + Ok(_) => panic!("failover without a checkpoint archive must be rejected"), + } + } + + fn test_controlled_checkpoint_ref(archive_prefix: &str, shard_id: &str) -> CheckpointRef { + let image_hex = "a".repeat(64); + CheckpointRef { + object_key: format!( + "{}/controlled/sha256/{image_hex}.image", + shard_archive_prefix(archive_prefix, &sanitize_shard_id(shard_id)) + ), + lsn: 0, + image_bytes: 1, + image_digest: format!("sha256:{image_hex}"), + digest: "0".repeat(64), + } + } + + #[test] + fn fresh_control_owned_open_requires_a_checkpoint_archive_before_acquire() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let result = Server::open_with_objects( + test_options(dir.path()), + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new("metadata/log")))], + )), + ); + match result { + Err(ServerError::Metadata(MetadError::InvalidPath(message))) => assert!( + message.contains("control-owned shard requires a metadata checkpoint archive"), + "unexpected deployment-gate error: {message}" + ), + Err(err) => panic!("unexpected controlled-open error: {err}"), + Ok(_) => panic!("control-owned open without an archive must be rejected"), + } + let record = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(record.epoch, 0, "deployment preflight must precede acquire"); + assert_eq!(record.owner, None); + assert_eq!(record.state, ShardState::Unassigned); + } + + #[test] + fn fresh_control_owned_open_requires_shared_log_before_any_side_effect() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let before = control.ensure_shard(shard.clone()).unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/checkpoint".to_owned()); + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard.as_str())); + let objects = MemoryObjectStore::new(); + + let result = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(objects.clone())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None)], + )), + ); + + assert!(matches!( + result, + Err(ServerError::Control(ControlError::InvalidOptions(message))) + if message.contains("control-owned shard") + && message.contains("requires synchronous shared_log") + )); + assert_eq!(control.get_shard(&shard).unwrap(), before); + assert!(!shard_meta_dir.exists()); + assert_eq!(objects.object_count(), 0); + } + + #[test] + fn failover_without_refs_still_requires_a_checkpoint_archive() { + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard, NodeId::new("node-a")) + .unwrap(); + control.mark_serving(&lease, None, None, 0).unwrap(); + control.release(&lease).unwrap(); + + assert_failover_without_checkpoint_archive_is_rejected(control, true); + } + + #[test] + fn failover_with_only_a_log_ref_still_requires_a_checkpoint_archive() { + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard, NodeId::new("node-a")) + .unwrap(); + let log = LogRef { + segments: vec![LogSegmentRef { + segment_key: "meta/log/segment-1".to_owned(), + first_lsn: 1, + last_lsn: 1, + digest: "digest-1".to_owned(), + }], + durable_lsn: 1, + digest: "digest-1".to_owned(), + }; + control.mark_serving(&lease, None, Some(log), 1).unwrap(); + control.release(&lease).unwrap(); + + assert_failover_without_checkpoint_archive_is_rejected(control, true); + } + + #[test] + fn failover_requires_checkpoint_identity_before_acquire_or_object_io() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard.clone(), NodeId::new("node-a")) + .unwrap(); + control.mark_serving(&lease, None, None, 0).unwrap(); + control.release(&lease).unwrap(); + + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/checkpoint".to_owned()); + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard.as_str())); + let objects = MemoryObjectStore::new(); + let result = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(objects.clone())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new("metadata/log")))], + )), + ); + + assert!(matches!( + result, + Err(ServerError::Control(ControlError::InvalidOptions(message))) + if message.contains("no durable checkpoint identity") + )); + let record = control.get_shard(&shard).unwrap(); + assert_eq!(record.epoch, 1, "checkpoint preflight must precede acquire"); + assert_eq!(record.owner, None); + assert!(!shard_meta_dir.exists()); + assert_eq!( + objects.object_count(), + 0, + "preflight must precede object I/O" + ); + } + + #[test] + fn failover_rejects_cross_prefix_checkpoint_identity_before_acquire() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard.clone(), NodeId::new("node-a")) + .unwrap(); + let mut checkpoint = test_controlled_checkpoint_ref("metadata/checkpoint", shard.as_str()); + checkpoint.object_key = + checkpoint + .object_key + .replacen("metadata/checkpoint/", "metadata/another-shard/", 1); + control + .mark_serving(&lease, Some(checkpoint), None, 0) + .unwrap(); + control.release(&lease).unwrap(); + + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/checkpoint".to_owned()); + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard.as_str())); + let objects = MemoryObjectStore::new(); + let result = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(objects.clone())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new("metadata/log")))], + )), + ); + + assert!(matches!( + result, + Err(ServerError::Metadata(MetadError::Codec(message))) + if message.contains("does not match archive prefix and image digest") + )); + let record = control.get_shard(&shard).unwrap(); + assert_eq!(record.epoch, 1, "identity preflight must precede acquire"); + assert_eq!(record.owner, None); + assert!(!shard_meta_dir.exists()); + assert_eq!( + objects.object_count(), + 0, + "preflight must precede object I/O" + ); + } + + #[test] + fn failover_rejects_cross_prefix_log_identity_before_acquire() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard.clone(), NodeId::new("node-a")) + .unwrap(); + let checkpoint = test_controlled_checkpoint_ref("metadata/checkpoint", shard.as_str()); + let tail_digest = "b".repeat(64); + let log = LogRef { + segments: vec![LogSegmentRef { + segment_key: format!( + "metadata/log/mount_2__/log/00000000000000000001-00000000000000000001-{tail_digest}.segment" + ), + first_lsn: 1, + last_lsn: 1, + digest: tail_digest.clone(), + }], + durable_lsn: 1, + digest: tail_digest, + }; + control + .mark_serving(&lease, Some(checkpoint), Some(log), 1) + .unwrap(); + control.release(&lease).unwrap(); + + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/checkpoint".to_owned()); + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard.as_str())); + let objects = MemoryObjectStore::new(); + let result = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(objects.clone())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new("metadata/log")))], + )), + ); + + assert!(matches!( + result, + Err(ServerError::Metadata(MetadError::Codec(message))) + if message.contains("outside archive prefix") + )); + let record = control.get_shard(&shard).unwrap(); + assert_eq!(record.epoch, 1, "log preflight must precede acquire"); + assert_eq!(record.owner, None); + assert!(!shard_meta_dir.exists()); + assert_eq!( + objects.object_count(), + 0, + "log preflight must precede object I/O" + ); + } + + #[test] + fn control_log_segment_identity_binds_loaded_range_and_tail_digest() { + let command = MetadataCommand { + request_id: b"control-segment-identity".to_vec(), + kind: CommandKind::RegisterNamespaceIndex, + read_version: Version::new(1).unwrap(), + commit_version: Version::new(2).unwrap(), + primary_family: RecordFamily::System, + primary_key: b"control-segment-identity".to_vec(), + predicates: Vec::new(), + mutations: vec![Mutation { + family: RecordFamily::System, + key: b"control-segment-identity".to_vec(), + op: MutationOp::Put, + value: Some(Value(b"value".to_vec())), + }], + watch: Vec::new(), + }; + let entry = MetadataLogEntry::seal( + "mount-1:/", + 1, + 1, + command, + CommitResult { + commit_version: Version::new(2).unwrap(), + applied_mutations: 1, + watch_events: 0, + }, + METADATA_LOG_ZERO_DIGEST, + ) + .unwrap(); + let segment = MetadataLogSegment::seal(vec![entry]).unwrap(); + let reference = LogSegmentRef { + segment_key: "metadata/log/mount_1__/log/segment".to_owned(), + first_lsn: segment.first_lsn, + last_lsn: segment.last_lsn, + digest: hex_digest(&segment.last_digest), + }; + validate_control_log_segment_identity(&reference, &segment).unwrap(); + + let mut wrong_range = reference.clone(); + wrong_range.last_lsn += 1; + assert!(matches!( + validate_control_log_segment_identity(&wrong_range, &segment), + Err(ServerError::Metadata(MetadError::Codec(message))) + if message.contains("does not match control identity") + )); + + let mut wrong_digest = reference; + wrong_digest.digest = "c".repeat(64); + assert!(matches!( + validate_control_log_segment_identity(&wrong_digest, &segment), + Err(ServerError::Metadata(MetadError::Codec(message))) + if message.contains("does not match control identity") + )); + } + + #[test] + fn failover_requires_shared_log_at_a_valid_zero_lsn_checkpoint() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard.clone(), NodeId::new("node-a")) + .unwrap(); + let checkpoint = test_controlled_checkpoint_ref("metadata/checkpoint", shard.as_str()); + control + .mark_serving(&lease, Some(checkpoint), None, 0) + .unwrap(); + control.release(&lease).unwrap(); + let before = control.get_shard(&shard).unwrap(); + assert_eq!(before.state, ShardState::Unassigned); + assert_eq!(before.durable_lsn, 0); + assert!(before.log.is_none()); + + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/checkpoint".to_owned()); + let shard_meta_dir = options.meta_path.join(sanitize_shard_id(shard.as_str())); + let objects = MemoryObjectStore::new(); + let result = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(objects.clone())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1).with_renewal(None)], + )), + ); + + assert!(matches!( + result, + Err(ServerError::Control(ControlError::InvalidOptions(message))) + if message.contains("control-owned shard") + && message.contains("requires synchronous shared_log") + )); + let after = control.get_shard(&shard).unwrap(); + assert_eq!(after.epoch, before.epoch); + assert_eq!(after.owner, before.owner); + assert_eq!(after.state, before.state); + assert_eq!(after.checkpoint, before.checkpoint); + assert!(!shard_meta_dir.exists()); + assert_eq!(objects.object_count(), 0); + } + + #[test] + fn failover_with_durable_log_requires_shared_log_before_acquire() { + let dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let shard = ShardId::new("mount-1:/"); + let lease = control + .acquire_unassigned(shard.clone(), NodeId::new("node-a")) + .unwrap(); + let log = LogRef { + segments: vec![LogSegmentRef { + segment_key: "meta/log/segment-1".to_owned(), + first_lsn: 1, + last_lsn: 1, + digest: "digest-1".to_owned(), + }], + durable_lsn: 1, + digest: "digest-1".to_owned(), + }; + let checkpoint = test_controlled_checkpoint_ref("metadata/checkpoint", shard.as_str()); + control + .mark_serving(&lease, Some(checkpoint), Some(log), 1) + .unwrap(); + control.release(&lease).unwrap(); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/checkpoint".to_owned()); + + let result = Server::open_with_objects( + options, + ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1).with_renewal(None)], + )), + ); + + assert!(matches!( + result, + Err(ServerError::Control(ControlError::InvalidOptions(message))) + if message.contains("control-owned shard") + && message.contains("requires synchronous shared_log") + )); + let record = control.get_shard(&shard).unwrap(); + assert_eq!(record.epoch, 1, "shared-log preflight must precede acquire"); + assert_eq!(record.owner, None); + assert!( + !dir.path().join(sanitize_shard_id(shard.as_str())).exists(), + "shared-log preflight must precede local metadata creation and object publication" + ); + } + + #[cfg(feature = "etcd")] + #[test] + fn configured_etcd_control_store_expires_session_and_allows_failover() { + let endpoints = match env::var("NOKV_ETCD_ENDPOINTS") { + Ok(raw) if !raw.trim().is_empty() => raw .split(',') .map(str::trim) .filter(|endpoint| !endpoint.is_empty()) @@ -1671,14 +4651,29 @@ pub(crate) mod tests { .with_key_prefix(key_prefix.clone()) .with_lease_ttl_seconds(1) }; + let control = Arc::new( + nokv_control::EtcdControlStore::connect(etcd_options()) + .expect("connect etcd control store"), + ); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); let first_dir = tempdir().unwrap(); let mut first_options = test_options(first_dir.path()); - first_options.control = Some(ServerControlOptions { - store: ServerControlStoreOptions::Etcd(etcd_options()), - shard_owner: ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None), - }); - let first = Server::open(first_options).unwrap(); + first_options.metadata_checkpoint_archive_prefix = + Some("metadata/etcd-failover".to_owned()); + let first = Server::open_with_objects( + first_options, + objects.clone(), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/etcd-failover-log", + )))], + )), + ) + .unwrap(); let first_state = first.shard_owner_state().unwrap().unwrap(); assert_eq!(first_state.node_id.as_str(), "node-a"); assert_eq!(first_state.epoch, 1); @@ -1687,12 +4682,20 @@ pub(crate) mod tests { let deadline = Instant::now() + Duration::from_secs(8); let second = loop { let mut second_options = test_options(second_dir.path()); - second_options.control = Some(ServerControlOptions { - store: ServerControlStoreOptions::Etcd(etcd_options()), - shard_owner: ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) - .with_renewal(None), - }); - match Server::open(second_options) { + second_options.metadata_checkpoint_archive_prefix = + Some("metadata/etcd-failover".to_owned()); + match Server::open_with_objects( + second_options, + objects.clone(), + Some(( + control.clone(), + vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1) + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "metadata/etcd-failover-log", + )))], + )), + ) { Ok(server) => break server, Err(ServerError::Control(ControlError::ShardAlreadyOwned { .. })) if Instant::now() < deadline => @@ -1716,8 +4719,8 @@ pub(crate) mod tests { fn shard_owner_log_ref_publish_updates_control_record() { let dir = tempdir().unwrap(); let control = Arc::new(InMemoryControlStore::new()); - let server = Server::open_with_control( - test_options(dir.path()), + let server = open_memory_controlled( + dir.path(), control.clone(), vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None)], ) @@ -1725,11 +4728,11 @@ pub(crate) mod tests { let log = LogRef { segments: vec![LogSegmentRef { segment_key: "meta/shared-log/log/segment-1".to_owned(), - first_lsn: 40, - last_lsn: 42, + first_lsn: 1, + last_lsn: 3, digest: "abc123".to_owned(), }], - durable_lsn: 42, + durable_lsn: 3, digest: "abc123".to_owned(), }; @@ -1739,23 +4742,186 @@ pub(crate) mod tests { .unwrap(); assert_eq!(state.log, Some(log.clone())); - assert_eq!(state.durable_lsn, 42); + assert_eq!(state.durable_lsn, 3); let record = control .get_shard(&nokv_control::ShardId::new("mount-1:/")) .unwrap(); assert_eq!(record.log, Some(log)); - assert_eq!(record.durable_lsn, 42); + assert_eq!(record.durable_lsn, 3); assert!(server.stats_json().contains( - "\"log\":{\"segments\":[{\"segment_key\":\"meta/shared-log/log/segment-1\",\"first_lsn\":40,\"last_lsn\":42,\"digest\":\"abc123\"}],\"durable_lsn\":42" + "\"log\":{\"segments\":[{\"segment_key\":\"meta/shared-log/log/segment-1\",\"first_lsn\":1,\"last_lsn\":3,\"digest\":\"abc123\"}],\"durable_lsn\":3" + )); + } + + #[test] + fn prepare_retry_flushes_allocator_reservation_before_control_ack() { + let objects = BlockingPutStore::new(); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + HoltMetadataStore::open_memory().unwrap(), + objects.clone(), + ); + service + .bootstrap_root(DEFAULT_ROOT_MODE, 1000, 1000) + .unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let owner = ServerShardOwner::acquire( + Arc::clone(&control) as Arc, + ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None), + &service, + ) + .unwrap(); + // Initialize the claim outside the log; subsequent prepare-only calls + // produce no metadata command until the allocator boundary is crossed. + service + .prepare_artifact_create( + InodeId::root(), + DentryName::new(b"claim-warmup.bin".to_vec()).unwrap(), + ) + .unwrap(); + service + .enable_sync_metadata_log(MetadataLogSyncConfig::new( + "metadata/allocator-ack-log", + "mount-1:/", + 1, + 0, + METADATA_LOG_ZERO_DIGEST, + )) + .unwrap(); + objects.fail_puts_containing("metadata/allocator-ack-log/log/"); + + let mut first_error = None; + for _ in 0..4096 { + match service.prepare_artifact_create( + InodeId::root(), + DentryName::new(b"prepare-only.bin".to_vec()).unwrap(), + ) { + Ok(_) => {} + Err(err) => { + first_error = Some(err); + break; + } + } + } + assert!(matches!( + first_error, + Some(MetadError::SyncLogArchiveFailed { + committed: true, + .. + }) )); + assert_eq!(service.sync_metadata_log_snapshot().unwrap().durable_lsn, 0); + assert_eq!( + control + .get_shard(&ShardId::new("mount-1:/")) + .unwrap() + .durable_lsn, + 0 + ); + + objects.clear_put_failures(); + service + .prepare_artifact_create( + InodeId::root(), + DentryName::new(b"retry-after-archive.bin".to_vec()).unwrap(), + ) + .unwrap(); + let local = service.sync_metadata_log_snapshot().unwrap(); + assert_eq!(local.durable_lsn, 2); + // This is the same publication helper the RPC response path invokes. + // A successful external ACK occurs only after this owner-fenced CAS. + let published = Server::publish_owner_latest_log_ref(&service, &owner) + .unwrap() + .unwrap(); + assert_eq!(published.durable_lsn, local.durable_lsn); + let durable = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(durable.durable_lsn, 2); + assert_eq!(durable.log.as_ref().unwrap().segments.len(), 2); + owner.release().unwrap(); + } + + #[test] + fn latest_log_snapshot_and_publication_are_owner_ordered() { + let dir = tempdir().unwrap(); + let control = Arc::new(BlockingServingControl::new()); + let objects = ConfiguredObjectStore::Memory(Arc::new(MemoryObjectStore::new())); + let mut options = test_options(dir.path()); + options.metadata_checkpoint_archive_prefix = Some("metadata/ordered-log-ck".to_owned()); + let server = Arc::new( + Server::open_with_objects( + options, + objects, + Some(( + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") + .with_renewal(None) + .with_shared_log(Some(crate::ServerSharedLogOptions::new( + "meta/ordered-log", + )))], + )), + ) + .unwrap(), + ); + server + .service() + .create_dir_path("/first", 0o755, 1000, 1000) + .unwrap(); + + control.arm_mark(); + let first_server = Arc::clone(&server); + let first_publish = thread::spawn(move || first_server.publish_latest_metadata_log_ref()); + control.wait_until_mark_reached(); + + server + .service() + .create_dir_path("/second", 0o755, 1000, 1000) + .unwrap(); + let newest_lsn = server + .service() + .sync_metadata_log_snapshot() + .unwrap() + .durable_lsn; + let (sent, received) = std::sync::mpsc::channel(); + let second_server = Arc::clone(&server); + let second_publish = thread::spawn(move || { + let result = second_server.publish_latest_metadata_log_ref(); + sent.send(result).unwrap(); + }); + + // The first control CAS is deliberately blocked. The newer publisher + // must not reach the control store (or complete) until the older + // snapshot publication leaves the owner gate. + if let Ok(premature) = received.recv_timeout(Duration::from_millis(200)) { + control.release_mark(); + let _ = first_publish.join(); + let _ = second_publish.join(); + panic!("newer log publication escaped the owner gate: {premature:?}"); + } + control.release_mark(); + let first_error = first_publish.join().unwrap().unwrap_err(); + assert!( + first_error + .to_string() + .contains("authoritative recovery refs do not exactly cover"), + "unexpected stale publication result: {first_error}" + ); + received + .recv_timeout(Duration::from_secs(2)) + .expect("newer publication should finish after the gate opens") + .unwrap(); + second_publish.join().unwrap(); + + let record = control.get_shard(&ShardId::new("mount-1:/")).unwrap(); + assert_eq!(record.durable_lsn, newest_lsn); + assert_eq!(record.log.unwrap().durable_lsn, newest_lsn); } #[test] fn owner_arms_lease_deadline_when_renewal_enabled() { let dir = tempdir().unwrap(); let control = Arc::new(InMemoryControlStore::new()); - let server = Server::open_with_control( - test_options(dir.path()), + let server = open_memory_controlled( + dir.path(), control, vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], ) @@ -1766,12 +4932,64 @@ pub(crate) mod tests { ); } + #[test] + fn mark_serving_does_not_extend_local_deadline_but_renew_does() { + let control: Arc = Arc::new(InMemoryControlStore::new()); + let service = NoKvFs::new( + MountId::new(1).unwrap(), + ServerMetadataStore::direct(HoltMetadataStore::open_memory().unwrap()), + MemoryObjectStore::new(), + ); + service + .bootstrap_root(DEFAULT_ROOT_MODE, 1000, 1000) + .unwrap(); + service.set_clock_override_ms(1_000); + let owner = ServerShardOwner::acquire( + control, + ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(Some( + ServerShardOwnerRenewalOptions { + interval: Duration::from_millis(10), + run_immediately: false, + lease_ttl: Duration::from_millis(100), + }, + )), + &service, + ) + .unwrap(); + assert_eq!(service.lease_deadline_ms(), 1_100); + + service.set_clock_override_ms(1_050); + owner.mark_serving(&service).unwrap(); + assert_eq!( + service.lease_deadline_ms(), + 1_100, + "mark_serving is not a keepalive and must not extend the local fence" + ); + + service.set_clock_override_ms(1_075); + owner.renew(&service).unwrap(); + assert_eq!( + service.lease_deadline_ms(), + 1_175, + "only a successful control keepalive may extend the deadline" + ); + service.set_clock_override_ms(1_175); + assert!(matches!( + service.verify_owner_lease(), + Err(MetadError::LeaseExpired { + now_ms: 1_175, + deadline_ms: 1_175, + }) + )); + owner.release().unwrap(); + } + #[test] fn owner_without_renewal_has_no_lease_deadline() { let dir = tempdir().unwrap(); let control = Arc::new(InMemoryControlStore::new()); - let server = Server::open_with_control( - test_options(dir.path()), + let server = open_memory_controlled( + dir.path(), control, vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None)], ) @@ -1789,8 +5007,8 @@ pub(crate) mod tests { let control = Arc::new(InMemoryControlStore::new()); let shard = nokv_control::ShardId::new("mount-1:/"); { - let _server = Server::open_with_control( - test_options(dir.path()), + let _server = open_memory_controlled( + dir.path(), control.clone(), vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None)], ) @@ -1808,24 +5026,91 @@ pub(crate) mod tests { } #[test] - fn stale_owner_renew_observes_new_epoch_and_fences_commits() { + fn fresh_open_after_release_requires_explicit_failover() { let first_dir = tempdir().unwrap(); let control = Arc::new(InMemoryControlStore::new()); - let first = Server::open_with_control( - test_options(first_dir.path()), + let shard = ShardId::new("mount-1:/"); + let first = open_memory_controlled( + first_dir.path(), control.clone(), - vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None)], ) .unwrap(); + drop(first); let second_dir = tempdir().unwrap(); - let _second = Server::open_with_control( - test_options(second_dir.path()), - control, - vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1)], + let result = open_memory_controlled( + second_dir.path(), + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-b").with_renewal(None)], + ); + assert!(matches!( + result, + Err(ServerError::Control( + ControlError::FreshAcquireRequiresFailover { epoch: 2, .. } + )) + )); + let record = control.get_shard(&shard).unwrap(); + assert_eq!(record.epoch, 2); + assert_eq!(record.owner, None, "rejected fresh lease must be released"); + assert_eq!(record.state, ShardState::Unassigned); + } + + #[test] + fn never_served_fresh_shard_can_retry_after_checkpoint_publication_failure() { + let first_dir = tempdir().unwrap(); + let control = Arc::new(FailingCheckpointPublishControl::new()); + let shard = ShardId::new("mount-1:/"); + control.fail_next_marks(1); + + let first = open_memory_controlled( + first_dir.path(), + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a").with_renewal(None)], + ); + assert!(matches!( + first, + Err(ServerError::Control(ControlError::Backend(message))) + if message.contains("injected checkpoint publish failure") + )); + let abandoned = control.get_shard(&shard).unwrap(); + assert_eq!(abandoned.epoch, 1); + assert_eq!(abandoned.owner, None); + assert!(!abandoned.ever_served); + assert!(abandoned.checkpoint.is_none()); + assert!(abandoned.log.is_none()); + assert_eq!(abandoned.durable_lsn, 0); + + let second_dir = tempdir().unwrap(); + let second = open_memory_controlled( + second_dir.path(), + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-b").with_renewal(None)], + ) + .unwrap(); + let resurrected = control.get_shard(&shard).unwrap(); + assert_eq!(resurrected.epoch, 2); + assert_eq!(resurrected.state, ShardState::Serving); + assert!(resurrected.ever_served); + assert!(resurrected.checkpoint.is_some()); + drop(second); + } + + #[test] + fn stale_owner_renew_observes_new_epoch_and_fences_commits() { + let first_dir = tempdir().unwrap(); + let control = Arc::new(InMemoryControlStore::new()); + let first = open_memory_controlled( + first_dir.path(), + control.clone(), + vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a")], ) .unwrap(); + let successor = control + .acquire_after_failure(ShardId::new("mount-1:/"), NodeId::new("node-b"), 1) + .unwrap(); + let err = first.renew_shard_owner_lease().unwrap_err(); assert!(matches!( err, @@ -1841,14 +5126,15 @@ pub(crate) mod tests { required_epoch: 2 }) )); + control.release(&successor).unwrap(); } #[test] fn shard_owner_auto_renewal_reports_success() { let dir = tempdir().unwrap(); let control = Arc::new(InMemoryControlStore::new()); - let server = Server::open_with_control( - test_options(dir.path()), + let server = open_memory_controlled( + dir.path(), control, vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") .with_renewal(Some(fast_renewal_options(true)))], @@ -1873,21 +5159,17 @@ pub(crate) mod tests { fn shard_owner_auto_renewal_detects_failover_and_fences_commits() { let first_dir = tempdir().unwrap(); let control = Arc::new(InMemoryControlStore::new()); - let first = Server::open_with_control( - test_options(first_dir.path()), + let first = open_memory_controlled( + first_dir.path(), control.clone(), vec![ServerShardOwnerOptions::fresh("mount-1:/", "node-a") .with_renewal(Some(fast_renewal_options(false)))], ) .unwrap(); - let second_dir = tempdir().unwrap(); - let _second = Server::open_with_control( - test_options(second_dir.path()), - control, - vec![ServerShardOwnerOptions::failover("mount-1:/", "node-b", 1).with_renewal(None)], - ) - .unwrap(); + let successor = control + .acquire_after_failure(ShardId::new("mount-1:/"), NodeId::new("node-b"), 1) + .unwrap(); wait_until(|| { first @@ -1906,5 +5188,6 @@ pub(crate) mod tests { required_epoch: 2 }) )); + control.release(&successor).unwrap(); } } diff --git a/crates/nokv/src/bin/nokv/workbench_mcp.rs b/crates/nokv/src/bin/nokv/workbench_mcp.rs index a54f41735..c1433ff75 100644 --- a/crates/nokv/src/bin/nokv/workbench_mcp.rs +++ b/crates/nokv/src/bin/nokv/workbench_mcp.rs @@ -2843,6 +2843,20 @@ fn client_error(err: ClientError) -> WorkbenchToolError { json!({"root_path": root_path}), ) } + ClientError::Metadata(MetadError::ForkRetentionActive { + snapshot_id, + fork_root, + borrower, + }) => WorkbenchToolError::typed( + "ForkRetentionActive", + err.to_string(), + false, + json!({ + "snapshot_id": snapshot_id, + "fork_root": fork_root.get(), + "borrower": borrower.get(), + }), + ), ClientError::Metadata(MetadError::SnapshotRenewContended { snapshot_id, attempts, diff --git a/scripts/run-object-gc-fence-live-e2e.sh b/scripts/run-object-gc-fence-live-e2e.sh new file mode 100755 index 000000000..edb50040d --- /dev/null +++ b/scripts/run-object-gc-fence-live-e2e.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# +# Run the durable object-reference/GC fence acceptance against live RustFS. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +RUSTFS_BIN="${NOKV_RUSTFS_BIN:-rustfs}" +AWS_BIN="${NOKV_AWS_BIN:-aws}" +RUSTFS_ADDRESS="${NOKV_OBJECT_GC_E2E_RUSTFS_ADDRESS:-127.0.0.1:9050}" +RUSTFS_CONSOLE_ADDRESS="${NOKV_OBJECT_GC_E2E_RUSTFS_CONSOLE_ADDRESS:-127.0.0.1:9051}" +RUSTFS_ENDPOINT="${NOKV_OBJECT_GC_E2E_RUSTFS_ENDPOINT:-http://${RUSTFS_ADDRESS}}" +RUSTFS_BUCKET="${NOKV_OBJECT_GC_E2E_RUSTFS_BUCKET:-nokv-object-gc-fence-$$}" +RUSTFS_ACCESS_KEY="${NOKV_OBJECT_GC_E2E_RUSTFS_ACCESS_KEY:-rustfsadmin}" +RUSTFS_SECRET_KEY="${NOKV_OBJECT_GC_E2E_RUSTFS_SECRET_KEY:-rustfsadmin}" +RUSTFS_BUFFER_PROFILE="${NOKV_OBJECT_GC_E2E_RUSTFS_BUFFER_PROFILE:-AiTraining}" +RUSTFS_RUN_PREFIX="${NOKV_OBJECT_GC_E2E_RUN_PREFIX:-nokv-object-gc-fence/$(date -u +%Y%m%dT%H%M%SZ)-$$}" +CARGO_TARGET_DIR_OVERRIDE="${NOKV_OBJECT_GC_E2E_CARGO_TARGET_DIR:-${CARGO_TARGET_DIR:-}}" +EXTERNAL_RUSTFS="${NOKV_OBJECT_GC_E2E_EXTERNAL_RUSTFS:-0}" + +RUSTFS_DATA_DIR="${NOKV_OBJECT_GC_E2E_RUSTFS_DATA_DIR:-}" +RUSTFS_LOG="${NOKV_OBJECT_GC_E2E_RUSTFS_LOG:-}" +RUSTFS_PID="" +OWN_DATA_DIR=0 + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "error: required command not found: $cmd" >&2 + exit 127 + fi +} + +cleanup() { + local status=$? + if [[ -n "$RUSTFS_PID" ]] && kill -0 "$RUSTFS_PID" >/dev/null 2>&1; then + kill "$RUSTFS_PID" >/dev/null 2>&1 || true + wait "$RUSTFS_PID" >/dev/null 2>&1 || true + fi + if [[ "$status" -ne 0 && -n "$RUSTFS_LOG" && -f "$RUSTFS_LOG" ]]; then + echo "---- RustFS log tail ----" >&2 + tail -80 "$RUSTFS_LOG" >&2 || true + echo "-------------------------" >&2 + fi + if [[ "$OWN_DATA_DIR" -eq 1 && "${NOKV_OBJECT_GC_E2E_KEEP_DATA:-0}" != "1" ]]; then + rm -rf "$RUSTFS_DATA_DIR" + elif [[ -n "$RUSTFS_DATA_DIR" ]]; then + echo "RustFS data directory: $RUSTFS_DATA_DIR" >&2 + fi + exit "$status" +} + +wait_for_rustfs() { + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if [[ -n "$RUSTFS_PID" ]] && ! kill -0 "$RUSTFS_PID" >/dev/null 2>&1; then + echo "error: RustFS exited before becoming ready" >&2 + return 1 + fi + if curl -fsS --max-time 2 "$RUSTFS_ENDPOINT" >/dev/null 2>&1 \ + || curl -sS -I --max-time 2 "$RUSTFS_ENDPOINT" >/dev/null 2>&1; then + return 0 + fi + sleep 0.25 + done + echo "error: timed out waiting for RustFS at $RUSTFS_ENDPOINT" >&2 + return 1 +} + +create_bucket() { + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ + "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ + s3api create-bucket --bucket "$RUSTFS_BUCKET" >/dev/null 2>&1; then + return 0 + fi + if AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \ + "$AWS_BIN" --endpoint-url "$RUSTFS_ENDPOINT" \ + s3api head-bucket --bucket "$RUSTFS_BUCKET" >/dev/null 2>&1; then + return 0 + fi + sleep 0.5 + done + echo "error: failed to create or find bucket '$RUSTFS_BUCKET'" >&2 + return 1 +} + +if [[ "$EXTERNAL_RUSTFS" != "1" ]]; then + require_cmd "$RUSTFS_BIN" +fi +require_cmd "$AWS_BIN" +require_cmd cargo +require_cmd curl + +if [[ "$EXTERNAL_RUSTFS" != "1" ]]; then + if [[ -z "$RUSTFS_DATA_DIR" ]]; then + RUSTFS_DATA_DIR="$(mktemp -d "${TMPDIR:-/tmp}/nokv-object-gc-e2e.XXXXXX")" + OWN_DATA_DIR=1 + else + mkdir -p "$RUSTFS_DATA_DIR" + fi + if [[ -z "$RUSTFS_LOG" ]]; then + RUSTFS_LOG="$RUSTFS_DATA_DIR/rustfs.log" + fi +fi + +trap cleanup EXIT INT TERM + +if [[ "$EXTERNAL_RUSTFS" == "1" ]]; then + echo "Using external RustFS at $RUSTFS_ENDPOINT" +else + echo "Starting RustFS at $RUSTFS_ENDPOINT" + RUSTFS_ACCESS_KEY="$RUSTFS_ACCESS_KEY" \ + RUSTFS_SECRET_KEY="$RUSTFS_SECRET_KEY" \ + "$RUSTFS_BIN" server \ + --address "$RUSTFS_ADDRESS" \ + --console-enable \ + --console-address "$RUSTFS_CONSOLE_ADDRESS" \ + --buffer-profile "$RUSTFS_BUFFER_PROFILE" \ + "$RUSTFS_DATA_DIR" >"$RUSTFS_LOG" 2>&1 & + RUSTFS_PID=$! +fi + +wait_for_rustfs +create_bucket + +echo "Running object-reference/GC fence live acceptance" +test_args=( + test + -p nokv-client + --test object_gc_fence_live + -- + --ignored + --nocapture + --test-threads=1 +) + +( + cd "$ROOT_DIR" + if [[ -n "$CARGO_TARGET_DIR_OVERRIDE" ]]; then + CARGO_TARGET_DIR="$CARGO_TARGET_DIR_OVERRIDE" \ + NOKV_OBJECT_GC_LIVE_ENDPOINT="$RUSTFS_ENDPOINT" \ + NOKV_OBJECT_GC_LIVE_BUCKET="$RUSTFS_BUCKET" \ + NOKV_OBJECT_GC_LIVE_RUN_PREFIX="$RUSTFS_RUN_PREFIX" \ + NOKV_OBJECT_GC_LIVE_ACCESS_KEY="$RUSTFS_ACCESS_KEY" \ + NOKV_OBJECT_GC_LIVE_SECRET_KEY="$RUSTFS_SECRET_KEY" \ + cargo "${test_args[@]}" + else + NOKV_OBJECT_GC_LIVE_ENDPOINT="$RUSTFS_ENDPOINT" \ + NOKV_OBJECT_GC_LIVE_BUCKET="$RUSTFS_BUCKET" \ + NOKV_OBJECT_GC_LIVE_RUN_PREFIX="$RUSTFS_RUN_PREFIX" \ + NOKV_OBJECT_GC_LIVE_ACCESS_KEY="$RUSTFS_ACCESS_KEY" \ + NOKV_OBJECT_GC_LIVE_SECRET_KEY="$RUSTFS_SECRET_KEY" \ + cargo "${test_args[@]}" + fi +)