From 8d0a6ec66f7237f73f4b792ec161646b5f5926bc Mon Sep 17 00:00:00 2001 From: wchwawa Date: Sat, 11 Jul 2026 14:13:59 +1000 Subject: [PATCH] feat(workbench): restore checkpoints to COW forks Signed-off-by: wchwawa --- crates/nokv-client/src/file_client.rs | 360 +- crates/nokv-client/src/lib.rs | 8 +- crates/nokv-client/src/service.rs | 110 +- crates/nokv-client/src/service_tests.rs | 194 +- crates/nokv-client/src/wire.rs | 35 + crates/nokv-meta/src/command.rs | 2 + crates/nokv-meta/src/holtstore.rs | 64 +- crates/nokv-meta/src/holtstore/families.rs | 1 - crates/nokv-meta/src/holtstore/tests.rs | 157 +- crates/nokv-meta/src/layout/codec.rs | 121 +- crates/nokv-meta/src/layout/mod.rs | 248 +- crates/nokv-meta/src/lib.rs | 10 +- crates/nokv-meta/src/log.rs | 2 + crates/nokv-meta/src/service.rs | 417 ++- crates/nokv-meta/src/service/checkpoint.rs | 31 +- crates/nokv-meta/src/service/clone.rs | 2524 +++++++++++++- crates/nokv-meta/src/service/gc.rs | 2135 +++++++++++- crates/nokv-meta/src/service/lifecycle.rs | 8 + crates/nokv-meta/src/service/namespace.rs | 322 +- crates/nokv-meta/src/service/publish.rs | 166 +- crates/nokv-meta/src/service/read.rs | 315 +- crates/nokv-meta/src/service/rollback.rs | 136 +- crates/nokv-meta/src/service/snapshot.rs | 6 +- crates/nokv-meta/src/service/xattr.rs | 28 +- crates/nokv-meta/src/service_tests.rs | 3056 +++++++++++++++-- crates/nokv-protocol/src/lib.rs | 133 + crates/nokv-server/src/http.rs | 38 +- crates/nokv-server/src/rpc/mod.rs | 60 +- crates/nokv-server/src/rpc/wire.rs | 29 + crates/nokv-types/src/lib.rs | 7 +- crates/nokv-types/src/types.rs | 45 +- crates/nokv/src/bin/nokv.rs | 149 +- crates/nokv/src/bin/nokv/workbench_mcp.rs | 315 +- .../CHECKPOINT_RESTORE_LIVE_E2E.md | 138 + scripts/lingtai-workbench/README.md | 16 +- .../checkpoint_restore_live_e2e.py | 2260 ++++++++++++ .../checkpoint_restore_live_e2e_test.py | 257 ++ scripts/lingtai-workbench/up.sh | 5 +- 38 files changed, 13075 insertions(+), 833 deletions(-) create mode 100644 scripts/lingtai-workbench/CHECKPOINT_RESTORE_LIVE_E2E.md create mode 100755 scripts/lingtai-workbench/checkpoint_restore_live_e2e.py create mode 100755 scripts/lingtai-workbench/checkpoint_restore_live_e2e_test.py diff --git a/crates/nokv-client/src/file_client.rs b/crates/nokv-client/src/file_client.rs index 68885b956..0429a8da7 100644 --- a/crates/nokv-client/src/file_client.rs +++ b/crates/nokv-client/src/file_client.rs @@ -1,5 +1,5 @@ use std::collections::HashSet; -use std::io::Read; +use std::io::{Read, Seek, SeekFrom}; use std::net::SocketAddr; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -31,6 +31,7 @@ const MAX_READ_PIPELINES: usize = 1024; const MAX_READ_PLAN_CACHE_ENTRIES: usize = 4096; const MAX_BATCH_READ_WORKERS: usize = 32; const MIN_RANGE_READAHEAD_BYTES: usize = DEFAULT_BLOCK_SIZE / 4; +const MAX_ARTIFACT_OBJECT_GC_REFRESHES: usize = 3; struct BatchReadTask { index: usize, @@ -1728,48 +1729,19 @@ where bytes: Vec, metadata: ArtifactMetadata, ) -> Result { - let prepared = self.metadata.prepare_artifact_path(path, false)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_body(&prepared, &bytes, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result.entry), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_bytes(path, &bytes, metadata, false, None) + .map(|result| result.entry) } - pub fn put_artifact_from_reader( + pub fn put_artifact_from_reader( &self, path: &str, - reader: R, + mut reader: R, metadata: ArtifactMetadata, ) -> Result { let prepared = self.metadata.prepare_artifact_path(path, false)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_reader(&prepared, reader, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result.entry), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_reader(prepared, &mut reader, metadata) + .map(|result| result.entry) } pub fn put_artifact_replace( @@ -1778,48 +1750,17 @@ where bytes: Vec, metadata: ArtifactMetadata, ) -> Result { - let prepared = self.metadata.prepare_artifact_path(path, true)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_body(&prepared, &bytes, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_bytes(path, &bytes, metadata, true, None) } - pub fn put_artifact_replace_from_reader( + pub fn put_artifact_replace_from_reader( &self, path: &str, - reader: R, + mut reader: R, metadata: ArtifactMetadata, ) -> Result { let prepared = self.metadata.prepare_artifact_path(path, true)?; - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_reader(&prepared, reader, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result), - Err(err) => { - self.objects - .delete_staged(&staged) - .map_err(ClientError::Object)?; - Err(err) - } - } + self.put_artifact_reader(prepared, &mut reader, metadata) } /// Publish `delta` at the current end of `path` as a delta generation: only @@ -1863,7 +1804,7 @@ where let new_size = base_size .checked_add(delta.len() as u64) .ok_or_else(|| ClientError::Protocol("append size exceeds u64".to_owned()))?; - let prepared = self + let mut prepared = self .metadata .prepare_artifact_path(path, true) .map_err(|err| fold_append_prepare_not_found(err, expected_generation))?; @@ -1872,63 +1813,88 @@ where // would be wrong, so fail before writing any blocks. return Err(artifact_write_conflict()); } - let dirty_ranges = if delta.is_empty() { - Vec::new() - } else { - vec![ChunkWriteRange { - logical_offset: base_size, - bytes: delta.into(), - }] - }; - let written = match self.objects.write_ranges_with_block_index_base( - dirty_ranges, - ChunkWriteOptions { - manifest_id: metadata.manifest_id.clone(), - mount: prepared.mount, - inode: prepared.inode.get(), - generation: prepared.generation, - chunk_size: DEFAULT_CHUNK_SIZE, - block_size: DEFAULT_BLOCK_SIZE, - }, - 0, - ) { - Ok(written) => written, - Err(err) => { - cleanup_staged_write_error(&self.objects, &err)?; - return Err(ClientError::Object(err)); - } - }; - self.record_staged_write_stats(&written); - let staged = written.staged_objects()?; - let session = PublishArtifactStagedSession { - parent: prepared.parent, - name: prepared.name.clone(), - producer: metadata.producer, - digest_uri: metadata.digest_uri, - content_type: metadata.content_type, - manifest_id: written.manifest_id.clone(), - size: new_size, - chunks: written.chunk_manifests(), - staged: staged.clone(), - mode: metadata.mode, - uid: metadata.uid, - gid: metadata.gid, - }; - match self - .metadata - .publish_prepared_artifact_staged_session(prepared, session) - { - Ok(result) => Ok(AppendOutcome { - new_size: result.entry.attr.size, - generation: artifact_body_generation(&result.entry), - created: false, - }), - Err(err) => { - // Best-effort cleanup: a failed staged-object delete must not - // replace the publish error, whose classification - // (is_artifact_write_conflict) drives the caller's retry loop. - let _ = self.objects.delete_staged(&staged); + let mut refreshes = 0; + loop { + let dirty_ranges = if delta.is_empty() { + Vec::new() + } else { + vec![ChunkWriteRange { + logical_offset: base_size, + bytes: delta.clone().into(), + }] + }; + let written = match self.objects.write_ranges_with_block_index_base( + dirty_ranges, + ChunkWriteOptions { + manifest_id: metadata.manifest_id.clone(), + mount: prepared.mount, + inode: prepared.inode.get(), + generation: prepared.generation, + chunk_size: DEFAULT_CHUNK_SIZE, + block_size: DEFAULT_BLOCK_SIZE, + }, + 0, + ) { + Ok(written) => written, + Err(err) => { + cleanup_staged_write_error(&self.objects, &err)?; + return Err(ClientError::Object(err)); + } + }; + self.record_staged_write_stats(&written); + let staged = written.staged_objects()?; + let session = PublishArtifactStagedSession { + parent: prepared.parent, + name: prepared.name.clone(), + producer: metadata.producer.clone(), + digest_uri: metadata.digest_uri.clone(), + content_type: metadata.content_type.clone(), + manifest_id: written.manifest_id.clone(), + size: new_size, + chunks: written.chunk_manifests(), + staged: staged.clone(), + mode: metadata.mode, + uid: metadata.uid, + gid: metadata.gid, + }; + match self + .metadata + .publish_prepared_artifact_staged_session(prepared.clone(), session) + { + Ok(result) => { + return Ok(AppendOutcome { + new_size: result.entry.attr.size, + generation: artifact_body_generation(&result.entry), + created: false, + }); + } Err(err) + if is_stale_prepared_object_gc_epoch(&err) + && refreshes < MAX_ARTIFACT_OBJECT_GC_REFRESHES => + { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + prepared = self + .metadata + .refresh_prepared_artifact_object_gc_epoch(prepared)?; + refreshes += 1; + } + Err(err) if is_stale_prepared_object_gc_epoch(&err) => { + // A caller may safely restart a stale publish only after + // proving this unattached generation was removed. + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + return Err(err); + } + Err(err) => { + // Preserve the metadata error used by the caller's retry + // classifier; this cleanup is best-effort only after the + // client has decided not to mint another generation. + let _ = self.objects.delete_staged(&staged); + return Err(err); + } } } } @@ -1944,25 +1910,97 @@ where metadata: ArtifactMetadata, expected_generation: u64, ) -> Result { - let prepared = self.metadata.prepare_artifact_path(path, true)?; - if prepared.old_generation != Some(expected_generation) { + self.put_artifact_bytes(path, &bytes, metadata, true, Some(expected_generation)) + } + + /// Execute the replayable, self-contained artifact transaction. A stale + /// object-GC epoch never changes the namespace proof: the client removes + /// the unreachable generation, refreshes that exact prepared mutation, + /// fully restages under its new generation, and only then republishes. + fn put_artifact_bytes( + &self, + path: &str, + bytes: &[u8], + metadata: ArtifactMetadata, + replace: bool, + expected_generation: Option, + ) -> Result { + let mut prepared = self.metadata.prepare_artifact_path(path, replace)?; + if expected_generation.is_some() && prepared.old_generation != expected_generation { return Err(artifact_write_conflict()); } - let mode = metadata.mode; - let uid = metadata.uid; - let gid = metadata.gid; - let (body, chunks, staged) = self.stage_artifact_body(&prepared, &bytes, metadata)?; - match self - .metadata - .publish_prepared_artifact(prepared, body, chunks, mode, uid, gid) - { - Ok(result) => Ok(result), - Err(err) => { - // Best-effort cleanup: a failed staged-object delete must not - // replace the publish error, whose classification - // (is_artifact_write_conflict) drives the caller's retry loop. - let _ = self.objects.delete_staged(&staged); - Err(err) + let mut refreshes = 0; + loop { + let (body, chunks, staged) = + self.stage_artifact_body(&prepared, bytes, metadata.clone())?; + match self.metadata.publish_prepared_artifact( + prepared.clone(), + body, + chunks, + metadata.mode, + metadata.uid, + metadata.gid, + ) { + Ok(result) => return Ok(result), + Err(err) => { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + if is_stale_prepared_object_gc_epoch(&err) + && refreshes < MAX_ARTIFACT_OBJECT_GC_REFRESHES + { + prepared = self + .metadata + .refresh_prepared_artifact_object_gc_epoch(prepared)?; + refreshes += 1; + continue; + } + return Err(err); + } + } + } + } + + fn put_artifact_reader( + &self, + mut prepared: ClientPreparedArtifact, + reader: &mut R, + metadata: ArtifactMetadata, + ) -> Result { + let initial_position = reader + .stream_position() + .map_err(|err| ClientError::Io(err.to_string()))?; + let mut refreshes = 0; + loop { + reader + .seek(SeekFrom::Start(initial_position)) + .map_err(|err| ClientError::Io(err.to_string()))?; + let (body, chunks, staged) = + self.stage_artifact_reader(&prepared, &mut *reader, metadata.clone())?; + match self.metadata.publish_prepared_artifact( + prepared.clone(), + body, + chunks, + metadata.mode, + metadata.uid, + metadata.gid, + ) { + Ok(result) => return Ok(result), + Err(err) => { + self.objects + .delete_staged(&staged) + .map_err(ClientError::Object)?; + if is_stale_prepared_object_gc_epoch(&err) + && refreshes < MAX_ARTIFACT_OBJECT_GC_REFRESHES + { + prepared = self + .metadata + .refresh_prepared_artifact_object_gc_epoch(prepared)?; + refreshes += 1; + continue; + } + return Err(err); + } } } } @@ -2244,6 +2282,21 @@ pub fn is_artifact_write_conflict(err: &ClientError) -> bool { ) } +/// True when re-running an artifact write from its read/prepare boundary is +/// safe. In addition to namespace/generation conflicts, an object-GC epoch +/// change is retryable because the failed generation was never attached and +/// its staged objects have been cleaned up by the client write path. +pub fn is_artifact_write_retryable(err: &ClientError) -> bool { + is_artifact_write_conflict(err) || is_stale_prepared_object_gc_epoch(err) +} + +fn is_stale_prepared_object_gc_epoch(err: &ClientError) -> bool { + matches!( + err, + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { .. }) + ) +} + // Client-side guard failures surface as the same predicate error a server-side // publish CAS loss decodes to, so one discriminant covers both. fn artifact_write_conflict() -> ClientError { @@ -2676,6 +2729,23 @@ mod tests { ))); } + #[test] + fn artifact_write_retryable_includes_stale_object_gc_epoch() { + let stale = + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { + expected: 7, + current: 9, + }); + assert!(!is_artifact_write_conflict(&stale)); + assert!(is_artifact_write_retryable(&stale)); + assert!(is_artifact_write_retryable(&ClientError::Metadata( + nokv_meta::MetadError::Metadata(nokv_meta::MetadataError::PredicateFailed), + ))); + assert!(!is_artifact_write_retryable(&ClientError::Metadata( + nokv_meta::MetadError::NotFile, + ))); + } + /// A file deleted between `lookup` and `prepare` inside `append_artifact` /// must classify as a retryable conflict when no generation is pinned, so /// the caller's retry loop re-reads and takes the create branch. diff --git a/crates/nokv-client/src/lib.rs b/crates/nokv-client/src/lib.rs index fd5a17458..985d289e4 100644 --- a/crates/nokv-client/src/lib.rs +++ b/crates/nokv-client/src/lib.rs @@ -26,15 +26,15 @@ pub use artifact::{ ArtifactRepositoryOptions, }; pub use file_client::{ - is_artifact_write_conflict, AppendOutcome, NoKvFsClient, PathRangeReadRequest, PathReadRange, - PreparedPathRangeBatch, + is_artifact_write_conflict, is_artifact_write_retryable, AppendOutcome, NoKvFsClient, + PathRangeReadRequest, PathReadRange, PreparedPathRangeBatch, }; pub use nokv_object::{DataFabricReadStats, ObjectReadPlan}; pub use nokv_protocol::{decode_name_cursor, encode_name_cursor}; pub use service::{ ClientPreparedArtifact, ClientPreparedArtifactRecovery, CloneOutcome, MetadataClient, - MetadataClientOptions, PathLayoutOpen, PathLayoutOpenRequest, SnapshotOutcome, - SnapshotPinStatus, + MetadataClientOptions, PathLayoutOpen, PathLayoutOpenRequest, RestoreOutcome, RestoreState, + SnapshotOutcome, SnapshotPinStatus, }; #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/crates/nokv-client/src/service.rs b/crates/nokv-client/src/service.rs index d22556ce2..f77300483 100644 --- a/crates/nokv-client/src/service.rs +++ b/crates/nokv-client/src/service.rs @@ -18,7 +18,7 @@ use nokv_meta::{ NamespaceReadItem, NamespaceReadOptions, NamespaceReadPage, NamespaceRecordCount, NamespaceRecordType, NamespaceSchema, NamespaceSort, NamespaceSortDirection, NamespaceSortField, PublishArtifactStagedSession, RecordCountProvenance, RenameReplaceResult, - SnapshotRenewOutcome, SubtreeDelta, UpdateAttr, XattrSetMode, + RestoreInitialization, SnapshotRenewOutcome, SubtreeDelta, UpdateAttr, XattrSetMode, }; use nokv_object::ObjectReadPlan; use nokv_protocol::{ @@ -38,7 +38,7 @@ use nokv_protocol::{ WireNamespaceReadOptions, WireNamespaceReadPage, WireNamespaceRecordCount, WireNamespaceRecordType, WireNamespaceSchema, WireNamespaceSort, WireNamespaceSortDirection, WireNamespaceSortField, WireOpenPathReadPlanRequest, WireRecordCountProvenance, - WireSnapshotRenewOutcome, + WireRestoreInitialization, WireRestoreInitializationFile, WireSnapshotRenewOutcome, }; use nokv_types::{ AdvisoryLock, AdvisoryLockRequest, BodyDescriptor, ChunkManifest, DentryName, FileType, @@ -110,12 +110,13 @@ pub struct ClientPreparedArtifact { pub replace: bool, pub dentry_version: Option, pub old_generation: Option, + pub object_gc_claim_version: u64, } /// Stable reconstruction fields persisted by clients that journal a prepared -/// artifact outside `nokv-client`. Future publish-fencing fields belong on -/// [`ClientPreparedArtifact`] so external journals can adopt them explicitly -/// without breaking compilation in the protocol change that introduces them. +/// artifact outside `nokv-client`. New publish-fencing fields belong on +/// [`ClientPreparedArtifact`] and default to a fail-closed value here until the +/// external journal explicitly adopts them. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ClientPreparedArtifactRecovery { pub mount: u64, @@ -145,8 +146,14 @@ impl ClientPreparedArtifact { replace: recovery.replace, dentry_version: recovery.dentry_version, old_generation: recovery.old_generation, + object_gc_claim_version: 0, } } + + pub fn with_object_gc_claim_version(mut self, object_gc_claim_version: u64) -> Self { + self.object_gc_claim_version = object_gc_claim_version; + self + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -195,6 +202,22 @@ pub struct SnapshotOutcome { pub lease_expires_unix_ms: u64, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RestoreState { + Complete, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreOutcome { + pub operation_id: String, + pub state: RestoreState, + pub source_root: InodeId, + pub destination_root: InodeId, + pub snapshot_id: u64, + pub read_version: u64, + pub cleanup_pending: bool, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct SnapshotPinStatus { pub pin: Option, @@ -1576,6 +1599,71 @@ impl MetadataClient { } } + pub fn restore_subtree_path_to_fork( + &self, + source: &str, + snapshot_id: u64, + destination: &str, + ) -> Result { + self.restore_subtree_path_to_fork_initialized( + source, + snapshot_id, + destination, + RestoreInitialization::default(), + ) + } + + pub fn restore_subtree_path_to_fork_initialized( + &self, + source: &str, + snapshot_id: u64, + destination: &str, + initialization: RestoreInitialization, + ) -> Result { + self.ensure_same_shard_paths(source, destination)?; + match self.call(MetadataRpcRequest::RestoreSubtreePathToFork { + source_path: source.to_owned(), + snapshot_id, + destination_path: destination.to_owned(), + initialization: WireRestoreInitialization { + remove_relative_paths: initialization.remove_relative_paths, + files: initialization + .files + .into_iter() + .map(|file| WireRestoreInitializationFile { + relative_path: file.relative_path, + bytes: file.bytes, + content_type: file.content_type, + mode: file.mode, + uid: file.uid, + gid: file.gid, + }) + .collect(), + }, + })? { + MetadataRpcResult::Restore { outcome } => { + let state = match outcome.state.as_str() { + "complete" => RestoreState::Complete, + other => { + return Err(ClientError::Protocol(format!( + "unknown restore state {other}" + ))) + } + }; + Ok(RestoreOutcome { + operation_id: outcome.operation_id, + state, + source_root: inode_id(outcome.source_root)?, + destination_root: inode_id(outcome.destination_root)?, + snapshot_id: outcome.snapshot_id, + read_version: outcome.read_version, + cleanup_pending: outcome.cleanup_pending, + }) + } + other => Err(unexpected_result(other)), + } + } + pub fn snapshot_pin( &self, root_path: &str, @@ -1943,6 +2031,18 @@ impl MetadataClient { } } + pub fn refresh_prepared_artifact_object_gc_epoch( + &self, + prepared: ClientPreparedArtifact, + ) -> Result { + match self.call(MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { + prepared: prepared_artifact_to_wire(&prepared)?, + })? { + MetadataRpcResult::PreparedArtifact { prepared } => wire_prepared_artifact(prepared), + other => Err(unexpected_result(other)), + } + } + pub fn publish_prepared_artifact( &self, prepared: ClientPreparedArtifact, diff --git a/crates/nokv-client/src/service_tests.rs b/crates/nokv-client/src/service_tests.rs index 70358931e..4cc9eca22 100644 --- a/crates/nokv-client/src/service_tests.rs +++ b/crates/nokv-client/src/service_tests.rs @@ -608,7 +608,7 @@ fn service_create_file_prepared_uses_single_frame() { other => panic!("unexpected request: {other:?}"), } response_body( - r#"{"ok":true,"result":{"type":"created_prepared_artifact","entry":{"dentry":{"parent":2,"name_hex":"636865636b706f696e742e62696e","child":40,"child_type":"file","attr_generation":7},"attr":{"inode":40,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":0,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":null},"prepared":{"mount":1,"parent":2,"name":"checkpoint.bin","inode":40,"generation":8,"mtime_ms":8,"ctime_ms":8,"replace":true,"dentry_version":7,"old_generation":null}}}"#, + r#"{"ok":true,"result":{"type":"created_prepared_artifact","entry":{"dentry":{"parent":2,"name_hex":"636865636b706f696e742e62696e","child":40,"child_type":"file","attr_generation":7},"attr":{"inode":40,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":0,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":null},"prepared":{"mount":1,"parent":2,"name":"checkpoint.bin","inode":40,"generation":8,"mtime_ms":8,"ctime_ms":8,"replace":true,"dentry_version":7,"old_generation":null,"object_gc_claim_version":12}}}"#, ) }); let client = MetadataClient::connect(addr); @@ -625,6 +625,43 @@ fn service_create_file_prepared_uses_single_frame() { assert_eq!(created.prepared.inode.get(), 40); assert_eq!(created.prepared.generation, 8); assert!(created.prepared.replace); + assert_eq!(created.prepared.object_gc_claim_version, 12); +} + +#[test] +fn service_refresh_prepared_upload_mints_generation_and_object_gc_epoch() { + let addr = serve_one_request(|request| { + match request { + MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared } => { + assert_eq!(prepared.inode, 42); + assert_eq!(prepared.generation, 7); + assert_eq!(prepared.object_gc_claim_version, 12); + } + other => panic!("unexpected request: {other:?}"), + } + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":2,"name":"artifact.bin","inode":42,"generation":13,"mtime_ms":1700000000001,"ctime_ms":1700000000001,"replace":true,"dentry_version":7,"old_generation":6,"object_gc_claim_version":14}}}"#, + ) + }); + let client = MetadataClient::connect(addr); + let refreshed = client + .refresh_prepared_artifact_object_gc_epoch(ClientPreparedArtifact { + mount: 1, + parent: InodeId::new(2).unwrap(), + name: DentryName::new(b"artifact.bin".to_vec()).unwrap(), + path: None, + inode: InodeId::new(42).unwrap(), + generation: 7, + mtime_ms: 1_700_000_000_000, + ctime_ms: 1_700_000_000_000, + replace: true, + dentry_version: Some(7), + old_generation: Some(6), + object_gc_claim_version: 12, + }) + .unwrap(); + assert_eq!(refreshed.generation, 13); + assert_eq!(refreshed.object_gc_claim_version, 14); } #[test] @@ -2456,7 +2493,7 @@ fn service_file_client_uploads_blocks_then_publishes_metadata() { let store = MemoryObjectStore::new(); let addr = serve_many(vec![ response_body( - r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null}}}"#, + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, ), response_body( r#"{"ok":true,"result":{"type":"rename_replace","entry":{"dentry":{"parent":1,"name_hex":"61727469666163742e62696e","child":42,"child_type":"file","attr_generation":7},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":11,"generation":7,"mtime_ms":7,"ctime_ms":7},"body":{"producer":"unit-test","digest_uri":"sha256:demo","size":11,"content_type":"application/octet-stream","manifest_id":"artifact.bin","generation":7,"base_generation":0,"chunk_size":67108864,"block_size":4194304}},"replaced":null}}"#, @@ -2480,12 +2517,137 @@ fn service_file_client_uploads_blocks_then_publishes_metadata() { ); } +#[test] +fn service_file_client_refreshes_and_restages_after_object_gc_epoch_change() { + let store = MemoryObjectStore::new(); + let addr = serve_request_sequence(vec![ + Box::new(|request| { + assert_eq!( + request, + MetadataRpcRequest::PrepareArtifactPath { + path: "/artifact.bin".to_owned(), + replace: false, + } + ); + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","path":"/artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::PublishPreparedArtifact { prepared, body, .. } = request else { + panic!("expected initial prepared publish") + }; + assert_eq!(prepared.parent, 1); + assert_eq!(prepared.path.as_deref(), Some("/artifact.bin")); + assert_eq!(prepared.inode, 42); + assert_eq!(prepared.generation, 7); + assert_eq!(prepared.object_gc_claim_version, 12); + assert_eq!(body.generation, 7); + response_body( + r#"{"ok":false,"error":"prepared artifact object-GC epoch 12 is stale; current epoch is 14","error_kind":{"type":"stale_prepared_artifact_object_gc_epoch","expected":12,"current":14}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared } = request + else { + panic!("expected prepared object-GC refresh") + }; + assert_eq!(prepared.parent, 1); + assert_eq!(prepared.name, "artifact.bin"); + assert_eq!(prepared.path.as_deref(), Some("/artifact.bin")); + assert_eq!(prepared.inode, 42); + assert_eq!(prepared.generation, 7); + assert_eq!(prepared.object_gc_claim_version, 12); + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","path":"/artifact.bin","inode":42,"generation":13,"mtime_ms":1700000000001,"ctime_ms":1700000000001,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":14}}}"#, + ) + }), + Box::new(|request| { + let MetadataRpcRequest::PublishPreparedArtifact { prepared, body, .. } = request else { + panic!("expected refreshed prepared publish") + }; + assert_eq!(prepared.parent, 1); + assert_eq!(prepared.path.as_deref(), Some("/artifact.bin")); + assert_eq!(prepared.inode, 42); + assert_eq!(prepared.generation, 13); + assert_eq!(prepared.object_gc_claim_version, 14); + assert_eq!(body.generation, 13); + response_body( + r#"{"ok":true,"result":{"type":"rename_replace","entry":{"dentry":{"parent":1,"name_hex":"61727469666163742e62696e","child":42,"child_type":"file","attr_generation":13},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":11,"generation":13,"mtime_ms":13,"ctime_ms":13},"body":{"producer":"unit-test","digest_uri":"sha256:artifact.bin","size":11,"content_type":"application/octet-stream","manifest_id":"artifact.bin","generation":13,"base_generation":0,"chunk_size":67108864,"block_size":4194304}},"replaced":null}}"#, + ) + }), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + + let entry = client + .put_artifact( + "/artifact.bin", + b"hello world".to_vec(), + artifact_metadata("artifact.bin"), + ) + .unwrap(); + + assert_eq!(entry.body.unwrap().generation, 13); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_none(), + "the stale unattached generation must be removed before refresh" + ); + assert!( + store + .head(&ObjectKey::new("blocks/1/42/13/0/0").unwrap()) + .unwrap() + .is_some(), + "the refreshed generation must be fully restaged before publish" + ); +} + +#[test] +fn service_file_client_replays_seekable_reader_from_its_initial_position_after_refresh() { + let store = MemoryObjectStore::new(); + let addr = serve_many(vec![ + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, + ), + response_body( + r#"{"ok":false,"error":"prepared artifact object-GC epoch 12 is stale; current epoch is 14","error_kind":{"type":"stale_prepared_artifact_object_gc_epoch","expected":12,"current":14}}"#, + ), + response_body( + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":13,"mtime_ms":1700000000001,"ctime_ms":1700000000001,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":14}}}"#, + ), + response_body( + r#"{"ok":true,"result":{"type":"rename_replace","entry":{"dentry":{"parent":1,"name_hex":"61727469666163742e62696e","child":42,"child_type":"file","attr_generation":13},"attr":{"inode":42,"file_type":"file","mode":420,"uid":1000,"gid":1000,"rdev":0,"nlink":1,"size":11,"generation":13,"mtime_ms":13,"ctime_ms":13},"body":{"producer":"unit-test","digest_uri":"sha256:artifact.bin","size":11,"content_type":"application/octet-stream","manifest_id":"artifact.bin","generation":13,"base_generation":0,"chunk_size":67108864,"block_size":4194304}},"replaced":null}}"#, + ), + ]); + let client = NoKvFsClient::connect(addr, store.clone()); + let mut reader = std::io::Cursor::new(b"skip--hello world".to_vec()); + reader.set_position(6); + + let entry = client + .put_artifact_from_reader("/artifact.bin", reader, artifact_metadata("artifact.bin")) + .unwrap(); + + assert_eq!(entry.attr.size, 11); + assert_eq!( + store + .get(&ObjectKey::new("blocks/1/42/13/0/0").unwrap(), None,) + .unwrap(), + b"hello world" + ); + assert!(store + .head(&ObjectKey::new("blocks/1/42/7/0/0").unwrap()) + .unwrap() + .is_none()); +} + #[test] fn service_file_client_cleans_staged_blocks_after_publish_failure() { let store = MemoryObjectStore::new(); let addr = serve_many(vec![ response_body( - r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null}}}"#, + r#"{"ok":true,"result":{"type":"prepared_artifact","prepared":{"mount":1,"parent":1,"name":"artifact.bin","inode":42,"generation":7,"mtime_ms":1700000000000,"ctime_ms":1700000000000,"replace":false,"dentry_version":null,"old_generation":null,"object_gc_claim_version":12}}}"#, ), response_body( r#"{"ok":false,"error":"metadata command predicate failed","error_kind":{"type":"predicate_failed"}}"#, @@ -2534,6 +2696,32 @@ fn service_clone_subtree_path_sends_typed_rpc_and_maps_outcome() { assert_eq!(outcome.snapshot_id, 7); } +#[test] +fn service_restore_to_fork_sends_typed_rpc_and_maps_outcome() { + let addr = serve_one_request(|request| { + assert_eq!( + request, + MetadataRpcRequest::RestoreSubtreePathToFork { + source_path: "/base".to_owned(), + snapshot_id: 7, + destination_path: "/restored".to_owned(), + initialization: nokv_protocol::WireRestoreInitialization::default(), + } + ); + response_body( + r#"{"ok":true,"result":{"type":"restore","outcome":{"operation_id":"restore-abc","state":"complete","source_root":2,"destination_root":99,"snapshot_id":7,"read_version":6,"cleanup_pending":false}}}"#, + ) + }); + let client = MetadataClient::connect(addr); + let outcome = client + .restore_subtree_path_to_fork("/base", 7, "/restored") + .unwrap(); + assert_eq!(outcome.operation_id, "restore-abc"); + assert_eq!(outcome.state, RestoreState::Complete); + assert_eq!(outcome.source_root.get(), 2); + assert_eq!(outcome.destination_root.get(), 99); +} + #[test] fn service_diff_subtrees_sends_typed_rpc_and_maps_deltas() { let addr = serve_one_request(|request| { diff --git a/crates/nokv-client/src/wire.rs b/crates/nokv-client/src/wire.rs index 32bc93c76..a413ec1df 100644 --- a/crates/nokv-client/src/wire.rs +++ b/crates/nokv-client/src/wire.rs @@ -86,6 +86,7 @@ pub(crate) fn wire_prepared_artifact( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, }) } @@ -104,6 +105,7 @@ pub(crate) fn prepared_artifact_to_wire( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, }) } @@ -262,6 +264,24 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr dest_shard, }), WireMetadataError::GraftPoint => ClientError::Metadata(nokv_meta::MetadError::GraftPoint), + WireMetadataError::RestoreInProgress => { + ClientError::Metadata(nokv_meta::MetadError::RestoreInProgress) + } + WireMetadataError::RestoreResourceLimit { + resource, + limit, + actual, + } => ClientError::Metadata(nokv_meta::MetadError::RestoreResourceLimit { + resource, + limit, + actual, + }), + WireMetadataError::StalePreparedArtifactObjectGcEpoch { expected, current } => { + ClientError::Metadata(nokv_meta::MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current, + }) + } WireMetadataError::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, @@ -300,6 +320,21 @@ pub(crate) fn client_error_from_wire_error(error: WireMetadataError) -> ClientEr snapshot_id, attempts, }), + WireMetadataError::RestoreHardlinkUnsupported { inode } => match inode_id(inode) { + Ok(inode) => { + ClientError::Metadata(nokv_meta::MetadError::RestoreHardlinkUnsupported { inode }) + } + Err(err) => err, + }, + WireMetadataError::RestoreCrossShardUnsupported { inode } => match inode_id(inode) { + Ok(inode) => { + ClientError::Metadata(nokv_meta::MetadError::RestoreCrossShardUnsupported { inode }) + } + Err(err) => err, + }, + WireMetadataError::RestoreDestinationConflict { destination } => { + ClientError::Metadata(nokv_meta::MetadError::RestoreDestinationConflict { destination }) + } WireMetadataError::SyncLogArchiveFailed { committed, message } => { ClientError::Metadata(nokv_meta::MetadError::SyncLogArchiveFailed { committed, diff --git a/crates/nokv-meta/src/command.rs b/crates/nokv-meta/src/command.rs index c421980e6..1088d7dda 100644 --- a/crates/nokv-meta/src/command.rs +++ b/crates/nokv-meta/src/command.rs @@ -53,6 +53,8 @@ pub enum CommandKind { WatchSubtree, CleanupObjects, RegisterNamespaceIndex, + /// Build metadata below an unreachable restore-to-fork staging root. + StageDetachedRestore, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/nokv-meta/src/holtstore.rs b/crates/nokv-meta/src/holtstore.rs index 99fb6b1fc..4e155989d 100644 --- a/crates/nokv-meta/src/holtstore.rs +++ b/crates/nokv-meta/src/holtstore.rs @@ -45,11 +45,11 @@ pub struct HoltMetadataStore { #[derive(Default)] struct HistoryRetentionState { /// Ordinary commands share the read side from planning through durable - /// apply. Snapshot pins and fork-base bindings take the write side, making + /// apply. Snapshot pins and preparing fork-base holds take the write side, making /// their lifetime transition indivisible from the counters used by planners. planning_fence: RwLock<()>, active_snapshot_pins: AtomicU64, - active_fork_bindings: AtomicU64, + active_fork_base_holds: AtomicU64, /// Monotonic process-local sequence for durable holder transitions. It is /// protected by `planning_fence` and serves as the seqlock/CAS token for a /// service-computed retention floor. @@ -72,19 +72,19 @@ type TestHook = Arc; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] struct HistoryRetentionDelta { snapshot_pins: i64, - fork_bindings: i64, + fork_base_holds: i64, } impl HistoryRetentionDelta { fn is_zero(self) -> bool { - self.snapshot_pins == 0 && self.fork_bindings == 0 + self.snapshot_pins == 0 && self.fork_base_holds == 0 } } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] struct RecoveredHistoryRetentionState { active_snapshot_pins: u64, - active_fork_bindings: u64, + active_fork_base_holds: u64, } #[derive(Default)] @@ -203,7 +203,7 @@ impl HistoryRetentionState { Self { planning_fence: RwLock::new(()), active_snapshot_pins: AtomicU64::new(recovered.active_snapshot_pins), - active_fork_bindings: AtomicU64::new(recovered.active_fork_bindings), + active_fork_base_holds: AtomicU64::new(recovered.active_fork_base_holds), epoch: AtomicU64::new(1), max_applied_commit_version: AtomicU64::new(0), #[cfg(test)] @@ -216,8 +216,8 @@ impl HistoryRetentionState { fn install_recovered(&self, recovered: RecoveredHistoryRetentionState) { self.active_snapshot_pins .store(recovered.active_snapshot_pins, Ordering::Release); - self.active_fork_bindings - .store(recovered.active_fork_bindings, Ordering::Release); + self.active_fork_base_holds + .store(recovered.active_fork_base_holds, Ordering::Release); self.epoch.fetch_add(1, Ordering::AcqRel); // A checkpoint install replaces the DB while holding the exclusive // planning fence, so there is no surviving pre-install plan to order. @@ -226,7 +226,7 @@ impl HistoryRetentionState { fn has_active_hold(&self) -> bool { self.active_snapshot_pins.load(Ordering::Acquire) > 0 - || self.active_fork_bindings.load(Ordering::Acquire) > 0 + || self.active_fork_base_holds.load(Ordering::Acquire) > 0 } fn apply_delta(&self, delta: HistoryRetentionDelta) { @@ -234,7 +234,7 @@ impl HistoryRetentionState { return; } apply_counter_delta(&self.active_snapshot_pins, delta.snapshot_pins); - apply_counter_delta(&self.active_fork_bindings, delta.fork_bindings); + apply_counter_delta(&self.active_fork_base_holds, delta.fork_base_holds); self.epoch.fetch_add(1, Ordering::AcqRel); } @@ -1320,10 +1320,9 @@ impl HoltMetadataStore { mutations: &[PlannedMutation], ) -> Result<(HistoryRetentionDelta, bool), MetadataError> { let mut final_states = Vec::<(RecordFamily, Vec, bool)>::new(); - for planned in mutations - .iter() - .filter(|planned| is_history_retention_family(planned.mutation.family)) - { + for planned in mutations.iter().filter(|planned| { + is_history_retention_key(planned.mutation.family, &planned.mutation.key) + }) { let new_active = planned.mutation.op == MutationOp::Put; if let Some((_, _, active)) = final_states.iter_mut().find(|(family, key, _)| { *family == planned.mutation.family @@ -1349,7 +1348,9 @@ impl HoltMetadataStore { let family_delta = i64::from(new_active) - i64::from(old_active); match family { RecordFamily::Snapshot => delta.snapshot_pins += family_delta, - RecordFamily::ForkBinding => delta.fork_bindings += family_delta, + RecordFamily::System if is_fork_base_hold_key(&key) => { + delta.fork_base_holds += family_delta; + } _ => unreachable!("retention family filter accepts only durable holds"), } } @@ -1540,12 +1541,20 @@ fn recover_history_retention_state( db: &DB, ) -> Result { Ok(RecoveredHistoryRetentionState { - active_snapshot_pins: count_active_records(db, SNAPSHOT_CURRENT_TREE)?, - active_fork_bindings: count_active_records(db, FORK_BINDING_CURRENT_TREE)?, + active_snapshot_pins: count_active_records_matching(db, SNAPSHOT_CURRENT_TREE, |_| true)?, + active_fork_base_holds: count_active_records_matching( + db, + SYSTEM_CURRENT_TREE, + is_fork_base_hold_key, + )?, }) } -fn count_active_records(db: &DB, tree_name: &str) -> Result { +fn count_active_records_matching( + db: &DB, + tree_name: &str, + mut key_matches: impl FnMut(&[u8]) -> bool, +) -> Result { let tree = match db.open_tree(tree_name) { Ok(tree) => tree, Err(HoltError::TreeNotFound { .. }) => return Ok(0), @@ -1553,30 +1562,37 @@ fn count_active_records(db: &DB, tree_name: &str) -> Result }; let mut total = 0_u64; for entry in tree.range() { - let RangeEntry::Key { value, .. } = entry.map_err(to_backend_error)? else { + let RangeEntry::Key { key, value, .. } = entry.map_err(to_backend_error)? else { continue; }; - if decode_current_value(&value)?.1.is_some() { + if key_matches(&key) && decode_current_value(&value)?.1.is_some() { total += 1; } } Ok(total) } -fn is_history_retention_family(family: RecordFamily) -> bool { - matches!(family, RecordFamily::Snapshot | RecordFamily::ForkBinding) +fn is_fork_base_hold_key(key: &[u8]) -> bool { + key.get(8..) + .is_some_and(|suffix| suffix.starts_with(b"fork-base-hold\0")) +} + +fn is_history_retention_key(family: RecordFamily, key: &[u8]) -> bool { + family == RecordFamily::Snapshot + || (family == RecordFamily::System && is_fork_base_hold_key(key)) } fn command_mutates_history_retention(command: &MetadataCommand) -> bool { command .mutations .iter() - .any(|mutation| is_history_retention_family(mutation.family)) + .any(|mutation| is_history_retention_key(mutation.family, &mutation.key)) } fn command_advances_history_version(command: &MetadataCommand) -> bool { command.mutations.iter().any(|mutation| { - family_requires_history(mutation.family) && !is_history_retention_family(mutation.family) + family_requires_history(mutation.family) + && !is_history_retention_key(mutation.family, &mutation.key) }) } diff --git a/crates/nokv-meta/src/holtstore/families.rs b/crates/nokv-meta/src/holtstore/families.rs index 7baf5f551..21f8c2281 100644 --- a/crates/nokv-meta/src/holtstore/families.rs +++ b/crates/nokv-meta/src/holtstore/families.rs @@ -68,6 +68,5 @@ pub(super) fn family_requires_history(family: RecordFamily) -> bool { | RecordFamily::Watch | RecordFamily::Gc | RecordFamily::ForkBinding - | RecordFamily::ForkShadow ) } diff --git a/crates/nokv-meta/src/holtstore/tests.rs b/crates/nokv-meta/src/holtstore/tests.rs index 0f4e0f820..3573ceb8e 100644 --- a/crates/nokv-meta/src/holtstore/tests.rs +++ b/crates/nokv-meta/src/holtstore/tests.rs @@ -90,13 +90,10 @@ fn snapshot_pin_command(request_id: &[u8], commit: u64) -> MetadataCommand { retention_put_command(RecordFamily::Snapshot, b"snapshot/1", request_id, commit) } -fn fork_binding_command(request_id: &[u8], commit: u64) -> MetadataCommand { - retention_put_command( - RecordFamily::ForkBinding, - b"fork-binding/1", - request_id, - commit, - ) +const FORK_BASE_HOLD_KEY: &[u8] = b"\0\0\0\0\0\0\0\x01fork-base-hold\0restore-operation-1"; + +fn fork_base_hold_command(request_id: &[u8], commit: u64) -> MetadataCommand { + retention_put_command(RecordFamily::System, FORK_BASE_HOLD_KEY, request_id, commit) } fn retention_put_command( @@ -479,21 +476,21 @@ fn independent_batch_isolates_snapshot_retention_changes() { } #[test] -fn independent_batch_orders_fork_binding_retention_transitions() { +fn independent_batch_orders_fork_base_hold_retention_transitions() { let store = HoltMetadataStore::open_memory().unwrap(); store .commit_metadata(put_command(b"dir/a", b"req-1", b"value-a", 2)) .unwrap(); let results = store.commit_independent_batch(&[ - fork_binding_command(b"fork-1", 3), + fork_base_hold_command(b"fork-1", 3), replace_command(b"dir/a", b"req-2", b"value-b", 2, 4), ]); assert!(results.iter().all(Result::is_ok)); assert_eq!( store .history_retention - .active_fork_bindings + .active_fork_base_holds .load(Ordering::Acquire), 1 ); @@ -512,8 +509,8 @@ fn independent_batch_orders_fork_binding_retention_transitions() { let history_before_release = store.metadata_store_stats().history_write_total; let results = store.commit_independent_batch(&[ retention_delete_command( - RecordFamily::ForkBinding, - b"fork-binding/1", + RecordFamily::System, + FORK_BASE_HOLD_KEY, b"fork-retire", 3, 5, @@ -524,7 +521,7 @@ fn independent_batch_orders_fork_binding_retention_transitions() { assert_eq!( store .history_retention - .active_fork_bindings + .active_fork_base_holds .load(Ordering::Acquire), 0 ); @@ -596,13 +593,13 @@ fn snapshot_pin_rejects_a_read_version_older_than_an_applied_writer() { } #[test] -fn fork_binding_retains_point_reads_and_snapshot_scans() { +fn fork_base_hold_retains_point_reads_and_snapshot_scans() { let store = HoltMetadataStore::open_memory().unwrap(); store .commit_metadata(put_command(b"dir/a", b"req-1", b"value-a", 2)) .unwrap(); store - .commit_metadata(fork_binding_command(b"fork-1", 3)) + .commit_metadata(fork_base_hold_command(b"fork-1", 3)) .unwrap(); store .commit_metadata(replace_command(b"dir/a", b"req-2", b"value-b", 2, 4)) @@ -638,6 +635,45 @@ fn fork_binding_retains_point_reads_and_snapshot_scans() { ); } +#[test] +fn ordinary_fork_binding_does_not_hold_history_retention() { + let store = HoltMetadataStore::open_memory().unwrap(); + store + .commit_metadata(put_command(b"dir/a", b"req-1", b"value-a", 2)) + .unwrap(); + store + .commit_metadata(retention_put_command( + RecordFamily::ForkBinding, + b"fork-binding/1", + b"fork-binding", + 3, + )) + .unwrap(); + store + .commit_metadata(replace_command(b"dir/a", b"req-2", b"value-b", 2, 4)) + .unwrap(); + + assert_eq!( + store + .history_retention + .active_fork_base_holds + .load(Ordering::Acquire), + 0 + ); + assert_eq!(store.metadata_store_stats().history_write_total, 0); + assert_eq!( + store + .get( + RecordFamily::Dentry, + b"dir/a", + version(2), + ReadPurpose::Snapshot, + ) + .unwrap(), + None + ); +} + #[test] fn retention_counts_change_only_after_successful_commands() { let store = HoltMetadataStore::open_memory().unwrap(); @@ -811,7 +847,7 @@ fn checkpoint_install_and_file_reopen_restore_all_retention_state() { .commit_metadata(snapshot_pin_command(b"snapshot-1", 2)) .unwrap(); source - .commit_metadata(fork_binding_command(b"fork-1", 3)) + .commit_metadata(fork_base_hold_command(b"fork-1", 3)) .unwrap(); let image = source.export_checkpoint_image().unwrap(); @@ -821,7 +857,7 @@ fn checkpoint_install_and_file_reopen_restore_all_retention_state() { assert_eq!( restored .history_retention - .active_fork_bindings + .active_fork_base_holds .load(Ordering::Acquire), 1 ); @@ -841,7 +877,7 @@ fn checkpoint_install_and_file_reopen_restore_all_retention_state() { .commit_metadata(snapshot_pin_command(b"snapshot-1", 2)) .unwrap(); store - .commit_metadata(fork_binding_command(b"fork-1", 3)) + .commit_metadata(fork_base_hold_command(b"fork-1", 3)) .unwrap(); } let reopened = HoltMetadataStore::open_file(path).unwrap(); @@ -849,7 +885,7 @@ fn checkpoint_install_and_file_reopen_restore_all_retention_state() { assert_eq!( reopened .history_retention - .active_fork_bindings + .active_fork_base_holds .load(Ordering::Acquire), 1 ); @@ -1595,6 +1631,89 @@ fn pruning_last_history_record_removes_candidate_index_key() { .is_none()); } +#[test] +fn fork_shadow_history_scan_and_prune_use_candidate_index() { + let store = HoltMetadataStore::open_memory().unwrap(); + let key = b"shadow/parent/child"; + let put = MetadataCommand { + request_id: b"fork-shadow-put".to_vec(), + kind: CommandKind::CreateFile, + read_version: version(1), + commit_version: version(2), + primary_family: RecordFamily::ForkShadow, + primary_key: key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::ForkShadow, + key: key.to_vec(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::ForkShadow, + key: key.to_vec(), + op: MutationOp::Put, + value: Some(Value(b"shadow-value".to_vec())), + }], + watch: Vec::new(), + }; + store.commit_metadata(put).unwrap(); + store + .commit_metadata(snapshot_pin_command(b"fork-shadow-snapshot", 3)) + .unwrap(); + let delete = MetadataCommand { + request_id: b"fork-shadow-delete".to_vec(), + kind: CommandKind::RemoveFile, + read_version: version(3), + commit_version: version(4), + primary_family: RecordFamily::ForkShadow, + primary_key: key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::ForkShadow, + key: key.to_vec(), + predicate: Predicate::Exists, + }], + mutations: vec![Mutation { + family: RecordFamily::ForkShadow, + key: key.to_vec(), + op: MutationOp::Delete, + value: None, + }], + watch: Vec::new(), + }; + store.commit_metadata(delete).unwrap(); + + assert_eq!( + store + .scan(ScanRequest { + family: RecordFamily::ForkShadow, + prefix: b"shadow/parent/".to_vec(), + start_after: None, + version: version(3), + limit: 10, + purpose: ReadPurpose::Snapshot, + }) + .unwrap() + .into_iter() + .map(|item| (item.key, item.value)) + .collect::>(), + vec![(key.to_vec(), Value(b"shadow-value".to_vec()))] + ); + + let outcome = store + .prune_history(HistoryPruneRequest { + retain_from: None, + retention_epoch: store.history_retention_epoch().unwrap(), + limit: 100, + }) + .unwrap(); + assert_eq!(outcome.removed, 2); + assert!(store + .history_key_index_tree() + .unwrap() + .get(&history_index_key(RecordFamily::ForkShadow, key)) + .unwrap() + .is_none()); +} + #[test] fn stale_prune_plan_cannot_remove_history_added_by_concurrent_write() { let store = HoltMetadataStore::open_memory().unwrap(); diff --git a/crates/nokv-meta/src/layout/codec.rs b/crates/nokv-meta/src/layout/codec.rs index 4b5880c28..79d5d398c 100644 --- a/crates/nokv-meta/src/layout/codec.rs +++ b/crates/nokv-meta/src/layout/codec.rs @@ -2,13 +2,15 @@ use std::fmt; use nokv_types::{ BlockDescriptor, BodyDescriptor, ChunkManifest, DentryName, DentryProjection, DentryRecord, - FileType, ForkBinding, InodeAttr, InodeId, ObjectGcRecord, SliceManifest, SnapshotPin, - WatchEvent, WatchEventKind, + FileType, ForkBinding, ForkBindingState, InodeAttr, InodeId, ObjectGcRecord, SliceManifest, + SnapshotPin, WatchEvent, WatchEventKind, }; #[derive(Clone, Debug, PartialEq, Eq)] pub enum CodecError { Truncated, + InvalidMagic(&'static str), + UnsupportedVersion { codec: &'static str, version: u8 }, InvalidFileType(u8), InvalidWatchEventKind(u8), InvalidOptionTag(u8), @@ -312,24 +314,68 @@ pub fn decode_snapshot_pin(bytes: &[u8]) -> Result { Ok(pin) } +const FORK_BINDING_MAGIC: &[u8; 4] = b"NKVF"; +const FORK_BINDING_CODEC_VERSION: u8 = 1; + pub fn encode_fork_binding(binding: &ForkBinding) -> Vec { - let mut out = Vec::with_capacity(40); + let mut out = Vec::with_capacity(122 + binding.destination_path.len()); + out.extend_from_slice(FORK_BINDING_MAGIC); + out.push(FORK_BINDING_CODEC_VERSION); push_u64(&mut out, binding.fork_root.get()); push_u64(&mut out, binding.source_root.get()); push_u64(&mut out, binding.pinned_read_version); push_u64(&mut out, binding.snapshot_id); push_u64(&mut out, binding.created_version); + push_u64(&mut out, binding.base_ref_set_id); + out.extend_from_slice(&binding.operation_digest); + out.extend_from_slice(&binding.initialization_digest); + out.push(match binding.state { + ForkBindingState::Preparing => 1, + ForkBindingState::Complete => 2, + ForkBindingState::Releasing => 3, + }); + put_string(&mut out, &binding.destination_path); out } pub fn decode_fork_binding(bytes: &[u8]) -> Result { let mut input = Decoder::new(bytes); + if input.fixed::<4>()? != *FORK_BINDING_MAGIC { + return Err(CodecError::InvalidMagic("fork binding")); + } + let version = input.u8()?; + if version != FORK_BINDING_CODEC_VERSION { + return Err(CodecError::UnsupportedVersion { + codec: "fork binding", + version, + }); + } + let fork_root = inode(input.u64()?)?; + let source_root = inode(input.u64()?)?; + let pinned_read_version = input.u64()?; + let snapshot_id = input.u64()?; + let created_version = input.u64()?; + let base_ref_set_id = input.u64()?; + let operation_digest = input.fixed::<32>()?; + let initialization_digest = input.fixed::<32>()?; + let state = match input.u8()? { + 1 => ForkBindingState::Preparing, + 2 => ForkBindingState::Complete, + 3 => ForkBindingState::Releasing, + tag => return Err(CodecError::InvalidOptionTag(tag)), + }; + let destination_path = input.string()?; let binding = ForkBinding { - fork_root: inode(input.u64()?)?, - source_root: inode(input.u64()?)?, - pinned_read_version: input.u64()?, - snapshot_id: input.u64()?, - created_version: input.u64()?, + fork_root, + source_root, + pinned_read_version, + snapshot_id, + created_version, + base_ref_set_id, + operation_digest, + initialization_digest, + state, + destination_path, }; input.finish()?; Ok(binding) @@ -617,6 +663,10 @@ impl<'a> Decoder<'a> { Ok(u64::from_be_bytes(bytes.try_into().unwrap())) } + fn fixed(&mut self) -> Result<[u8; N], CodecError> { + Ok(self.take(N)?.try_into().expect("fixed decoder width")) + } + fn bytes(&mut self) -> Result<&'a [u8], CodecError> { let len = self.u32()? as usize; self.take(len) @@ -649,6 +699,10 @@ impl fmt::Display for CodecError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Truncated => write!(f, "encoded metadata value is truncated"), + Self::InvalidMagic(codec) => write!(f, "invalid {codec} codec magic"), + Self::UnsupportedVersion { codec, version } => { + write!(f, "unsupported {codec} codec version {version}") + } Self::InvalidFileType(tag) => write!(f, "invalid file type tag {tag}"), Self::InvalidWatchEventKind(tag) => write!(f, "invalid watch event kind tag {tag}"), Self::InvalidOptionTag(tag) => write!(f, "invalid optional value tag {tag}"), @@ -675,6 +729,11 @@ mod tests { pinned_read_version: 1234, snapshot_id: 5678, created_version: 5679, + base_ref_set_id: 5680, + operation_digest: [7; 32], + initialization_digest: [8; 32], + state: ForkBindingState::Complete, + destination_path: "/restored".to_owned(), }; assert_eq!( decode_fork_binding(&encode_fork_binding(&binding)).unwrap(), @@ -686,6 +745,52 @@ mod tests { assert!(decode_fork_binding(&extra).is_err()); } + #[test] + fn fork_binding_rejects_legacy_unversioned_layout() { + let mut legacy = Vec::new(); + push_u64(&mut legacy, 42); + push_u64(&mut legacy, 7); + push_u64(&mut legacy, 1234); + push_u64(&mut legacy, 5678); + push_u64(&mut legacy, 5679); + push_u64(&mut legacy, 5680); + legacy.extend_from_slice(&[7; 32]); + legacy.extend_from_slice(&[8; 32]); + legacy.push(2); + put_string(&mut legacy, "/restored"); + + assert_eq!( + decode_fork_binding(&legacy), + Err(CodecError::InvalidMagic("fork binding")) + ); + } + + #[test] + fn fork_binding_rejects_unknown_codec_version() { + let binding = ForkBinding { + fork_root: InodeId::new(42).unwrap(), + source_root: InodeId::new(7).unwrap(), + pinned_read_version: 1234, + snapshot_id: 5678, + created_version: 5679, + base_ref_set_id: 5680, + operation_digest: [7; 32], + initialization_digest: [8; 32], + state: ForkBindingState::Complete, + destination_path: "/restored".to_owned(), + }; + let mut encoded = encode_fork_binding(&binding); + encoded[FORK_BINDING_MAGIC.len()] = 99; + + assert_eq!( + decode_fork_binding(&encoded), + Err(CodecError::UnsupportedVersion { + codec: "fork binding", + version: 99, + }) + ); + } + #[test] fn fork_shadow_round_trips() { let source = InodeId::new(99).unwrap(); diff --git a/crates/nokv-meta/src/layout/mod.rs b/crates/nokv-meta/src/layout/mod.rs index 77110106b..a1b4886e3 100644 --- a/crates/nokv-meta/src/layout/mod.rs +++ b/crates/nokv-meta/src/layout/mod.rs @@ -30,6 +30,231 @@ pub fn allocator_key(mount: MountId) -> Vec { out } +pub fn restore_operation_key(mount: MountId, operation_digest: &[u8; 32]) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 18 + operation_digest.len()); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"restore-operation\0"); + out.extend_from_slice(operation_digest); + out +} + +pub fn restore_path_index_set_prefix(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH * 2 + 20); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"restore-path-index\0"); + push_u64(&mut out, base_ref_set_id); + out.push(PATH_INDEX_DELIMITER); + out +} + +pub fn restore_path_index_key( + mount: MountId, + base_ref_set_id: u64, + parent: InodeId, + name: &DentryName, +) -> Vec { + let mut out = restore_path_index_prefix(mount, base_ref_set_id, parent); + out.extend_from_slice(name.as_bytes()); + out +} + +pub fn restore_path_index_prefix(mount: MountId, base_ref_set_id: u64, parent: InodeId) -> Vec { + let mut out = restore_path_index_set_prefix(mount, base_ref_set_id); + push_u64(&mut out, parent.get()); + out.push(PATH_INDEX_DELIMITER); + out +} + +pub fn fork_base_ref_owner_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 20); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"fork-base-ref-owner\0"); + out +} + +pub fn fork_base_ref_set_prefix(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = fork_base_ref_owner_prefix(mount); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn fork_base_ref_owner_key( + mount: MountId, + base_ref_set_id: u64, + object_digest: &[u8; 32], +) -> Vec { + let mut out = fork_base_ref_set_prefix(mount, base_ref_set_id); + out.extend_from_slice(object_digest); + out +} + +pub fn fork_base_ref_digest_from_owner_key( + mount: MountId, + base_ref_set_id: u64, + key: &[u8], +) -> Option<[u8; 32]> { + let prefix = fork_base_ref_set_prefix(mount, base_ref_set_id); + let suffix = key.strip_prefix(prefix.as_slice())?; + if suffix.len() != 32 { + return None; + } + suffix.try_into().ok() +} + +pub fn fork_base_ref_inverse_prefix(mount: MountId, object_digest: &[u8; 32]) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 22 + object_digest.len()); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"fork-base-ref-inverse\0"); + out.extend_from_slice(object_digest); + out +} + +pub fn fork_base_ref_inverse_key( + mount: MountId, + object_digest: &[u8; 32], + base_ref_set_id: u64, +) -> Vec { + let mut out = fork_base_ref_inverse_prefix(mount, object_digest); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn fork_base_ref_release_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 22); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"fork-base-ref-release\0"); + out +} + +pub fn fork_base_ref_release_key(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = fork_base_ref_release_prefix(mount); + push_u64(&mut out, base_ref_set_id); + out +} + +/// Durable round-robin cursor for a releasing fork's owner rows. Live borrower +/// rows may have to remain after the fork root is removed (for example, when a +/// hardlink or rename moved the inode elsewhere), so a release worker must not +/// repeatedly stop at the same retained prefix page and starve later rows. +pub fn fork_base_ref_release_cursor_key(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH * 2 + 21); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"fork-base-ref-cursor\0"); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn fork_base_hold_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 15); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"fork-base-hold\0"); + out +} + +pub fn fork_base_hold_key( + mount: MountId, + pinned_read_version: u64, + base_ref_set_id: u64, +) -> Vec { + let mut out = fork_base_hold_prefix(mount); + push_u64(&mut out, pinned_read_version); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn fork_base_hold_read_version(mount: MountId, key: &[u8]) -> Option { + let prefix = fork_base_hold_prefix(mount); + let suffix = key.strip_prefix(prefix.as_slice())?; + let raw = suffix.get(..8)?; + Some(u64::from_be_bytes(raw.try_into().ok()?)) +} + +pub fn object_gc_claim_key(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 16); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"object-gc-claim\0"); + out +} + +pub fn path_index_gc_cursor_key(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 21); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"path-index-gc-cursor\0"); + out +} + +pub fn object_gc_quarantine_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 21); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"object-gc-quarantine\0"); + out +} + +pub fn object_gc_quarantine_key(mount: MountId, record_digest: &[u8; 32]) -> Vec { + let mut out = object_gc_quarantine_prefix(mount); + out.extend_from_slice(record_digest); + out +} + +pub fn fork_base_release_quarantine_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 29); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"fork-base-release-quarantine\0"); + out +} + +pub fn fork_base_release_quarantine_key(mount: MountId, job_digest: &[u8; 32]) -> Vec { + let mut out = fork_base_release_quarantine_prefix(mount); + out.extend_from_slice(job_digest); + out +} + +pub fn restore_staging_cleanup_prefix(mount: MountId) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH + 24); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"restore-staging-cleanup\0"); + out +} + +pub fn restore_staging_cleanup_key(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = restore_staging_cleanup_prefix(mount); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn restore_staging_clean_key(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH * 2 + 22); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"restore-staging-clean\0"); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn restore_staging_member_prefix(mount: MountId, base_ref_set_id: u64) -> Vec { + let mut out = Vec::with_capacity(U64_WIDTH * 2 + 23); + push_u64(&mut out, mount.get()); + out.extend_from_slice(b"restore-staging-member\0"); + push_u64(&mut out, base_ref_set_id); + out +} + +pub fn restore_staging_member_key(mount: MountId, base_ref_set_id: u64, inode: InodeId) -> Vec { + let mut out = restore_staging_member_prefix(mount, base_ref_set_id); + push_u64(&mut out, inode.get()); + out +} + +pub fn restore_staging_member_inode( + mount: MountId, + base_ref_set_id: u64, + key: &[u8], +) -> Option { + let prefix = restore_staging_member_prefix(mount, base_ref_set_id); + let suffix = key.strip_prefix(prefix.as_slice())?; + let raw: [u8; U64_WIDTH] = suffix.try_into().ok()?; + InodeId::new(u64::from_be_bytes(raw)).ok() +} + pub fn inode_key(mount: MountId, inode: InodeId) -> Vec { let mut out = inode_prefix(mount); push_u64(&mut out, inode.get()); @@ -189,10 +414,16 @@ pub fn fork_binding_key(mount: MountId, fork_root: InodeId) -> Vec { out } -pub fn fork_shadow_key(mount: MountId, fork_inode: InodeId) -> Vec { +pub fn fork_shadow_prefix(mount: MountId, parent: InodeId) -> Vec { let mut out = Vec::with_capacity(U64_WIDTH * 2); push_u64(&mut out, mount.get()); - push_u64(&mut out, fork_inode.get()); + push_u64(&mut out, parent.get()); + out +} + +pub fn fork_shadow_key(mount: MountId, parent: InodeId, name: &DentryName) -> Vec { + let mut out = fork_shadow_prefix(mount, parent); + out.extend_from_slice(name.as_bytes()); out } @@ -220,6 +451,19 @@ pub fn gc_object_key( out } +pub fn gc_released_base_ref_key( + mount: MountId, + enqueue_version: u64, + base_ref_set_id: u64, + object_digest: &[u8; 32], +) -> Vec { + let mut out = gc_queue_prefix(mount); + push_u64(&mut out, enqueue_version); + push_u64(&mut out, base_ref_set_id); + out.extend_from_slice(object_digest); + out +} + pub fn history_key(family: RecordFamily, user_key: &[u8], commit_version: u64) -> Vec { let mut out = history_prefix(family, user_key); push_u64(&mut out, u64::MAX - commit_version); diff --git a/crates/nokv-meta/src/lib.rs b/crates/nokv-meta/src/lib.rs index 504bdaa47..e31ccaf3a 100644 --- a/crates/nokv-meta/src/lib.rs +++ b/crates/nokv-meta/src/lib.rs @@ -30,9 +30,9 @@ pub use log::{ METADATA_LOG_ZERO_DIGEST, }; pub use service::{ - BodyReadPlan, CheckpointHandle, CheckpointShard, CloneHandle, CreateInDirPathBatch, - CreatedPreparedArtifact, DanglingBlock, DentryWithAttr, FsckReport, MetadError, - MetadataArchiveConfig, MetadataBackupOutcome, MetadataLogArchiveConfig, + restore_operation_id, BodyReadPlan, CheckpointHandle, CheckpointShard, CloneHandle, + CreateInDirPathBatch, CreatedPreparedArtifact, DanglingBlock, DentryWithAttr, FsckReport, + MetadError, MetadataArchiveConfig, MetadataBackupOutcome, MetadataLogArchiveConfig, MetadataLogRestoreOutcome, MetadataLogSegmentArchiveOutcome, MetadataLogSegmentPointer, MetadataLogSyncConfig, MetadataLogSyncSnapshot, MetadataRestoreOutcome, MetadataServiceStats, NamespaceAggregateGroup, NamespaceAggregateMeasure, NamespaceAggregateOp, @@ -50,6 +50,6 @@ pub use service::{ ObjectTransferStats, OpenPathReadPlan, OpenPathReadPlanRequest, PendingObjectCleanupOutcome, PreparedArtifact, PublishArtifact, PublishArtifactRange, PublishArtifactSession, PublishArtifactStagedSession, ReadDirPlusPage, RecordCountProvenance, RenameReplaceResult, - SnapshotReapOutcome, SnapshotRenewOutcome, SubtreeDelta, SubtreeDeltaKind, UpdateAttr, - XattrSetMode, DEFAULT_SNAPSHOT_LEASE_MS, + RestoreInitialization, RestoreInitializationFile, SnapshotReapOutcome, SnapshotRenewOutcome, + SubtreeDelta, SubtreeDeltaKind, UpdateAttr, XattrSetMode, DEFAULT_SNAPSHOT_LEASE_MS, }; diff --git a/crates/nokv-meta/src/log.rs b/crates/nokv-meta/src/log.rs index 3e0d1c04c..096ab3e9a 100644 --- a/crates/nokv-meta/src/log.rs +++ b/crates/nokv-meta/src/log.rs @@ -660,6 +660,7 @@ fn command_kind_tag(kind: CommandKind) -> u8 { CommandKind::WatchSubtree => 20, CommandKind::CleanupObjects => 21, CommandKind::RegisterNamespaceIndex => 22, + CommandKind::StageDetachedRestore => 23, } } @@ -687,6 +688,7 @@ fn command_kind_from_tag(tag: u8) -> Result { 20 => Ok(CommandKind::WatchSubtree), 21 => Ok(CommandKind::CleanupObjects), 22 => Ok(CommandKind::RegisterNamespaceIndex), + 23 => Ok(CommandKind::StageDetachedRestore), tag => Err(MetadataLogError::InvalidCommandKind(tag)), } } diff --git a/crates/nokv-meta/src/service.rs b/crates/nokv-meta/src/service.rs index 4850b9657..4d39fb2d9 100644 --- a/crates/nokv-meta/src/service.rs +++ b/crates/nokv-meta/src/service.rs @@ -50,20 +50,29 @@ use crate::command::{ CommandKind, CommitResult, DelimitedScanItem, DelimitedScanRequest, HistoryPruneOutcome, HistoryPruneRequest, KeyScanRequest, MetadataCommand, MetadataError, MetadataStore, MetadataStoreStats, MetadataStoreStatsProvider, Mutation, MutationOp, Predicate, PredicateRef, - ReadPurpose, ScanRequest, Value, Version, WatchProjection, + ReadPurpose, ScanItem, ScanRequest, Value, Version, WatchProjection, }; use crate::layout::{ allocator_key, chunk_manifest_key, chunk_manifest_prefix, decode_allocator_state, - decode_body_descriptor, decode_chunk_manifest, decode_dentry_projection, decode_inode_attr, - decode_object_gc_record, decode_path_index_catalog, decode_path_index_row, decode_snapshot_pin, - decode_watch_event, dentry_key, dentry_mount_prefix, dentry_prefix, encode_allocator_state, - encode_body_descriptor, encode_chunk_manifest, encode_dentry_projection, encode_inode_attr, - encode_object_gc_record, encode_path_index_catalog, encode_path_index_row, encode_snapshot_pin, - encode_watch_event, gc_object_key, gc_queue_prefix, inode_key, path_index_catalog_key, - path_index_key, path_index_prefix, path_index_row_key, path_index_row_prefix, snapshot_pin_key, - snapshot_pin_prefix, watch_log_key, watch_log_prefix, xattr_key, xattr_prefix, - PathIndexCatalogRecord, PathIndexFieldRecord, PathIndexRowRecord, PathIndexValueRecord, - PATH_INDEX_DELIMITER, + decode_body_descriptor, decode_chunk_manifest, decode_dentry_projection, decode_fork_binding, + decode_inode_attr, decode_object_gc_record, decode_path_index_catalog, decode_path_index_row, + decode_snapshot_pin, decode_watch_event, dentry_key, dentry_mount_prefix, dentry_prefix, + encode_allocator_state, encode_body_descriptor, encode_chunk_manifest, + encode_dentry_projection, encode_fork_binding, encode_inode_attr, encode_object_gc_record, + encode_path_index_catalog, encode_path_index_row, encode_snapshot_pin, encode_watch_event, + fork_base_hold_key, fork_base_hold_prefix, fork_base_hold_read_version, + fork_base_ref_digest_from_owner_key, fork_base_ref_inverse_key, fork_base_ref_inverse_prefix, + fork_base_ref_owner_key, fork_base_ref_release_cursor_key, fork_base_ref_release_key, + fork_base_ref_release_prefix, fork_base_ref_set_prefix, fork_base_release_quarantine_key, + fork_binding_key, fork_shadow_key, fork_shadow_prefix, gc_object_key, gc_queue_prefix, + gc_released_base_ref_key, inode_key, object_gc_claim_key, object_gc_quarantine_key, + path_index_catalog_key, path_index_gc_cursor_key, path_index_key, path_index_prefix, + path_index_row_key, path_index_row_prefix, restore_operation_key, restore_path_index_key, + restore_path_index_set_prefix, restore_staging_clean_key, restore_staging_cleanup_key, + restore_staging_cleanup_prefix, restore_staging_member_inode, restore_staging_member_key, + restore_staging_member_prefix, snapshot_pin_key, snapshot_pin_prefix, watch_log_key, + watch_log_prefix, xattr_key, xattr_prefix, PathIndexCatalogRecord, PathIndexFieldRecord, + PathIndexRowRecord, PathIndexValueRecord, PATH_INDEX_DELIMITER, }; use nokv_object::{ plan_chunk_manifest_reads, BlockReadOptions, ChunkStore, ChunkWriteOptions, ChunkWriteRange, @@ -72,9 +81,10 @@ use nokv_object::{ }; use nokv_types::{ parse_absolute_path, AdvisoryLock, BlockDescriptor, BodyDescriptor, ChunkManifest, DentryName, - DentryProjection, DentryRecord, FileType, InodeAttr, InodeId, ModelError, MountId, - ObjectGcRecord, PathError, PathMetadata, ReadLease, RecordFamily, SliceManifest, SnapshotPin, - SpecialNodeSpec, WatchCursor, WatchEvent, WatchEventKind, WatchRecord, + DentryProjection, DentryRecord, FileType, ForkBinding, ForkBindingState, InodeAttr, InodeId, + ModelError, MountId, ObjectGcRecord, PathError, PathMetadata, ReadLease, RecordFamily, + SliceManifest, SnapshotPin, SpecialNodeSpec, WatchCursor, WatchEvent, WatchEventKind, + WatchRecord, }; use sha2::{Digest, Sha256}; @@ -110,6 +120,262 @@ const PATH_INDEX_VALIDATION_CACHE_MAX_ENTRIES_PER_SHARD: usize = PATH_INDEX_VALIDATION_CACHE_MAX_ENTRIES / PATH_CACHE_SHARD_COUNT; pub(crate) const DEFAULT_READ_LEASE_MS: u64 = 3_600_000; +/// Stable request identity for restore-to-fork. It deliberately depends only on +/// the canonical public request, so a completed retry remains resolvable after +/// the source path or caller-owned snapshot pin disappears. +pub fn restore_operation_id( + source_path: &str, + snapshot_id: u64, + destination_path: &str, +) -> Result { + let source = canonical_path(&parse_absolute_path(source_path)?)?; + let destination = canonical_path(&parse_absolute_path(destination_path)?)?; + Ok(format!( + "restore-{}", + hex_digest(&restore_operation_digest( + &source, + snapshot_id, + &destination + )) + )) +} + +fn restore_operation_digest( + source_path: &str, + snapshot_id: u64, + destination_path: &str, +) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-to-fork-request-v2\0"); + hasher.update(source_path.as_bytes()); + hasher.update([0]); + hasher.update(snapshot_id.to_be_bytes()); + hasher.update(destination_path.as_bytes()); + hasher.finalize().into() +} + +fn hex_digest(bytes: &[u8; 32]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(64); + for byte in bytes { + write!(&mut out, "{byte:02x}").expect("writing into String cannot fail"); + } + out +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RestoreStagingCleanupDisposition { + ResetForRetry, + Discard, + ForgetMembers, +} + +fn encode_restore_staging_cleanup( + disposition: RestoreStagingCleanupDisposition, + binding: &ForkBinding, +) -> Vec { + let encoded = encode_fork_binding(binding); + let mut value = Vec::with_capacity(1 + encoded.len()); + value.push(match disposition { + RestoreStagingCleanupDisposition::ResetForRetry => 1, + RestoreStagingCleanupDisposition::Discard => 2, + RestoreStagingCleanupDisposition::ForgetMembers => 3, + }); + value.extend_from_slice(&encoded); + value +} + +fn decode_restore_staging_cleanup( + value: &[u8], +) -> Result<(RestoreStagingCleanupDisposition, ForkBinding), MetadError> { + let Some((tag, binding)) = value.split_first() else { + return Err(MetadError::Codec( + "empty restore staging cleanup record".to_owned(), + )); + }; + let disposition = match tag { + 1 => RestoreStagingCleanupDisposition::ResetForRetry, + 2 => RestoreStagingCleanupDisposition::Discard, + 3 => RestoreStagingCleanupDisposition::ForgetMembers, + _ => { + return Err(MetadError::Codec(format!( + "invalid restore staging cleanup disposition {tag}" + ))) + } + }; + let binding = decode_fork_binding(binding).map_err(|err| MetadError::Codec(err.to_string()))?; + Ok((disposition, binding)) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ForkBaseRef { + owner_inode: InodeId, + owner_generation: u64, + read_version: u64, + size: u64, + object_key: String, + digest_uri: String, +} + +fn encode_fork_base_ref(reference: &ForkBaseRef) -> Result, MetadError> { + let key_len = u32::try_from(reference.object_key.len()) + .map_err(|_| MetadError::Codec("fork base object key is too long".to_owned()))?; + let digest_len = u32::try_from(reference.digest_uri.len()) + .map_err(|_| MetadError::Codec("fork base digest is too long".to_owned()))?; + let mut out = Vec::with_capacity(40 + reference.object_key.len() + reference.digest_uri.len()); + out.extend_from_slice(&reference.owner_inode.get().to_be_bytes()); + out.extend_from_slice(&reference.owner_generation.to_be_bytes()); + out.extend_from_slice(&reference.read_version.to_be_bytes()); + out.extend_from_slice(&reference.size.to_be_bytes()); + out.extend_from_slice(&key_len.to_be_bytes()); + out.extend_from_slice(reference.object_key.as_bytes()); + out.extend_from_slice(&digest_len.to_be_bytes()); + out.extend_from_slice(reference.digest_uri.as_bytes()); + Ok(out) +} + +fn decode_fork_base_ref(bytes: &[u8]) -> Result { + fn take<'a>(input: &mut &'a [u8], len: usize) -> Result<&'a [u8], MetadError> { + let Some((head, tail)) = input.split_at_checked(len) else { + return Err(MetadError::Codec("fork base ref is truncated".to_owned())); + }; + *input = tail; + Ok(head) + } + let mut input = bytes; + let owner_inode = InodeId::new(u64::from_be_bytes( + take(&mut input, 8)?.try_into().expect("u64 width"), + ))?; + let owner_generation = u64::from_be_bytes(take(&mut input, 8)?.try_into().expect("u64 width")); + let read_version = u64::from_be_bytes(take(&mut input, 8)?.try_into().expect("u64 width")); + let size = u64::from_be_bytes(take(&mut input, 8)?.try_into().expect("u64 width")); + let key_len = u32::from_be_bytes(take(&mut input, 4)?.try_into().expect("u32 width")) as usize; + let object_key = String::from_utf8(take(&mut input, key_len)?.to_vec()) + .map_err(|_| MetadError::Codec("fork base object key is not utf-8".to_owned()))?; + let digest_len = + u32::from_be_bytes(take(&mut input, 4)?.try_into().expect("u32 width")) as usize; + let digest_uri = String::from_utf8(take(&mut input, digest_len)?.to_vec()) + .map_err(|_| MetadError::Codec("fork base digest is not utf-8".to_owned()))?; + if !input.is_empty() { + return Err(MetadError::Codec( + "fork base ref has trailing bytes".to_owned(), + )); + } + Ok(ForkBaseRef { + owner_inode, + owner_generation, + read_version, + size, + object_key, + digest_uri, + }) +} + +fn encode_restore_shadow_inverse(base_ref_set_id: u64, projection: &DentryProjection) -> Vec { + let encoded = encode_dentry_projection(projection); + let mut out = Vec::with_capacity(8 + encoded.len()); + out.extend_from_slice(&base_ref_set_id.to_be_bytes()); + out.extend_from_slice(&encoded); + out +} + +fn decode_restore_shadow_inverse(bytes: &[u8]) -> Result<(u64, DentryProjection), MetadError> { + let (set, projection) = bytes + .split_at_checked(8) + .ok_or_else(|| MetadError::Codec("restore shadow inverse is truncated".to_owned()))?; + let base_ref_set_id = u64::from_be_bytes(set.try_into().expect("u64 width")); + let projection = + decode_dentry_projection(projection).map_err(|err| MetadError::Codec(err.to_string()))?; + Ok((base_ref_set_id, projection)) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum ObjectGcClaim { + Open, + Deleting { + owner_epoch: u64, + operation_token: u64, + gc_record_key: Vec, + gc_record_version: u64, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ObjectReferenceMutation { + claim_version: Version, +} + +impl ObjectReferenceMutation { + pub(super) fn from_version(claim_version: Version) -> Self { + Self { claim_version } + } + + pub(super) fn version(self) -> Version { + self.claim_version + } + + pub(super) fn predicate(self, mount: MountId) -> PredicateRef { + PredicateRef { + family: RecordFamily::System, + key: object_gc_claim_key(mount), + predicate: Predicate::VersionEquals(self.claim_version), + } + } +} + +fn encode_object_gc_claim(claim: &ObjectGcClaim) -> Result, MetadError> { + match claim { + ObjectGcClaim::Open => Ok(vec![1]), + ObjectGcClaim::Deleting { + owner_epoch, + operation_token, + gc_record_key, + gc_record_version, + } => { + let key_len = u32::try_from(gc_record_key.len()) + .map_err(|_| MetadError::Codec("object GC record key is too long".to_owned()))?; + let mut out = Vec::with_capacity(29 + gc_record_key.len()); + out.push(2); + out.extend_from_slice(&owner_epoch.to_be_bytes()); + out.extend_from_slice(&operation_token.to_be_bytes()); + out.extend_from_slice(&gc_record_version.to_be_bytes()); + out.extend_from_slice(&key_len.to_be_bytes()); + out.extend_from_slice(gc_record_key); + Ok(out) + } + } +} + +fn decode_object_gc_claim(bytes: &[u8]) -> Result { + match bytes { + [1] => Ok(ObjectGcClaim::Open), + [2, rest @ ..] if rest.len() >= 28 => { + let owner_epoch = u64::from_be_bytes(rest[..8].try_into().expect("u64 width")); + let operation_token = u64::from_be_bytes(rest[8..16].try_into().expect("u64 width")); + let gc_record_version = u64::from_be_bytes(rest[16..24].try_into().expect("u64 width")); + let key_len = u32::from_be_bytes(rest[24..28].try_into().expect("u32 width")) as usize; + if rest.len() != 28 + key_len { + return Err(MetadError::Codec( + "object GC claim record length mismatch".to_owned(), + )); + } + Ok(ObjectGcClaim::Deleting { + owner_epoch, + operation_token, + gc_record_key: rest[28..].to_vec(), + gc_record_version, + }) + } + _ => Err(MetadError::Codec( + "invalid durable object GC claim record".to_owned(), + )), + } +} + +fn object_key_digest(object_key: &str) -> [u8; 32] { + Sha256::digest(object_key.as_bytes()).into() +} + // Families folded into the fallback allocator rebuild when the durable // `allocator_key` System record is absent. Each row contributes its commit // version to the recovered high-water (`last_commit_version`), and the Inode / @@ -201,6 +467,7 @@ struct ReplaceProjectionCommit<'a> { old_generation: Option, version: Version, path_index: Option>, + object_reference: ObjectReferenceMutation, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -309,6 +576,10 @@ pub struct PreparedArtifact { pub replace: bool, pub dentry_version: Option, pub old_generation: Option, + /// Durable object-GC Open epoch captured before upload/planning. Publish + /// must CAS this exact version so an intervening delete cycle cannot make a + /// newly committed manifest point at an object that GC already removed. + pub object_gc_claim_version: u64, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -317,6 +588,24 @@ pub struct CreatedPreparedArtifact { pub prepared: PreparedArtifact, } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RestoreInitialization { + /// Snapshot-root-relative files to remove before destination attach. + pub remove_relative_paths: Vec, + /// Snapshot-root-relative files to create or replace before attach. + pub files: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreInitializationFile { + pub relative_path: String, + pub bytes: Vec, + pub content_type: String, + pub mode: u32, + pub uid: u32, + pub gid: u32, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct ObjectTransferStats { pub object_puts: u64, @@ -434,21 +723,39 @@ pub struct RenameReplaceResult { pub replaced: Option, } -/// Handle to a writable copy-on-write fork produced by [`NoKvFs::clone_subtree`]. +/// Handle to a writable copy-on-write fork produced by the path-based clone API. /// /// `root` is the new namespace root: it sees every file and directory the source /// subtree had at clone time and shares the source's object blocks (no data copy) -/// until the fork diverges on write. `snapshot_id` is a retained snapshot pin on -/// the source that holds the GC retention floor down so the shared base blocks are -/// protected while the fork references them. Retire it with -/// [`NoKvFs::retire_snapshot`] once the fork's own divergent state no longer needs -/// the source's base objects (typically when the fork is deleted). +/// until the fork diverges on write. `snapshot_id` identifies the construction +/// checkpoint; it is not the destination's lifetime owner. Once attached, durable +/// fork-base references protect every shared object independently, so retiring or +/// expiring that checkpoint cannot make the destination unreadable. Removing the +/// destination releases those base references through bounded GC. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct CloneHandle { pub root: InodeId, pub snapshot_id: u64, } +/// Durable state returned by a restore-to-fork operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RestoreState { + Complete, +} + +/// Result of restoring a snapshot into a new copy-on-write workbench. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RestoreOutcome { + pub operation_id: String, + pub state: RestoreState, + pub source_root: InodeId, + pub destination_root: InodeId, + pub snapshot_id: u64, + pub read_version: u64, + pub cleanup_pending: bool, +} + /// How a path differs between two subtrees, as reported by /// [`NoKvFs::diff_subtrees`]. Directions are relative to the diff arguments /// `diff_subtrees(a_root, b_root)`: `Added` exists only under `b_root`, `Removed` @@ -499,6 +806,13 @@ pub enum MetadError { bytes: u64, }, InvalidPreparedArtifact(String), + /// A prepared upload crossed an object-GC delete epoch. The caller may + /// refresh the prepared upload identity, but must mint a new generation and + /// fully restage the body before retrying the metadata publish. + StalePreparedArtifactObjectGcEpoch { + expected: u64, + current: u64, + }, InvalidQuery(String), StaleBodyGeneration { expected: u64, @@ -549,6 +863,29 @@ pub enum MetadError { /// point), NOT `EXDEV` — there is no copy+unlink fallback that would be /// correct here. GraftPoint, + /// The destination already exists but was not created by the same restore. + RestoreDestinationConflict { + destination: String, + }, + /// An identical restore is still preparing or cleaning its staging tree. + RestoreInProgress, + /// A restore was rejected before staging because one bounded metadata + /// resource would exceed the implementation's atomic-command limit. + RestoreResourceLimit { + resource: String, + limit: u64, + actual: u64, + }, + /// The restore subtree contains a non-directory inode with links outside the + /// tree; copying/deleting it without a link graph would corrupt that inode. + RestoreHardlinkUnsupported { + inode: InodeId, + }, + /// Cross-shard grafts require a distributed transaction and are rejected by + /// the single-shard restore implementation. + RestoreCrossShardUnsupported { + inode: InodeId, + }, SnapshotLeaseExpired { snapshot_id: u64, lease_expires_unix_ms: u64, @@ -587,6 +924,13 @@ pub struct NoKvFs { objects: O, allocator_gate: Mutex<()>, backup_gate: Mutex<()>, + /// Serializes restore-to-fork construction inside the active shard owner. + /// Durable predicates remain authoritative across processes; this gate makes + /// an orphaned operation hold unambiguously recoverable after restart. + restore_gate: Mutex<()>, + /// Prevents a live worker/manual GC call from mistaking another in-process + /// worker's durable Deleting/Pruning claim for crash recovery. + object_gc_gate: Mutex<()>, /// Serializes owner-epoch installs/observes against in-flight commits. /// Commits hold a read guard across their fence check + durable apply; /// epoch changes take the write guard. This closes the TOCTOU where a @@ -945,6 +1289,11 @@ fn validate_prepared_artifact( body: &BodyDescriptor, chunks: &[ChunkManifest], ) -> Result<(), MetadError> { + if prepared.object_gc_claim_version == 0 { + return Err(MetadError::InvalidPreparedArtifact( + "prepared artifact is missing a durable mutation epoch".to_owned(), + )); + } if body.generation != prepared.generation { return Err(MetadError::InvalidPreparedArtifact(format!( "body generation {} does not match prepared generation {}", @@ -1432,6 +1781,7 @@ fn kind_name(kind: CommandKind) -> &'static [u8] { CommandKind::WatchSubtree => b"watch-subtree", CommandKind::CleanupObjects => b"cleanup-objects", CommandKind::RegisterNamespaceIndex => b"register-namespace-index", + CommandKind::StageDetachedRestore => b"stage-detached-restore", } } @@ -1498,6 +1848,10 @@ impl fmt::Display for MetadError { Self::InvalidPreparedArtifact(err) => { write!(f, "invalid prepared artifact: {err}") } + Self::StalePreparedArtifactObjectGcEpoch { expected, current } => write!( + f, + "prepared artifact object-GC epoch {expected} is stale; current epoch is {current}" + ), Self::InvalidQuery(err) => write!(f, "invalid namespace query: {err}"), Self::StaleBodyGeneration { expected, current } => write!( f, @@ -1553,6 +1907,29 @@ impl fmt::Display for MetadError { f, "path is a cross-shard graft point; use unregister-graft" ), + Self::RestoreDestinationConflict { destination } => write!( + f, + "restore destination already exists for another operation: {destination}" + ), + Self::RestoreInProgress => write!(f, "restore operation is still in progress"), + Self::RestoreResourceLimit { + resource, + limit, + actual, + } => write!( + f, + "restore resource {resource} exceeds limit {limit} (actual {actual})" + ), + Self::RestoreHardlinkUnsupported { inode } => write!( + f, + "restore does not support hard-linked inode {}", + inode.get() + ), + Self::RestoreCrossShardUnsupported { inode } => write!( + f, + "restore does not support cross-shard inode {}", + inode.get() + ), Self::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, diff --git a/crates/nokv-meta/src/service/checkpoint.rs b/crates/nokv-meta/src/service/checkpoint.rs index ab22433d2..0be54b239 100644 --- a/crates/nokv-meta/src/service/checkpoint.rs +++ b/crates/nokv-meta/src/service/checkpoint.rs @@ -52,6 +52,7 @@ where "checkpoint requires at least one shard".to_owned(), )); } + let object_reference = self.begin_object_reference_mutation()?; // Stage every shard's objects + build its (projection, chunks). Track the // staged object sets so a mid-stage failure reclaims them instead of @@ -70,6 +71,10 @@ where } }; let version = Version::new(prepared.generation)?; + if prepared.object_gc_claim_version != object_reference.version().get() { + self.abort_staged_checkpoint(&staged_sets); + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } let request = PublishArtifact { parent, name: shard.name.clone(), @@ -116,7 +121,12 @@ where // One atomic commit makes every shard visible together. let commit_version = self.next_version()?; - if let Err(err) = self.commit_checkpoint_shards(parent, &staged_artifacts, commit_version) { + if let Err(err) = self.commit_checkpoint_shards( + parent, + &staged_artifacts, + commit_version, + object_reference, + ) { self.abort_staged_checkpoint(&staged_sets); return Err(err); } @@ -137,12 +147,14 @@ where parent: InodeId, shards: &[(DentryProjection, Vec)], commit_version: Version, + object_reference: ObjectReferenceMutation, ) -> Result<(), MetadError> { let mut predicates = vec![PredicateRef { family: RecordFamily::Inode, key: inode_key(self.mount, parent), predicate: Predicate::Exists, }]; + predicates.push(object_reference.predicate(self.mount)); let mut mutations = Vec::new(); for (proj, chunks) in shards { let inode = proj.attr.inode; @@ -186,6 +198,21 @@ where } } } + let watch = shards + .iter() + .flat_map(|(projection, _)| { + self.watch_projection( + parent, + WatchEvent { + kind: WatchEventKind::PublishArtifact, + parent: Some(parent), + name: Some(projection.dentry.name.clone()), + inode: projection.attr.inode, + version: commit_version.get(), + }, + ) + }) + .collect(); self.commit_metadata(MetadataCommand { request_id: request_id(b"publish-checkpoint", self.mount, parent, commit_version), kind: CommandKind::PublishArtifact, @@ -195,7 +222,7 @@ where primary_key: dentry_prefix(self.mount, parent), predicates, mutations, - watch: Vec::new(), + watch, })?; Ok(()) } diff --git a/crates/nokv-meta/src/service/clone.rs b/crates/nokv-meta/src/service/clone.rs index 54a142560..19dc4051e 100644 --- a/crates/nokv-meta/src/service/clone.rs +++ b/crates/nokv-meta/src/service/clone.rs @@ -1,9 +1,47 @@ use super::*; +const RESTORE_CLONE_BATCH_ENTRIES: usize = 64; +const MAX_RESTORE_COMMAND_ITEMS: usize = 4096; +const MAX_RESTORE_COMMAND_BYTES: usize = 8 * 1024 * 1024; +pub(super) const MAX_RESTORE_SUBTREE_ENTRIES: usize = 1_000_000; + /// One node of the source subtree paired with the fork inode it copies into. struct CloneFrame { src_inode: InodeId, dst_inode: InodeId, + relative_components: Vec, + ancestor_markers: Vec, +} + +pub(super) struct RestorePathIndexContext { + pub(super) source_root_components: Vec, + pub(super) base_ref_set_id: u64, + pub(super) track_staging_members: bool, +} + +struct CloneChildrenContext<'a> { + src_parent: InodeId, + dst_parent: InodeId, + relative_parent: &'a [DentryName], + ancestor_markers: &'a [DentryProjection], + read_version: Version, + path_index: Option<&'a RestorePathIndexContext>, +} + +pub(super) struct DirectoryPathProof { + pub(super) inode: InodeId, + pub(super) predicates: Vec, +} + +struct RestoreHoldRequest<'a> { + restored_root: InodeId, + source_root: InodeId, + pin: &'a SnapshotPin, + pin_version: Version, + operation_digest: [u8; 32], + destination_path: String, + source_path_predicates: &'a [PredicateRef], + initialization_digest: [u8; 32], } impl NoKvFs @@ -11,6 +49,989 @@ where M: MetadataStore, O: ObjectStore, { + /// Restore a retained snapshot into a new writable copy-on-write subtree. + /// + /// The source remains unchanged. The destination is attached exactly once; + /// retrying the same `(source, snapshot, destination)` returns the existing + /// fork, while an unrelated existing destination fails loudly. The source + /// checkpoint stabilizes construction; once attached, the [`ForkBinding`] + /// lifecycle and its exact durable base references protect shared objects + /// independently of the caller-owned snapshot lease. + pub fn restore_subtree_path_to_fork( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + ) -> Result { + self.restore_subtree_path_to_fork_initialized( + source_path, + snapshot_id, + destination_path, + RestoreInitialization::default(), + ) + } + + pub fn restore_subtree_path_to_fork_initialized( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + initialization: RestoreInitialization, + ) -> Result { + self.restore_subtree_path_to_fork_impl( + source_path, + snapshot_id, + destination_path, + initialization, + false, + ) + } + + #[cfg(test)] + pub(super) fn restore_subtree_path_to_fork_leave_preparing( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + initialization: RestoreInitialization, + ) -> Result<(), MetadError> { + self.restore_subtree_path_to_fork_impl( + source_path, + snapshot_id, + destination_path, + initialization, + true, + ) + .map(|_| ()) + } + + fn restore_subtree_path_to_fork_impl( + &self, + source_path: &str, + snapshot_id: u64, + destination_path: &str, + initialization: RestoreInitialization, + stop_before_attach: bool, + ) -> Result { + let _restore = self + .restore_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let source_components = parse_absolute_path(source_path)?; + let source_path = canonical_path(&source_components)?; + let destination_components = parse_absolute_path(destination_path)?; + let destination_path = canonical_path(&destination_components)?; + const MAX_RESTORE_PATH_BYTES: usize = 4096; + if source_path.len().saturating_add(destination_path.len()) > MAX_RESTORE_PATH_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore source and destination path bytes".to_owned(), + limit: MAX_RESTORE_PATH_BYTES as u64, + actual: source_path.len().saturating_add(destination_path.len()) as u64, + }); + } + let (initialization, initialization_digest) = + canonical_restore_initialization(initialization)?; + let operation_digest = + restore_operation_digest(&source_path, snapshot_id, &destination_path); + let operation_id = restore_operation_id(&source_path, snapshot_id, &destination_path)?; + + // Terminal lookup precedes source/pin validation. The durable completed + // binding is sufficient to answer an identical retry even after the + // source or caller-owned snapshot pin has been retired. + if let Some(outcome) = self.existing_restore_outcome( + &destination_path, + &operation_id, + operation_digest, + initialization_digest, + )? { + return Ok(outcome); + } + let Some((destination_name, destination_parent_components)) = + destination_components.split_last() + else { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + }; + self.resolve_directory_path_proof(destination_parent_components)?; + if self.lookup_path(&destination_path)?.is_some() { + if let Some(stale) = self.restore_binding_by_operation(operation_digest)? { + if stale.state == ForkBindingState::Preparing + && !self.binding_root_is_visible(&stale)? + { + self.schedule_restore_staging_cleanup( + &stale, + RestoreStagingCleanupDisposition::Discard, + )?; + } + } + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + // A crash-left Preparing hold is the recovery capability: it retains the + // historical source version even after the caller pin/source path is + // removed. Reset partial staging while preserving that hold, then rebuild + // from its durable source root/read version. + let resume = if let Some(stale) = self.restore_binding_by_operation(operation_digest)? { + if stale.state == ForkBindingState::Releasing { + return Err(MetadError::RestoreInProgress); + } + if stale.initialization_digest != initialization_digest + || stale.state != ForkBindingState::Preparing + || self.binding_root_is_visible(&stale)? + { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path, + }); + } + if !self.claim_clean_restore_staging(&stale)? { + self.schedule_restore_staging_cleanup( + &stale, + RestoreStagingCleanupDisposition::ResetForRetry, + )?; + return Err(MetadError::RestoreInProgress); + } + if let Err(err) = self.preflight_snapshot_subtree_restore( + stale.source_root, + Version::new(stale.pinned_read_version)?, + &initialization, + ) { + if is_deterministic_restore_preflight_error(&err) { + self.schedule_restore_staging_cleanup( + &stale, + RestoreStagingCleanupDisposition::Discard, + )?; + } + return Err(err); + } + Some(stale) + } else { + None + }; + + let (source_root, read_version, restored_root, binding) = if let Some(binding) = resume { + ( + binding.source_root, + binding.pinned_read_version, + binding.fork_root, + binding, + ) + } else { + let mut prepared = None; + for attempt in 0..8 { + let source_proof = self.resolve_directory_path_proof(&source_components)?; + let source_root = source_proof.inode; + self.ensure_snapshot_id_shard(snapshot_id, source_root)?; + let (pin, pin_version) = self + .versioned_snapshot_pin(snapshot_id)? + .ok_or(MetadError::NotFound)?; + if pin.root != source_root { + return Err(MetadError::SnapshotRootMismatch { + snapshot_id, + expected_root: source_root, + actual_root: Some(pin.root), + actual_shard: self.shard_index(), + }); + } + self.ensure_snapshot_pin_live(&pin)?; + self.preflight_snapshot_subtree_restore( + source_root, + Version::new(pin.read_version)?, + &initialization, + )?; + let restored_root = self.next_inode()?; + match self.create_restore_hold(RestoreHoldRequest { + restored_root, + source_root, + pin: &pin, + pin_version, + operation_digest, + destination_path: destination_path.clone(), + source_path_predicates: &source_proof.predicates, + initialization_digest, + }) { + Ok(binding) => { + prepared = Some((source_root, pin.read_version, restored_root, binding)); + break; + } + Err(MetadError::Metadata(MetadataError::PredicateFailed)) if attempt < 7 => { + continue; + } + Err(err) => return Err(err), + } + } + prepared.ok_or(MetadError::Metadata(MetadataError::PredicateFailed))? + }; + let path_index = RestorePathIndexContext { + source_root_components: source_components.clone(), + base_ref_set_id: binding.base_ref_set_id, + track_staging_members: true, + }; + if let Err(err) = self.materialize_subtree_at_root( + source_root, + restored_root, + Version::new(read_version)?, + Some(&path_index), + ) { + self.schedule_restore_staging_cleanup( + &binding, + RestoreStagingCleanupDisposition::ResetForRetry, + )?; + return Err(err); + } + if let Err(err) = self.apply_restore_initialization( + restored_root, + binding.base_ref_set_id, + &initialization, + ) { + self.schedule_restore_staging_cleanup( + &binding, + RestoreStagingCleanupDisposition::ResetForRetry, + )?; + return Err(err); + } + let base_refs = match self.subtree_base_refs(restored_root, read_version) { + Ok(references) => references, + Err(err) => { + self.schedule_restore_staging_cleanup( + &binding, + RestoreStagingCleanupDisposition::ResetForRetry, + )?; + return Err(err); + } + }; + if let Err(err) = self.persist_fork_base_refs(&binding, &base_refs) { + self.schedule_restore_staging_cleanup( + &binding, + RestoreStagingCleanupDisposition::ResetForRetry, + )?; + return Err(err); + } + if stop_before_attach { + return Err(MetadError::SyncLogArchiveFailed { + committed: false, + message: "test stop before restore attach".to_owned(), + }); + } + let mut attach = Err(MetadError::Metadata(MetadataError::PredicateFailed)); + for _ in 0..8 { + let destination_proof = + match self.resolve_directory_path_proof(destination_parent_components) { + Ok(proof) => proof, + Err(err) => { + attach = Err(err); + break; + } + }; + attach = self.attach_restored_root( + restored_root, + &binding, + destination_proof.inode, + destination_name.clone(), + &destination_proof.predicates, + ); + if !matches!( + attach, + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + ) { + break; + } + } + if let Err(err) = attach { + // A racing identical request may have attached first. Reconcile the + // durable destination before returning the predicate failure. + if let Some(outcome) = self.existing_restore_outcome( + &destination_path, + &operation_id, + operation_digest, + initialization_digest, + )? { + return Ok(outcome); + } + let err = if self.lookup_path(&destination_path)?.is_some() { + MetadError::RestoreDestinationConflict { + destination: destination_path.clone(), + } + } else { + err + }; + self.schedule_restore_staging_cleanup( + &binding, + RestoreStagingCleanupDisposition::Discard, + )?; + return Err(err); + } + + Ok(RestoreOutcome { + operation_id, + state: RestoreState::Complete, + source_root, + destination_root: restored_root, + snapshot_id, + read_version, + cleanup_pending: false, + }) + } + + fn existing_restore_outcome( + &self, + destination_path: &str, + operation_id: &str, + operation_digest: [u8; 32], + initialization_digest: [u8; 32], + ) -> Result, MetadError> { + let Some(binding) = self.restore_binding_by_operation(operation_digest)? else { + return Ok(None); + }; + if binding.initialization_digest != initialization_digest { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path.to_owned(), + }); + } + if binding.state == ForkBindingState::Preparing { + return Ok(None); + } + if binding.state == ForkBindingState::Releasing { + return Err(MetadError::RestoreInProgress); + } + let Some(destination) = self.lookup_path(destination_path)? else { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path.to_owned(), + }); + }; + if destination.attr.file_type != FileType::Directory { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path.to_owned(), + }); + } + if destination.attr.inode != binding.fork_root + || binding.destination_path != destination_path + || binding.operation_digest != operation_digest + || binding.state != ForkBindingState::Complete + { + return Err(MetadError::RestoreDestinationConflict { + destination: destination_path.to_owned(), + }); + } + Ok(Some(RestoreOutcome { + operation_id: operation_id.to_owned(), + state: RestoreState::Complete, + source_root: binding.source_root, + destination_root: binding.fork_root, + snapshot_id: binding.snapshot_id, + read_version: binding.pinned_read_version, + cleanup_pending: false, + })) + } + + pub(super) fn versioned_snapshot_pin( + &self, + snapshot_id: u64, + ) -> Result, MetadError> { + self.metadata + .get_versioned( + RecordFamily::Snapshot, + &snapshot_pin_key(self.mount, snapshot_id), + self.read_version()?, + ReadPurpose::UserStrong, + )? + .map(|item| { + decode_snapshot_pin(&item.value.0) + .map(|pin| (pin, item.version)) + .map_err(|err| MetadError::Codec(err.to_string())) + }) + .transpose() + } + + fn restore_binding_by_operation( + &self, + operation_digest: [u8; 32], + ) -> Result, MetadError> { + self.metadata + .get( + RecordFamily::System, + &restore_operation_key(self.mount, &operation_digest), + self.read_version()?, + ReadPurpose::UserStrong, + )? + .map(|value| { + decode_fork_binding(&value.0).map_err(|err| MetadError::Codec(err.to_string())) + }) + .transpose() + } + + pub(super) fn schedule_restore_staging_cleanup( + &self, + binding: &ForkBinding, + disposition: RestoreStagingCleanupDisposition, + ) -> Result<(), MetadError> { + let cleanup_key = restore_staging_cleanup_key(self.mount, binding.base_ref_set_id); + if let Some(existing) = self.metadata.get_versioned( + RecordFamily::System, + &cleanup_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + let (current_disposition, current_binding) = + decode_restore_staging_cleanup(&existing.value.0)?; + if current_binding.operation_digest != binding.operation_digest { + return Err(MetadError::RestoreDestinationConflict { + destination: binding.destination_path.clone(), + }); + } + if current_disposition == disposition + || current_disposition == RestoreStagingCleanupDisposition::Discard + || disposition != RestoreStagingCleanupDisposition::Discard + { + return Ok(()); + } + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"upgrade-restore-staging-cleanup", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cleanup_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::VersionEquals(existing.version), + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cleanup_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_cleanup( + RestoreStagingCleanupDisposition::Discard, + ¤t_binding, + ))), + }], + watch: Vec::new(), + })?; + return Ok(()); + } + + let binding_key = fork_binding_key(self.mount, binding.fork_root); + let operation_key = restore_operation_key(self.mount, &binding.operation_digest); + let read_version = self.read_version()?; + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let mut releasing = decode_fork_binding(&binding_item.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if releasing.operation_digest != binding.operation_digest { + return Err(MetadError::RestoreDestinationConflict { + destination: binding.destination_path.clone(), + }); + } + let operation = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if releasing.state == ForkBindingState::Releasing { + return Err(MetadError::RestoreInProgress); + } + if releasing.state != ForkBindingState::Preparing { + return Err(MetadError::RestoreDestinationConflict { + destination: binding.destination_path.clone(), + }); + } + releasing.state = ForkBindingState::Releasing; + let clean_key = restore_staging_clean_key(self.mount, binding.base_ref_set_id); + let clean = self.metadata.get_versioned( + RecordFamily::System, + &clean_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let version = self.next_version()?; + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(binding_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation.version), + }, + PredicateRef { + family: RecordFamily::System, + key: cleanup_key.clone(), + predicate: Predicate::NotExists, + }, + ]; + let mut mutations = vec![ + Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&releasing))), + }, + Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&releasing))), + }, + Mutation { + family: RecordFamily::System, + key: cleanup_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_cleanup( + disposition, + &releasing, + ))), + }, + ]; + if let Some(clean) = clean { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: clean_key.clone(), + predicate: Predicate::VersionEquals(clean.version), + }); + mutations.push(delete_mutation(RecordFamily::System, clean_key)); + } + match self.commit_metadata(MetadataCommand { + request_id: request_id( + b"schedule-restore-staging-cleanup", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: restore_staging_cleanup_key(self.mount, binding.base_ref_set_id), + predicates, + mutations, + watch: Vec::new(), + }) { + Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => Ok(()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + if self + .metadata + .get( + RecordFamily::System, + &restore_staging_cleanup_key(self.mount, binding.base_ref_set_id), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .is_some() => + { + Ok(()) + } + Err(err) => Err(err), + } + } + + fn claim_clean_restore_staging(&self, binding: &ForkBinding) -> Result { + let clean_key = restore_staging_clean_key(self.mount, binding.base_ref_set_id); + let read_version = self.read_version()?; + let Some(clean) = self.metadata.get_versioned( + RecordFamily::System, + &clean_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(false); + }; + let binding_key = fork_binding_key(self.mount, binding.fork_root); + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"claim-clean-restore-staging", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: clean_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key, + predicate: Predicate::VersionEquals(binding_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: clean_key.clone(), + predicate: Predicate::VersionEquals(clean.version), + }, + ], + mutations: vec![delete_mutation(RecordFamily::System, clean_key)], + watch: Vec::new(), + })?; + Ok(true) + } + + fn binding_root_is_visible(&self, binding: &ForkBinding) -> Result { + Ok(self + .lookup_path(&binding.destination_path)? + .is_some_and(|entry| entry.attr.inode == binding.fork_root)) + } + + pub(super) fn resolve_directory_path_proof( + &self, + components: &[DentryName], + ) -> Result { + let version = self.read_version()?; + let mut current = InodeId::root(); + let mut predicates = vec![PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, current), + predicate: Predicate::Exists, + }]; + for name in components { + let (entry, dentry_version) = self + .lookup_plus_at_version_for_purpose( + current, + name, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if entry.attr.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + predicates.push(PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key(self.mount, current, name), + predicate: Predicate::VersionEquals(dentry_version), + }); + current = entry.attr.inode; + } + Ok(DirectoryPathProof { + inode: current, + predicates, + }) + } + + fn create_restore_hold( + &self, + request: RestoreHoldRequest<'_>, + ) -> Result { + let RestoreHoldRequest { + restored_root, + source_root, + pin, + pin_version, + operation_digest, + destination_path, + source_path_predicates, + initialization_digest, + } = request; + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let key = fork_binding_key(self.mount, restored_root); + let operation_key = restore_operation_key(self.mount, &operation_digest); + let binding = ForkBinding { + fork_root: restored_root, + source_root, + pinned_read_version: pin.read_version, + snapshot_id: pin.snapshot_id, + created_version: version.get(), + base_ref_set_id: version.get(), + operation_digest, + initialization_digest, + state: ForkBindingState::Preparing, + destination_path, + }; + let hold_key = fork_base_hold_key( + self.mount, + binding.pinned_read_version, + binding.base_ref_set_id, + ); + let mut predicates = source_path_predicates.to_vec(); + predicates.push(object_reference.predicate(self.mount)); + predicates.extend([ + PredicateRef { + family: RecordFamily::Snapshot, + key: snapshot_pin_key(self.mount, pin.snapshot_id), + predicate: Predicate::VersionEquals(pin_version), + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::System, + key: fork_base_ref_set_prefix(self.mount, binding.base_ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: hold_key.clone(), + predicate: Predicate::NotExists, + }, + ]); + let committed = self.commit_metadata(MetadataCommand { + request_id: request_id(b"restore-to-fork-hold", self.mount, restored_root, version), + kind: CommandKind::SnapshotSubtree, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::ForkBinding, + primary_key: key.clone(), + predicates, + mutations: vec![ + Mutation { + family: RecordFamily::ForkBinding, + key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&binding))), + }, + Mutation { + family: RecordFamily::System, + key: operation_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&binding))), + }, + Mutation { + family: RecordFamily::System, + key: hold_key, + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }, + ], + watch: Vec::new(), + }); + match committed { + Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => { + let durable = self + .metadata + .get( + RecordFamily::System, + &operation_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "restore hold committed without operation index".to_owned(), + ) + })?; + if decode_fork_binding(&durable.0) + .map_err(|err| MetadError::Codec(err.to_string()))? + != binding + { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + Ok(binding) + } + Err(err) => Err(err), + } + } + + fn attach_restored_root( + &self, + restored_root: InodeId, + binding: &ForkBinding, + destination_parent: InodeId, + destination_name: DentryName, + destination_path_predicates: &[PredicateRef], + ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let read_version = predecessor(version)?; + let Some(mut attr) = self.get_attr_at_version_for_purpose( + restored_root, + read_version, + ReadPurpose::WritePlanLocal, + )? + else { + return Err(MetadError::NotFound); + }; + attr.ctime_ms = self.now_ms(); + let projection = projection(destination_parent, destination_name.clone(), attr, None); + let dentry = dentry_key(self.mount, destination_parent, &destination_name); + let binding_key = fork_binding_key(self.mount, restored_root); + let operation_key = restore_operation_key(self.mount, &binding.operation_digest); + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let durable_binding = decode_fork_binding(&binding_item.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if &durable_binding != binding { + return Err(MetadError::RestoreDestinationConflict { + destination: String::from_utf8_lossy(destination_name.as_bytes()).into_owned(), + }); + } + let binding_version = binding_item.version; + let operation_item = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if decode_fork_binding(&operation_item.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))? + != *binding + { + return Err(MetadError::RestoreDestinationConflict { + destination: binding.destination_path.clone(), + }); + } + let hold_key = fork_base_hold_key( + self.mount, + binding.pinned_read_version, + binding.base_ref_set_id, + ); + let hold = self + .metadata + .get_versioned( + RecordFamily::System, + &hold_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("restore hold index is missing".to_owned()))?; + let mut completed_binding = durable_binding; + completed_binding.state = ForkBindingState::Complete; + let staging_cleanup_key = restore_staging_cleanup_key(self.mount, binding.base_ref_set_id); + let mut predicates = destination_path_predicates.to_vec(); + predicates.push(object_reference.predicate(self.mount)); + predicates.extend([ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, destination_parent), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(binding_version), + }, + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: hold_key.clone(), + predicate: Predicate::VersionEquals(hold.version), + }, + PredicateRef { + family: RecordFamily::System, + key: staging_cleanup_key.clone(), + predicate: Predicate::NotExists, + }, + ]); + let command = MetadataCommand { + request_id: request_id( + b"restore-to-fork-attach", + self.mount, + restored_root, + version, + ), + kind: CommandKind::CreateDir, + read_version, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry.clone(), + predicates, + mutations: vec![ + put_projection_mutation(RecordFamily::Dentry, dentry, &projection), + Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&completed_binding))), + }, + Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&completed_binding))), + }, + delete_mutation(RecordFamily::System, hold_key), + Mutation { + family: RecordFamily::System, + key: staging_cleanup_key, + op: MutationOp::Put, + value: Some(Value(encode_restore_staging_cleanup( + RestoreStagingCleanupDisposition::ForgetMembers, + &completed_binding, + ))), + }, + ], + watch: self + .watch_projection( + destination_parent, + WatchEvent { + kind: WatchEventKind::Create, + parent: Some(destination_parent), + name: Some(destination_name), + inode: restored_root, + version: version.get(), + }, + ) + .into_iter() + .collect(), + }; + self.commit_metadata(command)?; + Ok(()) + } + /// Create a writable copy-on-write fork of the directory subtree rooted at /// `src_root`. /// @@ -30,11 +1051,11 @@ where /// fork's own inode, producing new object keys; the borrowed source blocks are /// left untouched (a borrower never GCs another namespace's blocks). /// - /// The clone retains a snapshot pin on the source ([`CloneHandle::snapshot_id`]) - /// so the GC retention floor protects the shared base blocks while the fork - /// references them. Retire it with [`NoKvFs::retire_snapshot`] when the fork no - /// longer needs the source's base objects (typically when the fork is deleted). - pub fn clone_subtree(&self, src_root: InodeId) -> Result { + /// This low-level detached constructor keeps its construction pin until an + /// internal caller attaches or discards the tree. Public path-based clones use + /// the durable restore-to-fork lifecycle instead. + #[cfg(test)] + pub(super) fn clone_subtree(&self, src_root: InodeId) -> Result { // Pin the source first so the read version we copy from is stable and the // shared base objects are GC-protected from the moment the fork exists. let pin = self.snapshot_subtree(src_root)?; @@ -46,76 +1067,789 @@ where }) } - /// Materialize the directory subtree rooted at `src_root`, **as seen at - /// `read_version`**, into a brand-new detached namespace root and return that - /// root inode. - /// - /// This is the shared copy-on-write walk behind both [`NoKvFs::clone_subtree`] - /// (which materializes the *current* state) and [`NoKvFs::rollback_subtree`] - /// (which materializes a *prior snapshot's* state). Every node is reproduced - /// under a fresh inode while keeping each file body's `generation`, so the new - /// tree's chunk manifests reference the same `blocks/{mount}/{inode}/{generation}/...` - /// object keys as the source captured at `read_version` — the bodies are shared, - /// not copied, so this is zero-copy in data and O(entries) in metadata (one - /// commit per descendant). The returned root is detached: no - /// dentry names it yet, so the caller must link it (clone) or graft it over an - /// existing root (rollback). - /// - /// `read_version` must be a stable, GC-protected read version (a snapshot pin's - /// `read_version`), otherwise the source blocks the new manifests borrow could be - /// reclaimed out from under it. - /// - /// Snapshot-aware metadata scans merge current keys with the derived retained- - /// history key index, so deleted entries are enumerated without a whole-history - /// scan or a second point-read reconstruction pass. - pub(super) fn materialize_subtree_at( + /// Materialize the directory subtree rooted at `src_root`, **as seen at + /// `read_version`**, into a brand-new detached namespace root and return that + /// root inode. + /// + /// This convenience wrapper reserves a destination root and delegates + /// to the same copy-on-write walk used by restore and rollback. Every node is reproduced + /// under a fresh inode while keeping each file body's `generation`, so the new + /// tree's chunk manifests reference the same `blocks/{mount}/{inode}/{generation}/...` + /// object keys as the source captured at `read_version` — the bodies are shared, + /// not copied, so this is zero-copy in data and O(entries) in metadata (one + /// commit per descendant). The returned root is detached: no + /// dentry names it yet, so the caller must link it (clone) or graft it over an + /// existing root (rollback). + /// + /// `read_version` must be a stable, GC-protected read version (a snapshot pin's + /// `read_version`), otherwise the source blocks the new manifests borrow could be + /// reclaimed out from under it. + /// + /// Snapshot-aware metadata scans merge current keys with the derived retained- + /// history key index, so deleted entries are enumerated without a whole-history + /// scan or a second point-read reconstruction pass. + pub(super) fn materialize_subtree_at( + &self, + src_root: InodeId, + read_version: Version, + ) -> Result { + let dst_root = self.next_inode()?; + self.materialize_subtree_at_root(src_root, dst_root, read_version, None)?; + Ok(dst_root) + } + + /// Materialize into a pre-reserved root. Restore-to-fork reserves this inode + /// before installing its durable operation hold, so history/object retention + /// is protected for the entire materialization window. + pub(super) fn materialize_subtree_at_root( + &self, + src_root: InodeId, + dst_root: InodeId, + read_version: Version, + path_index: Option<&RestorePathIndexContext>, + ) -> Result<(), MetadError> { + let Some(src_attr) = + self.get_attr_at_version_for_purpose(src_root, read_version, ReadPurpose::Snapshot)? + else { + return Err(MetadError::NotFound); + }; + if src_attr.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + + let mut root_attr = src_attr; + root_attr.inode = dst_root; + self.commit_root_inode( + &root_attr, + path_index + .filter(|context| context.track_staging_members) + .map(|context| context.base_ref_set_id), + )?; + self.copy_inode_xattrs(src_root, dst_root, read_version)?; + + // Breadth-first so a fork parent always exists before its children (the + // create path predicates on the parent inode existing). Each directory's + // children are materialized in a single batched commit, so a clone costs + // one commit per source directory rather than one per entry. + let mut queue = vec![CloneFrame { + src_inode: src_root, + dst_inode: dst_root, + relative_components: Vec::new(), + ancestor_markers: Vec::new(), + }]; + while let Some(frame) = queue.pop() { + let children = self.list_dir_at_version(frame.src_inode, read_version)?; + self.validate_restore_entries(&children)?; + if !children.is_empty() { + let context = CloneChildrenContext { + src_parent: frame.src_inode, + dst_parent: frame.dst_inode, + relative_parent: &frame.relative_components, + ancestor_markers: &frame.ancestor_markers, + read_version, + path_index, + }; + let mut sub_frames = self.clone_children_into(&context, &children)?; + queue.append(&mut sub_frames); + } + } + + Ok(()) + } + + /// Validate every deterministic restore constraint before creating a durable + /// hold or detached inode. This keeps invalid hardlinks, cross-shard grafts, + /// initialization paths, and oversized metadata commands from leaking a + /// resumable Preparing operation that can never succeed. + pub(super) fn preflight_snapshot_subtree_restore( + &self, + root: InodeId, + version: Version, + initialization: &RestoreInitialization, + ) -> Result<(), MetadError> { + let attr = self + .get_attr_at_version_for_purpose(root, version, ReadPurpose::Snapshot)? + .ok_or(MetadError::NotFound)?; + if attr.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + self.preflight_snapshot_xattrs(root, version)?; + let mut queue = vec![root]; + let mut entries_seen = 0usize; + while let Some(directory) = queue.pop() { + let children = self.list_dir_at_version(directory, version)?; + self.validate_restore_entries(&children)?; + entries_seen = entries_seen.saturating_add(children.len()); + check_restore_resource( + "snapshot subtree entries", + MAX_RESTORE_SUBTREE_ENTRIES, + entries_seen, + )?; + for batch in children.chunks(RESTORE_CLONE_BATCH_ENTRIES) { + let mut items = 2usize.saturating_add(batch.len()); + let mut bytes = 256usize; + for child in batch { + items = items.saturating_add(2); + bytes = bytes + .saturating_add(child.dentry.name.as_bytes().len()) + .saturating_add(encode_inode_attr(&child.attr).len()) + .saturating_add( + encode_dentry_projection(&projection( + directory, + child.dentry.name.clone(), + child.attr.clone(), + child.body.clone(), + )) + .len(), + ); + if let Some(body) = &child.body { + let chunks = self.chunk_manifests_for_body_at_version( + child.attr.inode, + body, + version, + ReadPurpose::Snapshot, + )?; + items = items.saturating_add(1).saturating_add(chunks.len()); + bytes = bytes.saturating_add(encode_body_descriptor(body).len()); + for chunk in &chunks { + bytes = bytes.saturating_add(encode_chunk_manifest(chunk).len()); + } + } + self.preflight_snapshot_xattrs(child.attr.inode, version)?; + if child.attr.file_type == FileType::Directory { + queue.push(child.attr.inode); + } + } + check_restore_resource( + "detached clone batch items", + MAX_RESTORE_COMMAND_ITEMS, + items, + )?; + check_restore_resource( + "detached clone batch encoded bytes", + MAX_RESTORE_COMMAND_BYTES, + bytes, + )?; + } + } + self.preflight_restore_initialization(root, version, initialization) + } + + fn preflight_snapshot_xattrs( + &self, + inode: InodeId, + version: Version, + ) -> Result<(), MetadError> { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::Xattr, + prefix: xattr_prefix(self.mount, inode), + start_after: None, + version, + limit: 0, + purpose: ReadPurpose::Snapshot, + })?; + for batch in rows.chunks(128) { + let items = 2usize.saturating_add(batch.len().saturating_mul(2)); + let bytes = batch.iter().fold(256usize, |total, row| { + total + .saturating_add(row.key.len().saturating_mul(2)) + .saturating_add(row.value.0.len()) + }); + check_restore_resource( + "detached xattr batch items", + MAX_RESTORE_COMMAND_ITEMS, + items, + )?; + check_restore_resource( + "detached xattr batch encoded bytes", + MAX_RESTORE_COMMAND_BYTES, + bytes, + )?; + } + Ok(()) + } + + fn preflight_restore_initialization( + &self, + root: InodeId, + version: Version, + initialization: &RestoreInitialization, + ) -> Result<(), MetadError> { + for relative_path in &initialization.remove_relative_paths { + let components = restore_relative_components(relative_path)?; + let Some((name, parents)) = components.split_last() else { + return Err(MetadError::InvalidPath( + "restore initialization cannot remove its root".to_owned(), + )); + }; + let parent = match self.resolve_components_as_directory_from_at_version_for_purpose( + root, + parents, + version, + ReadPurpose::Snapshot, + ) { + Ok(parent) => parent, + Err(MetadError::NotFound) => continue, + Err(err) => return Err(err), + }; + if self + .lookup_plus_at_version_for_purpose(parent, name, version, ReadPurpose::Snapshot)? + .is_some_and(|(entry, _)| entry.attr.file_type == FileType::Directory) + { + return Err(MetadError::InvalidPath(format!( + "restore initialization remove path is a directory: {relative_path}" + ))); + } + } + for file in &initialization.files { + let components = restore_relative_components(&file.relative_path)?; + let Some((name, parents)) = components.split_last() else { + return Err(MetadError::InvalidPath( + "restore initialization cannot write its root".to_owned(), + )); + }; + let parent = self.resolve_components_as_directory_from_at_version_for_purpose( + root, + parents, + version, + ReadPurpose::Snapshot, + )?; + if self + .lookup_plus_at_version_for_purpose(parent, name, version, ReadPurpose::Snapshot)? + .is_some_and(|(entry, _)| entry.attr.file_type != FileType::File) + { + return Err(MetadError::InvalidPath(format!( + "restore initialization write path is not a file: {}", + file.relative_path + ))); + } + } + Ok(()) + } + + pub(super) fn validate_restore_entries( + &self, + entries: &[DentryWithAttr], + ) -> Result<(), MetadError> { + for entry in entries { + if entry.attr.inode.shard_index() != self.shard_index { + return Err(MetadError::RestoreCrossShardUnsupported { + inode: entry.attr.inode, + }); + } + if entry.attr.file_type != FileType::Directory && entry.attr.nlink > 1 { + return Err(MetadError::RestoreHardlinkUnsupported { + inode: entry.attr.inode, + }); + } + } + Ok(()) + } + + pub(super) fn subtree_base_refs( + &self, + root: InodeId, + read_version: u64, + ) -> Result, MetadError> { + let version = self.read_version()?; + let mut references = BTreeMap::new(); + let mut queue = vec![root]; + while let Some(dir) = queue.pop() { + for child in + self.read_dir_plus_at_version_for_purpose(dir, version, ReadPurpose::Snapshot)? + { + if child.attr.file_type == FileType::Directory { + queue.push(child.attr.inode); + } else if let Some(body) = &child.body { + let manifests = self.chunk_manifests_for_body_at_version( + child.attr.inode, + body, + version, + ReadPurpose::Snapshot, + )?; + for block in manifests + .iter() + .flat_map(|manifest| &manifest.slices) + .flat_map(|slice| &slice.blocks) + { + if self.owns_block_object_key( + child.attr.inode, + body.generation, + &block.object_key, + ) { + continue; + } + references + .entry(block.object_key.clone()) + .or_insert_with(|| ForkBaseRef { + owner_inode: child.attr.inode, + owner_generation: body.generation, + read_version, + size: block.len, + object_key: block.object_key.clone(), + digest_uri: block.digest_uri.clone(), + }); + } + } + } + } + Ok(references.into_values().collect()) + } + + pub(super) fn persist_fork_base_refs( + &self, + binding: &ForkBinding, + references: &[ForkBaseRef], + ) -> Result<(), MetadError> { + const BATCH_SIZE: usize = 128; + if references.is_empty() { + return Ok(()); + } + let binding_key = fork_binding_key(self.mount, binding.fork_root); + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if decode_fork_binding(&binding_item.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))? + != *binding + { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + for chunk in references.chunks(BATCH_SIZE) { + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(binding_item.version), + }, + ]; + let mut mutations = Vec::with_capacity(chunk.len() * 2); + for reference in chunk { + let digest = object_key_digest(&reference.object_key); + let owner_key = + fork_base_ref_owner_key(self.mount, binding.base_ref_set_id, &digest); + let inverse_key = + fork_base_ref_inverse_key(self.mount, &digest, binding.base_ref_set_id); + predicates.push(PredicateRef { + family: RecordFamily::System, + key: owner_key.clone(), + predicate: Predicate::NotExists, + }); + predicates.push(PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: owner_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_base_ref(reference)?)), + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: inverse_key, + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }); + } + let command = MetadataCommand { + request_id: request_id( + b"persist-fork-base-refs", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: fork_base_ref_set_prefix(self.mount, binding.base_ref_set_id), + predicates, + mutations, + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => { + for reference in chunk { + let digest = object_key_digest(&reference.object_key); + let owner_key = + fork_base_ref_owner_key(self.mount, binding.base_ref_set_id, &digest); + let value = self + .metadata + .get( + RecordFamily::System, + &owner_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("fork base-ref commit lost a row".to_owned()) + })?; + if decode_fork_base_ref(&value.0)? != *reference { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + if self + .metadata + .get( + RecordFamily::System, + &fork_base_ref_inverse_key( + self.mount, + &digest, + binding.base_ref_set_id, + ), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .is_none() + { + return Err(MetadError::Codec( + "fork base-ref commit lost its inverse row".to_owned(), + )); + } + } + } + Err(err) => return Err(err), + } + } + Ok(()) + } + + fn apply_restore_initialization( + &self, + root: InodeId, + base_ref_set_id: u64, + initialization: &RestoreInitialization, + ) -> Result<(), MetadError> { + for relative_path in &initialization.remove_relative_paths { + let components = restore_relative_components(relative_path)?; + let Some((name, parents)) = components.split_last() else { + return Err(MetadError::InvalidPath( + "restore initialization cannot remove its root".to_owned(), + )); + }; + let parent = match self.resolve_components_as_directory_from_at_version( + root, + parents, + self.read_version()?, + ) { + Ok(parent) => parent, + Err(MetadError::NotFound) => continue, + Err(err) => return Err(err), + }; + let Some(entry) = self.lookup_plus(parent, name)? else { + continue; + }; + if entry.attr.file_type == FileType::Directory { + return Err(MetadError::InvalidPath(format!( + "restore initialization remove path is a directory: {relative_path}" + ))); + } + self.remove_detached_restore_file(parent, name)?; + } + for file in &initialization.files { + let components = restore_relative_components(&file.relative_path)?; + let Some((name, parents)) = components.split_last() else { + return Err(MetadError::InvalidPath( + "restore initialization cannot write its root".to_owned(), + )); + }; + let parent = self.resolve_components_as_directory_from_at_version( + root, + parents, + self.read_version()?, + )?; + let request = PublishArtifact { + parent, + name: name.clone(), + producer: "nokv-restore-initialization".to_owned(), + digest_uri: body_digest_uri(&file.bytes), + content_type: file.content_type.clone(), + manifest_id: format!("restore-init/{}", file.relative_path), + bytes: file.bytes.clone(), + mode: file.mode, + uid: file.uid, + gid: file.gid, + }; + self.publish_detached_restore_file(request, base_ref_set_id)?; + } + Ok(()) + } + + fn remove_detached_restore_file( + &self, + parent: InodeId, + name: &DentryName, + ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let read_version = predecessor(version)?; + let (entry, dentry_version) = self + .lookup_plus_at_version_for_purpose( + parent, + name, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if entry.attr.file_type == FileType::Directory { + return Err(MetadError::NotFile); + } + let inode_key = inode_key(self.mount, entry.attr.inode); + let inode = self + .metadata + .get_versioned( + RecordFamily::Inode, + &inode_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let dentry_key = dentry_key(self.mount, parent, name); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_key.clone(), + predicate: Predicate::VersionEquals(dentry_version), + }, + PredicateRef { + family: RecordFamily::Inode, + key: inode_key.clone(), + predicate: Predicate::VersionEquals(inode.version), + }, + ]; + let mut mutations = vec![ + delete_mutation(RecordFamily::Dentry, dentry_key.clone()), + delete_mutation(RecordFamily::Inode, inode_key), + ]; + for row in self.metadata.scan(ScanRequest { + family: RecordFamily::Xattr, + prefix: xattr_prefix(self.mount, entry.attr.inode), + start_after: None, + version: read_version, + limit: 0, + purpose: ReadPurpose::WritePlanLocal, + })? { + predicates.push(PredicateRef { + family: RecordFamily::Xattr, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::Xattr, row.key)); + } + if let Some(body) = entry.body { + mutations.extend(self.chunk_manifest_delete_and_gc_mutations( + entry.attr.inode, + body.generation, + version, + &HashSet::new(), + )?); + } + let command = MetadataCommand { + request_id: request_id( + b"restore-initialization-remove", + self.mount, + entry.attr.inode, + version, + ), + kind: CommandKind::StageDetachedRestore, + read_version, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry_key, + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore initialization remove")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn publish_detached_restore_file( &self, - src_root: InodeId, - read_version: Version, - ) -> Result { - let Some(src_attr) = - self.get_attr_at_version_for_purpose(src_root, read_version, ReadPurpose::Snapshot)? - else { - return Err(MetadError::NotFound); - }; - if src_attr.file_type != FileType::Directory { - return Err(MetadError::NotDirectory); + request: PublishArtifact, + base_ref_set_id: u64, + ) -> Result<(), MetadError> { + let existing = self.lookup_plus_for_write_plan(request.parent, &request.name)?; + if existing + .as_ref() + .is_some_and(|(entry, _)| entry.attr.file_type != FileType::File) + { + return Err(MetadError::NotFile); } - - let dst_root = self.next_inode()?; - let root_version = self.next_version()?; - let root_attr = directory_attr( - dst_root, - src_attr.mode, - src_attr.uid, - src_attr.gid, - root_version.get(), + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; + let inode = match &existing { + Some((entry, _)) => entry.attr.inode, + None => self.next_inode()?, + }; + let StagedArtifactBody { + body, + chunks, + old_chunks: _, + staged, + } = self.stage_artifact_body(&request, inode, version)?; + let attr = InodeAttr { + inode, + file_type: FileType::File, + mode: request.mode, + uid: request.uid, + gid: request.gid, + rdev: 0, + nlink: existing + .as_ref() + .map_or(FileType::File.initial_link_count(), |(entry, _)| { + entry.attr.nlink + }), + size: body.size, + generation: version.get(), + mtime_ms: self.now_ms(), + ctime_ms: self.now_ms(), + }; + let projection = projection(request.parent, request.name, attr, Some(body)); + let dentry = dentry_key( + self.mount, + projection.dentry.parent, + &projection.dentry.name, ); - self.commit_root_inode(&root_attr, root_version)?; - self.copy_inode_xattrs(src_root, dst_root, read_version)?; - - // Breadth-first so a fork parent always exists before its children (the - // create path predicates on the parent inode existing). Each directory's - // children are materialized in a single batched commit, so a clone costs - // one commit per source directory rather than one per entry. - let mut queue = vec![CloneFrame { - src_inode: src_root, - dst_inode: dst_root, - }]; - while let Some(frame) = queue.pop() { - let children = self.list_dir_at_version(frame.src_inode, read_version)?; - if !children.is_empty() { - let mut sub_frames = - self.clone_children_into(frame.dst_inode, &children, read_version)?; - queue.append(&mut sub_frames); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, projection.dentry.parent), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: existing + .as_ref() + .map_or(Predicate::NotExists, |(_, version)| { + Predicate::VersionEquals(*version) + }), + }, + ]; + let mut mutations = vec![ + Mutation { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(&projection.attr))), + }, + put_projection_mutation(RecordFamily::Dentry, dentry.clone(), &projection), + ]; + if let Some((old, _)) = &existing { + let old_inode_key = inode_key(self.mount, inode); + let old_inode = self + .metadata + .get_versioned( + RecordFamily::Inode, + &old_inode_key, + predecessor(version)?, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: old_inode_key, + predicate: Predicate::VersionEquals(old_inode.version), + }); + if let Some(old_body) = &old.body { + mutations.extend(self.collapse_chain_gc_mutations( + inode, + old_body.generation, + &[], + version, + &chunk_object_keys(&chunks), + )?); } + } else { + predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, inode), + predicate: Predicate::NotExists, + }); + let member = restore_staging_member_key(self.mount, base_ref_set_id, inode); + predicates.push(PredicateRef { + family: RecordFamily::System, + key: member.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: member, + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }); } - - Ok(dst_root) + if let Some(body) = &projection.body { + mutations.push(Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key( + self.mount, + inode, + body.generation, + BODY_SUMMARY_CHUNK_INDEX, + ), + op: MutationOp::Put, + value: Some(Value(encode_body_descriptor(body))), + }); + mutations.extend(chunks.iter().map(|chunk| Mutation { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_key(self.mount, inode, body.generation, chunk.chunk_index), + op: MutationOp::Put, + value: Some(Value(encode_chunk_manifest(chunk))), + })); + } + let command = MetadataCommand { + request_id: request_id( + b"restore-initialization-publish", + self.mount, + inode, + version, + ), + kind: CommandKind::StageDetachedRestore, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::Dentry, + primary_key: dentry, + predicates, + mutations, + watch: Vec::new(), + }; + if let Err(err) = + validate_restore_command_bounds(&command, "restore initialization publish") + { + self.cleanup_staged_objects(&staged)?; + return Err(err); + } + let committed = self.commit_metadata(command); + if let Err(source) = committed { + self.cleanup_staged_objects(&staged)?; + return Err(MetadError::PublishArtifactFailed { + source: Box::new(source), + staged, + }); + } + Ok(()) } /// Enumerate the children of `dir` at `version` through snapshot-aware scan. - fn list_dir_at_version( + pub(super) fn list_dir_at_version( &self, dir: InodeId, version: Version, @@ -124,7 +1858,8 @@ where } /// Path variant of [`NoKvFs::clone_subtree`]. - pub fn clone_subtree_path(&self, path: &str) -> Result { + #[cfg(test)] + pub(super) fn clone_subtree_path(&self, path: &str) -> Result { let root = self.resolve_directory_path(path)?; self.clone_subtree(root) } @@ -132,70 +1867,28 @@ where /// Clone the subtree at `src_path` and link the resulting fork root into the /// namespace at `dst_path`, so the fork is a usable, navigable directory. /// - /// This is [`NoKvFs::clone_subtree_path`] plus a single dentry that attaches the - /// detached fork root under `dst_path`'s parent. The fork shares the source's - /// object blocks until it diverges on write (see [`NoKvFs::clone_subtree`]) and - /// the returned [`CloneHandle`] carries the retained snapshot pin exactly as the - /// detached clone does. `dst_path`'s parent must already exist and `dst_path` - /// itself must be free. + /// The construction checkpoint stabilizes the source read only. The attached + /// destination receives durable fork-base references before it becomes visible, + /// so source checkpoint expiry/retirement does not affect its lifetime. + /// `dst_path`'s parent must already exist and `dst_path` itself must be free. pub fn clone_subtree_path_into( &self, src_path: &str, dst_path: &str, ) -> Result { - let src_root = self.resolve_directory_path(src_path)?; - let (dst_parent, dst_name) = self.resolve_parent_path(dst_path)?; - let handle = self.clone_subtree(src_root)?; - self.link_clone_root(handle.root, dst_parent, dst_name)?; - Ok(handle) - } - - /// Attach an existing (detached) fork root inode under `dst_parent` as - /// `dst_name`, committing only the directory dentry that names it. The inode - /// already exists from [`NoKvFs::clone_subtree`]; this just makes it reachable. - fn link_clone_root( - &self, - root: InodeId, - dst_parent: InodeId, - dst_name: DentryName, - ) -> Result<(), MetadError> { - let version = self.next_version()?; - let read_version = predecessor(version)?; - let Some(mut attr) = - self.get_attr_at_version_for_purpose(root, read_version, ReadPurpose::WritePlanLocal)? - else { - return Err(MetadError::NotFound); + let pin = self.snapshot_subtree_path(src_path)?; + let restored = match self.restore_subtree_path_to_fork(src_path, pin.snapshot_id, dst_path) + { + Ok(restored) => restored, + Err(MetadError::RestoreDestinationConflict { .. }) => { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + Err(err) => return Err(err), }; - attr.ctime_ms = current_time_ms(); - let projection = projection(dst_parent, dst_name, attr, None); - let dentry = dentry_key(self.mount, dst_parent, &projection.dentry.name); - self.commit_metadata(MetadataCommand { - request_id: request_id(b"clone-subtree-link", self.mount, root, version), - kind: CommandKind::CreateDir, - read_version, - commit_version: version, - primary_family: RecordFamily::Dentry, - primary_key: dentry.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, dst_parent), - predicate: Predicate::Exists, - }, - PredicateRef { - family: RecordFamily::Dentry, - key: dentry.clone(), - predicate: Predicate::NotExists, - }, - ], - mutations: vec![put_projection_mutation( - RecordFamily::Dentry, - dentry, - &projection, - )], - watch: Vec::new(), - })?; - Ok(()) + Ok(CloneHandle { + root: restored.destination_root, + snapshot_id: pin.snapshot_id, + }) } /// Report the path-level differences between two subtrees as a flat list of @@ -309,59 +2002,104 @@ where .collect()) } - fn commit_root_inode(&self, attr: &InodeAttr, version: Version) -> Result<(), MetadError> { + fn commit_root_inode( + &self, + attr: &InodeAttr, + staging_set_id: Option, + ) -> Result<(), MetadError> { + let object_reference = self.begin_object_reference_mutation()?; + let version = self.next_version()?; let key = inode_key(self.mount, attr.inode); + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Inode, + key: key.clone(), + predicate: Predicate::NotExists, + }, + ]; + let mut mutations = vec![Mutation { + family: RecordFamily::Inode, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_inode_attr(attr))), + }]; + if let Some(base_ref_set_id) = staging_set_id { + let member = restore_staging_member_key(self.mount, base_ref_set_id, attr.inode); + predicates.push(PredicateRef { + family: RecordFamily::System, + key: member.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: member, + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }); + } self.commit_metadata(MetadataCommand { request_id: request_id(b"clone-subtree-root", self.mount, attr.inode, version), - kind: CommandKind::CreateDir, + kind: CommandKind::StageDetachedRestore, read_version: predecessor(version)?, commit_version: version, primary_family: RecordFamily::Inode, primary_key: key.clone(), - predicates: vec![PredicateRef { - family: RecordFamily::Inode, - key: key.clone(), - predicate: Predicate::NotExists, - }], - mutations: vec![Mutation { - family: RecordFamily::Inode, - key, - op: MutationOp::Put, - value: Some(Value(encode_inode_attr(attr))), - }], + predicates, + mutations, watch: Vec::new(), })?; Ok(()) } - /// Materialize every `child` of one source directory into the fork under - /// `dst_parent` in a **single batched commit**, keeping each file body's - /// generation (so chunk manifests keep referencing the source's object - /// blocks). Returns the child directories to recurse into. - /// - /// One commit for a whole directory's children rather than one per child is - /// what keeps clone's per-commit overhead from dominating: cost scales with the - /// number of *directories*, not the number of entries. + /// Materialize one source directory in fixed-size metadata batches. A wide + /// directory therefore cannot create an unbounded atomic command, while each + /// child remains unreachable until the final restore attach. fn clone_children_into( &self, - dst_parent: InodeId, + context: &CloneChildrenContext<'_>, + children: &[DentryWithAttr], + ) -> Result, MetadError> { + let mut sub_frames = Vec::new(); + for batch in children.chunks(RESTORE_CLONE_BATCH_ENTRIES) { + sub_frames.extend(self.clone_children_batch(context, batch)?); + } + Ok(sub_frames) + } + + fn clone_children_batch( + &self, + context: &CloneChildrenContext<'_>, children: &[DentryWithAttr], - read_version: Version, ) -> Result, MetadError> { + let CloneChildrenContext { + src_parent, + dst_parent, + relative_parent, + ancestor_markers, + read_version, + path_index, + } = *context; + let object_reference = self.begin_object_reference_mutation()?; let commit_version = self.next_version()?; - let mut predicates = vec![PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, dst_parent), - predicate: Predicate::Exists, - }]; + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, dst_parent), + predicate: Predicate::Exists, + }, + ]; let mut mutations = Vec::new(); - let mut watch = Vec::new(); let mut sub_frames = Vec::new(); // Xattrs are copied after the batch commit (so the dst inodes exist); most // entries have none, so this is usually empty. let mut xattr_copies = Vec::new(); + let mut planned_shadow_keys = HashSet::new(); for child in children { + let mut relative_components = relative_parent.to_vec(); + relative_components.push(child.dentry.name.clone()); let dst_inode = self.next_inode()?; let mut attr = child.attr.clone(); attr.inode = dst_inode; @@ -401,6 +2139,149 @@ where value: Some(Value(encode_inode_attr(&proj.attr))), }); mutations.push(put_projection_mutation(RecordFamily::Dentry, dentry, &proj)); + if let Some(context) = path_index { + if context.track_staging_members { + let member = + restore_staging_member_key(self.mount, context.base_ref_set_id, dst_inode); + predicates.push(PredicateRef { + family: RecordFamily::System, + key: member.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: member, + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }); + } + let mut source_components = context.source_root_components.clone(); + source_components.extend(relative_components.iter().cloned()); + let source_index_key = path_index_key(self.mount, &source_components); + let source_index = self.metadata.get_versioned( + RecordFamily::PathIndex, + &source_index_key, + read_version, + ReadPurpose::Snapshot, + )?; + let source_dentry = self.lookup_plus_at_version_for_purpose( + src_parent, + &child.dentry.name, + read_version, + ReadPurpose::Snapshot, + )?; + let source_shadow = self.metadata.get_versioned( + RecordFamily::ForkShadow, + &fork_shadow_key(self.mount, src_parent, &child.dentry.name), + read_version, + ReadPurpose::Snapshot, + )?; + let source_is_indexed = match source_dentry.as_ref() { + Some((canonical, dentry_version)) => { + let live_indexed = source_index + .as_ref() + .map(|index| { + let indexed = decode_dentry_projection(&index.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + Ok::<_, MetadError>( + index.version == *dentry_version + && DentryWithAttr::from(indexed) == *canonical + && canonical == child, + ) + }) + .transpose()? + .unwrap_or(false); + let shadow_indexed = source_shadow + .as_ref() + .map(|shadow| { + let (_, indexed) = decode_restore_shadow_inverse(&shadow.value.0)?; + Ok::<_, MetadError>( + DentryWithAttr::from(indexed) == *canonical + && canonical == child, + ) + }) + .transpose()? + .unwrap_or(false); + live_indexed || shadow_indexed + } + None => false, + }; + if source_is_indexed { + for marker in ancestor_markers.iter().chain(std::iter::once(&proj)) { + let shadow_key = restore_path_index_key( + self.mount, + context.base_ref_set_id, + marker.dentry.parent, + &marker.dentry.name, + ); + if !planned_shadow_keys.insert(shadow_key.clone()) { + continue; + } + match self.metadata.get( + RecordFamily::System, + &shadow_key, + predecessor(commit_version)?, + ReadPurpose::WritePlanLocal, + )? { + Some(value) => { + let existing = decode_dentry_projection(&value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if existing.attr.inode != marker.attr.inode + || existing.dentry.parent != marker.dentry.parent + || existing.dentry.name != marker.dentry.name + { + return Err(MetadError::Metadata( + MetadataError::PredicateFailed, + )); + } + } + None => { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: shadow_key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(put_projection_mutation( + RecordFamily::System, + shadow_key, + marker, + )); + } + } + let inverse_key = + fork_shadow_key(self.mount, marker.dentry.parent, &marker.dentry.name); + let inverse_value = + encode_restore_shadow_inverse(context.base_ref_set_id, marker); + match self.metadata.get_versioned( + RecordFamily::ForkShadow, + &inverse_key, + predecessor(commit_version)?, + ReadPurpose::WritePlanLocal, + )? { + Some(item) => { + if item.value.0 != inverse_value { + return Err(MetadError::Metadata( + MetadataError::PredicateFailed, + )); + } + } + None => { + predicates.push(PredicateRef { + family: RecordFamily::ForkShadow, + key: inverse_key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::ForkShadow, + key: inverse_key, + op: MutationOp::Put, + value: Some(Value(inverse_value)), + }); + } + } + } + } + } if let Some(body) = &proj.body { mutations.push(Mutation { family: RecordFamily::ChunkManifest, @@ -427,43 +2308,37 @@ where }); } } - if let Some(event) = self.watch_projection( - dst_parent, - WatchEvent { - kind: create_watch_kind(clone_command_kind(child.attr.file_type)), - parent: Some(dst_parent), - name: Some(child.dentry.name.clone()), - inode: dst_inode, - version: commit_version.get(), - }, - ) { - watch.push(event); - } xattr_copies.push((child.attr.inode, dst_inode)); if child.attr.file_type == FileType::Directory { + let mut child_ancestors = ancestor_markers.to_vec(); + child_ancestors.push(proj.clone()); sub_frames.push(CloneFrame { src_inode: child.attr.inode, dst_inode, + relative_components, + ancestor_markers: child_ancestors, }); } } - self.commit_metadata(MetadataCommand { + let command = MetadataCommand { request_id: request_id( b"clone-subtree-batch", self.mount, dst_parent, commit_version, ), - kind: CommandKind::CreateFiles, + kind: CommandKind::StageDetachedRestore, read_version: predecessor(commit_version)?, commit_version, primary_family: RecordFamily::Dentry, primary_key: dentry_prefix(self.mount, dst_parent), predicates, mutations, - watch, - })?; + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "detached clone batch")?; + self.commit_metadata(command)?; for (src, dst) in xattr_copies { self.copy_inode_xattrs(src, dst, read_version)?; @@ -487,27 +2362,58 @@ where limit: 0, purpose: ReadPurpose::Snapshot, })?; - for row in rows { - let Some(name) = row.key.strip_prefix(prefix.as_slice()) else { - continue; + for chunk in rows.chunks(128) { + let object_reference = self.begin_object_reference_mutation()?; + let commit_version = self.next_version()?; + let mut predicates = vec![ + object_reference.predicate(self.mount), + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, dst_inode), + predicate: Predicate::Exists, + }, + ]; + let mut mutations = Vec::with_capacity(chunk.len()); + for row in chunk { + let name = row.key.strip_prefix(prefix.as_slice()).ok_or_else(|| { + MetadError::Codec("source xattr escaped its inode prefix".to_owned()) + })?; + let key = xattr_key(self.mount, dst_inode, name); + predicates.push(PredicateRef { + family: RecordFamily::Xattr, + key: key.clone(), + predicate: Predicate::NotExists, + }); + mutations.push(Mutation { + family: RecordFamily::Xattr, + key, + op: MutationOp::Put, + value: Some(row.value.clone()), + }); + } + let command = MetadataCommand { + request_id: request_id( + b"clone-detached-xattrs", + self.mount, + dst_inode, + commit_version, + ), + kind: CommandKind::StageDetachedRestore, + read_version: predecessor(commit_version)?, + commit_version, + primary_family: RecordFamily::Xattr, + primary_key: xattr_prefix(self.mount, dst_inode), + predicates, + mutations, + watch: Vec::new(), }; - self.set_xattr(dst_inode, name, row.value.0, XattrSetMode::Create)?; + validate_restore_command_bounds(&command, "detached xattr batch")?; + self.commit_metadata(command)?; } Ok(()) } } -fn clone_command_kind(file_type: FileType) -> CommandKind { - match file_type { - FileType::Directory => CommandKind::CreateDir, - FileType::Symlink => CommandKind::CreateSymlink, - FileType::File => CommandKind::CreateFile, - FileType::NamedPipe | FileType::CharDevice | FileType::BlockDevice | FileType::Socket => { - CommandKind::CreateSpecialNode - } - } -} - /// Two non-directory entries are equivalent when they have the same type and, for /// content-bearing nodes, the same size, content generation, and digest. A /// divergent write bumps the generation, so a rewritten file is never equivalent @@ -538,3 +2444,249 @@ fn child_path(prefix: &str, name: &[u8]) -> Result { .map_err(|_| MetadError::InvalidPath("subtree diff requires utf-8 names".to_owned()))?; Ok(format!("{prefix}/{name}")) } + +fn restore_relative_components(path: &str) -> Result, MetadError> { + if path.is_empty() || path.starts_with('/') { + return Err(MetadError::InvalidPath(format!( + "restore initialization path must be non-empty and relative: {path}" + ))); + } + parse_absolute_path(&format!("/{path}")).map_err(Into::into) +} + +fn canonical_restore_relative_path(path: &str) -> Result { + let components = restore_relative_components(path)?; + let mut canonical = String::new(); + for (index, component) in components.iter().enumerate() { + if index != 0 { + canonical.push('/'); + } + canonical.push_str(std::str::from_utf8(component.as_bytes()).map_err(|_| { + MetadError::InvalidPath("restore initialization path is not utf-8".to_owned()) + })?); + } + Ok(canonical) +} + +fn canonical_restore_initialization( + mut initialization: RestoreInitialization, +) -> Result<(RestoreInitialization, [u8; 32]), MetadError> { + const MAX_INITIALIZATION_ENTRIES: usize = 1024; + const MAX_INITIALIZATION_BYTES: usize = 8 * 1024 * 1024; + const MAX_RELATIVE_PATH_BYTES: usize = 4096; + if initialization + .remove_relative_paths + .len() + .saturating_add(initialization.files.len()) + > MAX_INITIALIZATION_ENTRIES + { + return Err(MetadError::RestoreResourceLimit { + resource: "restore initialization entries".to_owned(), + limit: MAX_INITIALIZATION_ENTRIES as u64, + actual: initialization + .remove_relative_paths + .len() + .saturating_add(initialization.files.len()) as u64, + }); + } + for path in &mut initialization.remove_relative_paths { + *path = canonical_restore_relative_path(path)?; + } + for file in &mut initialization.files { + file.relative_path = canonical_restore_relative_path(&file.relative_path)?; + } + initialization.remove_relative_paths.sort(); + initialization + .files + .sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + let total_bytes = initialization + .remove_relative_paths + .iter() + .map(String::len) + .chain(initialization.files.iter().map(|file| { + file.relative_path + .len() + .saturating_add(file.bytes.len()) + .saturating_add(file.content_type.len()) + })) + .fold(0usize, usize::saturating_add); + if total_bytes > MAX_INITIALIZATION_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: "restore initialization bytes".to_owned(), + limit: MAX_INITIALIZATION_BYTES as u64, + actual: total_bytes as u64, + }); + } + if initialization + .remove_relative_paths + .iter() + .chain(initialization.files.iter().map(|file| &file.relative_path)) + .any(|path| path.len() > MAX_RELATIVE_PATH_BYTES) + { + let actual = initialization + .remove_relative_paths + .iter() + .chain(initialization.files.iter().map(|file| &file.relative_path)) + .map(String::len) + .max() + .unwrap_or(0); + return Err(MetadError::RestoreResourceLimit { + resource: "restore initialization relative path bytes".to_owned(), + limit: MAX_RELATIVE_PATH_BYTES as u64, + actual: actual as u64, + }); + } + if initialization + .remove_relative_paths + .windows(2) + .any(|paths| paths[0] == paths[1]) + || initialization + .files + .windows(2) + .any(|files| files[0].relative_path == files[1].relative_path) + { + return Err(MetadError::InvalidPath( + "restore initialization contains duplicate paths".to_owned(), + )); + } + let mut all_paths = initialization + .remove_relative_paths + .iter() + .chain(initialization.files.iter().map(|file| &file.relative_path)) + .collect::>(); + all_paths.sort(); + if all_paths.windows(2).any(|paths| { + paths[1] + .as_bytes() + .strip_prefix(paths[0].as_bytes()) + .is_some_and(|suffix| suffix.first() == Some(&b'/')) + }) { + return Err(MetadError::InvalidPath( + "restore initialization paths cannot overlap as ancestor and descendant".to_owned(), + )); + } + let removals = initialization + .remove_relative_paths + .iter() + .map(String::as_str) + .collect::>(); + if initialization + .files + .iter() + .any(|file| removals.contains(file.relative_path.as_str())) + { + return Err(MetadError::InvalidPath( + "restore initialization cannot remove and write the same path".to_owned(), + )); + } + + fn hash_bytes(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); + } + let mut hasher = Sha256::new(); + hasher.update(b"nokv-restore-initialization-v1\0"); + hasher.update((initialization.remove_relative_paths.len() as u64).to_be_bytes()); + for path in &initialization.remove_relative_paths { + hash_bytes(&mut hasher, path.as_bytes()); + } + hasher.update((initialization.files.len() as u64).to_be_bytes()); + for file in &initialization.files { + hash_bytes(&mut hasher, file.relative_path.as_bytes()); + hash_bytes(&mut hasher, &file.bytes); + hash_bytes(&mut hasher, file.content_type.as_bytes()); + hasher.update(file.mode.to_be_bytes()); + hasher.update(file.uid.to_be_bytes()); + hasher.update(file.gid.to_be_bytes()); + } + Ok((initialization, hasher.finalize().into())) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct RestoreCommandShape { + pub(super) item_count: usize, + pub(super) encoded_bytes: usize, +} + +pub(super) fn restore_command_shape(command: &MetadataCommand) -> RestoreCommandShape { + let item_count = command + .predicates + .len() + .saturating_add(command.mutations.len()) + .saturating_add(command.watch.len()); + let mut encoded_bytes = command + .request_id + .len() + .saturating_add(command.primary_key.len()); + for predicate in &command.predicates { + encoded_bytes = encoded_bytes + .saturating_add(predicate.key.len()) + .saturating_add(24); + } + for mutation in &command.mutations { + encoded_bytes = encoded_bytes + .saturating_add(mutation.key.len()) + .saturating_add(mutation.value.as_ref().map_or(0, |value| value.0.len())) + .saturating_add(24); + } + for projection in &command.watch { + encoded_bytes = encoded_bytes + .saturating_add(projection.key.len()) + .saturating_add(projection.event.len()) + .saturating_add(16); + } + RestoreCommandShape { + item_count, + encoded_bytes, + } +} + +pub(super) fn validate_restore_command_bounds( + command: &MetadataCommand, + resource: &str, +) -> Result<(), MetadError> { + let shape = restore_command_shape(command); + if shape.item_count > MAX_RESTORE_COMMAND_ITEMS { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} items"), + limit: MAX_RESTORE_COMMAND_ITEMS as u64, + actual: shape.item_count as u64, + }); + } + if shape.encoded_bytes > MAX_RESTORE_COMMAND_BYTES { + return Err(MetadError::RestoreResourceLimit { + resource: format!("{resource} encoded bytes"), + limit: MAX_RESTORE_COMMAND_BYTES as u64, + actual: shape.encoded_bytes as u64, + }); + } + Ok(()) +} + +pub(super) fn check_restore_resource( + resource: &str, + limit: usize, + actual: usize, +) -> Result<(), MetadError> { + if actual <= limit { + return Ok(()); + } + Err(MetadError::RestoreResourceLimit { + resource: resource.to_owned(), + limit: limit as u64, + actual: actual as u64, + }) +} + +fn is_deterministic_restore_preflight_error(error: &MetadError) -> bool { + matches!( + error, + MetadError::InvalidPath(_) + | MetadError::NotFound + | MetadError::NotDirectory + | MetadError::NotFile + | MetadError::RestoreHardlinkUnsupported { .. } + | MetadError::RestoreCrossShardUnsupported { .. } + | MetadError::RestoreResourceLimit { .. } + ) +} diff --git a/crates/nokv-meta/src/service/gc.rs b/crates/nokv-meta/src/service/gc.rs index edb38e283..0fb539cad 100644 --- a/crates/nokv-meta/src/service/gc.rs +++ b/crates/nokv-meta/src/service/gc.rs @@ -1,16 +1,1737 @@ +use super::clone::validate_restore_command_bounds; use super::*; +use std::collections::HashMap; use std::time::Duration; +struct ObjectRetention { + version_floor: Option, +} + +struct BaseRefReleaseGuard { + family: RecordFamily, + key: Vec, + version: Version, + binding: Option<(Vec, Version)>, + operation: Option<(Vec, Version)>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ObjectGcClaimProgress { + Open, + Pending, +} + impl NoKvFs where M: MetadataStore, O: ObjectStore, { + pub(super) fn ensure_object_gc_claim_record(&self) -> Result<(), MetadError> { + let key = object_gc_claim_key(self.mount); + if let Some(value) = self.metadata.get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + decode_object_gc_claim(&value.0)?; + return Ok(()); + } + let version = self.next_version()?; + let result = self.commit_metadata(MetadataCommand { + request_id: request_id( + b"initialize-object-gc-claim", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_claim(&ObjectGcClaim::Open)?)), + }], + watch: Vec::new(), + }); + match result { + Ok(_) + | Err(MetadError::Metadata(MetadataError::PredicateFailed)) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => { + let value = self + .metadata + .get( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec( + "object GC claim initialization was not durable".to_owned(), + ) + })?; + decode_object_gc_claim(&value.0)?; + Ok(()) + } + Err(err) => Err(err), + } + } + + /// Capture the durable Open epoch before any upload or historical planning. + /// Callers must carry this exact version into the metadata commit that makes + /// an object reference durable; reading a newer Open epoch at commit time + /// would permit an intervening GC delete cycle to go unnoticed. + pub(super) fn begin_object_reference_mutation( + &self, + ) -> Result { + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &object_gc_claim_key(self.mount), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + let claim = decode_object_gc_claim(&item.value.0)?; + let current_epoch = self.epoch.load(Ordering::Relaxed); + let claim_owner = match &claim { + ObjectGcClaim::Open => None, + ObjectGcClaim::Deleting { owner_epoch, .. } => Some(*owner_epoch), + }; + if claim_owner.is_some_and(|owner| owner > current_epoch) { + return Err(MetadError::StaleOwnerEpoch { + owner_epoch: current_epoch, + required_epoch: claim_owner.expect("future claim owner exists"), + }); + } + match claim { + ObjectGcClaim::Open => Ok(ObjectReferenceMutation { + claim_version: item.version, + }), + ObjectGcClaim::Deleting { .. } => { + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + } + } + } + + fn recover_object_gc_claim_locked( + &self, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result { + let key = object_gc_claim_key(self.mount); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + match decode_object_gc_claim(&item.value.0)? { + ObjectGcClaim::Open => Ok(ObjectGcClaimProgress::Open), + ObjectGcClaim::Deleting { + owner_epoch, + gc_record_key, + gc_record_version, + .. + } => { + let current_epoch = self.epoch.load(Ordering::Relaxed); + if owner_epoch > current_epoch { + return Err(MetadError::StaleOwnerEpoch { + owner_epoch: current_epoch, + required_epoch: owner_epoch, + }); + } + let Some(row) = self.metadata.get_versioned( + RecordFamily::Gc, + &gc_record_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + else { + self.release_object_gc_claim(item.version, None)?; + return Ok(ObjectGcClaimProgress::Open); + }; + if row.version.get() != gc_record_version { + self.release_object_gc_claim(item.version, None)?; + return Ok(ObjectGcClaimProgress::Open); + } + let row = crate::command::ScanItem { + key: gc_record_key, + value: row.value, + version: row.version, + }; + let record = match decode_object_gc_record(&row.value.0) { + Ok(record) => record, + Err(err) => { + self.quarantine_claimed_gc_row( + item.version, + &row, + &err.to_string(), + outcome, + )?; + return Ok(ObjectGcClaimProgress::Open); + } + }; + self.finish_claimed_gc_row(item.version, &row, &record, outcome) + } + } + } + + fn acquire_object_gc_claim( + &self, + row: &crate::command::ScanItem, + ) -> Result { + let open = self.begin_object_reference_mutation()?; + self.transition_object_gc_claim( + open.claim_version, + &ObjectGcClaim::Deleting { + owner_epoch: self.epoch.load(Ordering::Relaxed), + operation_token: open.claim_version.get(), + gc_record_key: row.key.clone(), + gc_record_version: row.version.get(), + }, + Some((&row.key, row.version)), + b"claim-object-delete", + ) + } + + fn transition_object_gc_claim( + &self, + expected_claim_version: Version, + next: &ObjectGcClaim, + gc_record: Option<(&[u8], Version)>, + request_domain: &[u8], + ) -> Result { + let key = object_gc_claim_key(self.mount); + let version = self.next_version()?; + let encoded = encode_object_gc_claim(next)?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(expected_claim_version), + }]; + if let Some((gc_key, gc_version)) = gc_record { + predicates.push(PredicateRef { + family: RecordFamily::Gc, + key: gc_key.to_vec(), + predicate: Predicate::VersionEquals(gc_version), + }); + } + let command = MetadataCommand { + request_id: request_id(request_domain, self.mount, InodeId::root(), version), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates, + mutations: vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encoded.clone())), + }], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + result @ (Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + })) => { + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("object GC claim transition was lost".to_owned()) + })?; + if item.version != version || item.value.0 != encoded { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + match result { + Ok(_) => Ok(item.version), + Err(err) => Err(err), + } + } + Err(err) => Err(err), + } + } + + fn release_object_gc_claim( + &self, + expected_claim_version: Version, + delete_gc_record: Option<(&[u8], Version)>, + ) -> Result<(), MetadError> { + let key = object_gc_claim_key(self.mount); + let version = self.next_version()?; + let encoded = encode_object_gc_claim(&ObjectGcClaim::Open)?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: key.clone(), + predicate: Predicate::VersionEquals(expected_claim_version), + }]; + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encoded.clone())), + }]; + if let Some((gc_key, gc_version)) = delete_gc_record { + predicates.push(PredicateRef { + family: RecordFamily::Gc, + key: gc_key.to_vec(), + predicate: Predicate::VersionEquals(gc_version), + }); + mutations.push(delete_mutation(RecordFamily::Gc, gc_key.to_vec())); + } + let command = MetadataCommand { + request_id: request_id( + b"release-object-gc-claim", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + match self.commit_metadata(command) { + result @ (Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + })) => { + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| MetadError::Codec("object GC claim was lost".to_owned()))?; + if item.version != version + || decode_object_gc_claim(&item.value.0)? != ObjectGcClaim::Open + { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + match result { + Ok(_) => Ok(()), + Err(err) => Err(err), + } + } + Err(err) => Err(err), + } + } + + fn delete_gc_row_under_durable_claim( + &self, + row: &crate::command::ScanItem, + record: &ObjectGcRecord, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result { + let claim_version = match self.acquire_object_gc_claim(row) { + Ok(version) => version, + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + return Ok(ObjectGcClaimProgress::Pending); + } + Err(err) => return Err(err), + }; + self.finish_claimed_gc_row(claim_version, row, record, outcome) + } + + fn finish_claimed_gc_row( + &self, + claim_version: Version, + row: &crate::command::ScanItem, + record: &ObjectGcRecord, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result { + let protected = match self.object_delete_is_protected(record) { + Ok(protected) => protected, + Err(err) => { + self.quarantine_claimed_gc_row(claim_version, row, &err.to_string(), outcome)?; + return Ok(ObjectGcClaimProgress::Open); + } + }; + if protected { + outcome.blocked_by_snapshots += 1; + self.release_object_gc_claim(claim_version, None)?; + return Ok(ObjectGcClaimProgress::Open); + } + self.ensure_owner_epoch_current()?; + let object_key = match ObjectKey::new(record.object_key.clone()) { + Ok(key) => key, + Err(err) => { + self.quarantine_claimed_gc_row(claim_version, row, &err.to_string(), outcome)?; + return Ok(ObjectGcClaimProgress::Open); + } + }; + outcome.attempted += 1; + let deletion = self.objects.delete(&object_key); + match deletion { + Ok(true) => outcome.deleted += 1, + Ok(false) => outcome.missing += 1, + Err(err) => { + let _ = self.release_object_gc_claim(claim_version, None); + return Err(err.into()); + } + } + self.release_object_gc_claim(claim_version, Some((&row.key, row.version)))?; + outcome.records_removed += 1; + Ok(ObjectGcClaimProgress::Open) + } + + fn quarantine_claimed_gc_row( + &self, + claim_version: Version, + row: &crate::command::ScanItem, + reason: &str, + outcome: &mut PendingObjectCleanupOutcome, + ) -> Result<(), MetadError> { + let claim_key = object_gc_claim_key(self.mount); + let digest: [u8; 32] = Sha256::digest(&row.key).into(); + let quarantine_key = object_gc_quarantine_key(self.mount, &digest); + let version = self.next_version()?; + let mut value = Vec::new(); + let reason = reason.as_bytes(); + value.extend_from_slice(&(row.key.len() as u32).to_be_bytes()); + value.extend_from_slice(&row.key); + value.extend_from_slice(&(row.value.0.len() as u32).to_be_bytes()); + value.extend_from_slice(&row.value.0); + let reason_len = reason.len().min(4096); + value.extend_from_slice(&(reason_len as u32).to_be_bytes()); + value.extend_from_slice(&reason[..reason_len]); + let command = MetadataCommand { + request_id: request_id( + b"quarantine-object-gc-row", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: claim_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim_version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: quarantine_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + Mutation { + family: RecordFamily::System, + key: claim_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_claim(&ObjectGcClaim::Open)?)), + }, + delete_mutation(RecordFamily::Gc, row.key.clone()), + Mutation { + family: RecordFamily::System, + key: quarantine_key, + op: MutationOp::Put, + value: Some(Value(value)), + }, + ], + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) => { + outcome.records_removed += 1; + Ok(()) + } + Err(err) => Err(err), + } + } + + fn object_delete_is_protected(&self, record: &ObjectGcRecord) -> Result { + if self + .object_retention()? + .version_floor + .is_some_and(|floor| floor.get() < record.enqueue_version) + { + return Ok(true); + } + let digest = object_key_digest(&record.object_key); + if !self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: fork_base_ref_inverse_prefix(self.mount, &digest), + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .is_empty() + { + return Ok(true); + } + let Some(body) = self.body_descriptor(record.inode)? else { + return Ok(false); + }; + let manifests = self.chunk_manifests_for_body_at_version( + record.inode, + &body, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + Ok(manifests + .iter() + .flat_map(|manifest| &manifest.slices) + .flat_map(|slice| &slice.blocks) + .any(|block| block.object_key == record.object_key)) + } + + fn release_releasing_fork_base_refs(&self, limit: usize) -> Result<(), MetadError> { + let mut remaining = limit.max(1); + for row in self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: fork_base_ref_release_prefix(self.mount), + start_after: None, + version: self.read_version()?, + limit: remaining, + purpose: ReadPurpose::WritePlanLocal, + })? { + let binding = match decode_fork_binding(&row.value.0) { + Ok(binding) => binding, + Err(err) => { + self.quarantine_base_ref_release_job(&row, &err.to_string())?; + continue; + } + }; + let binding_key = fork_binding_key(self.mount, binding.fork_root); + let binding_item = self.metadata.get_versioned( + RecordFamily::ForkBinding, + &binding_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + let live_binding = binding_item + .as_ref() + .map(|item| { + decode_fork_binding(&item.value.0) + .map_err(|err| MetadError::Codec(err.to_string())) + }) + .transpose()? + .filter(|live| { + live.state == ForkBindingState::Releasing + && live.base_ref_set_id == binding.base_ref_set_id + }); + let operation_key = restore_operation_key(self.mount, &binding.operation_digest); + let operation = if live_binding.is_some() { + self.metadata.get_versioned( + RecordFamily::System, + &operation_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + } else { + None + }; + let guard = BaseRefReleaseGuard { + family: RecordFamily::System, + key: row.key, + version: row.version, + binding: live_binding + .and_then(|_| binding_item.map(|item| (binding_key, item.version))), + operation: operation.map(|item| (operation_key, item.version)), + }; + let released = match self.release_base_ref_page(&binding, &guard, remaining) { + Ok(released) => released, + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => continue, + Err(MetadError::Codec(reason)) => { + self.quarantine_base_ref_release_job( + &crate::command::ScanItem { + key: guard.key.clone(), + value: Value(encode_fork_binding(&binding)), + version: guard.version, + }, + &reason, + )?; + continue; + } + Err(err) => return Err(err), + }; + remaining = remaining.saturating_sub(released); + if remaining == 0 { + return Ok(()); + } + } + Ok(()) + } + + fn quarantine_base_ref_release_job( + &self, + row: &crate::command::ScanItem, + reason: &str, + ) -> Result<(), MetadError> { + let digest: [u8; 32] = Sha256::digest(&row.key).into(); + let quarantine_key = fork_base_release_quarantine_key(self.mount, &digest); + let version = self.next_version()?; + let reason = reason.as_bytes(); + let reason_len = reason.len().min(4096); + let mut value = Vec::new(); + value.extend_from_slice(&(row.key.len() as u32).to_be_bytes()); + value.extend_from_slice(&row.key); + value.extend_from_slice(&(row.value.0.len() as u32).to_be_bytes()); + value.extend_from_slice(&row.value.0); + value.extend_from_slice(&(reason_len as u32).to_be_bytes()); + value.extend_from_slice(&reason[..reason_len]); + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"quarantine-fork-base-release", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: row.key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: quarantine_key.clone(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![ + delete_mutation(RecordFamily::System, row.key.clone()), + Mutation { + family: RecordFamily::System, + key: quarantine_key, + op: MutationOp::Put, + value: Some(Value(value)), + }, + ], + watch: Vec::new(), + })?; + Ok(()) + } + + fn release_base_ref_page( + &self, + binding: &ForkBinding, + guard: &BaseRefReleaseGuard, + limit: usize, + ) -> Result { + let read_version = self.read_version()?; + let owner_prefix = fork_base_ref_set_prefix(self.mount, binding.base_ref_set_id); + let cursor_key = fork_base_ref_release_cursor_key(self.mount, binding.base_ref_set_id); + let cursor = self.metadata.get_versioned( + RecordFamily::System, + &cursor_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let start_after = cursor.as_ref().map(|item| item.value.0.clone()); + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: owner_prefix.clone(), + start_after, + version: read_version, + limit: limit.max(1), + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + if let Some(cursor) = cursor { + self.reset_base_ref_release_cursor(guard, &cursor_key, cursor.version)?; + return Ok(1); + } + let released = self.release_restore_path_index_page(binding, guard, limit)?; + if released != 0 { + return Ok(released); + } + self.finalize_base_ref_release(guard)?; + return Ok(0); + } + + // A base-ref belongs to the restored borrower inode, not to the fork + // root's pathname. Renames and hardlinks can therefore outlive removal + // of the root binding. Keep every owner/inverse row while the borrower's + // current manifest still names that object; final unlink or a + // self-contained body replacement removes the manifest and makes the + // row releasable on a later pass. + let references = rows + .iter() + .map(|row| decode_fork_base_ref(&row.value.0)) + .collect::, _>>()?; + let mut required_by_owner = HashMap::<(InodeId, u64), HashSet>::new(); + for reference in &references { + required_by_owner + .entry((reference.owner_inode, reference.owner_generation)) + .or_default() + .insert(reference.object_key.clone()); + } + let mut retained = HashSet::new(); + for ((owner_inode, owner_generation), required) in required_by_owner { + retained.extend(self.current_manifest_borrowed_object_keys( + owner_inode, + owner_generation, + read_version, + &required, + )?); + } + + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: guard.family, + key: guard.key.clone(), + predicate: Predicate::VersionEquals(guard.version), + }]; + predicates.push(PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: cursor.as_ref().map_or(Predicate::NotExists, |item| { + Predicate::VersionEquals(item.version) + }), + }); + let mut mutations = Vec::with_capacity(rows.len() * 3 + 1); + mutations.push(Mutation { + family: RecordFamily::System, + key: cursor_key, + op: MutationOp::Put, + value: Some(Value( + rows.last() + .expect("non-empty base-ref release page") + .key + .clone(), + )), + }); + for (row, reference) in rows.iter().zip(references) { + if retained.contains(&reference.object_key) { + continue; + } + let digest = + fork_base_ref_digest_from_owner_key(self.mount, binding.base_ref_set_id, &row.key) + .ok_or_else(|| { + MetadError::Codec("invalid fork base-ref owner key".to_owned()) + })?; + let inverse_key = + fork_base_ref_inverse_key(self.mount, &digest, binding.base_ref_set_id); + let inverse = match self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + Some(inverse) => inverse, + None => { + self.repair_missing_fork_base_inverse(binding, guard, row, &inverse_key)?; + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + }; + let gc_key = gc_released_base_ref_key( + self.mount, + version.get(), + binding.base_ref_set_id, + &digest, + ); + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: gc_key.clone(), + predicate: Predicate::NotExists, + }, + ]); + mutations.extend([ + delete_mutation(RecordFamily::System, row.key.clone()), + delete_mutation(RecordFamily::System, inverse_key), + Mutation { + family: RecordFamily::Gc, + key: gc_key, + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&ObjectGcRecord { + inode: reference.owner_inode, + generation: reference.owner_generation, + object_key: reference.object_key, + size: reference.size, + digest_uri: reference.digest_uri, + enqueue_version: version.get(), + enqueue_unix_ms: self.now_ms(), + }))), + }, + ]); + } + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"release-fork-base-ref-page", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: rows[0].key.clone(), + predicates, + mutations, + watch: Vec::new(), + })?; + // The cursor is advanced even when the whole page is retained. This is + // bounded useful work and prevents an escaped borrower at the beginning + // of the key range from starving releasable rows later in the set. A + // subsequent empty tail pass resets the cursor and starts another round. + Ok(rows.len()) + } + + fn reset_base_ref_release_cursor( + &self, + guard: &BaseRefReleaseGuard, + cursor_key: &[u8], + cursor_version: Version, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"reset-fork-base-ref-release-cursor", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.to_vec(), + predicates: vec![ + PredicateRef { + family: guard.family, + key: guard.key.clone(), + predicate: Predicate::VersionEquals(guard.version), + }, + PredicateRef { + family: RecordFamily::System, + key: cursor_key.to_vec(), + predicate: Predicate::VersionEquals(cursor_version), + }, + ], + mutations: vec![delete_mutation(RecordFamily::System, cursor_key.to_vec())], + watch: Vec::new(), + })?; + Ok(()) + } + + fn current_manifest_borrowed_object_keys( + &self, + owner_inode: InodeId, + owner_generation: u64, + version: Version, + required: &HashSet, + ) -> Result, MetadError> { + const MANIFEST_SCAN_PAGE: usize = 128; + let prefix = chunk_manifest_prefix(self.mount, owner_inode, owner_generation); + let mut start_after = None; + let mut retained = HashSet::new(); + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: MANIFEST_SCAN_PAGE, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + break; + } + let exhausted = rows.len() < MANIFEST_SCAN_PAGE; + start_after = rows.last().map(|row| row.key.clone()); + for row in rows { + if chunk_index_from_manifest_key(&row.key)? == BODY_SUMMARY_CHUNK_INDEX { + continue; + } + let manifest = decode_chunk_manifest(&row.value.0) + .map_err(|err| MetadataError::Backend(err.to_string()))?; + for block in manifest.slices.iter().flat_map(|slice| slice.blocks.iter()) { + if required.contains(&block.object_key) { + retained.insert(block.object_key.clone()); + } + } + } + if retained.len() == required.len() || exhausted { + break; + } + } + Ok(retained) + } + + fn release_restore_path_index_page( + &self, + binding: &ForkBinding, + guard: &BaseRefReleaseGuard, + limit: usize, + ) -> Result { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_path_index_set_prefix(self.mount, binding.base_ref_set_id), + start_after: None, + version: self.read_version()?, + limit: limit.max(1), + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(0); + } + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: guard.family, + key: guard.key.clone(), + predicate: Predicate::VersionEquals(guard.version), + }]; + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in &rows { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + let marker = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + let inverse_key = + fork_shadow_key(self.mount, marker.dentry.parent, &marker.dentry.name); + let inverse = self.metadata.get_versioned( + RecordFamily::ForkShadow, + &inverse_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + if let Some(inverse) = inverse { + let (inverse_set, _) = decode_restore_shadow_inverse(&inverse.value.0)?; + if inverse_set == binding.base_ref_set_id { + predicates.push(PredicateRef { + family: RecordFamily::ForkShadow, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }); + mutations.push(delete_mutation(RecordFamily::ForkShadow, inverse_key)); + } + } + } + let released = rows.len(); + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"release-restore-path-index-page", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: rows[0].key.clone(), + predicates, + mutations, + watch: Vec::new(), + })?; + Ok(released) + } + + fn repair_missing_fork_base_inverse( + &self, + binding: &ForkBinding, + guard: &BaseRefReleaseGuard, + owner: &crate::command::ScanItem, + inverse_key: &[u8], + ) -> Result<(), MetadError> { + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"repair-fork-base-ref-inverse", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: inverse_key.to_vec(), + predicates: vec![ + PredicateRef { + family: guard.family, + key: guard.key.clone(), + predicate: Predicate::VersionEquals(guard.version), + }, + PredicateRef { + family: RecordFamily::System, + key: owner.key.clone(), + predicate: Predicate::VersionEquals(owner.version), + }, + PredicateRef { + family: RecordFamily::System, + key: inverse_key.to_vec(), + predicate: Predicate::NotExists, + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: inverse_key.to_vec(), + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }], + watch: Vec::new(), + })?; + Ok(()) + } + + fn finalize_base_ref_release(&self, guard: &BaseRefReleaseGuard) -> Result<(), MetadError> { + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: guard.family, + key: guard.key.clone(), + predicate: Predicate::VersionEquals(guard.version), + }]; + let mut mutations = vec![delete_mutation(guard.family, guard.key.clone())]; + if let Some((binding_key, binding_version)) = &guard.binding { + predicates.push(PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(*binding_version), + }); + mutations.push(delete_mutation( + RecordFamily::ForkBinding, + binding_key.clone(), + )); + } + if let Some((operation_key, operation_version)) = &guard.operation { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(*operation_version), + }); + mutations.push(delete_mutation(RecordFamily::System, operation_key.clone())); + } + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"complete-fork-base-ref-release", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: guard.family, + primary_key: guard.key.clone(), + predicates, + mutations, + watch: Vec::new(), + })?; + Ok(()) + } + pub fn cleanup_staged_objects( &self, - staged: &StagedObjectSet, - ) -> Result { - self.objects.delete_staged(staged).map_err(Into::into) + staged: &StagedObjectSet, + ) -> Result { + self.objects.delete_staged(staged).map_err(Into::into) + } + + fn advance_pending_restore_staging_cleanup(&self, limit: usize) -> Result { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_staging_cleanup_prefix(self.mount), + start_after: None, + version: self.read_version()?, + limit: limit.clamp(1, 128), + purpose: ReadPurpose::WritePlanLocal, + })?; + let mut advanced = 0; + for state in rows { + match self.advance_restore_staging_cleanup_state(&state, limit) { + Ok(()) | Err(MetadError::Metadata(MetadataError::PredicateFailed)) => {} + Err(err) => return Err(err), + } + advanced += 1; + } + Ok(advanced) + } + + fn advance_restore_staging_cleanup_state( + &self, + state: &ScanItem, + limit: usize, + ) -> Result<(), MetadError> { + let (disposition, binding) = decode_restore_staging_cleanup(&state.value.0)?; + let member_prefix = restore_staging_member_prefix(self.mount, binding.base_ref_set_id); + let member = self + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: member_prefix, + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .into_iter() + .next(); + if let Some(member) = member { + if disposition == RestoreStagingCleanupDisposition::ForgetMembers { + return self.forget_restore_staging_member(state, &member); + } + return self.cleanup_restore_staging_member(state, &binding, &member); + } + if disposition != RestoreStagingCleanupDisposition::ForgetMembers { + if self.delete_restore_staging_base_ref_page(state, &binding, limit)? != 0 { + return Ok(()); + } + if self.delete_restore_staging_path_index_page(state, &binding, limit)? != 0 { + return Ok(()); + } + } + self.finalize_restore_staging_cleanup(state, disposition, &binding) + } + + fn forget_restore_staging_member( + &self, + state: &ScanItem, + member: &ScanItem, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + self.commit_metadata(MetadataCommand { + request_id: request_id( + b"forget-restore-staging-member", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: member.key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: state.key.clone(), + predicate: Predicate::VersionEquals(state.version), + }, + PredicateRef { + family: RecordFamily::System, + key: member.key.clone(), + predicate: Predicate::VersionEquals(member.version), + }, + ], + mutations: vec![delete_mutation(RecordFamily::System, member.key.clone())], + watch: Vec::new(), + })?; + Ok(()) + } + + fn cleanup_restore_staging_member( + &self, + state: &ScanItem, + binding: &ForkBinding, + member: &ScanItem, + ) -> Result<(), MetadError> { + let inode = restore_staging_member_inode(self.mount, binding.base_ref_set_id, &member.key) + .ok_or_else(|| MetadError::Codec("invalid restore staging member key".to_owned()))?; + if self.cleanup_restore_staging_family_page( + state, + inode, + RecordFamily::Dentry, + dentry_prefix(self.mount, inode), + )? { + return Ok(()); + } + if self.cleanup_restore_staging_family_page( + state, + inode, + RecordFamily::Xattr, + xattr_prefix(self.mount, inode), + )? { + return Ok(()); + } + let inode_key = inode_key(self.mount, inode); + let attr = self.metadata.get_versioned( + RecordFamily::Inode, + &inode_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )?; + if let Some(attr) = &attr { + let attr_value = decode_inode_attr(&attr.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + let manifests = self.metadata.scan(ScanRequest { + family: RecordFamily::ChunkManifest, + prefix: chunk_manifest_prefix(self.mount, inode, attr_value.generation), + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })?; + if let Some(manifest) = manifests.first() { + let version = self.next_version()?; + let command = self.restore_staging_manifest_cleanup_command( + &state.key, + state.version, + inode, + attr_value.generation, + manifest, + version, + self.now_ms(), + )?; + validate_restore_command_bounds(&command, "restore staging manifest cleanup")?; + self.commit_metadata(command)?; + return Ok(()); + } + } + self.finish_restore_staging_member(state, binding, member, inode, attr.as_ref()) + } + + #[allow(clippy::too_many_arguments)] + fn restore_staging_manifest_cleanup_command( + &self, + cleanup_key: &[u8], + cleanup_version: Version, + inode: InodeId, + generation: u64, + row: &ScanItem, + version: Version, + enqueue_unix_ms: u64, + ) -> Result { + let chunk_index = chunk_index_from_manifest_key(&row.key)?; + let mut mutations = Vec::new(); + if chunk_index != BODY_SUMMARY_CHUNK_INDEX { + let manifest = decode_chunk_manifest(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + for (block_index, block) in manifest + .slices + .iter() + .flat_map(|slice| slice.blocks.iter()) + .enumerate() + { + if !self.owns_block_object_key(inode, generation, &block.object_key) { + continue; + } + mutations.push(Mutation { + family: RecordFamily::Gc, + key: gc_object_key( + self.mount, + version.get(), + inode, + generation, + chunk_index, + block_index as u64, + ), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&ObjectGcRecord { + inode, + generation, + object_key: block.object_key.clone(), + size: block.len, + digest_uri: block.digest_uri.clone(), + enqueue_version: version.get(), + enqueue_unix_ms, + }))), + }); + } + } + mutations.push(delete_mutation( + RecordFamily::ChunkManifest, + row.key.clone(), + )); + Ok(MetadataCommand { + request_id: request_id( + b"restore-staging-cleanup-manifest", + self.mount, + inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::ChunkManifest, + primary_key: row.key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: cleanup_key.to_vec(), + predicate: Predicate::VersionEquals(cleanup_version), + }, + PredicateRef { + family: RecordFamily::ChunkManifest, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }, + ], + mutations, + watch: Vec::new(), + }) + } + + fn cleanup_restore_staging_family_page( + &self, + state: &ScanItem, + inode: InodeId, + family: RecordFamily, + prefix: Vec, + ) -> Result { + const PAGE: usize = 64; + let rows = self.metadata.scan(ScanRequest { + family, + prefix, + start_after: None, + version: self.read_version()?, + limit: PAGE, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(false); + } + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: state.key.clone(), + predicate: Predicate::VersionEquals(state.version), + }]; + predicates.extend(rows.iter().map(|row| PredicateRef { + family, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + })); + let command = MetadataCommand { + request_id: request_id( + b"cleanup-restore-staging-family", + self.mount, + inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: family, + primary_key: rows[0].key.clone(), + predicates, + mutations: rows + .iter() + .map(|row| delete_mutation(family, row.key.clone())) + .collect(), + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore staging family cleanup")?; + self.commit_metadata(command)?; + Ok(true) + } + + fn finish_restore_staging_member( + &self, + state: &ScanItem, + binding: &ForkBinding, + member: &ScanItem, + inode: InodeId, + attr: Option<&crate::command::ReadItem>, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + let inode_key = inode_key(self.mount, inode); + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: state.key.clone(), + predicate: Predicate::VersionEquals(state.version), + }, + PredicateRef { + family: RecordFamily::System, + key: member.key.clone(), + predicate: Predicate::VersionEquals(member.version), + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry_prefix(self.mount, inode), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::Xattr, + key: xattr_prefix(self.mount, inode), + predicate: Predicate::PrefixEmpty, + }, + ]; + if let Some(attr) = attr { + predicates.push(PredicateRef { + family: RecordFamily::Inode, + key: inode_key.clone(), + predicate: Predicate::VersionEquals(attr.version), + }); + let decoded = decode_inode_attr(&attr.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + predicates.push(PredicateRef { + family: RecordFamily::ChunkManifest, + key: chunk_manifest_prefix(self.mount, inode, decoded.generation), + predicate: Predicate::PrefixEmpty, + }); + } + let mut mutations = vec![delete_mutation(RecordFamily::System, member.key.clone())]; + if attr.is_some() { + mutations.push(delete_mutation(RecordFamily::Inode, inode_key)); + } + let command = MetadataCommand { + request_id: request_id( + b"finish-restore-staging-member", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: member.key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore staging member finish")?; + self.commit_metadata(command)?; + Ok(()) + } + + fn delete_restore_staging_base_ref_page( + &self, + state: &ScanItem, + binding: &ForkBinding, + limit: usize, + ) -> Result { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: fork_base_ref_set_prefix(self.mount, binding.base_ref_set_id), + start_after: None, + version: self.read_version()?, + limit: limit.clamp(1, 128), + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(0); + } + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: state.key.clone(), + predicate: Predicate::VersionEquals(state.version), + }]; + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in &rows { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + let digest = + fork_base_ref_digest_from_owner_key(self.mount, binding.base_ref_set_id, &row.key) + .ok_or_else(|| MetadError::Codec("invalid staging base-ref key".to_owned()))?; + let inverse_key = + fork_base_ref_inverse_key(self.mount, &digest, binding.base_ref_set_id); + if let Some(inverse) = self.metadata.get_versioned( + RecordFamily::System, + &inverse_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }); + mutations.push(delete_mutation(RecordFamily::System, inverse_key)); + } + } + let command = MetadataCommand { + request_id: request_id( + b"delete-restore-staging-base-refs", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: rows[0].key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore staging base-ref cleanup")?; + self.commit_metadata(command)?; + Ok(rows.len()) + } + + fn delete_restore_staging_path_index_page( + &self, + state: &ScanItem, + binding: &ForkBinding, + limit: usize, + ) -> Result { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_path_index_set_prefix(self.mount, binding.base_ref_set_id), + start_after: None, + version: self.read_version()?, + limit: limit.clamp(1, 128), + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + return Ok(0); + } + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: state.key.clone(), + predicate: Predicate::VersionEquals(state.version), + }]; + let mut mutations = Vec::with_capacity(rows.len() * 2); + for row in &rows { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }); + mutations.push(delete_mutation(RecordFamily::System, row.key.clone())); + let marker = decode_dentry_projection(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + let inverse_key = + fork_shadow_key(self.mount, marker.dentry.parent, &marker.dentry.name); + if let Some(inverse) = self.metadata.get_versioned( + RecordFamily::ForkShadow, + &inverse_key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? { + let (set_id, _) = decode_restore_shadow_inverse(&inverse.value.0)?; + if set_id == binding.base_ref_set_id { + predicates.push(PredicateRef { + family: RecordFamily::ForkShadow, + key: inverse_key.clone(), + predicate: Predicate::VersionEquals(inverse.version), + }); + mutations.push(delete_mutation(RecordFamily::ForkShadow, inverse_key)); + } + } + } + let command = MetadataCommand { + request_id: request_id( + b"delete-restore-staging-path-index", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: rows[0].key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore staging path-index cleanup")?; + self.commit_metadata(command)?; + Ok(rows.len()) + } + + fn finalize_restore_staging_cleanup( + &self, + state: &ScanItem, + disposition: RestoreStagingCleanupDisposition, + binding: &ForkBinding, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::System, + key: state.key.clone(), + predicate: Predicate::VersionEquals(state.version), + }, + PredicateRef { + family: RecordFamily::System, + key: restore_staging_member_prefix(self.mount, binding.base_ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + ]; + let mut mutations = vec![delete_mutation(RecordFamily::System, state.key.clone())]; + if disposition != RestoreStagingCleanupDisposition::ForgetMembers { + predicates.extend([ + PredicateRef { + family: RecordFamily::System, + key: fork_base_ref_set_prefix(self.mount, binding.base_ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + PredicateRef { + family: RecordFamily::System, + key: restore_path_index_set_prefix(self.mount, binding.base_ref_set_id), + predicate: Predicate::PrefixEmpty, + }, + ]); + let binding_key = fork_binding_key(self.mount, binding.fork_root); + let operation_key = restore_operation_key(self.mount, &binding.operation_digest); + let read_version = self.read_version()?; + let binding_item = self + .metadata + .get_versioned( + RecordFamily::ForkBinding, + &binding_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + let operation = self + .metadata + .get_versioned( + RecordFamily::System, + &operation_key, + read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + predicates.extend([ + PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + predicate: Predicate::VersionEquals(binding_item.version), + }, + PredicateRef { + family: RecordFamily::System, + key: operation_key.clone(), + predicate: Predicate::VersionEquals(operation.version), + }, + ]); + match disposition { + RestoreStagingCleanupDisposition::ResetForRetry => { + let mut prepared = binding.clone(); + prepared.state = ForkBindingState::Preparing; + let clean_key = restore_staging_clean_key(self.mount, binding.base_ref_set_id); + predicates.push(PredicateRef { + family: RecordFamily::System, + key: clean_key.clone(), + predicate: Predicate::NotExists, + }); + mutations.extend([ + Mutation { + family: RecordFamily::ForkBinding, + key: binding_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&prepared))), + }, + Mutation { + family: RecordFamily::System, + key: operation_key, + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(&prepared))), + }, + Mutation { + family: RecordFamily::System, + key: clean_key, + op: MutationOp::Put, + value: Some(Value(Vec::new())), + }, + ]); + } + RestoreStagingCleanupDisposition::Discard => { + let hold_key = fork_base_hold_key( + self.mount, + binding.pinned_read_version, + binding.base_ref_set_id, + ); + if let Some(hold) = self.metadata.get_versioned( + RecordFamily::System, + &hold_key, + read_version, + ReadPurpose::WritePlanLocal, + )? { + predicates.push(PredicateRef { + family: RecordFamily::System, + key: hold_key.clone(), + predicate: Predicate::VersionEquals(hold.version), + }); + mutations.push(delete_mutation(RecordFamily::System, hold_key)); + } + mutations.push(delete_mutation(RecordFamily::ForkBinding, binding_key)); + mutations.push(delete_mutation(RecordFamily::System, operation_key)); + } + RestoreStagingCleanupDisposition::ForgetMembers => unreachable!(), + } + } + let command = MetadataCommand { + request_id: request_id( + b"finalize-restore-staging-cleanup", + self.mount, + binding.fork_root, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: state.key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + validate_restore_command_bounds(&command, "restore staging cleanup finalization")?; + self.commit_metadata(command)?; + Ok(()) } pub fn cleanup_pending_objects( @@ -25,90 +1746,327 @@ where limit: usize, read_lease_grace: Duration, ) -> Result { - // Reap expired snapshot pins first so the retention floor reflects only - // live snapshots before deciding what is reclaimable. - let snapshot_reap = self.reclaim_expired_snapshot_pins(limit)?; + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let mut outcome = PendingObjectCleanupOutcome::default(); + if self.recover_object_gc_claim_locked(&mut outcome)? == ObjectGcClaimProgress::Pending { + return Ok(outcome); + } + self.advance_pending_restore_staging_cleanup(limit)?; + outcome.snapshot_reap = self.reclaim_expired_snapshot_pins(limit)?; + self.release_releasing_fork_base_refs(limit)?; + self.cleanup_stale_path_index_page(limit)?; let now_ms = self.now_ms(); let grace_ms = duration_millis_u64(read_lease_grace); - let version = self.read_version()?; let rows = self.metadata.scan(ScanRequest { family: RecordFamily::Gc, prefix: gc_queue_prefix(self.mount), start_after: None, - version, + version: self.read_version()?, limit, purpose: ReadPurpose::UserStrong, })?; - if rows.is_empty() { - return Ok(PendingObjectCleanupOutcome { - snapshot_reap, - ..PendingObjectCleanupOutcome::default() - }); - } - let retention_floor = self.history_retention_floor()?; - - let mut outcome = PendingObjectCleanupOutcome { - snapshot_reap, - scanned: rows.len(), - blocked_by_snapshots: 0, - blocked_by_read_leases: 0, - attempted: 0, - deleted: 0, - missing: 0, - records_removed: 0, - }; - let mut cleaned_keys = Vec::with_capacity(rows.len()); + outcome.scanned += rows.len(); for row in rows { - let record = decode_object_gc_record(&row.value.0) - .map_err(|err| MetadError::Codec(err.to_string()))?; - if retention_floor.is_some_and(|floor| floor.get() < record.enqueue_version) { - outcome.blocked_by_snapshots += 1; - continue; - } + let record = match decode_object_gc_record(&row.value.0) { + Ok(record) => record, + Err(err) => { + let claim_version = self.acquire_object_gc_claim(&row)?; + self.quarantine_claimed_gc_row( + claim_version, + &row, + &err.to_string(), + &mut outcome, + )?; + continue; + } + }; if now_ms < record.enqueue_unix_ms.saturating_add(grace_ms) { outcome.blocked_by_read_leases += 1; continue; } - let key = ObjectKey::new(record.object_key)?; - outcome.attempted += 1; - if self.objects.delete(&key)? { - outcome.deleted += 1; - } else { - outcome.missing += 1; + if self.delete_gc_row_under_durable_claim(&row, &record, &mut outcome)? + == ObjectGcClaimProgress::Pending + { + break; } - cleaned_keys.push(row.key); } + Ok(outcome) + } - if cleaned_keys.is_empty() { - return Ok(outcome); + fn cleanup_stale_path_index_page(&self, limit: usize) -> Result { + const MAX_PAGE: usize = 128; + let page_size = limit.clamp(1, MAX_PAGE); + let cursor_key = path_index_gc_cursor_key(self.mount); + let read_version = self.read_version()?; + let cursor = self.metadata.get_versioned( + RecordFamily::System, + &cursor_key, + read_version, + ReadPurpose::WritePlanLocal, + )?; + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::PathIndex, + prefix: path_index_prefix(self.mount, &[]), + start_after: cursor.as_ref().map(|item| item.value.0.clone()), + version: read_version, + limit: page_size, + purpose: ReadPurpose::WritePlanLocal, + })?; + if rows.is_empty() { + if let Some(cursor) = cursor { + self.reset_path_index_gc_cursor(&cursor_key, cursor.version)?; + return Ok(1); + } + return Ok(0); + } + let stale = rows + .iter() + .map(|row| { + self.path_index_row_is_stale(row, read_version) + .map(|stale| (row, stale)) + }) + .collect::, _>>()? + .into_iter() + .filter_map(|(row, stale)| stale.then_some(row)) + .collect::>(); + let last_key = rows + .last() + .expect("non-empty path-index GC page") + .key + .clone(); + let version = self.next_version()?; + let mut predicates = vec![PredicateRef { + family: RecordFamily::System, + key: cursor_key.clone(), + predicate: cursor.as_ref().map_or(Predicate::NotExists, |item| { + Predicate::VersionEquals(item.version) + }), + }]; + predicates.extend(stale.iter().map(|row| PredicateRef { + family: RecordFamily::PathIndex, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + })); + let mut mutations = vec![Mutation { + family: RecordFamily::System, + key: cursor_key.clone(), + op: MutationOp::Put, + value: Some(Value(last_key.clone())), + }]; + mutations.extend( + stale + .iter() + .map(|row| delete_mutation(RecordFamily::PathIndex, row.key.clone())), + ); + let command = MetadataCommand { + request_id: request_id( + b"cleanup-stale-path-index-page", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.clone(), + predicates, + mutations, + watch: Vec::new(), + }; + match self.commit_metadata(command) { + Ok(_) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => Ok(rows.len()), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) => { + // One concurrent PathIndex update must not block unrelated stale + // rows in this page. Delete each candidate with its captured + // version, then advance the durable cursor independently. + for row in stale { + self.delete_stale_path_index_row(row)?; + } + self.advance_path_index_gc_cursor(&cursor_key, cursor.as_ref(), &last_key)?; + Ok(rows.len()) + } + Err(err) => Err(err), } + } - let commit_version = self.next_version()?; - let records_removed = cleaned_keys.len(); - self.commit_metadata(MetadataCommand { + fn path_index_row_is_stale( + &self, + row: &crate::command::ScanItem, + version: Version, + ) -> Result { + let prefix = path_index_prefix(self.mount, &[]); + let Some(suffix) = row.key.strip_prefix(prefix.as_slice()) else { + return Ok(true); + }; + let components = suffix + .split(|byte| *byte == PATH_INDEX_DELIMITER) + .map(|component| { + DentryName::new(component.to_vec()) + .map_err(|err| MetadError::InvalidPath(err.to_string())) + }) + .collect::, _>>(); + let components = match components { + Ok(components) if !components.is_empty() => components, + Ok(_) | Err(_) => return Ok(true), + }; + let (name, parents) = components + .split_last() + .expect("non-empty PathIndex components"); + let parent = match self.resolve_components_as_directory_from_at_version_for_purpose( + InodeId::root(), + parents, + version, + ReadPurpose::WritePlanLocal, + ) { + Ok(parent) => parent, + Err(MetadError::NotFound | MetadError::NotDirectory) => return Ok(true), + Err(err) => return Err(err), + }; + let Some((canonical, dentry_version)) = self.lookup_plus_at_version_for_purpose( + parent, + name, + version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(true); + }; + let indexed = decode_dentry_projection(&row.value.0) + .map(DentryWithAttr::from) + .map_err(|err| MetadError::Codec(err.to_string()))?; + Ok(row.version != dentry_version || indexed != canonical) + } + + fn delete_stale_path_index_row( + &self, + row: &crate::command::ScanItem, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + match self.commit_metadata(MetadataCommand { request_id: request_id( - b"cleanup-objects", + b"cleanup-stale-path-index-row", self.mount, InodeId::root(), - commit_version, + version, ), kind: CommandKind::CleanupObjects, - read_version: predecessor(commit_version)?, - commit_version, - primary_family: RecordFamily::Gc, - primary_key: gc_queue_prefix(self.mount), - predicates: Vec::new(), - mutations: cleaned_keys - .into_iter() - .map(|key| delete_mutation(RecordFamily::Gc, key)) - .collect(), + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::PathIndex, + primary_key: row.key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::PathIndex, + key: row.key.clone(), + predicate: Predicate::VersionEquals(row.version), + }], + mutations: vec![delete_mutation(RecordFamily::PathIndex, row.key.clone())], watch: Vec::new(), - })?; - outcome.records_removed = records_removed; - Ok(outcome) + }) { + Ok(_) + | Err(MetadError::Metadata(MetadataError::PredicateFailed)) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => Ok(()), + Err(err) => Err(err), + } + } + + fn advance_path_index_gc_cursor( + &self, + cursor_key: &[u8], + cursor: Option<&crate::command::ReadItem>, + last_key: &[u8], + ) -> Result<(), MetadError> { + let version = self.next_version()?; + match self.commit_metadata(MetadataCommand { + request_id: request_id( + b"advance-path-index-gc-cursor", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: cursor_key.to_vec(), + predicate: cursor.map_or(Predicate::NotExists, |item| { + Predicate::VersionEquals(item.version) + }), + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: cursor_key.to_vec(), + op: MutationOp::Put, + value: Some(Value(last_key.to_vec())), + }], + watch: Vec::new(), + }) { + Ok(_) + | Err(MetadError::Metadata(MetadataError::PredicateFailed)) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => Ok(()), + Err(err) => Err(err), + } + } + + fn reset_path_index_gc_cursor( + &self, + cursor_key: &[u8], + cursor_version: Version, + ) -> Result<(), MetadError> { + let version = self.next_version()?; + match self.commit_metadata(MetadataCommand { + request_id: request_id( + b"reset-path-index-gc-cursor", + self.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version)?, + commit_version: version, + primary_family: RecordFamily::System, + primary_key: cursor_key.to_vec(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: cursor_key.to_vec(), + predicate: Predicate::VersionEquals(cursor_version), + }], + mutations: vec![delete_mutation(RecordFamily::System, cursor_key.to_vec())], + watch: Vec::new(), + }) { + Ok(_) + | Err(MetadError::Metadata(MetadataError::PredicateFailed)) + | Err(MetadError::SyncLogArchiveFailed { + committed: true, .. + }) => Ok(()), + Err(err) => Err(err), + } } pub fn cleanup_history(&self, limit: usize) -> Result { + let _gate = self + .object_gc_gate + .lock() + .unwrap_or_else(|err| err.into_inner()); + let mut recovery_outcome = PendingObjectCleanupOutcome::default(); + if self.recover_object_gc_claim_locked(&mut recovery_outcome)? + == ObjectGcClaimProgress::Pending + { + return Ok(HistoryPruneOutcome::default()); + } const RETENTION_RETRIES: usize = 8; for attempt in 0..RETENTION_RETRIES { let before = self.metadata.history_retention_epoch()?; @@ -122,7 +2080,9 @@ where retention_epoch: after, limit, }) { - Err(MetadataError::PredicateFailed) if attempt + 1 < RETENTION_RETRIES => continue, + Err(MetadataError::PredicateFailed) if attempt + 1 < RETENTION_RETRIES => { + continue; + } result => return result.map_err(Into::into), } } @@ -174,9 +2134,60 @@ where let version = Version::new(pin.read_version)?; floor = Some(floor.map_or(version, |floor| floor.min(version))); } + if let Some(version) = self.preparing_fork_hold_floor()? { + floor = Some(floor.map_or(version, |floor| floor.min(version))); + } Ok(floor) } + /// Object GC can use exact completed fork references. Preparing forks still + /// conservatively hold their whole read version because materialization has + /// not yet produced the final object-key set. + fn object_retention(&self) -> Result { + let now_ms = self.now_ms(); + let mut version_floor: Option = None; + for row in self.metadata.scan(ScanRequest { + family: RecordFamily::Snapshot, + prefix: snapshot_pin_prefix(self.mount), + start_after: None, + version: self.read_version()?, + limit: 0, + purpose: ReadPurpose::UserStrong, + })? { + let pin = decode_snapshot_pin(&row.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if now_ms < pin.lease_expires_unix_ms { + let version = Version::new(pin.read_version)?; + version_floor = Some(version_floor.map_or(version, |floor| floor.min(version))); + } + } + if let Some(version) = self.preparing_fork_hold_floor()? { + version_floor = Some(version_floor.map_or(version, |floor| floor.min(version))); + } + Ok(ObjectRetention { version_floor }) + } + + fn preparing_fork_hold_floor(&self) -> Result, MetadError> { + self.metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: fork_base_hold_prefix(self.mount), + start_after: None, + version: self.read_version()?, + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + })? + .first() + .map(|row| { + let read_version = + fork_base_hold_read_version(self.mount, &row.key).ok_or_else(|| { + MetadError::Codec("invalid preparing fork hold key".to_owned()) + })?; + Version::new(read_version).map_err(Into::into) + }) + .transpose() + } + /// Delete pin records whose lease has expired. Every delete is fenced by the /// record version observed by the scan, so an old reaper pass cannot remove /// a pin that changed after it was selected. @@ -343,7 +2354,7 @@ where enqueue_version: Version, retained_object_keys: &HashSet, ) -> Result, MetadError> { - let enqueue_unix_ms = current_time_ms(); + let enqueue_unix_ms = self.now_ms(); let rows = self.metadata.scan(ScanRequest { family: RecordFamily::ChunkManifest, prefix: chunk_manifest_prefix(self.mount, inode, generation), @@ -465,7 +2476,7 @@ where enqueue_version: Version, retained_object_keys: &HashSet, ) -> Vec { - let enqueue_unix_ms = current_time_ms(); + let enqueue_unix_ms = self.now_ms(); let mut mutations = vec![delete_mutation( RecordFamily::ChunkManifest, chunk_manifest_key(self.mount, inode, generation, BODY_SUMMARY_CHUNK_INDEX), diff --git a/crates/nokv-meta/src/service/lifecycle.rs b/crates/nokv-meta/src/service/lifecycle.rs index 8597e8370..c34ebb829 100644 --- a/crates/nokv-meta/src/service/lifecycle.rs +++ b/crates/nokv-meta/src/service/lifecycle.rs @@ -13,6 +13,8 @@ where objects, allocator_gate: Mutex::new(()), backup_gate: Mutex::new(()), + restore_gate: Mutex::new(()), + object_gc_gate: Mutex::new(()), epoch_fence: RwLock::new(()), path_resolution_cache: new_path_resolution_cache_shards(), path_index_lookup_cache: new_path_index_lookup_cache_shards(), @@ -88,6 +90,10 @@ where /// and the allocator floor is then seeded into this shard's high-bit subspace /// exactly like [`Self::with_shard_index`]. So callers must NOT chain /// `.with_shard_index(...)` after `open_existing`; pass the index here. + /// + /// This constructor only reconstructs process-local state. Durable GC and + /// restore cleanup is resumed by the ordinary cleanup worker after the + /// server has acquired ownership and installed its durability policy. pub fn open_existing( mount: MountId, metadata: M, @@ -110,6 +116,8 @@ where objects, allocator_gate: Mutex::new(()), backup_gate: Mutex::new(()), + restore_gate: Mutex::new(()), + object_gc_gate: Mutex::new(()), epoch_fence: RwLock::new(()), path_resolution_cache: new_path_resolution_cache_shards(), path_index_lookup_cache: new_path_index_lookup_cache_shards(), diff --git a/crates/nokv-meta/src/service/namespace.rs b/crates/nokv-meta/src/service/namespace.rs index 17d9c31a0..11e22beb8 100644 --- a/crates/nokv-meta/src/service/namespace.rs +++ b/crates/nokv-meta/src/service/namespace.rs @@ -28,6 +28,7 @@ where O: ObjectStore, { pub fn bootstrap_root(&self, mode: u32, uid: u32, gid: u32) -> Result { + self.ensure_object_gc_claim_record()?; let version = self.next_version()?; let root = directory_attr(InodeId::root(), mode, uid, gid, version.get()); let command = MetadataCommand { @@ -254,6 +255,7 @@ where uid: u32, gid: u32, ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; if target.is_empty() || target.contains(&0) { return Err(MetadError::InvalidPath( "symlink target must be non-empty and must not contain NUL".to_owned(), @@ -300,6 +302,7 @@ where &projection, &chunks, version, + object_reference, ) { return Err(MetadError::PublishArtifactFailed { source: Box::new(err), @@ -474,6 +477,7 @@ where name: &DentryName, changes: UpdateAttr, ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let (entry, dentry_version) = self .lookup_plus_for_write_plan(parent, name)? .ok_or(MetadError::NotFound)?; @@ -568,6 +572,7 @@ where old_generation, version, path_index: None, + object_reference, })?; Ok(projection.into()) } @@ -625,7 +630,19 @@ where op: MutationOp::Put, value: Some(Value(encode_inode_attr(&attr))), }], - watch: Vec::new(), + watch: self + .watch_projection( + InodeId::root(), + WatchEvent { + kind: WatchEventKind::UpdateAttr, + parent: None, + name: None, + inode: InodeId::root(), + version: version.get(), + }, + ) + .into_iter() + .collect(), })?; Ok(attr) } @@ -691,6 +708,9 @@ where uid: u32, gid: u32, ) -> Result, MetadError> { + if names.is_empty() { + return Ok(Vec::new()); + } let parent_components = parse_absolute_path(parent_path)?; let parent = self.resolve_components_as_directory(&parent_components)?; self.create_files_in_dir_with_parent_components(parent, names, mode, uid, gid) @@ -704,11 +724,11 @@ where uid: u32, gid: u32, ) -> Result, MetadError> { - let parent_components = parse_absolute_path(parent_path)?; - let parent = self.resolve_components_as_directory(&parent_components)?; if names.is_empty() { return Ok(Vec::new()); } + let parent_components = parse_absolute_path(parent_path)?; + let parent = self.resolve_components_as_directory(&parent_components)?; ensure_unique_names(&names)?; let version = self.next_version()?; let inodes = self.next_inodes(names.len())?; @@ -985,6 +1005,28 @@ where { mutations.push(delete_mutation(RecordFamily::PathIndex, path_index)); } + let restore_shadow = match path_components { + Some(components) => self.restore_shadow_key_for_entry( + components, + parent, + name, + &entry, + predecessor(version)?, + )?, + None => self.restore_shadow_key_for_inode_entry( + parent, + name, + &entry, + predecessor(version)?, + )?, + }; + if let Some((shadow, _)) = restore_shadow { + mutations.push(delete_mutation(RecordFamily::System, shadow)); + mutations.push(delete_mutation( + RecordFamily::ForkShadow, + fork_shadow_key(self.mount, parent, name), + )); + } if canonical_attr.nlink == 0 { return Err(MetadError::InvalidPath( "inode link count is already zero".to_owned(), @@ -1183,11 +1225,133 @@ where delete_mutation(RecordFamily::Dentry, source_key.clone()), delete_mutation(RecordFamily::Inode, inode_key(self.mount, entry.attr.inode)), ]; + let binding_key = fork_binding_key(self.mount, entry.attr.inode); + let binding = self.metadata.get_versioned( + RecordFamily::ForkBinding, + &binding_key, + predecessor(version)?, + ReadPurpose::WritePlanLocal, + )?; + let releasing_binding = binding + .as_ref() + .map(|item| { + let mut binding = decode_fork_binding(&item.value.0) + .map_err(|err| MetadError::Codec(err.to_string()))?; + if binding.state != ForkBindingState::Complete { + return Err(MetadError::RestoreInProgress); + } + binding.state = ForkBindingState::Releasing; + Ok::<_, MetadError>(binding) + }) + .transpose()?; + let operation = releasing_binding + .as_ref() + .map(|binding| { + let key = restore_operation_key(self.mount, &binding.operation_digest); + Ok::<_, MetadError>( + self.metadata + .get_versioned( + RecordFamily::System, + &key, + predecessor(version)?, + ReadPurpose::WritePlanLocal, + )? + .map(|item| (key, item.version)), + ) + }) + .transpose()? + .flatten(); + let release_key = releasing_binding + .as_ref() + .map(|binding| fork_base_ref_release_key(self.mount, binding.base_ref_set_id)); + if let Some(releasing) = &releasing_binding { + // The namespace disappears atomically with Complete -> Releasing, + // while the durable binding and inverse rows continue protecting + // shared objects until the GC worker pages the ref set out. + mutations.push(Mutation { + family: RecordFamily::ForkBinding, + key: binding_key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(releasing))), + }); + mutations.push(Mutation { + family: RecordFamily::System, + key: release_key + .as_ref() + .expect("releasing binding has a release key") + .clone(), + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(releasing))), + }); + if let Some((key, _)) = &operation { + mutations.push(Mutation { + family: RecordFamily::System, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_fork_binding(releasing))), + }); + } + } if let Some(path_index) = self.live_path_index_key_for_entry(path_components, parent, name, &entry, version)? { mutations.push(delete_mutation(RecordFamily::PathIndex, path_index)); } + let restore_shadow = match path_components { + Some(components) => self.restore_shadow_key_for_entry( + components, + parent, + name, + &entry, + predecessor(version)?, + )?, + None => self.restore_shadow_key_for_inode_entry( + parent, + name, + &entry, + predecessor(version)?, + )?, + }; + if let Some((shadow, _)) = restore_shadow { + mutations.push(delete_mutation(RecordFamily::System, shadow)); + mutations.push(delete_mutation( + RecordFamily::ForkShadow, + fork_shadow_key(self.mount, parent, name), + )); + } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Dentry, + key: source_key.clone(), + predicate: Predicate::VersionEquals(dentry_version), + }, + PredicateRef { + family: RecordFamily::Dentry, + key: child_prefix, + predicate: Predicate::PrefixEmpty, + }, + ]; + if let Some(binding) = binding { + predicates.push(PredicateRef { + family: RecordFamily::ForkBinding, + key: binding_key, + predicate: Predicate::VersionEquals(binding.version), + }); + } + if let Some(key) = release_key { + predicates.push(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::NotExists, + }); + } + if let Some((key, version)) = operation { + predicates.push(PredicateRef { + family: RecordFamily::System, + key, + predicate: Predicate::VersionEquals(version), + }); + } let command = MetadataCommand { request_id: request_id(b"remove-empty-dir", self.mount, entry.attr.inode, version), kind: CommandKind::RemoveEmptyDir, @@ -1195,18 +1359,7 @@ where commit_version: version, primary_family: RecordFamily::Dentry, primary_key: source_key.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Dentry, - key: source_key.clone(), - predicate: Predicate::VersionEquals(dentry_version), - }, - PredicateRef { - family: RecordFamily::Dentry, - key: child_prefix, - predicate: Predicate::PrefixEmpty, - }, - ], + predicates, mutations, watch: self .watch_projection( @@ -1386,6 +1539,7 @@ where entry.attr.inode.shard_index() != self.shard_index() } + #[allow(clippy::too_many_arguments)] pub(super) fn rename_inner( &self, parent: InodeId, @@ -1443,7 +1597,7 @@ where let destination_key = dentry_key(self.mount, new_parent, &new_name); let projection = projection( new_parent, - new_name, + new_name.clone(), source.attr.clone(), source.body.clone(), ); @@ -1484,13 +1638,29 @@ where value: Some(Value(encode_dentry_projection(&projection))), }, ]; - if let Some(source_path) = self.live_path_index_key_for_entry( + let live_source_path = self.live_path_index_key_for_entry( path_index.map(|(source, _)| source), parent, name, &source, version, - )? { + )?; + let restore_shadow = match path_index { + Some((source_components, _)) => self.restore_shadow_key_for_entry( + source_components, + parent, + name, + &source, + predecessor(version)?, + )?, + None => self.restore_shadow_key_for_inode_entry( + parent, + name, + &source, + predecessor(version)?, + )?, + }; + if let Some(source_path) = live_source_path.as_ref() { let destination_path = path_index .map(|(_, destination)| path_index_key(self.mount, destination)) .ok_or_else(|| { @@ -1498,13 +1668,68 @@ where "live source path index requires destination path context".to_owned(), ) })?; - mutations.push(delete_mutation(RecordFamily::PathIndex, source_path)); + mutations.push(delete_mutation( + RecordFamily::PathIndex, + source_path.clone(), + )); mutations.push(put_projection_mutation( RecordFamily::PathIndex, destination_path, &projection, )); } + if let Some((source_shadow, base_ref_set_id)) = restore_shadow { + mutations.push(delete_mutation(RecordFamily::System, source_shadow)); + mutations.push(delete_mutation( + RecordFamily::ForkShadow, + fork_shadow_key(self.mount, parent, name), + )); + if live_source_path.is_none() { + let destination_shadow = match path_index { + Some((_, destination_components)) => self.restore_shadow_destination_key( + destination_components, + base_ref_set_id, + new_parent, + &new_name, + predecessor(version)?, + )?, + None => self.restore_shadow_destination_key_for_parent( + base_ref_set_id, + new_parent, + &new_name, + predecessor(version)?, + )?, + }; + if let Some(destination_shadow) = destination_shadow { + mutations.push(put_projection_mutation( + RecordFamily::System, + destination_shadow, + &projection, + )); + mutations.push(Mutation { + family: RecordFamily::ForkShadow, + key: fork_shadow_key(self.mount, new_parent, &new_name), + op: MutationOp::Put, + value: Some(Value(encode_restore_shadow_inverse( + base_ref_set_id, + &projection, + ))), + }); + } else { + let (_, destination_components) = path_index.ok_or_else(|| { + MetadataError::Backend( + "moving indexed restore entry outside its binding requires path context" + .to_owned(), + ) + })?; + mutations.push(put_projection_mutation( + RecordFamily::PathIndex, + path_index_key(self.mount, destination_components), + &projection, + )); + } + } + } if let Some(replaced) = &replaced { let replaced_inode_key = inode_key(self.mount, replaced.attr.inode); let Some(replaced_inode_item) = self.metadata.get_versioned( @@ -1707,7 +1932,14 @@ where projection: &DentryProjection, version: Version, ) -> Result<(), MetadError> { - self.commit_create_projection_with_chunks(kind, projection, &[], version) + self.commit_create_projection_with_chunks_and_path_index( + kind, + projection, + &[], + version, + None, + None, + ) } pub(super) fn commit_create_projections_with_path_indexes( @@ -1798,7 +2030,7 @@ where watch.push(event); } } - Ok(MetadataCommand { + let command = MetadataCommand { request_id: request_id(kind_name(kind), self.mount, parent, version), kind, read_version: predecessor(version)?, @@ -1808,7 +2040,8 @@ where predicates, mutations, watch, - }) + }; + Ok(command) } /// Build the cross-shard graft command. Unlike every other create path this @@ -1824,7 +2057,7 @@ where ) -> Result { let parent = projection.dentry.parent; let dentry = dentry_key(self.mount, parent, &projection.dentry.name); - Ok(MetadataCommand { + let command = MetadataCommand { request_id: request_id( kind_name(CommandKind::CreateDir), self.mount, @@ -1868,7 +2101,8 @@ where ) .into_iter() .collect(), - }) + }; + Ok(command) } pub(super) fn commit_create_projection_with_chunks( @@ -1877,12 +2111,19 @@ where projection: &DentryProjection, chunks: &[ChunkManifest], version: Version, + object_reference: ObjectReferenceMutation, ) -> Result<(), MetadError> { self.commit_create_projection_with_chunks_and_path_index( - kind, projection, chunks, version, None, + kind, + projection, + chunks, + version, + None, + Some(object_reference), ) } + #[allow(clippy::too_many_arguments)] pub(super) fn commit_create_projection_with_chunks_and_path_index( &self, kind: CommandKind, @@ -1890,6 +2131,7 @@ where chunks: &[ChunkManifest], version: Version, path_index: Option>, + object_reference: Option, ) -> Result<(), MetadError> { let inode = projection.attr.inode; let dentry = dentry_key( @@ -1934,6 +2176,21 @@ where }); } } + let mut predicates = vec![ + PredicateRef { + family: RecordFamily::Inode, + key: inode_key(self.mount, projection.dentry.parent), + predicate: Predicate::Exists, + }, + PredicateRef { + family: RecordFamily::Dentry, + key: dentry.clone(), + predicate: Predicate::NotExists, + }, + ]; + if let Some(object_reference) = object_reference { + predicates.push(object_reference.predicate(self.mount)); + } self.commit_metadata(MetadataCommand { request_id: request_id(kind_name(kind), self.mount, inode, version), kind, @@ -1941,18 +2198,7 @@ where commit_version: version, primary_family: RecordFamily::Dentry, primary_key: dentry.clone(), - predicates: vec![ - PredicateRef { - family: RecordFamily::Inode, - key: inode_key(self.mount, projection.dentry.parent), - predicate: Predicate::Exists, - }, - PredicateRef { - family: RecordFamily::Dentry, - key: dentry, - predicate: Predicate::NotExists, - }, - ], + predicates, mutations, watch: self .watch_projection( @@ -1984,6 +2230,7 @@ where old_generation, version, path_index, + object_reference, } = commit; let inode = projection.attr.inode; let dentry = dentry_key( @@ -2013,6 +2260,7 @@ where predicate: Predicate::Exists, }, ]; + predicates.push(object_reference.predicate(self.mount)); let mut mutations = vec![Mutation { family: RecordFamily::Inode, key: inode_key(self.mount, inode), diff --git a/crates/nokv-meta/src/service/publish.rs b/crates/nokv-meta/src/service/publish.rs index a7744de6b..2187e5081 100644 --- a/crates/nokv-meta/src/service/publish.rs +++ b/crates/nokv-meta/src/service/publish.rs @@ -16,6 +16,7 @@ where O: ObjectStore, { pub fn publish_artifact(&self, request: PublishArtifact) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let version = self.next_version()?; let inode = self.next_inode()?; let StagedArtifactBody { @@ -44,6 +45,7 @@ where &projection, &chunks, version, + object_reference, ) { return Err(MetadError::PublishArtifactFailed { source: Box::new(err), @@ -57,6 +59,7 @@ where &self, request: PublishArtifact, ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let (existing, dentry_version) = self .lookup_plus_for_write_plan(request.parent, &request.name)? .ok_or(MetadError::NotFound)?; @@ -95,6 +98,7 @@ where old_generation, version, path_index: None, + object_reference, }) { return Err(MetadError::PublishArtifactFailed { source: Box::new(err), @@ -111,6 +115,16 @@ where &self, parent: InodeId, name: DentryName, + ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; + self.prepare_artifact_create_with_object_reference(parent, name, object_reference) + } + + fn prepare_artifact_create_with_object_reference( + &self, + parent: InodeId, + name: DentryName, + object_reference: ObjectReferenceMutation, ) -> Result { let Some(parent_attr) = self.get_attr_at_version_for_purpose( parent, @@ -140,13 +154,16 @@ where replace: false, dentry_version: None, old_generation: None, + object_gc_claim_version: object_reference.version().get(), }) } pub fn prepare_artifact_create_path(&self, path: &str) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let components = parse_absolute_path(path)?; let (parent, name) = self.resolve_parent_path(path)?; - let mut prepared = self.prepare_artifact_create(parent, name)?; + let mut prepared = + self.prepare_artifact_create_with_object_reference(parent, name, object_reference)?; prepared.path = Some(canonical_path(&components)?); Ok(prepared) } @@ -155,6 +172,16 @@ where &self, parent: InodeId, name: DentryName, + ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; + self.prepare_artifact_replace_with_object_reference(parent, name, object_reference) + } + + fn prepare_artifact_replace_with_object_reference( + &self, + parent: InodeId, + name: DentryName, + object_reference: ObjectReferenceMutation, ) -> Result { let (existing, dentry_version) = self .lookup_plus_for_write_plan(parent, &name)? @@ -175,6 +202,7 @@ where replace: true, dentry_version: Some(dentry_version.get()), old_generation: existing.body.as_ref().map(|body| body.generation), + object_gc_claim_version: object_reference.version().get(), }) } @@ -182,13 +210,89 @@ where &self, path: &str, ) -> Result { + let object_reference = self.begin_object_reference_mutation()?; let (parent, name) = self.resolve_parent_path(path)?; let components = parse_absolute_path(path)?; - let mut prepared = self.prepare_artifact_replace(parent, name)?; + let mut prepared = + self.prepare_artifact_replace_with_object_reference(parent, name, object_reference)?; prepared.path = Some(canonical_path(&components)?); Ok(prepared) } + /// Refresh an unpublished prepared upload after an unrelated object-GC + /// epoch invalidated its object reference token. The namespace proof remains + /// fixed, while a fresh generation makes it + /// impossible to reuse objects uploaded under the stale GC epoch. Callers + /// must persist this returned identity before fully restaging the body. + pub fn refresh_prepared_artifact_object_gc_epoch( + &self, + mut prepared: PreparedArtifact, + ) -> Result { + if let Some(path) = prepared.path.as_deref() { + let components = parse_absolute_path(path)?; + let Some((name, parent_components)) = components.split_last() else { + return Err(MetadError::InvalidPreparedArtifact( + "prepared artifact path has no leaf".to_owned(), + )); + }; + if name != &prepared.name + || self.resolve_components_as_directory(parent_components)? != prepared.parent + { + return Err(MetadError::InvalidPreparedArtifact( + "prepared artifact path no longer matches its parent and name".to_owned(), + )); + } + } + + let current = self.lookup_plus_for_write_plan(prepared.parent, &prepared.name)?; + if prepared.replace { + let expected = Version::new(prepared.dentry_version.ok_or_else(|| { + MetadError::InvalidPreparedArtifact( + "replace artifact is missing dentry version".to_owned(), + ) + })?)?; + let Some((entry, version)) = current else { + return Err(MetadError::NotFound); + }; + if entry.attr.file_type != FileType::File { + return Err(MetadError::NotFile); + } + if entry.attr.inode != prepared.inode || version != expected { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + } else { + if prepared.dentry_version.is_some() || prepared.old_generation.is_some() { + return Err(MetadError::InvalidPreparedArtifact( + "create artifact must not carry replace state".to_owned(), + )); + } + if current.is_some() || self.get_attr(prepared.inode)?.is_some() { + return Err(MetadError::Metadata(MetadataError::PredicateFailed)); + } + let parent = self + .get_attr(prepared.parent)? + .ok_or(MetadError::NotFound)?; + if parent.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + } + + match self.ensure_prepared_object_gc_epoch(prepared.object_gc_claim_version) { + Ok(()) => return Ok(prepared), + Err(MetadError::StalePreparedArtifactObjectGcEpoch { .. }) => {} + Err(err) => return Err(err), + } + + let object_reference = self.begin_object_reference_mutation()?; + let generation = self.next_version()?; + let now_ms = current_time_ms(); + prepared.generation = generation.get(); + prepared.mtime_ms = now_ms; + prepared.ctime_ms = now_ms; + prepared.object_gc_claim_version = object_reference.version().get(); + Ok(prepared) + } + pub fn publish_prepared_artifact( &self, prepared: PreparedArtifact, @@ -223,6 +327,7 @@ where gid, } = request; validate_prepared_artifact(&prepared, &body, &chunks)?; + self.ensure_prepared_object_gc_epoch(prepared.object_gc_claim_version)?; let version = Version::new(prepared.generation)?; let mut attr = InodeAttr { inode: prepared.inode, @@ -276,6 +381,12 @@ where .map(|components| path_index_key(self.mount, &components)) }) .transpose()?, + object_reference: ObjectReferenceMutation::from_version(Version::new( + prepared.object_gc_claim_version, + )?), + }) + .map_err(|err| { + self.classify_prepared_object_gc_epoch_error(err, prepared.object_gc_claim_version) })?; Ok(RenameReplaceResult { entry: projection.into(), @@ -300,7 +411,13 @@ where .map(|components| path_index_key(self.mount, &components)) }) .transpose()?, - )?; + Some(ObjectReferenceMutation::from_version(Version::new( + prepared.object_gc_claim_version, + )?)), + ) + .map_err(|err| { + self.classify_prepared_object_gc_epoch_error(err, prepared.object_gc_claim_version) + })?; Ok(RenameReplaceResult { entry: projection.into(), replaced: None, @@ -308,6 +425,49 @@ where } } + fn ensure_prepared_object_gc_epoch(&self, expected: u64) -> Result<(), MetadError> { + let key = object_gc_claim_key(self.mount); + let item = self + .metadata + .get_versioned( + RecordFamily::System, + &key, + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadError::Codec("durable object GC claim is not initialized".to_owned()) + })?; + if item.version.get() == expected + && matches!(decode_object_gc_claim(&item.value.0)?, ObjectGcClaim::Open) + { + return Ok(()); + } + Err(MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current: item.version.get(), + }) + } + + fn classify_prepared_object_gc_epoch_error( + &self, + err: MetadError, + expected: u64, + ) -> MetadError { + if !matches!(err, MetadError::Metadata(MetadataError::PredicateFailed)) { + return err; + } + self.ensure_prepared_object_gc_epoch(expected) + .err() + .filter(|current| { + matches!( + current, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + ) + }) + .unwrap_or(err) + } + pub fn publish_prepared_artifact_session( &self, prepared: PreparedArtifact, diff --git a/crates/nokv-meta/src/service/read.rs b/crates/nokv-meta/src/service/read.rs index a1cf75a1d..0211b0e39 100644 --- a/crates/nokv-meta/src/service/read.rs +++ b/crates/nokv-meta/src/service/read.rs @@ -274,19 +274,64 @@ where ) -> Result { let requested = limit.max(1); let prefix = path_index_prefix(self.mount, components); - let mut start_after = after.map(|name| { - let mut marker = components.to_vec(); - marker.push(name.clone()); - path_index_prefix(self.mount, &marker) - }); + let cache_epoch = self.path_cache_epoch.load(Ordering::Acquire); + let (live, mut stale_rows) = self.collect_live_indexed_children( + parent, + &prefix, + after, + requested, + version, + cache_epoch, + )?; + let mut merged = live + .into_iter() + .map(|entry| (entry.dentry.name.as_bytes().to_vec(), entry)) + .collect::>(); + let (shadow, shadow_stale) = + self.collect_restore_indexed_children(parent, after, requested, version)?; + stale_rows = stale_rows.saturating_add(shadow_stale); + for entry in shadow { + merged + .entry(entry.dentry.name.as_bytes().to_vec()) + .or_insert(entry); + } + let mut entries = merged.into_values().collect::>(); + let next_cursor = if entries.len() > requested { + entries.truncate(requested); + entries.last().map(|entry| entry.dentry.name.clone()) + } else { + None + }; + self.read_dir_plus_total.fetch_add(1, Ordering::Relaxed); + self.read_dir_plus_entry_total + .fetch_add(entries.len() as u64, Ordering::Relaxed); + self.read_dir_plus_projection_hit_total + .fetch_add(entries.len() as u64, Ordering::Relaxed); + self.path_index_scan_stale_total + .fetch_add(stale_rows, Ordering::Relaxed); + Ok(ReadDirPlusPage { + entries, + next_cursor, + }) + } + + fn collect_live_indexed_children( + &self, + parent: InodeId, + prefix: &[u8], + after: Option<&DentryName>, + requested: usize, + version: Version, + cache_epoch: u64, + ) -> Result<(Vec, u64), MetadError> { + let mut start_after = after.map(|name| delimited_child_marker(prefix, name)); let scan_limit = requested.saturating_add(1); let mut entries = Vec::::with_capacity(scan_limit); let mut stale_rows = 0_u64; - let cache_epoch = self.path_cache_epoch.load(Ordering::Acquire); loop { let rows = self.metadata.scan_delimited(DelimitedScanRequest { family: RecordFamily::PathIndex, - prefix: prefix.clone(), + prefix: prefix.to_vec(), start_after: start_after.clone(), delimiter: PATH_INDEX_DELIMITER, version, @@ -301,7 +346,7 @@ where for item in rows { last_marker = Some(delimited_scan_marker(&item)); let Some(entry) = - self.indexed_path_child(parent, &prefix, item, version, cache_epoch)? + self.indexed_path_child(parent, prefix, item, version, cache_epoch)? else { stale_rows += 1; continue; @@ -319,23 +364,237 @@ where }; start_after = Some(marker); } - let next_cursor = if entries.len() > requested { - entries.truncate(requested); - entries.last().map(|entry| entry.dentry.name.clone()) - } else { - None + Ok((entries, stale_rows)) + } + + fn collect_restore_indexed_children( + &self, + parent: InodeId, + after: Option<&DentryName>, + requested: usize, + version: Version, + ) -> Result<(Vec, u64), MetadError> { + let prefix = fork_shadow_prefix(self.mount, parent); + let mut start_after = after.map(|name| fork_shadow_key(self.mount, parent, name)); + let scan_limit = requested.saturating_add(1); + let mut entries = Vec::with_capacity(scan_limit); + let mut stale_rows = 0_u64; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::ForkShadow, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: scan_limit, + purpose: ReadPurpose::UserStrong, + })?; + if rows.is_empty() { + break; + } + let exhausted = rows.len() < scan_limit; + let mut last_marker = None; + for item in rows { + last_marker = Some(item.key.clone()); + let Some(entry) = + self.restore_indexed_path_child(parent, &prefix, &item, version)? + else { + stale_rows += 1; + continue; + }; + entries.push(entry); + if entries.len() > requested { + break; + } + } + if entries.len() > requested || exhausted { + break; + } + let Some(marker) = last_marker else { + break; + }; + start_after = Some(marker); + } + Ok((entries, stale_rows)) + } + + pub(super) fn restore_shadow_key_for_entry( + &self, + components: &[DentryName], + parent: InodeId, + name: &DentryName, + entry: &DentryWithAttr, + version: Version, + ) -> Result, u64)>, MetadError> { + let Some((path_name, parent_components)) = components.split_last() else { + return Ok(None); }; - self.read_dir_plus_total.fetch_add(1, Ordering::Relaxed); - self.read_dir_plus_entry_total - .fetch_add(entries.len() as u64, Ordering::Relaxed); - self.read_dir_plus_projection_hit_total - .fetch_add(entries.len() as u64, Ordering::Relaxed); - self.path_index_scan_stale_total - .fetch_add(stale_rows, Ordering::Relaxed); - Ok(ReadDirPlusPage { - entries, - next_cursor, - }) + if path_name != name { + return Err(MetadError::InvalidPath( + "path-index entry name does not match path context".to_owned(), + )); + } + let _ = parent_components; + self.restore_shadow_key_for_inode_entry(parent, name, entry, version) + } + + pub(super) fn restore_shadow_key_for_inode_entry( + &self, + parent: InodeId, + name: &DentryName, + entry: &DentryWithAttr, + version: Version, + ) -> Result, u64)>, MetadError> { + let inverse_key = fork_shadow_key(self.mount, parent, name); + let Some(value) = self.metadata.get( + RecordFamily::ForkShadow, + &inverse_key, + version, + ReadPurpose::WritePlanLocal, + )? + else { + return Ok(None); + }; + let (base_ref_set_id, marker) = decode_restore_shadow_inverse(&value.0)?; + Ok((marker.attr.inode == entry.attr.inode + && marker.dentry.parent == parent + && marker.dentry.name == *name) + .then(|| { + ( + restore_path_index_key(self.mount, base_ref_set_id, parent, name), + base_ref_set_id, + ) + })) + } + + pub(super) fn restore_shadow_destination_key( + &self, + components: &[DentryName], + base_ref_set_id: u64, + parent: InodeId, + name: &DentryName, + version: Version, + ) -> Result>, MetadError> { + let Some((path_name, parent_components)) = components.split_last() else { + return Ok(None); + }; + if path_name != name { + return Err(MetadError::InvalidPath( + "path-index destination name does not match path context".to_owned(), + )); + } + let _ = (parent_components, version); + Ok(Some(restore_path_index_key( + self.mount, + base_ref_set_id, + parent, + name, + ))) + } + + pub(super) fn restore_shadow_destination_key_for_parent( + &self, + base_ref_set_id: u64, + parent: InodeId, + name: &DentryName, + version: Version, + ) -> Result>, MetadError> { + let _ = version; + Ok(Some(restore_path_index_key( + self.mount, + base_ref_set_id, + parent, + name, + ))) + } + + fn restore_indexed_path_child( + &self, + parent: InodeId, + prefix: &[u8], + item: &ScanItem, + version: Version, + ) -> Result, MetadError> { + let name = DentryName::new( + item.key + .strip_prefix(prefix) + .ok_or_else(|| { + MetadError::Codec("restore shadow escaped parent prefix".to_owned()) + })? + .to_vec(), + ) + .map_err(|err| MetadError::InvalidPath(err.to_string()))?; + let (_, indexed): (u64, DentryProjection) = decode_restore_shadow_inverse(&item.value.0)?; + let Some((canonical, _)) = self.lookup_plus_at_version(parent, &name, version)? else { + return Ok(None); + }; + if indexed.attr.inode != canonical.attr.inode + || indexed.dentry.parent != parent + || indexed.dentry.name != name + { + return Ok(None); + } + if canonical.attr.file_type == FileType::Directory + && !self.restore_shadow_subtree_has_index( + canonical.attr.inode, + version, + &mut HashSet::new(), + )? + { + return Ok(None); + } + Ok(Some(canonical)) + } + + fn restore_shadow_subtree_has_index( + &self, + parent: InodeId, + version: Version, + visited: &mut HashSet, + ) -> Result { + if !visited.insert(parent) { + return Ok(false); + } + let prefix = fork_shadow_prefix(self.mount, parent); + let mut start_after = None; + loop { + let rows = self.metadata.scan(ScanRequest { + family: RecordFamily::ForkShadow, + prefix: prefix.clone(), + start_after: start_after.clone(), + version, + limit: 128, + purpose: ReadPurpose::UserStrong, + })?; + if rows.is_empty() { + break; + } + let exhausted = rows.len() < 128; + start_after = rows.last().map(|row| row.key.clone()); + for row in rows { + let (_, marker) = decode_restore_shadow_inverse(&row.value.0)?; + let Some((canonical, _)) = + self.lookup_plus_at_version(parent, &marker.dentry.name, version)? + else { + continue; + }; + if canonical.attr.inode != marker.attr.inode { + continue; + } + if canonical.attr.file_type != FileType::Directory + || self.restore_shadow_subtree_has_index( + canonical.attr.inode, + version, + visited, + )? + { + return Ok(true); + } + } + if exhausted { + break; + } + } + Ok(false) } fn indexed_path_child( @@ -1473,3 +1732,11 @@ fn delimited_scan_marker(item: &DelimitedScanItem) -> Vec { DelimitedScanItem::CommonPrefix(prefix) => prefix.clone(), } } + +fn delimited_child_marker(prefix: &[u8], name: &DentryName) -> Vec { + let mut marker = Vec::with_capacity(prefix.len() + name.as_bytes().len() + 1); + marker.extend_from_slice(prefix); + marker.extend_from_slice(name.as_bytes()); + marker.push(PATH_INDEX_DELIMITER); + marker +} diff --git a/crates/nokv-meta/src/service/rollback.rs b/crates/nokv-meta/src/service/rollback.rs index 2d10d09ef..03608af88 100644 --- a/crates/nokv-meta/src/service/rollback.rs +++ b/crates/nokv-meta/src/service/rollback.rs @@ -69,11 +69,7 @@ where /// The net effect mirrors [`NoKvFs::owns_block_object_key`]: a block reachable /// from the live namespace is never reclaimed, a block reachable only from the /// discarded delta is. - pub fn rollback_subtree( - &self, - target_root: InodeId, - snapshot_id: u64, - ) -> Result<(), MetadError> { + fn rollback_subtree(&self, target_root: InodeId, snapshot_id: u64) -> Result<(), MetadError> { let pin = self .snapshot_pin(snapshot_id)? .ok_or(MetadError::NotFound)?; @@ -84,7 +80,26 @@ where target_root.get() ))); } + if self + .metadata + .get( + RecordFamily::ForkBinding, + &fork_binding_key(self.mount, target_root), + self.read_version()?, + ReadPurpose::WritePlanLocal, + )? + .is_some() + { + return Err(MetadError::InvalidPath( + "in-place rollback is not supported for a restore-to-fork destination".to_owned(), + )); + } let snapshot_version = Version::new(pin.read_version)?; + if self.subtree_contains_fork_state(target_root, snapshot_version, ReadPurpose::Snapshot)? { + return Err(MetadError::InvalidPath( + "in-place rollback is not supported for a restore-to-fork destination".to_owned(), + )); + } if self .get_attr_at_version_for_purpose( @@ -131,10 +146,119 @@ where target_path: &str, snapshot_id: u64, ) -> Result<(), MetadError> { - let target_root = self.resolve_directory_path(target_path)?; + let target_root = self.resolve_non_fork_rollback_path(target_path)?; self.rollback_subtree(target_root, snapshot_id) } + fn resolve_non_fork_rollback_path(&self, target_path: &str) -> Result { + let components = parse_absolute_path(target_path)?; + let version = self.read_version()?; + let mut current = InodeId::root(); + for name in components { + if self.has_fork_binding(current, version)? { + return Err(MetadError::InvalidPath( + "in-place rollback is not supported for a restore-to-fork destination" + .to_owned(), + )); + } + if self + .metadata + .get( + RecordFamily::ForkShadow, + &fork_shadow_key(self.mount, current, &name), + version, + ReadPurpose::WritePlanLocal, + )? + .is_some() + { + return Err(MetadError::InvalidPath( + "in-place rollback is not supported for a restore-to-fork destination" + .to_owned(), + )); + } + let (entry, _) = self + .lookup_plus_at_version_for_purpose( + current, + &name, + version, + ReadPurpose::WritePlanLocal, + )? + .ok_or(MetadError::NotFound)?; + if entry.attr.file_type != FileType::Directory { + return Err(MetadError::NotDirectory); + } + current = entry.attr.inode; + } + if self.has_fork_binding(current, version)? { + return Err(MetadError::InvalidPath( + "in-place rollback is not supported for a restore-to-fork destination".to_owned(), + )); + } + Ok(current) + } + + fn has_fork_binding(&self, inode: InodeId, version: Version) -> Result { + Ok(self + .metadata + .get( + RecordFamily::ForkBinding, + &fork_binding_key(self.mount, inode), + version, + ReadPurpose::WritePlanLocal, + )? + .is_some()) + } + + fn subtree_contains_fork_state( + &self, + root: InodeId, + version: Version, + purpose: ReadPurpose, + ) -> Result { + let mut queue = vec![root]; + while let Some(parent) = queue.pop() { + for child in self.read_dir_plus_at_version_for_purpose(parent, version, purpose)? { + if self + .metadata + .get( + RecordFamily::ForkShadow, + &fork_shadow_key(self.mount, parent, &child.dentry.name), + version, + purpose, + )? + .is_some() + { + return Ok(true); + } + if child.attr.file_type == FileType::Directory { + queue.push(child.attr.inode); + continue; + } + let Some(body) = &child.body else { + continue; + }; + let manifests = self.chunk_manifests_for_body_at_version( + child.attr.inode, + body, + version, + purpose, + )?; + let block_prefix = format!("blocks/{}/", self.mount.get()); + if chunk_object_keys(&manifests).iter().any(|object_key| { + object_key.starts_with(&block_prefix) + && !self.owns_block_object_key( + child.attr.inode, + body.generation, + object_key, + ) + }) { + return Ok(true); + } + } + } + Ok(false) + } + /// Collect every object key referenced by the materialized subtree's file /// bodies. These are the blocks the restored tree shares with the snapshot; the /// teardown must not reclaim them and the swap must cancel any pending GC for diff --git a/crates/nokv-meta/src/service/snapshot.rs b/crates/nokv-meta/src/service/snapshot.rs index 7e5a0b726..44745830b 100644 --- a/crates/nokv-meta/src/service/snapshot.rs +++ b/crates/nokv-meta/src/service/snapshot.rs @@ -20,10 +20,12 @@ where M: MetadataStore, O: ObjectStore, { + #[cfg(test)] pub(crate) fn snapshot_subtree(&self, root: InodeId) -> Result { self.snapshot_subtree_with_lease(root, DEFAULT_SNAPSHOT_LEASE_MS) } + #[cfg(test)] pub(crate) fn snapshot_subtree_with_lease( &self, root: InodeId, @@ -546,7 +548,7 @@ where Ok((u64::from(self.shard_index()) << SNAPSHOT_ID_LOCAL_BITS) | local) } - fn ensure_snapshot_id_shard( + pub(super) fn ensure_snapshot_id_shard( &self, snapshot_id: u64, expected_root: InodeId, @@ -563,7 +565,7 @@ where Ok(()) } - fn ensure_snapshot_pin_live(&self, pin: &SnapshotPin) -> Result<(), MetadError> { + pub(super) fn ensure_snapshot_pin_live(&self, pin: &SnapshotPin) -> Result<(), MetadError> { let now_ms = self.now_ms(); if now_ms >= pin.lease_expires_unix_ms { return Err(MetadError::SnapshotLeaseExpired { diff --git a/crates/nokv-meta/src/service/xattr.rs b/crates/nokv-meta/src/service/xattr.rs index ce2d581c4..35149d100 100644 --- a/crates/nokv-meta/src/service/xattr.rs +++ b/crates/nokv-meta/src/service/xattr.rs @@ -47,7 +47,19 @@ where op: MutationOp::Put, value: Some(Value(value)), }], - watch: Vec::new(), + watch: self + .watch_projection( + InodeId::root(), + WatchEvent { + kind: WatchEventKind::UpdateAttr, + parent: None, + name: None, + inode, + version: version.get(), + }, + ) + .into_iter() + .collect(), })?; Ok(()) } @@ -116,7 +128,19 @@ where op: MutationOp::Delete, value: None, }], - watch: Vec::new(), + watch: self + .watch_projection( + InodeId::root(), + WatchEvent { + kind: WatchEventKind::UpdateAttr, + parent: None, + name: None, + inode, + version: version.get(), + }, + ) + .into_iter() + .collect(), })?; Ok(()) } diff --git a/crates/nokv-meta/src/service_tests.rs b/crates/nokv-meta/src/service_tests.rs index 29bcda36c..aafaf5c2e 100644 --- a/crates/nokv-meta/src/service_tests.rs +++ b/crates/nokv-meta/src/service_tests.rs @@ -1,10 +1,14 @@ use super::*; -use crate::command::{ReadItem, ScanItem}; +use crate::command::{DelimitedScanItem, DelimitedScanRequest, ReadItem, ScanItem}; use crate::holtstore::HoltMetadataStore; +use crate::layout::{ + fork_base_release_quarantine_prefix, fork_binding_prefix, object_gc_quarantine_prefix, +}; use crate::{MetadataLogEntry, MetadataLogSegment, METADATA_LOG_ZERO_DIGEST}; use nokv_object::{MemoryObjectStore, ObjectBytes}; use nokv_types::{AdvisoryLockKind, AdvisoryLockRequest}; -use std::sync::{Arc, Barrier}; +use std::sync::{Arc, Barrier, Condvar}; +use std::time::{Duration, Instant}; #[derive(Clone)] struct SnapshotCommitBarrierStore { @@ -52,6 +56,7 @@ impl MetadataStore for SnapshotCommitBarrierStore { fn commit_metadata(&self, command: MetadataCommand) -> Result { if command.kind == self.kind + && command.primary_family == RecordFamily::Snapshot && self .remaining .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { @@ -129,6 +134,7 @@ impl MetadataStore for SnapshotPredicateOnceStore { fn commit_metadata(&self, command: MetadataCommand) -> Result { if command.kind == CommandKind::SnapshotSubtree + && command.primary_family == RecordFamily::Snapshot && self .remaining_failures .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { @@ -298,10 +304,355 @@ impl MetadataStoreStatsProvider for PurposeTrackingStore { } } +/// Test metastore that advances the durable Open object-reference claim in the +/// same atomic apply as every detached restore command. A second bounded stage +/// can commit only if it fetched the new claim version after the prior stage; +/// no wall-clock scheduling or worker thread is involved. +#[derive(Clone)] +struct RotatingObjectGcClaimStore { + inner: HoltMetadataStore, + rotations: Arc, +} + +impl RotatingObjectGcClaimStore { + fn new() -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + rotations: Arc::new(AtomicU64::new(0)), + } + } + + fn rotations(&self) -> u64 { + self.rotations.load(Ordering::Relaxed) + } +} + +impl MetadataStore for RotatingObjectGcClaimStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn scan_delimited( + &self, + request: DelimitedScanRequest, + ) -> Result, MetadataError> { + self.inner.scan_delimited(request) + } + + fn commit_metadata(&self, mut command: MetadataCommand) -> Result { + let claim_key = object_gc_claim_key(MountId::new(1).unwrap()); + let rotates_claim = command.kind == CommandKind::StageDetachedRestore + && command.predicates.iter().any(|predicate| { + predicate.family == RecordFamily::System && predicate.key == claim_key + }); + if rotates_claim { + let claim = self + .inner + .get_versioned( + RecordFamily::System, + &claim_key, + command.read_version, + ReadPurpose::WritePlanLocal, + )? + .ok_or_else(|| { + MetadataError::Backend( + "detached restore command references a missing object GC claim".to_owned(), + ) + })?; + command.mutations.push(Mutation { + family: RecordFamily::System, + key: claim_key, + op: MutationOp::Put, + value: Some(claim.value), + }); + } + let committed = self.inner.commit_metadata(command); + if rotates_claim && committed.is_ok() { + self.rotations.fetch_add(1, Ordering::Relaxed); + } + committed + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + commands + .iter() + .cloned() + .map(|command| self.commit_metadata(command)) + .collect() + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + +#[derive(Clone)] +struct PausingCommitStore { + inner: HoltMetadataStore, + gate: Arc<(Mutex, Condvar)>, +} + +#[derive(Default)] +struct PauseState { + point: Option, + reached: bool, + released: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PausePoint { + CreateFileCommit, + ObjectGcDeleting, + PathIndexGcCommit, +} + +impl PausingCommitStore { + fn new() -> Self { + Self { + inner: HoltMetadataStore::open_memory().unwrap(), + gate: Arc::new((Mutex::new(PauseState::default()), Condvar::new())), + } + } + + fn arm_create_file(&self) { + self.arm(PausePoint::CreateFileCommit); + } + + fn arm_object_gc_deleting(&self) { + self.arm(PausePoint::ObjectGcDeleting); + } + + fn arm_path_index_gc_commit(&self) { + self.arm(PausePoint::PathIndexGcCommit); + } + + fn arm(&self, point: PausePoint) { + let (lock, _) = &*self.gate; + *lock.lock().unwrap() = PauseState { + point: Some(point), + reached: false, + released: false, + }; + } + + fn pause_if(&self, point: PausePoint) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + if state.point != Some(point) { + return; + } + state.point = None; + state.reached = true; + changed.notify_all(); + while !state.released { + state = changed.wait(state).unwrap(); + } + } + + fn wait_until_reached(&self) { + assert!( + self.wait_until_reached_timeout(Duration::from_secs(10)), + "timed out waiting for deterministic metadata barrier" + ); + } + + fn wait_until_reached_timeout(&self, timeout: Duration) -> bool { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + let deadline = Instant::now() + timeout; + while !state.reached { + let now = Instant::now(); + if now >= deadline { + return false; + } + let (next, timed_out) = changed.wait_timeout(state, deadline - now).unwrap(); + state = next; + if timed_out.timed_out() && !state.reached { + return false; + } + } + true + } + + fn release(&self) { + let (lock, changed) = &*self.gate; + let mut state = lock.lock().unwrap(); + state.released = true; + changed.notify_all(); + } +} + +impl MetadataStore for PausingCommitStore { + fn get_versioned( + &self, + family: RecordFamily, + key: &[u8], + version: Version, + purpose: ReadPurpose, + ) -> Result, MetadataError> { + self.inner.get_versioned(family, key, version, purpose) + } + + fn scan(&self, request: ScanRequest) -> Result, MetadataError> { + self.inner.scan(request) + } + + fn scan_delimited( + &self, + request: DelimitedScanRequest, + ) -> Result, MetadataError> { + self.inner.scan_delimited(request) + } + + fn commit_metadata(&self, command: MetadataCommand) -> Result { + if command.primary_family == RecordFamily::System + && command.primary_key == path_index_gc_cursor_key(MountId::new(1).unwrap()) + { + self.pause_if(PausePoint::PathIndexGcCommit); + } + if matches!( + command.kind, + CommandKind::CreateFile | CommandKind::PublishArtifact + ) { + self.pause_if(PausePoint::CreateFileCommit); + } + let pause_after = command.primary_family == RecordFamily::System + && command.primary_key == object_gc_claim_key(MountId::new(1).unwrap()) + && command.mutations.iter().any(|mutation| { + mutation.family == RecordFamily::System + && mutation.key == command.primary_key + && mutation + .value + .as_ref() + .is_some_and(|value| value.0.first() == Some(&2)) + }); + let result = self.inner.commit_metadata(command); + if pause_after && result.is_ok() { + self.pause_if(PausePoint::ObjectGcDeleting); + } + result + } + + fn commit_independent_batch( + &self, + commands: &[MetadataCommand], + ) -> Vec> { + self.inner.commit_independent_batch(commands) + } + + fn committed_request_result( + &self, + request_id: &[u8], + ) -> Result, MetadataError> { + self.inner.committed_request_result(request_id) + } + + fn prune_history( + &self, + request: HistoryPruneRequest, + ) -> Result { + self.inner.prune_history(request) + } +} + fn service() -> NoKvFs { service_with_objects().0 } +fn drain_restore_staging_cleanup(service: &NoKvFs) +where + M: MetadataStore, + O: ObjectStore, +{ + for _ in 0..4096 { + let pending = service + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_staging_cleanup_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap(); + if pending.is_empty() { + return; + } + service.cleanup_pending_objects(128).unwrap(); + } + panic!("restore staging cleanup did not converge within the deterministic budget"); +} + +fn enqueue_gc_candidate(service: &NoKvFs, mut record: ObjectGcRecord) -> Vec +where + M: MetadataStore, + O: ObjectStore, +{ + let version = service.next_version().unwrap(); + record.enqueue_version = version.get(); + record.enqueue_unix_ms = service.now_ms(); + let key = gc_object_key( + service.mount, + version.get(), + record.inode, + record.generation, + 0, + 0, + ); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-enqueue-gc-candidate", + service.mount, + record.inode, + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: key.clone(), + op: MutationOp::Put, + value: Some(Value(encode_object_gc_record(&record))), + }], + watch: Vec::new(), + }) + .unwrap(); + key +} + fn service_with_objects() -> ( NoKvFs, MemoryObjectStore, @@ -331,8 +682,8 @@ fn artifact_request(name: DentryName, manifest_id: &str, bytes: &[u8]) -> Publis } } -fn publish_path_artifact( - service: &NoKvFs, +fn publish_path_artifact( + service: &NoKvFs, path: &str, manifest_id: &str, bytes: &[u8], @@ -1702,6 +2053,11 @@ fn directory_rename_leaves_descendant_path_index_as_derived_stale_cache() { .entry; let old_components = parse_absolute_path("/runs/checkpoint.bin").unwrap(); let old_key = path_index_key(MountId::new(1).unwrap(), &old_components); + publish_path_artifact(&service, "/valid.bin", "valid", b"valid"); + let valid_key = path_index_key( + MountId::new(1).unwrap(), + &parse_absolute_path("/valid.bin").unwrap(), + ); assert!(metadata .get( RecordFamily::PathIndex, @@ -1744,6 +2100,125 @@ fn directory_rename_leaves_descendant_path_index_as_derived_stale_cache() { service.lookup_path("/archive/checkpoint.bin").unwrap(), Some(artifact) ); + let before_gc = metadata.metadata_store_stats(); + service.cleanup_pending_objects(128).unwrap(); + let after_gc = metadata.metadata_store_stats(); + assert_eq!( + after_gc.atomic_apply_total - before_gc.atomic_apply_total, + 1, + "the no-conflict stale-index page must use one atomic fast-path apply" + ); + assert!(metadata + .get( + RecordFamily::PathIndex, + &old_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + assert!(metadata + .get( + RecordFamily::PathIndex, + &valid_key, + Version::new(u64::MAX).unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); +} + +#[test] +fn path_index_gc_conflict_does_not_block_other_stale_rows_and_later_converges() { + let metadata = PausingCommitStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + MemoryObjectStore::new(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/runs", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/runs/a.bin", "a", b"a"); + publish_path_artifact(&service, "/runs/b.bin", "b", b"b"); + let a_key = path_index_key(service.mount, &parse_absolute_path("/runs/a.bin").unwrap()); + let b_key = path_index_key(service.mount, &parse_absolute_path("/runs/b.bin").unwrap()); + service.rename_path("/runs", "/archive").unwrap(); + + metadata.arm_path_index_gc_commit(); + let cleanup = { + let service = Arc::clone(&service); + std::thread::spawn(move || service.cleanup_pending_objects(128)) + }; + metadata.wait_until_reached(); + let a = service + .metadata + .get_versioned( + RecordFamily::PathIndex, + &a_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"race-stale-path-index-gc".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::PathIndex, + primary_key: a_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::PathIndex, + key: a_key.clone(), + predicate: Predicate::VersionEquals(a.version), + }], + mutations: vec![Mutation { + family: RecordFamily::PathIndex, + key: a_key.clone(), + op: MutationOp::Put, + value: Some(a.value), + }], + watch: Vec::new(), + }) + .unwrap(); + metadata.release(); + cleanup.join().unwrap().unwrap(); + + assert!(service + .metadata + .get( + RecordFamily::PathIndex, + &a_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_some()); + assert!(service + .metadata + .get( + RecordFamily::PathIndex, + &b_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); + + service.cleanup_pending_objects(128).unwrap(); + service.cleanup_pending_objects(128).unwrap(); + assert!(service + .metadata + .get( + RecordFamily::PathIndex, + &a_key, + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .is_none()); } #[test] @@ -2316,6 +2791,7 @@ fn create_file_hot_path_write_attribution_is_bounded() { assert_eq!(after.history_write_total - before.history_write_total, 0); assert_eq!(after.watch_write_total - before.watch_write_total, 0); assert_eq!(after.dedupe_write_total - before.dedupe_write_total, 1); + assert_eq!(after.atomic_apply_total - before.atomic_apply_total, 1); } #[test] @@ -3044,7 +3520,7 @@ fn prepared_artifact_session_uploads_only_dirty_ranges_and_reuses_old_blocks() { let before_scan = service.metadata_store_stats(); let replaced = service .publish_prepared_artifact_session( - prepared, + prepared.clone(), PublishArtifactSession { parent: InodeId::root(), name, @@ -3101,10 +3577,10 @@ fn prepared_artifact_session_splits_noncontiguous_dirty_blocks() { service .publish_prepared_artifact_session( - prepared, + prepared.clone(), PublishArtifactSession { parent: InodeId::root(), - name, + name: name.clone(), producer: "unit-test".to_owned(), digest_uri: "unknown".to_owned(), content_type: "application/octet-stream".to_owned(), @@ -3759,6 +4235,316 @@ fn remove_file_queues_old_body_for_object_cleanup() { ); } +#[test] +fn durable_gc_claim_blocks_new_object_reference_planning_while_deleting() { + let metadata = PausingCommitStore::new(); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request(dname(b"old"), "old", b"old")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file_path("/old").unwrap(); + + metadata.arm_object_gc_deleting(); + let cleaner = { + let service = Arc::clone(&service); + std::thread::spawn(move || service.cleanup_pending_objects(1)) + }; + metadata.wait_until_reached(); + + assert!(matches!( + service.prepare_artifact_create(InodeId::root(), dname(b"new")), + Err(MetadError::Metadata(MetadataError::PredicateFailed)) + )); + metadata.release(); + let cleaned = cleaner.join().unwrap().unwrap(); + assert_eq!(cleaned.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); +} + +#[test] +fn object_upload_planned_before_gc_cannot_commit_after_delete_cycle() { + let metadata = PausingCommitStore::new(); + let objects = MemoryObjectStore::new(); + let service = Arc::new(NoKvFs::new( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + )); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let name = dname(b"racing.bin"); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "racing-manifest", + &[PublishArtifactRange { + offset: 0, + bytes: b"racing-body".to_vec(), + }], + 0, + ) + .unwrap(); + let chunks = written.chunk_manifests(); + let block = chunks[0].slices[0].blocks[0].clone(); + let staged = written.staged_objects().unwrap(); + enqueue_gc_candidate( + service.as_ref(), + ObjectGcRecord { + inode: prepared.inode, + generation: prepared.generation, + object_key: block.object_key.clone(), + size: block.len, + digest_uri: block.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + metadata.arm_create_file(); + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let publisher = { + let service = Arc::clone(&service); + let name = name.clone(); + std::thread::spawn(move || { + let result = service.publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:racing".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "racing-manifest".to_owned(), + size: 11, + chunks, + staged, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ); + result_tx.send(result).unwrap(); + }) + }; + if !metadata.wait_until_reached_timeout(Duration::from_secs(2)) { + metadata.release(); + let early = result_rx.recv_timeout(Duration::from_secs(1)); + if early.is_ok() { + publisher.join().unwrap(); + } + panic!("publisher exited or stalled before commit barrier: {early:?}"); + } + let cleanup = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.deleted, 1); + metadata.release(); + + let result = result_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + publisher.join().unwrap(); + assert!(matches!( + result, + Err(MetadError::PublishArtifactFailed { source, .. }) + if matches!(*source, MetadError::StalePreparedArtifactObjectGcEpoch { .. }) + )); + assert!(service.lookup_path("/racing.bin").unwrap().is_none()); + assert!(objects + .head(&ObjectKey::new(block.object_key).unwrap()) + .unwrap() + .is_none()); +} + +#[test] +fn gc_rescan_preserves_object_whose_manifest_committed_first() { + let (service, objects) = service_with_objects(); + let published = service + .publish_artifact(artifact_request(dname(b"live"), "live", b"live")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: published.attr.inode, + generation: body.generation, + object_key: object.as_str().to_owned(), + size: body.size, + digest_uri: body.digest_uri.clone(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let cleanup = service.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.blocked_by_snapshots, 1); + assert_eq!(cleanup.deleted, 0); + assert!(objects.head(&object).unwrap().is_some()); + assert_eq!( + service + .read_artifact(InodeId::root(), &dname(b"live")) + .unwrap(), + b"live" + ); +} + +#[test] +fn reopen_resumes_durable_object_gc_claim_after_crash() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let published = service + .publish_artifact(artifact_request(dname(b"old"), "old", b"old")) + .unwrap(); + let body = published.body.as_ref().unwrap(); + let object = block_key(published.attr.inode, body.generation, 0, 0); + service.remove_file_path("/old").unwrap(); + let gc_row = metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::WritePlanLocal, + }) + .unwrap() + .pop() + .unwrap(); + let claim_key = object_gc_claim_key(service.mount); + let claim = metadata + .get_versioned( + RecordFamily::System, + &claim_key, + service.read_version().unwrap(), + ReadPurpose::WritePlanLocal, + ) + .unwrap() + .unwrap(); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: request_id( + b"test-leave-object-gc-deleting", + service.mount, + InodeId::root(), + version, + ), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: claim_key.clone(), + predicates: vec![ + PredicateRef { + family: RecordFamily::System, + key: claim_key.clone(), + predicate: Predicate::VersionEquals(claim.version), + }, + PredicateRef { + family: RecordFamily::Gc, + key: gc_row.key.clone(), + predicate: Predicate::VersionEquals(gc_row.version), + }, + ], + mutations: vec![Mutation { + family: RecordFamily::System, + key: claim_key.clone(), + op: MutationOp::Put, + value: Some(Value( + encode_object_gc_claim(&ObjectGcClaim::Deleting { + owner_epoch: service.epoch.load(Ordering::Relaxed), + operation_token: claim.version.get(), + gc_record_key: gc_row.key, + gc_record_version: gc_row.version.get(), + }) + .unwrap(), + )), + }], + watch: Vec::new(), + }) + .unwrap(); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), + 0, + ) + .unwrap(); + assert!( + objects.head(&object).unwrap().is_some(), + "reopen must not mutate metadata or delete objects before ownership is established" + ); + let deferred_claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(matches!( + decode_object_gc_claim(&deferred_claim.0).unwrap(), + ObjectGcClaim::Deleting { .. } + )); + + // The ordinary cleanup pass resumes the durable claim after the server has + // established its ownership and durability policy. + let cleanup = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.deleted, 1); + let claim = metadata + .get( + RecordFamily::System, + &claim_key, + reopened.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert_eq!( + decode_object_gc_claim(&claim.0).unwrap(), + ObjectGcClaim::Open + ); + assert!(metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert!(objects.head(&object).unwrap().is_none()); + + let cleanup = reopened.cleanup_pending_objects(1).unwrap(); + assert_eq!(cleanup.deleted, 0); + assert!(metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert!(objects.head(&object).unwrap().is_none()); +} + #[test] fn read_lease_grace_blocks_recent_object_gc_records() { let (service, objects) = service_with_objects(); @@ -4125,6 +4911,60 @@ fn history_cleanup_keeps_snapshot_reads_until_snapshot_retired() { ); } +#[test] +fn history_cleanup_does_not_invalidate_a_prepared_object_upload() { + let (service, _) = service_with_objects(); + let name = dname(b"upload-during-history-prune.bin"); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + + service.cleanup_history(100).unwrap(); + + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) + .unwrap(); + assert_eq!( + refreshed, prepared, + "history pruning must not rotate the object-deletion epoch" + ); + let bytes = b"history-prune-independent"; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "history-prune-independent", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + service + .publish_prepared_artifact_staged_session( + prepared, + PublishArtifactStagedSession { + parent: InodeId::root(), + name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:history-prune-independent".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "history-prune-independent".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: written.staged_objects().unwrap(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + assert_eq!( + read_artifact_at_path(&service, "/upload-during-history-prune.bin"), + bytes + ); +} + #[test] fn remove_empty_dir_rejects_non_empty_directory() { let service = service(); @@ -4625,60 +5465,242 @@ fn failed_publish_returns_staged_objects_for_cleanup_and_does_not_reuse_identity assert!(next.attr.generation > first.attr.generation + 1); } -fn dname(raw: &[u8]) -> DentryName { - DentryName::new(raw.to_vec()).unwrap() -} - -fn block_count_for(objects: &MemoryObjectStore, inode: InodeId, generation: u64) -> usize { - // Count the published blocks the base file owns under its (inode, generation). - let mut count = 0; - let mut block = 0; - while objects - .head(&block_key(inode, generation, 0, block)) - .unwrap() - .is_some() - { - count += 1; - block += 1; - } - count -} - #[test] -fn clone_subtree_shares_base_blocks_diverges_on_write_and_keeps_gc_safe() { +fn prepared_publish_rejects_intervening_object_gc_epoch_and_reclaims_staging() { let (service, objects) = service_with_objects(); - // 1. Base namespace: /base with files a ("AAA..") and b ("BBB.."). - let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); - let a = publish_path_artifact(&service, "/base/a", "base/a", &vec![b'A'; 4096]); - let b = publish_path_artifact(&service, "/base/b", "base/b", &vec![b'B'; 4096]); - let a_gen = a.body.as_ref().unwrap().generation; - let b_gen = b.body.as_ref().unwrap().generation; - let a_block = block_key(a.attr.inode, a_gen, 0, 0); - let b_block = block_key(b.attr.inode, b_gen, 0, 0); - assert!(objects.head(&a_block).unwrap().is_some()); - assert!(objects.head(&b_block).unwrap().is_some()); - let objects_after_base = objects.object_count(); - - // 2. Writable O(1)-ish fork of /base. - let fork = service.clone_subtree_path("/base").unwrap(); - assert_ne!(fork.root, base.attr.inode); + let victim_name = dname(b"victim.bin"); + service + .publish_artifact(artifact_request( + victim_name.clone(), + "gc-victim", + b"victim", + )) + .unwrap(); + service.remove_file(InodeId::root(), &victim_name).unwrap(); - // 3. Sharing: the fork sees the base content, with NO duplicate blocks. - let fork_a = service - .lookup_plus(fork.root, &dname(b"a")) - .unwrap() + let name = dname(b"late-after-gc.bin"); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) .unwrap(); - let fork_b = service - .lookup_plus(fork.root, &dname(b"b")) - .unwrap() + let bytes = b"late-after-gc"; + let written = service + .stage_prepared_artifact_ranges( + &prepared, + "late-after-gc", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) .unwrap(); - assert_ne!( - fork_a.attr.inode, a.attr.inode, - "fork must use a fresh inode" - ); - // Shared files keep the source's content generation (the CoW sharing signal). - assert_eq!(fork_a.attr.generation, a_gen); - assert_eq!(fork_b.attr.generation, b_gen); + let staged_before = written.staged_objects().unwrap(); + for object in staged_before.objects() { + assert!(objects.head(&object.key).unwrap().is_some()); + } + + // Deleting the unrelated victim advances the durable claim through + // Open -> Deleting -> Open. The prepared write must not attach the newer + // Open epoch after upload; its original claim predicate is authoritative. + let cleanup = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(cleanup.deleted, 1); + let error = service + .publish_prepared_artifact_staged_session( + prepared.clone(), + PublishArtifactStagedSession { + parent: InodeId::root(), + name: name.clone(), + producer: "unit-test".to_owned(), + digest_uri: "sha256:late-after-gc".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "late-after-gc".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: staged_before, + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap_err(); + let staged = match error { + MetadError::PublishArtifactFailed { source, staged } => { + assert!(matches!( + *source, + MetadError::StalePreparedArtifactObjectGcEpoch { .. } + )); + staged + } + error => panic!("unexpected prepared publish error: {error:?}"), + }; + assert!(service.lookup_path("/late-after-gc.bin").unwrap().is_none()); + + let cleanup = service.cleanup_staged_objects(&staged).unwrap(); + assert_eq!(cleanup.deleted, staged.len()); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_none()); + } + + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) + .unwrap(); + assert!(refreshed.generation > prepared.generation); + assert_ne!( + refreshed.object_gc_claim_version, + prepared.object_gc_claim_version + ); + let rewritten = service + .stage_prepared_artifact_ranges( + &refreshed, + "late-after-gc", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let rewritten_staged = rewritten.staged_objects().unwrap(); + service + .publish_prepared_artifact_staged_session( + refreshed, + PublishArtifactStagedSession { + parent: InodeId::root(), + name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:late-after-gc-refreshed".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "late-after-gc".to_owned(), + size: bytes.len() as u64, + chunks: rewritten.chunk_manifests(), + staged: rewritten_staged.clone(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + assert_eq!(read_artifact_at_path(&service, "/late-after-gc.bin"), bytes); + for object in rewritten_staged.objects() { + assert!(objects.head(&object.key).unwrap().is_some()); + } +} + +#[test] +fn gc_before_first_upload_can_refresh_to_a_new_generation() { + let (service, objects) = service_with_objects(); + let victim_name = dname(b"first-upload-victim.bin"); + service + .publish_artifact(artifact_request( + victim_name.clone(), + "first-upload-victim", + b"victim", + )) + .unwrap(); + service.remove_file(InodeId::root(), &victim_name).unwrap(); + + let name = dname(b"first-upload.bin"); + let prepared = service + .prepare_artifact_create(InodeId::root(), name.clone()) + .unwrap(); + assert_eq!(service.cleanup_pending_objects(100).unwrap().deleted, 1); + let refreshed = service + .refresh_prepared_artifact_object_gc_epoch(prepared.clone()) + .unwrap(); + assert!(refreshed.generation > prepared.generation); + + let bytes = b"uploaded-only-after-refresh"; + let written = service + .stage_prepared_artifact_ranges( + &refreshed, + "first-upload", + &[PublishArtifactRange { + offset: 0, + bytes: bytes.to_vec(), + }], + 0, + ) + .unwrap(); + let staged = written.staged_objects().unwrap(); + service + .publish_prepared_artifact_staged_session( + refreshed, + PublishArtifactStagedSession { + parent: InodeId::root(), + name, + producer: "unit-test".to_owned(), + digest_uri: "sha256:first-upload".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "first-upload".to_owned(), + size: bytes.len() as u64, + chunks: written.chunk_manifests(), + staged: staged.clone(), + mode: 0o644, + uid: 1000, + gid: 1000, + }, + ) + .unwrap(); + assert_eq!(read_artifact_at_path(&service, "/first-upload.bin"), bytes); + for object in staged.objects() { + assert!(objects.head(&object.key).unwrap().is_some()); + } +} + +fn dname(raw: &[u8]) -> DentryName { + DentryName::new(raw.to_vec()).unwrap() +} + +fn block_count_for(objects: &MemoryObjectStore, inode: InodeId, generation: u64) -> usize { + // Count the published blocks the base file owns under its (inode, generation). + let mut count = 0; + let mut block = 0; + while objects + .head(&block_key(inode, generation, 0, block)) + .unwrap() + .is_some() + { + count += 1; + block += 1; + } + count +} + +#[test] +fn clone_subtree_shares_base_blocks_diverges_on_write_and_keeps_gc_safe() { + let (service, objects) = service_with_objects(); + // 1. Base namespace: /base with files a ("AAA..") and b ("BBB.."). + let base = service.create_dir_path("/base", 0o755, 1000, 1000).unwrap(); + let a = publish_path_artifact(&service, "/base/a", "base/a", &vec![b'A'; 4096]); + let b = publish_path_artifact(&service, "/base/b", "base/b", &vec![b'B'; 4096]); + let a_gen = a.body.as_ref().unwrap().generation; + let b_gen = b.body.as_ref().unwrap().generation; + let a_block = block_key(a.attr.inode, a_gen, 0, 0); + let b_block = block_key(b.attr.inode, b_gen, 0, 0); + assert!(objects.head(&a_block).unwrap().is_some()); + assert!(objects.head(&b_block).unwrap().is_some()); + let objects_after_base = objects.object_count(); + + // 2. Writable O(1)-ish fork of /base. + let fork = service.clone_subtree_path("/base").unwrap(); + assert_ne!(fork.root, base.attr.inode); + + // 3. Sharing: the fork sees the base content, with NO duplicate blocks. + let fork_a = service + .lookup_plus(fork.root, &dname(b"a")) + .unwrap() + .unwrap(); + let fork_b = service + .lookup_plus(fork.root, &dname(b"b")) + .unwrap() + .unwrap(); + assert_ne!( + fork_a.attr.inode, a.attr.inode, + "fork must use a fresh inode" + ); + // Shared files keep the source's content generation (the CoW sharing signal). + assert_eq!(fork_a.attr.generation, a_gen); + assert_eq!(fork_b.attr.generation, b_gen); assert_eq!(fork_b.body.as_ref().unwrap().generation, b_gen); assert_eq!( service.read_artifact(fork.root, &dname(b"a")).unwrap(), @@ -4889,188 +5911,1827 @@ fn clone_subtree_copies_nested_dirs_and_diff_reports_removed() { } #[test] -fn clone_subtree_path_rejects_non_directory() { +fn clone_subtree_path_rejects_non_directory() { + let service = service(); + publish_path_artifact(&service, "/file.bin", "file", b"bytes"); + assert!(matches!( + service.clone_subtree_path("/file.bin"), + Err(MetadError::NotDirectory) + )); +} + +fn read_artifact_at_path( + service: &NoKvFs, + path: &str, +) -> Vec { + let (parent, name) = service.resolve_parent_path(path).unwrap(); + service.read_artifact(parent, &name).unwrap() +} + +#[test] +fn rollback_subtree_restores_snapshot_shares_blocks_and_reclaims_delta() { + let (service, objects) = service_with_objects(); + // 1. Build /ws with files a="A1", b="B1", sub/c="C1" (real object data). + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/ws/sub", 0o755, 1000, 1000) + .unwrap(); + let a = publish_path_artifact(&service, "/ws/a", "ws/a", &vec![b'1'; 4096]); + let b = publish_path_artifact(&service, "/ws/b", "ws/b", &vec![b'2'; 4096]); + let c = publish_path_artifact(&service, "/ws/sub/c", "ws/sub/c", &vec![b'3'; 4096]); + let a_gen = a.body.as_ref().unwrap().generation; + let b_gen = b.body.as_ref().unwrap().generation; + let c_gen = c.body.as_ref().unwrap().generation; + let a1_block = block_key(a.attr.inode, a_gen, 0, 0); + let b1_block = block_key(b.attr.inode, b_gen, 0, 0); + let c1_block = block_key(c.attr.inode, c_gen, 0, 0); + assert!(objects.head(&a1_block).unwrap().is_some()); + assert!(objects.head(&b1_block).unwrap().is_some()); + assert!(objects.head(&c1_block).unwrap().is_some()); + + // 2. Snapshot /ws. + let snap = service.snapshot_subtree_path("/ws").unwrap(); + + // 3. Diverge /ws: rewrite a->"A2", add d="D1", delete b. + service + .replace_artifact(PublishArtifact { + parent: ws.attr.inode, + name: dname(b"a"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:a2".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "ws/a2".to_owned(), + bytes: vec![b'4'; 4096], + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let d = publish_path_artifact(&service, "/ws/d", "ws/d", &vec![b'5'; 4096]); + service.remove_file(ws.attr.inode, &dname(b"b")).unwrap(); + // Capture the delta's private blocks so we can assert their fate. + let a_diverged = service + .lookup_plus(ws.attr.inode, &dname(b"a")) + .unwrap() + .unwrap(); + let a2_block = block_key( + a_diverged.attr.inode, + a_diverged.body.as_ref().unwrap().generation, + 0, + 0, + ); + let d1_block = block_key(d.attr.inode, d.body.as_ref().unwrap().generation, 0, 0); + assert!(objects.head(&a2_block).unwrap().is_some()); + assert!(objects.head(&d1_block).unwrap().is_some()); + // Pre-rollback /ws is the diverged state. + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'4'; 4096]); + assert!(service + .lookup_plus(ws.attr.inode, &dname(b"b")) + .unwrap() + .is_none()); + + // 4. Roll /ws back to the snapshot. + service + .rollback_subtree_path("/ws", snap.snapshot_id) + .unwrap(); + + // 5. /ws now exactly matches the snapshot: a="A1", b="B1" (restored), sub/c="C1", + // and d is gone. The target keeps its inode identity. + assert_eq!( + service.resolve_directory_path("/ws").unwrap(), + ws.attr.inode, + "rollback keeps the target root's identity" + ); + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + assert_eq!( + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'3'; 4096] + ); + assert!( + service + .lookup_plus(ws.attr.inode, &dname(b"d")) + .unwrap() + .is_none(), + "the delta-only file d must be gone after rollback" + ); + + // 6. The rolled-back /ws is identical to a fresh clone of the snapshot: an empty + // diff in both directions. + let reference = service + .clone_subtree_path_into("/ws", "/reference") + .unwrap(); + assert!(service + .diff_subtrees(ws.attr.inode, reference.root) + .unwrap() + .is_empty()); + assert!(service + .diff_subtrees(reference.root, ws.attr.inode) + .unwrap() + .is_empty()); + + // 7. GC: the discarded delta's private blocks (A2, D1) are reclaimable, while the + // restored shared blocks (A1, B1, C1) survive and stay readable. The reference + // clone shares the snapshot's blocks, so retire its pin too before reclaiming. + assert!(service.retire_snapshot(snap.snapshot_id).unwrap()); + assert!(service.retire_snapshot(reference.snapshot_id).unwrap()); + let reclaim = service.cleanup_pending_objects(100).unwrap(); + assert!( + reclaim.deleted >= 2, + "delta-only blocks must be reclaimable" + ); + assert!( + objects.head(&a2_block).unwrap().is_none(), + "A2 must be reclaimed" + ); + assert!( + objects.head(&d1_block).unwrap().is_none(), + "D1 must be reclaimed" + ); + assert!( + objects.head(&a1_block).unwrap().is_some(), + "A1 must survive" + ); + assert!( + objects.head(&b1_block).unwrap().is_some(), + "B1 must survive" + ); + assert!( + objects.head(&c1_block).unwrap().is_some(), + "C1 must survive" + ); + // Restored content is still readable from the shared blocks after reclaim. + assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + assert_eq!( + read_artifact_at_path(&service, "/ws/sub/c"), + vec![b'3'; 4096] + ); +} + +#[test] +fn rollback_subtree_rejects_foreign_or_missing_snapshot() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/other", 0o755, 1000, 1000) + .unwrap(); + let snap = service.snapshot_subtree_path("/other").unwrap(); + + // A snapshot of /other cannot roll back /ws. + assert!(matches!( + service.rollback_subtree_path("/ws", snap.snapshot_id), + Err(MetadError::InvalidPath(_)) + )); + // An unknown snapshot id is not found. + assert!(matches!( + service.rollback_subtree_path("/ws", snap.snapshot_id + 9_999), + Err(MetadError::NotFound) + )); + // The rejected target is untouched and the legitimate one still works. + assert!(service + .rollback_subtree_path("/other", snap.snapshot_id) + .is_ok()); +} + +#[test] +fn restore_snapshot_to_fork_is_idempotent_and_holds_borrowed_objects() { + let (service, objects) = service_with_objects(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let original = publish_path_artifact(&service, "/ws/state", "ws/v1", b"version-one"); + let original_body = original.body.as_ref().unwrap(); + let original_block = block_key(original.attr.inode, original_body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + + service.remove_file_path("/ws/state").unwrap(); + publish_path_artifact(&service, "/ws/current", "ws/v2", b"version-two"); + + let first = service + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") + .unwrap(); + let retry = service + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") + .unwrap(); + assert_eq!(first, retry, "same restore request must be idempotent"); + assert_eq!(first.state, RestoreState::Complete); + assert_eq!( + read_artifact_at_path(&service, "/restored/state"), + b"version-one" + ); + assert!(service.lookup_path("/ws/state").unwrap().is_none()); + assert_eq!( + read_artifact_at_path(&service, "/ws/current"), + b"version-two" + ); + + let later = service.snapshot_subtree_path("/ws").unwrap(); + assert!(matches!( + service.restore_subtree_path_to_fork("/ws", later.snapshot_id, "/restored"), + Err(MetadError::RestoreDestinationConflict { .. }) + )); + + // The durable fork binding, not the caller's lease, owns retention after + // attach. Retiring both user-visible pins and running GC must not break the + // restored fork's borrowed v1 block. + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + assert!(service.retire_snapshot(later.snapshot_id).unwrap()); + let held = service.cleanup_pending_objects(100).unwrap(); + assert_eq!(held.deleted, 0, "held cleanup: {held:?}"); + assert!(held.blocked_by_snapshots >= 1, "held cleanup: {held:?}"); + assert!(objects.head(&original_block).unwrap().is_some()); + assert_eq!( + read_artifact_at_path(&service, "/restored/state"), + b"version-one" + ); + + // Removing the destination atomically releases its durable base reference; + // the already-enqueued source block can then be reclaimed. + service.remove_file_path("/restored/state").unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + let released = service.cleanup_pending_objects(100).unwrap(); + assert!(released.deleted >= 1, "released cleanup: {released:?}"); + assert!(objects.head(&original_block).unwrap().is_none()); +} + +#[test] +fn restore_borrowed_file_rename_escape_outlives_root_binding() { + let (service, objects) = service_with_objects(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/outside", 0o755, 1000, 1000) + .unwrap(); + let original = publish_path_artifact( + &service, + "/source/state", + "source/state", + b"borrowed-after-rename", + ); + let body = original.body.as_ref().unwrap(); + let block = block_key(original.attr.inode, body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + service + .rename_path("/restored/state", "/outside/state") + .unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + service.remove_file_path("/source/state").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + + for _ in 0..4 { + service.cleanup_pending_objects(128).unwrap(); + } + assert_eq!( + read_artifact_at_path(&service, "/outside/state"), + b"borrowed-after-rename" + ); + assert!(objects.head(&block).unwrap().is_some()); + + service.remove_file_path("/outside/state").unwrap(); + for _ in 0..6 { + service.cleanup_pending_objects(128).unwrap(); + } + assert!(objects.head(&block).unwrap().is_none()); +} + +#[test] +fn restore_borrowed_directory_rename_escape_outlives_root_binding() { + let (service, objects) = service_with_objects(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/nested", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/outside", 0o755, 1000, 1000) + .unwrap(); + let original = publish_path_artifact( + &service, + "/source/nested/state", + "source/nested/state", + b"borrowed-directory", + ); + let body = original.body.as_ref().unwrap(); + let block = block_key(original.attr.inode, body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + service + .rename_path("/restored/nested", "/outside/nested") + .unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + service.remove_file_path("/source/nested/state").unwrap(); + service.remove_empty_dir_path("/source/nested").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + + for _ in 0..4 { + service.cleanup_pending_objects(128).unwrap(); + } + assert_eq!( + read_artifact_at_path(&service, "/outside/nested/state"), + b"borrowed-directory" + ); + assert!(objects.head(&block).unwrap().is_some()); + + service.remove_file_path("/outside/nested/state").unwrap(); + service.remove_empty_dir_path("/outside/nested").unwrap(); + for _ in 0..6 { + service.cleanup_pending_objects(128).unwrap(); + } + assert!(objects.head(&block).unwrap().is_none()); +} + +#[test] +fn restore_borrowed_hardlink_escape_outlives_root_binding() { + let (service, objects) = service_with_objects(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + let outside = service + .create_dir_path("/outside", 0o755, 1000, 1000) + .unwrap(); + let original = publish_path_artifact( + &service, + "/source/state", + "source/state", + b"borrowed-hardlink", + ); + let body = original.body.as_ref().unwrap(); + let block = block_key(original.attr.inode, body.generation, 0, 0); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + let restored = service.lookup_path("/restored/state").unwrap().unwrap(); + + service + .link(restored.attr.inode, outside.attr.inode, dname(b"state")) + .unwrap(); + service.remove_file_path("/restored/state").unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + service.remove_file_path("/source/state").unwrap(); + service.remove_empty_dir_path("/source").unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + + for _ in 0..4 { + service.cleanup_pending_objects(128).unwrap(); + } + assert_eq!( + read_artifact_at_path(&service, "/outside/state"), + b"borrowed-hardlink" + ); + assert!(objects.head(&block).unwrap().is_some()); + + service.remove_file_path("/outside/state").unwrap(); + for _ in 0..6 { + service.cleanup_pending_objects(128).unwrap(); + } + assert!(objects.head(&block).unwrap().is_none()); +} + +#[test] +fn restore_to_fork_preserves_snapshot_path_index_membership_only() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/runs", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/runs/indexed.bin", + "source/indexed", + b"indexed", + ); + service + .create_file_path("/source/runs/plain.bin", 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .remove_file_path("/source/runs/indexed.bin") + .unwrap(); + + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + let indexed = service + .list_indexed_path_page("/restored/runs", None, 10) + .unwrap(); + assert_eq!( + indexed + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(), + vec![b"indexed.bin".as_slice()] + ); + let ordinary = service + .read_dir_plus_path_page("/restored/runs", None, 10) + .unwrap(); + assert_eq!(ordinary.entries.len(), 2); + assert!(ordinary + .entries + .iter() + .any(|entry| entry.dentry.name.as_bytes() == b"plain.bin")); +} + +#[test] +fn recursive_restore_reconstructs_snapshot_fork_shadow_after_rename_and_delete() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/indexed.bin", + "source/indexed", + b"indexed", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") + .unwrap(); + let restored_snapshot = service.snapshot_subtree_path("/restored").unwrap(); + + service + .rename_path("/restored/indexed.bin", "/restored/renamed.bin") + .unwrap(); + service.remove_file_path("/restored/renamed.bin").unwrap(); + service + .restore_subtree_path_to_fork("/restored", restored_snapshot.snapshot_id, "/copy") + .unwrap(); + + assert_eq!( + service + .list_indexed_path_page("/copy", None, 10) + .unwrap() + .entries + .into_iter() + .map(|entry| entry.dentry.name) + .collect::>(), + vec![dname(b"indexed.bin")] + ); + assert_eq!( + read_artifact_at_path(&service, "/copy/indexed.bin"), + b"indexed" + ); +} + +#[test] +fn recursive_restore_uses_historical_fork_shadow_after_source_root_release() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/indexed.bin", + "source/indexed", + b"indexed", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") + .unwrap(); + let restored_snapshot = service.snapshot_subtree_path("/restored").unwrap(); + assert!(service + .restore_subtree_path_to_fork_leave_preparing( + "/restored", + restored_snapshot.snapshot_id, + "/copy", + RestoreInitialization::default(), + ) + .is_err()); + + service.remove_file_path("/restored/indexed.bin").unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + for _ in 0..6 { + service.cleanup_pending_objects(128).unwrap(); + } + assert!(matches!( + service.restore_subtree_path_to_fork("/restored", restored_snapshot.snapshot_id, "/copy"), + Err(MetadError::RestoreInProgress) + )); + drain_restore_staging_cleanup(&service); + service + .restore_subtree_path_to_fork("/restored", restored_snapshot.snapshot_id, "/copy") + .unwrap(); + + assert_eq!( + service + .list_indexed_path_page("/copy", None, 10) + .unwrap() + .entries + .into_iter() + .map(|entry| entry.dentry.name) + .collect::>(), + vec![dname(b"indexed.bin")] + ); +} + +#[test] +fn restored_path_index_membership_survives_file_and_directory_moves() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_dir_path("/source/runs", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/runs/indexed.bin", + "source/indexed", + b"indexed", + ); + service + .create_file_path("/source/runs/plain.bin", 0o644, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + + service + .rename_path("/restored/runs/indexed.bin", "/restored/runs/renamed.bin") + .unwrap(); + assert_eq!( + service + .list_indexed_path_page("/restored/runs", None, 10) + .unwrap() + .entries[0] + .dentry + .name + .as_bytes(), + b"renamed.bin" + ); + + service + .rename_path("/restored/runs", "/restored/moved") + .unwrap(); + assert_eq!( + service + .list_indexed_path_page("/restored/moved", None, 10) + .unwrap() + .entries[0] + .dentry + .name + .as_bytes(), + b"renamed.bin" + ); + + service + .create_dir_path("/restored/other", 0o755, 1000, 1000) + .unwrap(); + service + .rename_path("/restored/moved/renamed.bin", "/restored/other/moved.bin") + .unwrap(); + let destination = service + .list_indexed_path_page("/restored/other", None, 10) + .unwrap(); + assert_eq!(destination.entries.len(), 1); + assert_eq!(destination.entries[0].dentry.name.as_bytes(), b"moved.bin"); + assert!(service + .list_indexed_path_page("/restored/moved", None, 10) + .unwrap() + .entries + .is_empty()); + assert!(service + .lookup_path("/restored/moved/plain.bin") + .unwrap() + .is_some()); + assert!(service + .list_indexed_path_page("/restored", None, 10) + .unwrap() + .entries + .iter() + .all(|entry| entry.dentry.name.as_bytes() != b"moved")); +} + +#[test] +fn restored_path_index_membership_survives_inode_api_rename() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/indexed.bin", + "source/indexed", + b"indexed", + ); + service + .create_dir_path("/source/runs", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/runs/nested.bin", + "source/runs/nested", + b"nested", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + service + .create_dir_path("/restored/destination", 0o755, 1000, 1000) + .unwrap(); + let restored = service.resolve_directory_path("/restored").unwrap(); + let destination = service + .resolve_directory_path("/restored/destination") + .unwrap(); + service + .rename( + restored, + &dname(b"indexed.bin"), + destination, + dname(b"moved.bin"), + ) + .unwrap(); + let indexed = service + .list_indexed_path_page("/restored/destination", None, 10) + .unwrap(); + assert_eq!(indexed.entries.len(), 1); + assert_eq!(indexed.entries[0].dentry.name.as_bytes(), b"moved.bin"); + + let outside = service + .create_dir_path("/outside", 0o755, 1000, 1000) + .unwrap(); + service + .rename( + destination, + &dname(b"moved.bin"), + outside.attr.inode, + dname(b"escaped.bin"), + ) + .unwrap(); + service + .rename( + restored, + &dname(b"runs"), + outside.attr.inode, + dname(b"escaped-runs"), + ) + .unwrap(); + let outside_indexed = service + .list_indexed_path_page("/outside", None, 10) + .unwrap(); + assert_eq!( + outside_indexed + .entries + .iter() + .map(|entry| entry.dentry.name.as_bytes()) + .collect::>(), + vec![b"escaped-runs".as_slice(), b"escaped.bin".as_slice()] + ); + assert_eq!( + service + .list_indexed_path_page("/outside/escaped-runs", None, 10) + .unwrap() + .entries[0] + .dentry + .name + .as_bytes(), + b"nested.bin" + ); +} + +#[test] +fn preparing_restore_path_index_is_hidden_and_rebuilt_after_reopen() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/indexed.bin", + "source/indexed", + b"indexed", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + assert!(service + .restore_subtree_path_to_fork_leave_preparing( + "/source", + snapshot.snapshot_id, + "/restored", + RestoreInitialization::default(), + ) + .is_err()); + let binding = metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_fork_binding(&row.value.0).unwrap()) + .next() + .unwrap(); + assert_eq!(binding.state, ForkBindingState::Preparing); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_path_index_set_prefix(service.mount, binding.base_ref_set_id,), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); + assert!(service.lookup_path("/restored").unwrap().is_none()); + assert_eq!( + service + .list_indexed_path_page("/source", None, 10) + .unwrap() + .entries + .len(), + 1 + ); + drop(service); + + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); + assert_eq!( + metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_path_index_set_prefix(reopened.mount, binding.base_ref_set_id,), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1, + "startup preserves hidden Preparing staging for an exact retry" + ); + assert!(reopened.lookup_path("/restored").unwrap().is_none()); + assert!(matches!( + reopened.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored"), + Err(MetadError::RestoreInProgress) + )); + drain_restore_staging_cleanup(&reopened); + reopened + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + assert_eq!( + reopened + .list_indexed_path_page("/restored", None, 10) + .unwrap() + .entries + .len(), + 1 + ); +} + +#[test] +fn preparing_restore_large_staging_reopens_o1_and_cleans_in_bounded_pages() { + const CHILDREN: usize = 160; + const XATTRS: usize = 160; + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let source = service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .create_files_in_dir_path( + "/source", + (0..CHILDREN) + .map(|index| dname(format!("child-{index:04}").as_bytes())) + .collect(), + 0o644, + 1000, + 1000, + ) + .unwrap(); + for index in 0..XATTRS { + service + .set_xattr( + source.attr.inode, + format!("user.wide-{index:04}").as_bytes(), + vec![index as u8; 60_000], + XattrSetMode::Create, + ) + .unwrap(); + } + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + assert!(service + .restore_subtree_path_to_fork_leave_preparing( + "/source", + snapshot.snapshot_id, + "/restored", + RestoreInitialization::default(), + ) + .is_err()); + drop(service); + + let before_open = metadata.metadata_store_stats(); + let reopened = + NoKvFs::open_existing(MountId::new(1).unwrap(), metadata.clone(), objects, 0).unwrap(); + let after_open = metadata.metadata_store_stats(); + assert!( + after_open + .scan_key_returned_total + .saturating_sub(before_open.scan_key_returned_total) + <= 128, + "readiness work must not scale with staging membership" + ); + let before_schedule = metadata.metadata_store_stats(); + assert!(matches!( + reopened.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored"), + Err(MetadError::RestoreInProgress) + )); + let after_schedule = metadata.metadata_store_stats(); + assert_eq!( + after_schedule.scan_key_returned_total - before_schedule.scan_key_returned_total, + 0, + "exact retry must schedule indexed cleanup without traversing staging" + ); + + let mut max_returned_per_pass = 0; + for _ in 0..4096 { + let pending = metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_staging_cleanup_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap(); + if pending.is_empty() { + break; + } + let before = metadata.metadata_store_stats(); + reopened.cleanup_pending_objects(1).unwrap(); + let after = metadata.metadata_store_stats(); + max_returned_per_pass = max_returned_per_pass.max( + after + .scan_key_returned_total + .saturating_sub(before.scan_key_returned_total), + ); + } + assert!( + max_returned_per_pass <= 80, + "bounded cleanup returned {max_returned_per_pass} keys in one pass" + ); + assert!(metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_staging_cleanup_prefix(reopened.mount), + start_after: None, + version: reopened.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + reopened + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + assert_eq!( + reopened.read_dir_plus_path("/restored").unwrap().len(), + CHILDREN + ); + assert_eq!( + reopened + .list_xattr(reopened.resolve_directory_path("/restored").unwrap()) + .unwrap() + .len(), + XATTRS + ); +} + +#[test] +fn restore_attach_conflict_discards_hidden_path_index_rows() { + let service = service(); + service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/indexed.bin", + "source/indexed", + b"indexed", + ); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + assert!(service + .restore_subtree_path_to_fork_leave_preparing( + "/source", + snapshot.snapshot_id, + "/restored", + RestoreInitialization::default(), + ) + .is_err()); + let binding = service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .into_iter() + .map(|row| decode_fork_binding(&row.value.0).unwrap()) + .next() + .unwrap(); + service + .create_dir_path("/restored", 0o755, 1000, 1000) + .unwrap(); + assert!(matches!( + service.restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored"), + Err(MetadError::RestoreDestinationConflict { .. }) + )); + drain_restore_staging_cleanup(&service); + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: restore_path_index_set_prefix(service.mount, binding.base_ref_set_id), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert!(service + .list_indexed_path_page("/restored", None, 10) + .unwrap() + .entries + .is_empty()); +} + +#[test] +fn restore_refreshes_object_gc_claim_between_bounded_staging_commits() { + let metadata = RotatingObjectGcClaimStore::new(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + let source = service + .create_dir_path("/source", 0o755, 1000, 1000) + .unwrap(); + service + .publish_artifact(PublishArtifact { + parent: source.attr.inode, + name: dname(b"state.bin"), + producer: "unit-test".to_owned(), + digest_uri: "sha256:stable".to_owned(), + content_type: "application/octet-stream".to_owned(), + manifest_id: "source/state".to_owned(), + bytes: b"stable".to_vec(), + mode: 0o644, + uid: 1000, + gid: 1000, + }) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/source").unwrap(); + + let outcome = service + .restore_subtree_path_to_fork("/source", snapshot.snapshot_id, "/restored") + .unwrap(); + assert_eq!(outcome.state, RestoreState::Complete); + assert!( + metadata.rotations() >= 2, + "restore must perform multiple bounded stages under independently refreshed claims" + ); + let restored = service.resolve_directory_path("/restored").unwrap(); + assert_eq!( + service + .read_artifact(restored, &dname(b"state.bin")) + .unwrap(), + b"stable" + ); +} + +#[test] +fn completed_restore_moved_from_original_destination_is_never_reclaimed_as_staging() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/ws/state", "ws/state", b"stable"); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + service + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") + .unwrap(); + service.rename_path("/restored", "/moved").unwrap(); + + assert!(matches!( + service.restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored"), + Err(MetadError::RestoreDestinationConflict { .. }) + )); + assert_eq!(read_artifact_at_path(&service, "/moved/state"), b"stable"); +} + +#[test] +fn completed_restore_retry_survives_source_and_pin_removal_after_reopen() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/ws/state", "ws/state", b"stable"); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let first = service + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") + .unwrap(); + service.remove_file_path("/ws/state").unwrap(); + service.remove_empty_dir_path("/ws").unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + drop(service); + + let reopened = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + let retry = reopened + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") + .unwrap(); + assert_eq!(retry, first); + assert_eq!( + read_artifact_at_path(&reopened, "/restored/state"), + b"stable" + ); +} + +#[test] +fn restore_initialization_is_complete_before_destination_attach() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/ws/metadata", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/ws/metadata/checkpoints.jsonl", + "registry", + b"source-alias\n", + ); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let manifest = br#"{"schema":"restore-test-v1"}"#.to_vec(); + + service + .restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + RestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: manifest.clone(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap(); + + assert!(service + .lookup_path("/restored/metadata/checkpoints.jsonl") + .unwrap() + .is_none()); + assert_eq!( + read_artifact_at_path(&service, "/restored/metadata/restore_manifest.json"), + manifest + ); + assert_eq!( + read_artifact_at_path(&service, "/ws/metadata/checkpoints.jsonl"), + b"source-alias\n" + ); +} + +#[test] +fn restore_initialization_digest_is_canonical_and_conflict_checked() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/ws/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let file = |path: &str, bytes: &[u8]| RestoreInitializationFile { + relative_path: path.to_owned(), + bytes: bytes.to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; + let first = service + .restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![file("metadata/b.json", b"b"), file("metadata/a.json", b"a")], + }, + ) + .unwrap(); + let retry = service + .restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![file("metadata/a.json", b"a"), file("metadata/b.json", b"b")], + }, + ) + .unwrap(); + assert_eq!(retry, first); + assert!(matches!( + service.restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![ + file("metadata/a.json", b"changed"), + file("metadata/b.json", b"b"), + ], + }, + ), + Err(MetadError::RestoreDestinationConflict { .. }) + )); +} + +#[test] +fn restore_initialization_rejects_duplicate_and_ancestor_overlaps_before_staging() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let file = |path: &str| RestoreInitializationFile { + relative_path: path.to_owned(), + bytes: b"value".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }; + let invalid = [ + RestoreInitialization { + remove_relative_paths: vec!["metadata".to_owned()], + files: vec![file("metadata/manifest.json")], + }, + RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![file("metadata"), file("metadata/manifest.json")], + }, + RestoreInitialization { + remove_relative_paths: vec!["metadata".to_owned(), "metadata/manifest.json".to_owned()], + files: Vec::new(), + }, + RestoreInitialization { + remove_relative_paths: vec!["metadata/manifest.json".to_owned()], + files: vec![file("metadata/manifest.json")], + }, + ]; + + for (index, initialization) in invalid.into_iter().enumerate() { + let destination = format!("/invalid-{index}"); + assert!(matches!( + service.restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + &destination, + initialization, + ), + Err(MetadError::InvalidPath(_)) + )); + assert!(service.lookup_path(&destination).unwrap().is_none()); + } + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); +} + +#[test] +fn malformed_gc_row_is_quarantined_once_without_starving_valid_candidate() { + let (service, objects) = service_with_objects(); + let mut malformed_key = gc_queue_prefix(service.mount); + malformed_key.extend_from_slice(&0u64.to_be_bytes()); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"malformed-gc-row".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::Gc, + primary_key: malformed_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::Gc, + key: malformed_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::Gc, + key: malformed_key, + op: MutationOp::Put, + value: Some(Value(b"not-an-object-gc-record".to_vec())), + }], + watch: Vec::new(), + }) + .unwrap(); + + let object = ObjectKey::new("blocks/1/900/1/0/0").unwrap(); + objects.put(&object, b"stale".to_vec()).unwrap(); + enqueue_gc_candidate( + &service, + ObjectGcRecord { + inode: InodeId::new(900).unwrap(), + generation: 1, + object_key: object.as_str().to_owned(), + size: 5, + digest_uri: "sha256:stale".to_owned(), + enqueue_version: 0, + enqueue_unix_ms: 0, + }, + ); + + let outcome = service.cleanup_pending_objects(10).unwrap(); + assert_eq!(outcome.scanned, 2); + assert_eq!(outcome.records_removed, 2); + assert_eq!(outcome.deleted, 1); + assert!(objects.head(&object).unwrap().is_none()); + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::Gc, + prefix: gc_queue_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: object_gc_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn malformed_base_ref_release_job_is_quarantined_without_starving_valid_job() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/ws/state", "base", b"stable"); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + service + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") + .unwrap(); + service.remove_file_path("/restored/state").unwrap(); + service.remove_empty_dir_path("/restored").unwrap(); + + let mut malformed_key = fork_base_ref_release_prefix(service.mount); + malformed_key.extend_from_slice(&0u64.to_be_bytes()); + let version = service.next_version().unwrap(); + service + .commit_metadata(MetadataCommand { + request_id: b"malformed-base-ref-release".to_vec(), + kind: CommandKind::CleanupObjects, + read_version: predecessor(version).unwrap(), + commit_version: version, + primary_family: RecordFamily::System, + primary_key: malformed_key.clone(), + predicates: vec![PredicateRef { + family: RecordFamily::System, + key: malformed_key.clone(), + predicate: Predicate::NotExists, + }], + mutations: vec![Mutation { + family: RecordFamily::System, + key: malformed_key, + op: MutationOp::Put, + value: Some(Value(b"not-a-fork-binding".to_vec())), + }], + watch: Vec::new(), + }) + .unwrap(); + + for _ in 0..4 { + service.cleanup_pending_objects(128).unwrap(); + if service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: fork_base_ref_release_prefix(service.mount), + start_after: None, + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty() + { + break; + } + } + assert!(service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: fork_base_ref_release_prefix(service.mount), + start_after: None, + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: fork_base_release_quarantine_prefix(service.mount), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn restore_initialization_size_limits_fail_before_durable_operation_creation() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let oversized = RestoreInitialization { + remove_relative_paths: Vec::new(), + files: vec![RestoreInitializationFile { + relative_path: "manifest.bin".to_owned(), + bytes: vec![0; 8 * 1024 * 1024 + 1], + content_type: "application/octet-stream".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }; + assert!(matches!( + service.restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/too-large", + oversized, + ), + Err(MetadError::RestoreResourceLimit { resource, .. }) + if resource == "restore initialization bytes" + )); + assert!(service.lookup_path("/too-large").unwrap().is_none()); + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); +} + +#[test] +fn restore_initialization_semantic_error_leaves_no_hold_inode_or_object() { + let (service, objects) = service_with_objects(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/ws/metadata", 0o755, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let inode_count_before = service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::Inode, + prefix: service.mount.get().to_be_bytes().to_vec(), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(); + let object_puts_before = service.object_stats().object_puts; + + assert!(matches!( + service.restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/invalid-semantic", + RestoreInitialization { + remove_relative_paths: vec!["metadata".to_owned()], + files: Vec::new(), + }, + ), + Err(MetadError::InvalidPath(message)) if message.contains("is a directory") + )); + assert!(service.lookup_path("/invalid-semantic").unwrap().is_none()); + assert!(service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::System, + prefix: fork_base_hold_prefix(service.mount), + start_after: None, + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert!(service + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 1, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .is_empty()); + assert_eq!( + service + .metadata + .scan_keys(KeyScanRequest { + family: RecordFamily::Inode, + prefix: service.mount.get().to_be_bytes().to_vec(), + start_after: None, + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap() + .len(), + inode_count_before + ); + assert_eq!(service.object_stats().object_puts, object_puts_before); + assert_eq!(objects.object_count(), 0); +} + +#[test] +fn detached_restore_initialization_emits_only_final_attach_watch() { let service = service(); - publish_path_artifact(&service, "/file.bin", "file", b"bytes"); - assert!(matches!( - service.clone_subtree_path("/file.bin"), - Err(MetadError::NotDirectory) - )); -} + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + service + .create_dir_path("/ws/metadata", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/ws/metadata/checkpoints.jsonl", + "registry", + b"source\n", + ); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let cursor = service.watch_subtree(InodeId::root()).unwrap(); -fn read_artifact_at_path( - service: &NoKvFs, - path: &str, -) -> Vec { - let (parent, name) = service.resolve_parent_path(path).unwrap(); - service.read_artifact(parent, &name).unwrap() + service + .restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + RestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"manifest".to_vec(), + content_type: "application/json".to_owned(), + mode: 0o644, + uid: 1000, + gid: 1000, + }], + }, + ) + .unwrap(); + + let records = service.replay_watch(InodeId::root(), cursor, 100).unwrap(); + assert_eq!(records.len(), 1, "records={records:?}"); + assert_eq!(records[0].event.kind, WatchEventKind::Create); + assert_eq!(records[0].event.parent, Some(InodeId::root())); + assert_eq!(records[0].event.name, Some(dname(b"restored"))); } #[test] -fn rollback_subtree_restores_snapshot_shares_blocks_and_reclaims_delta() { - let (service, objects) = service_with_objects(); - // 1. Build /ws with files a="A1", b="B1", sub/c="C1" (real object data). +fn restore_base_refs_are_paged_outside_the_binding_record() { + const BLOCKS: usize = 129; + let service = service(); let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let shards = (0..BLOCKS) + .map(|index| CheckpointShard { + name: dname(format!("block-{index:03}").as_bytes()), + bytes: vec![index as u8], + }) + .collect(); service - .create_dir_path("/ws/sub", 0o755, 1000, 1000) + .publish_checkpoint(ws.attr.inode, shards, 1000, 1000) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let outcome = service + .restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored") .unwrap(); - let a = publish_path_artifact(&service, "/ws/a", "ws/a", &vec![b'1'; 4096]); - let b = publish_path_artifact(&service, "/ws/b", "ws/b", &vec![b'2'; 4096]); - let c = publish_path_artifact(&service, "/ws/sub/c", "ws/sub/c", &vec![b'3'; 4096]); - let a_gen = a.body.as_ref().unwrap().generation; - let b_gen = b.body.as_ref().unwrap().generation; - let c_gen = c.body.as_ref().unwrap().generation; - let a1_block = block_key(a.attr.inode, a_gen, 0, 0); - let b1_block = block_key(b.attr.inode, b_gen, 0, 0); - let c1_block = block_key(c.attr.inode, c_gen, 0, 0); - assert!(objects.head(&a1_block).unwrap().is_some()); - assert!(objects.head(&b1_block).unwrap().is_some()); - assert!(objects.head(&c1_block).unwrap().is_some()); - // 2. Snapshot /ws. - let snap = service.snapshot_subtree_path("/ws").unwrap(); + let binding = service + .metadata + .get( + RecordFamily::ForkBinding, + &fork_binding_key(service.mount, outcome.destination_root), + service.read_version().unwrap(), + ReadPurpose::UserStrong, + ) + .unwrap() + .unwrap(); + assert!(binding.0.len() < 512, "binding bytes={}", binding.0.len()); + let refs = service + .metadata + .scan(ScanRequest { + family: RecordFamily::System, + prefix: crate::layout::fork_base_ref_owner_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) + .unwrap(); + assert_eq!(refs.len(), BLOCKS); + assert!(refs.iter().all(|row| row.value.0.len() < 1024)); +} - // 3. Diverge /ws: rewrite a->"A2", add d="D1", delete b. +#[test] +fn restore_initialization_recovers_before_attach_and_survives_immediate_reopen() { + let metadata = HoltMetadataStore::open_memory().unwrap(); + let objects = MemoryObjectStore::new(); + let service = NoKvFs::new(MountId::new(1).unwrap(), metadata.clone(), objects.clone()); + service.bootstrap_root(0o755, 1000, 1000).unwrap(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); service - .replace_artifact(PublishArtifact { - parent: ws.attr.inode, - name: dname(b"a"), - producer: "unit-test".to_owned(), - digest_uri: "sha256:a2".to_owned(), - content_type: "application/octet-stream".to_owned(), - manifest_id: "ws/a2".to_owned(), - bytes: vec![b'4'; 4096], + .create_dir_path("/ws/metadata", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/ws/metadata/checkpoints.jsonl", + "registry", + b"source-alias\n", + ); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + let initialization = RestoreInitialization { + remove_relative_paths: vec!["metadata/checkpoints.jsonl".to_owned()], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes: b"complete-manifest".to_vec(), + content_type: "application/json".to_owned(), mode: 0o644, uid: 1000, gid: 1000, - }) - .unwrap(); - let d = publish_path_artifact(&service, "/ws/d", "ws/d", &vec![b'5'; 4096]); - service.remove_file(ws.attr.inode, &dname(b"b")).unwrap(); - // Capture the delta's private blocks so we can assert their fate. - let a_diverged = service - .lookup_plus(ws.attr.inode, &dname(b"a")) - .unwrap() + }], + }; + assert!(service + .restore_subtree_path_to_fork_leave_preparing( + "/ws", + snapshot.snapshot_id, + "/restored", + initialization.clone(), + ) + .is_err()); + assert!(service.lookup_path("/restored").unwrap().is_none()); + service + .remove_file_path("/ws/metadata/checkpoints.jsonl") .unwrap(); - let a2_block = block_key( - a_diverged.attr.inode, - a_diverged.body.as_ref().unwrap().generation, - 0, + service.remove_empty_dir_path("/ws/metadata").unwrap(); + service.remove_empty_dir_path("/ws").unwrap(); + assert!(service.retire_snapshot(snapshot.snapshot_id).unwrap()); + drop(service); + + let reopened = NoKvFs::open_existing( + MountId::new(1).unwrap(), + metadata.clone(), + objects.clone(), 0, + ) + .unwrap(); + assert!(reopened.lookup_path("/restored").unwrap().is_none()); + assert!(matches!( + reopened.restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + initialization.clone(), + ), + Err(MetadError::RestoreInProgress) + )); + drain_restore_staging_cleanup(&reopened); + reopened + .restore_subtree_path_to_fork_initialized( + "/ws", + snapshot.snapshot_id, + "/restored", + initialization, + ) + .unwrap(); + drop(reopened); + + let completed = NoKvFs::open_existing(MountId::new(1).unwrap(), metadata, objects, 0).unwrap(); + assert!(completed + .lookup_path("/restored/metadata/checkpoints.jsonl") + .unwrap() + .is_none()); + assert_eq!( + read_artifact_at_path(&completed, "/restored/metadata/restore_manifest.json"), + b"complete-manifest" ); - let d1_block = block_key(d.attr.inode, d.body.as_ref().unwrap().generation, 0, 0); - assert!(objects.head(&a2_block).unwrap().is_some()); - assert!(objects.head(&d1_block).unwrap().is_some()); - // Pre-rollback /ws is the diverged state. - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'4'; 4096]); +} + +#[test] +fn restore_rejects_hardlinks_without_visible_or_staging_leaks() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let state = publish_path_artifact(&service, "/ws/state", "state", b"stable"); + service + .link(state.attr.inode, InodeId::root(), dname(b"outside")) + .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); + + assert!(matches!( + service.restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored"), + Err(MetadError::RestoreHardlinkUnsupported { inode }) if inode == state.attr.inode + )); + assert!(service.lookup_path("/restored").unwrap().is_none()); + assert_eq!(read_artifact_at_path(&service, "/ws/state"), b"stable"); + assert_eq!(read_artifact_at_path(&service, "/outside"), b"stable"); assert!(service - .lookup_plus(ws.attr.inode, &dname(b"b")) + .metadata + .scan(ScanRequest { + family: RecordFamily::ForkBinding, + prefix: fork_binding_prefix(service.mount), + start_after: None, + version: service.read_version().unwrap(), + limit: 0, + purpose: ReadPurpose::UserStrong, + }) .unwrap() - .is_none()); + .is_empty()); +} - // 4. Roll /ws back to the snapshot. +#[test] +fn restore_rejects_cross_shard_grafts_without_attach() { + let service = service(); + let ws = service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + let foreign = InodeId::compose(1, 42).unwrap(); service - .rollback_subtree_path("/ws", snap.snapshot_id) + .create_graft(ws.attr.inode, dname(b"foreign"), foreign, 0o755, 1000, 1000) .unwrap(); + let snapshot = service.snapshot_subtree_path("/ws").unwrap(); - // 5. /ws now exactly matches the snapshot: a="A1", b="B1" (restored), sub/c="C1", - // and d is gone. The target keeps its inode identity. - assert_eq!( - service.resolve_directory_path("/ws").unwrap(), - ws.attr.inode, - "rollback keeps the target root's identity" - ); - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + assert!(matches!( + service.restore_subtree_path_to_fork("/ws", snapshot.snapshot_id, "/restored"), + Err(MetadError::RestoreCrossShardUnsupported { inode }) if inode == foreign + )); + assert!(service.lookup_path("/restored").unwrap().is_none()); assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'3'; 4096] - ); - assert!( service - .lookup_plus(ws.attr.inode, &dname(b"d")) + .lookup_path("/ws/foreign") .unwrap() - .is_none(), - "the delta-only file d must be gone after rollback" + .unwrap() + .attr + .inode, + foreign ); +} - // 6. The rolled-back /ws is identical to a fresh clone of the snapshot: an empty - // diff in both directions. - let reference = service - .clone_subtree_path_into("/ws", "/reference") +#[test] +fn baseline_rollback_rejects_fork_backed_destination_without_mutation() { + let service = service(); + service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); + publish_path_artifact(&service, "/ws/state", "source", b"checkpoint"); + let source_snapshot = service.snapshot_subtree_path("/ws").unwrap(); + service + .restore_subtree_path_to_fork("/ws", source_snapshot.snapshot_id, "/restored") .unwrap(); - assert!(service - .diff_subtrees(ws.attr.inode, reference.root) - .unwrap() - .is_empty()); - assert!(service - .diff_subtrees(reference.root, ws.attr.inode) - .unwrap() - .is_empty()); + let restored_snapshot = service.snapshot_subtree_path("/restored").unwrap(); + publish_path_artifact(&service, "/restored/local", "local", b"local"); - // 7. GC: the discarded delta's private blocks (A2, D1) are reclaimable, while the - // restored shared blocks (A1, B1, C1) survive and stay readable. The reference - // clone shares the snapshot's blocks, so retire its pin too before reclaiming. - assert!(service.retire_snapshot(snap.snapshot_id).unwrap()); - assert!(service.retire_snapshot(reference.snapshot_id).unwrap()); - let reclaim = service.cleanup_pending_objects(100).unwrap(); - assert!( - reclaim.deleted >= 2, - "delta-only blocks must be reclaimable" - ); - assert!( - objects.head(&a2_block).unwrap().is_none(), - "A2 must be reclaimed" - ); - assert!( - objects.head(&d1_block).unwrap().is_none(), - "D1 must be reclaimed" - ); - assert!( - objects.head(&a1_block).unwrap().is_some(), - "A1 must survive" - ); - assert!( - objects.head(&b1_block).unwrap().is_some(), - "B1 must survive" - ); - assert!( - objects.head(&c1_block).unwrap().is_some(), - "C1 must survive" + assert!(matches!( + service.rollback_subtree_path("/restored", restored_snapshot.snapshot_id), + Err(MetadError::InvalidPath(message)) + if message.contains("restore-to-fork destination") + )); + assert_eq!( + read_artifact_at_path(&service, "/restored/state"), + b"checkpoint" ); - // Restored content is still readable from the shared blocks after reclaim. - assert_eq!(read_artifact_at_path(&service, "/ws/a"), vec![b'1'; 4096]); - assert_eq!(read_artifact_at_path(&service, "/ws/b"), vec![b'2'; 4096]); + assert_eq!(read_artifact_at_path(&service, "/restored/local"), b"local"); + + service.remove_file_path("/ws/state").unwrap(); + service.remove_empty_dir_path("/ws").unwrap(); + assert!(service + .retire_snapshot(source_snapshot.snapshot_id) + .unwrap()); + assert!(service + .retire_snapshot(restored_snapshot.snapshot_id) + .unwrap()); + for _ in 0..6 { + service.cleanup_pending_objects(128).unwrap(); + } assert_eq!( - read_artifact_at_path(&service, "/ws/sub/c"), - vec![b'3'; 4096] + read_artifact_at_path(&service, "/restored/state"), + b"checkpoint" ); } #[test] -fn rollback_subtree_rejects_foreign_or_missing_snapshot() { +fn baseline_rollback_rejects_nested_and_renamed_fork_subtrees() { let service = service(); - service.create_dir_path("/ws", 0o755, 1000, 1000).unwrap(); service - .create_dir_path("/other", 0o755, 1000, 1000) + .create_dir_path("/source", 0o755, 1000, 1000) .unwrap(); - let other_root = service.resolve_directory_path("/other").unwrap(); - let snap = service.snapshot_subtree_path("/other").unwrap(); + service + .create_dir_path("/source/nested", 0o755, 1000, 1000) + .unwrap(); + publish_path_artifact( + &service, + "/source/nested/state", + "source/nested", + b"checkpoint", + ); + let source_snapshot = service.snapshot_subtree_path("/source").unwrap(); + service + .restore_subtree_path_to_fork("/source", source_snapshot.snapshot_id, "/restored") + .unwrap(); + let nested_snapshot = service.snapshot_subtree_path("/restored/nested").unwrap(); + service.remove_file_path("/restored/nested/state").unwrap(); - // A snapshot of /other cannot roll back /ws. assert!(matches!( - service.rollback_subtree_path("/ws", snap.snapshot_id), - Err(MetadError::InvalidPath(_)) + service.rollback_subtree_path("/restored/nested", nested_snapshot.snapshot_id), + Err(MetadError::InvalidPath(message)) + if message.contains("restore-to-fork destination") )); - // An unknown snapshot id is not found. + service.rename_path("/restored/nested", "/escaped").unwrap(); assert!(matches!( - service.rollback_subtree_path("/ws", snap.snapshot_id + 9_999), - Err(MetadError::NotFound) + service.rollback_subtree_path("/escaped", nested_snapshot.snapshot_id), + Err(MetadError::InvalidPath(message)) + if message.contains("restore-to-fork destination") )); - // The rejected target is untouched and the legitimate one still works. - assert!(service - .rollback_subtree(other_root, snap.snapshot_id) - .is_ok()); + assert!(service.lookup_path("/escaped/state").unwrap().is_none()); } #[test] @@ -5385,7 +8046,6 @@ fn sync_metadata_log_archives_commit_before_recovery_ack() { METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - let key = b"sync-log-system-key".to_vec(); let value = b"sync-after-checkpoint".to_vec(); let commit_version = checkpoint.commit_version + 1; @@ -5458,7 +8118,6 @@ fn restore_metadata_with_sync_log_advances_allocator_after_replay() { METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - let post_checkpoint = service .create_dir_path("/runs/post-checkpoint", 0o755, 1000, 1000) .unwrap(); @@ -5526,7 +8185,6 @@ fn sync_metadata_log_archives_independent_batch_as_one_segment() { METADATA_LOG_ZERO_DIGEST, )) .unwrap(); - let results = service.create_file_batches_in_dir_path(vec![ CreateInDirPathBatch { parent_path: "/left".to_owned(), @@ -7118,7 +9776,9 @@ fn fallback_recovery_survives_command_dedupe_rows_on_real_store() { service .commit_metadata(MetadataCommand { request_id: probe_request.clone(), - kind: CommandKind::UpdateAttr, + // This command only seeds a CommandDedupe row for allocator + // recovery coverage; it does not mutate the live namespace. + kind: CommandKind::StageDetachedRestore, read_version: predecessor(probe_version).unwrap(), commit_version: probe_version, primary_family: RecordFamily::Inode, diff --git a/crates/nokv-protocol/src/lib.rs b/crates/nokv-protocol/src/lib.rs index ce060495c..e2d5263dc 100644 --- a/crates/nokv-protocol/src/lib.rs +++ b/crates/nokv-protocol/src/lib.rs @@ -293,6 +293,12 @@ pub enum MetadataRpcRequest { target_path: String, snapshot_id: u64, }, + RestoreSubtreePathToFork { + source_path: String, + snapshot_id: u64, + destination_path: String, + initialization: WireRestoreInitialization, + }, StatPathAtSnapshot { root_path: String, snapshot_id: u64, @@ -370,6 +376,9 @@ pub enum MetadataRpcRequest { path: String, replace: bool, }, + RefreshPreparedArtifactObjectGcEpoch { + prepared: WirePreparedArtifact, + }, PublishPreparedArtifact { prepared: WirePreparedArtifact, body: Box, @@ -469,6 +478,16 @@ pub enum WireMetadataError { /// The target dentry is a cross-shard graft point; remove/rename of it must /// go through the graft lifecycle. The client maps this to `EBUSY`. GraftPoint, + RestoreInProgress, + RestoreResourceLimit { + resource: String, + limit: u64, + actual: u64, + }, + StalePreparedArtifactObjectGcEpoch { + expected: u64, + current: u64, + }, SnapshotLeaseExpired { snapshot_id: u64, lease_expires_unix_ms: u64, @@ -487,6 +506,15 @@ pub enum WireMetadataError { snapshot_id: u64, attempts: usize, }, + RestoreHardlinkUnsupported { + inode: u64, + }, + RestoreCrossShardUnsupported { + inode: u64, + }, + RestoreDestinationConflict { + destination: String, + }, SyncLogArchiveFailed { committed: bool, message: String, @@ -587,6 +615,9 @@ pub enum MetadataRpcResult { root: u64, snapshot_id: u64, }, + Restore { + outcome: WireRestoreOutcome, + }, SubtreeDeltas { deltas: Vec, }, @@ -1129,6 +1160,23 @@ pub struct WirePreparedArtifact { pub replace: bool, pub dentry_version: Option, pub old_generation: Option, + pub object_gc_claim_version: u64, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct WireRestoreInitialization { + pub remove_relative_paths: Vec, + pub files: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct WireRestoreInitializationFile { + pub relative_path: String, + pub bytes: Vec, + pub content_type: String, + pub mode: u32, + pub uid: u32, + pub gid: u32, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -1192,6 +1240,17 @@ pub struct WireSnapshotPin { pub lease_expires_unix_ms: u64, } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct WireRestoreOutcome { + pub operation_id: String, + pub state: String, + pub source_root: u64, + pub destination_root: u64, + pub snapshot_id: u64, + pub read_version: u64, + pub cleanup_pending: bool, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(tag = "state", rename_all = "snake_case")] pub enum WireSnapshotRenewOutcome { @@ -1683,6 +1742,9 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { MetadataRpcRequest::RenamePath { source, .. } => RoutingKey::Path(source), MetadataRpcRequest::RenameReplacePath { source, .. } => RoutingKey::Path(source), MetadataRpcRequest::CloneSubtreePath { src_path, .. } => RoutingKey::Path(src_path), + MetadataRpcRequest::RestoreSubtreePathToFork { source_path, .. } => { + RoutingKey::Path(source_path) + } MetadataRpcRequest::DiffSubtrees { a_path, .. } => RoutingKey::Path(a_path), MetadataRpcRequest::RollbackSubtreePath { target_path, .. } => { RoutingKey::Path(target_path) @@ -1719,6 +1781,9 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { MetadataRpcRequest::PublishPreparedArtifact { prepared, .. } => { RoutingKey::Inode(prepared.parent) } + MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { prepared } => { + RoutingKey::Inode(prepared.parent) + } MetadataRpcRequest::PublishPreparedArtifactStagedSession { prepared, .. } => { RoutingKey::Inode(prepared.parent) } @@ -1734,6 +1799,74 @@ pub fn request_routing_key(request: &MetadataRpcRequest) -> RoutingKey<'_> { mod tests { use super::*; + #[test] + fn prepared_artifact_codec_preserves_object_gc_epoch() { + let prepared = WirePreparedArtifact { + mount: 1, + parent: 2, + name: "artifact.bin".to_owned(), + path: Some("/workspace/artifact.bin".to_owned()), + inode: 42, + generation: 7, + mtime_ms: 8, + ctime_ms: 9, + replace: true, + dentry_version: Some(6), + old_generation: Some(5), + object_gc_claim_version: 12, + }; + let mut encoded = Vec::new(); + prepared + .serialize(&mut rmp_serde::Serializer::new(&mut encoded).with_struct_map()) + .unwrap(); + let decoded: WirePreparedArtifact = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded, prepared); + + let request = MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { + prepared: prepared.clone(), + }; + assert_eq!( + decode_request(&encode_request(&request).unwrap()).unwrap(), + request + ); + } + + #[test] + fn prepared_artifact_codec_rejects_missing_object_gc_epoch() { + #[derive(Serialize)] + struct PreparedArtifactWithoutObjectGcEpoch { + mount: u64, + parent: u64, + name: String, + inode: u64, + generation: u64, + mtime_ms: u64, + ctime_ms: u64, + replace: bool, + dentry_version: Option, + old_generation: Option, + } + + let legacy = PreparedArtifactWithoutObjectGcEpoch { + mount: 1, + parent: 2, + name: "artifact.bin".to_owned(), + inode: 42, + generation: 7, + mtime_ms: 8, + ctime_ms: 9, + replace: false, + dentry_version: None, + old_generation: None, + }; + let mut encoded = Vec::new(); + legacy + .serialize(&mut rmp_serde::Serializer::new(&mut encoded).with_struct_map()) + .unwrap(); + + assert!(rmp_serde::from_slice::(&encoded).is_err()); + } + #[test] fn binary_codec_round_trips_metadata_request() { let request = MetadataRpcRequest::GrepPaths { diff --git a/crates/nokv-server/src/http.rs b/crates/nokv-server/src/http.rs index 0d50b43e0..82c4b7970 100644 --- a/crates/nokv-server/src/http.rs +++ b/crates/nokv-server/src/http.rs @@ -8,6 +8,8 @@ use crate::server::{Server, ServerError}; const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(5); const MAX_CONTROL_REQUEST_BYTES: usize = 1024 * 1024; +const DEFAULT_MANUAL_GC_LIMIT: usize = 1024; +const MAX_MANUAL_GC_LIMIT: usize = 16_384; #[derive(Clone, Debug, PartialEq, Eq)] struct HttpRequest { @@ -182,19 +184,28 @@ fn split_target(target: &str) -> (&str, Option<&str>) { fn gc_limit(query: Option<&str>) -> Result { let Some(query) = query else { - return Ok(usize::MAX); + return Ok(DEFAULT_MANUAL_GC_LIMIT); }; for pair in query.split('&') { let Some((key, value)) = pair.split_once('=') else { continue; }; if key == "limit" { - return value + let limit = value .parse::() - .map_err(|_| format!("invalid gc limit {value}")); + .map_err(|_| format!("invalid gc limit {value}"))?; + if limit == 0 { + return Ok(DEFAULT_MANUAL_GC_LIMIT); + } + if limit > MAX_MANUAL_GC_LIMIT { + return Err(format!( + "gc limit {limit} exceeds maximum {MAX_MANUAL_GC_LIMIT}" + )); + } + return Ok(limit); } } - Ok(usize::MAX) + Ok(DEFAULT_MANUAL_GC_LIMIT) } impl HttpResponse { @@ -285,6 +296,25 @@ mod tests { assert_eq!(response.status, "400 Bad Request"); } + #[test] + fn gc_limit_defaults_and_zero_are_bounded() { + assert_eq!(gc_limit(None).unwrap(), DEFAULT_MANUAL_GC_LIMIT); + assert_eq!(gc_limit(Some("limit=0")).unwrap(), DEFAULT_MANUAL_GC_LIMIT); + assert_eq!( + gc_limit(Some("unrelated=value")).unwrap(), + DEFAULT_MANUAL_GC_LIMIT + ); + } + + #[test] + fn gc_endpoint_rejects_limit_above_maximum() { + let server = test_server(); + let request = format!("GET /gc?limit={} HTTP/1.1", MAX_MANUAL_GC_LIMIT + 1); + let response = handle_parts(&server, &request, &[]); + assert_eq!(response.status, "400 Bad Request"); + assert!(response.body.contains("exceeds maximum")); + } + #[test] fn checkpoint_endpoint_triggers_metadata_snapshot() { let server = test_server(); diff --git a/crates/nokv-server/src/rpc/mod.rs b/crates/nokv-server/src/rpc/mod.rs index 468cbbba6..604da820b 100644 --- a/crates/nokv-server/src/rpc/mod.rs +++ b/crates/nokv-server/src/rpc/mod.rs @@ -17,12 +17,15 @@ pub(crate) use transport::{ read_frame, write_frame, MAX_FRAMED_RPC_WORKERS, MIN_FRAMED_RPC_WORKERS, }; -use nokv_meta::{MetadError, OpenPathReadPlanRequest, PublishArtifactStagedSession}; +use nokv_meta::{ + MetadError, OpenPathReadPlanRequest, PublishArtifactStagedSession, RestoreInitialization, + RestoreInitializationFile, +}; use nokv_protocol::{ decode_advisory_lock_kind, decode_file_type, decode_name_cursor, decode_request, decode_xattr_name, encode_envelope, encode_name_cursor, encode_xattr_name, MetadataRpcEnvelope, MetadataRpcRequest, MetadataRpcResult, WireAdvisoryLock, WireMetadataError, - WireOpenPathReadPlanRequest, WirePathMetadata, + WireOpenPathReadPlanRequest, WirePathMetadata, WireRestoreOutcome, }; use nokv_types::{AdvisoryLockRequest, SpecialNodeSpec}; @@ -693,6 +696,44 @@ fn execute(server: &Server, request: MetadataRpcRequest) -> Result { + let outcome = slot.service().restore_subtree_path_to_fork_initialized( + &source_path, + snapshot_id, + &destination_path, + RestoreInitialization { + remove_relative_paths: initialization.remove_relative_paths, + files: initialization + .files + .into_iter() + .map(|file| RestoreInitializationFile { + relative_path: file.relative_path, + bytes: file.bytes, + content_type: file.content_type, + mode: file.mode, + uid: file.uid, + gid: file.gid, + }) + .collect(), + }, + )?; + Ok(MetadataRpcResult::Restore { + outcome: WireRestoreOutcome { + operation_id: outcome.operation_id, + state: "complete".to_owned(), + source_root: outcome.source_root.get(), + destination_root: outcome.destination_root.get(), + snapshot_id: outcome.snapshot_id, + read_version: outcome.read_version, + cleanup_pending: outcome.cleanup_pending, + }, + }) + } MetadataRpcRequest::DiffSubtrees { a_path, b_path } => { let deltas = slot.service().diff_subtrees_path(&a_path, &b_path)?; Ok(MetadataRpcResult::SubtreeDeltas { @@ -931,6 +972,19 @@ fn execute(server: &Server, request: MetadataRpcRequest) -> Result { + if prepared.mount != slot.service().mount_id().get() { + return Err(ServerError::Metadata(MetadError::Codec( + "prepared artifact mount does not match server mount".to_owned(), + ))); + } + let prepared = slot + .service() + .refresh_prepared_artifact_object_gc_epoch(prepared_artifact(prepared)?)?; + Ok(MetadataRpcResult::PreparedArtifact { + prepared: wire_prepared_artifact(slot.service().mount_id(), &prepared), + }) + } MetadataRpcRequest::PublishPreparedArtifact { prepared, body, @@ -1102,11 +1156,13 @@ fn refreshes_metadata_view(request: &MetadataRpcRequest) -> bool { | MetadataRpcRequest::RenameReplacePath { .. } | MetadataRpcRequest::SnapshotSubtreePath { .. } | MetadataRpcRequest::CloneSubtreePath { .. } + | MetadataRpcRequest::RestoreSubtreePathToFork { .. } | MetadataRpcRequest::RollbackSubtreePath { .. } | MetadataRpcRequest::RetireSnapshot { .. } | MetadataRpcRequest::RenewSnapshot { .. } | MetadataRpcRequest::PrepareArtifact { .. } | MetadataRpcRequest::PrepareArtifactPath { .. } + | MetadataRpcRequest::RefreshPreparedArtifactObjectGcEpoch { .. } | MetadataRpcRequest::PublishPreparedArtifact { .. } | MetadataRpcRequest::PublishPreparedArtifactStagedSession { .. } => false, } diff --git a/crates/nokv-server/src/rpc/wire.rs b/crates/nokv-server/src/rpc/wire.rs index e4d053a2e..ed7a69c4a 100644 --- a/crates/nokv-server/src/rpc/wire.rs +++ b/crates/nokv-server/src/rpc/wire.rs @@ -134,6 +134,22 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { dest_shard: *dest_shard, }, MetadError::GraftPoint => WireMetadataError::GraftPoint, + MetadError::RestoreInProgress => WireMetadataError::RestoreInProgress, + MetadError::RestoreResourceLimit { + resource, + limit, + actual, + } => WireMetadataError::RestoreResourceLimit { + resource: resource.clone(), + limit: *limit, + actual: *actual, + }, + MetadError::StalePreparedArtifactObjectGcEpoch { expected, current } => { + WireMetadataError::StalePreparedArtifactObjectGcEpoch { + expected: *expected, + current: *current, + } + } MetadError::SnapshotLeaseExpired { snapshot_id, lease_expires_unix_ms, @@ -166,6 +182,17 @@ fn wire_metad_error(err: &MetadError) -> WireMetadataError { snapshot_id: *snapshot_id, attempts: *attempts, }, + MetadError::RestoreHardlinkUnsupported { inode } => { + WireMetadataError::RestoreHardlinkUnsupported { inode: inode.get() } + } + MetadError::RestoreCrossShardUnsupported { inode } => { + WireMetadataError::RestoreCrossShardUnsupported { inode: inode.get() } + } + MetadError::RestoreDestinationConflict { destination } => { + WireMetadataError::RestoreDestinationConflict { + destination: destination.clone(), + } + } other => WireMetadataError::Metadata { message: other.to_string(), }, @@ -240,6 +267,7 @@ pub(super) fn wire_prepared_artifact( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, } } @@ -258,6 +286,7 @@ pub(super) fn prepared_artifact( replace: prepared.replace, dentry_version: prepared.dentry_version, old_generation: prepared.old_generation, + object_gc_claim_version: prepared.object_gc_claim_version, }) } diff --git a/crates/nokv-types/src/lib.rs b/crates/nokv-types/src/lib.rs index 4f358c0b7..a86d481e9 100644 --- a/crates/nokv-types/src/lib.rs +++ b/crates/nokv-types/src/lib.rs @@ -13,7 +13,8 @@ pub use names::{parse_absolute_path, DentryName, NameError, PathError}; pub use shard::{ShardMap, ShardPrefix, ShardPrefixParseError, ShardRoute, DEFAULT_SHARD_INDEX}; pub use types::{ AdvisoryLock, AdvisoryLockKind, AdvisoryLockRequest, BlockDescriptor, BodyDescriptor, - ChunkManifest, DentryProjection, DentryRecord, FileType, ForkBinding, InodeAttr, InodeId, - ModelError, MountId, ObjectGcRecord, PathMetadata, ReadLease, RecordFamily, SliceManifest, - SnapshotPin, SpecialNodeSpec, WatchCursor, WatchEvent, WatchEventKind, WatchRecord, + ChunkManifest, DentryProjection, DentryRecord, FileType, ForkBinding, ForkBindingState, + InodeAttr, InodeId, ModelError, MountId, ObjectGcRecord, PathMetadata, ReadLease, RecordFamily, + SliceManifest, SnapshotPin, SpecialNodeSpec, WatchCursor, WatchEvent, WatchEventKind, + WatchRecord, }; diff --git a/crates/nokv-types/src/types.rs b/crates/nokv-types/src/types.rs index 380592639..f06ed3f4d 100644 --- a/crates/nokv-types/src/types.rs +++ b/crates/nokv-types/src/types.rs @@ -45,11 +45,11 @@ pub enum RecordFamily { Gc, CommandDedupe, History, - /// Anchors a lazy copy-on-write fork: maps a fork root to the source subtree - /// and the frozen read version it falls through to. Keyed by fork-root inode. + /// Durable lifecycle and exact base-reference owner for a + /// metadata-materialized copy-on-write fork. Keyed by fork-root inode. ForkBinding, - /// Durable `fork_inode -> source_inode` map for a lazy fork, so a bare-inode - /// read of an undiverged fork inode can recover the source. Keyed by fork inode. + /// Inverse index for borrowed PathIndex projections in a materialized fork, + /// keyed by destination parent inode and dentry name. ForkShadow, } @@ -217,17 +217,46 @@ pub struct ReadLease { pub lease_expires_unix_ms: u64, } -/// Anchor for a lazy copy-on-write fork (overlay/redirect-on-read clone): a -/// writable `fork_root` that overlays `source_root` as seen at -/// `pinned_read_version`. Undiverged reads fall through to the source at that -/// version; the retained `snapshot_id` keeps the shared base blocks GC-protected. +/// Lifecycle record for a metadata-materialized copy-on-write fork. +/// +/// The destination has its own inode and dentry tree; reads do not redirect to +/// the source namespace. File manifests may still reference source object blocks, +/// so `base_ref_set_id` owns the exact durable references that protect those +/// shared blocks, while ForkShadow indexes retain the corresponding borrowed +/// namespace projections. `snapshot_id` and `pinned_read_version` record the +/// construction checkpoint only; the snapshot lease is not the fork's lifetime +/// holder after the destination is attached. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ForkBindingState { + Preparing, + Complete, + /// The destination is no longer visible. Exact base-object references are + /// being released asynchronously before the durable binding is removed. + Releasing, +} + +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ForkBinding { pub fork_root: InodeId, pub source_root: InodeId, + /// Historical version used to materialize the destination metadata. pub pinned_read_version: u64, + /// Construction checkpoint identity retained for provenance and retries. pub snapshot_id: u64, pub created_version: u64, + /// Unique incarnation of the exact base-reference set owned by this fork. + /// Unlike `operation_digest`, a retried destination incarnation never + /// reuses this identifier. + pub base_ref_set_id: u64, + /// Stable digest of `(mount, source, snapshot, read-version, destination)`. + /// Zero is reserved for non-restore forks. + pub operation_digest: [u8; 32], + /// Digest of the canonical detached-tree initialization. It is checked + /// independently from the public operation id so manifest content may carry + /// that stable id without creating a hashing cycle. + pub initialization_digest: [u8; 32], + pub state: ForkBindingState, + pub destination_path: String, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] diff --git a/crates/nokv/src/bin/nokv.rs b/crates/nokv/src/bin/nokv.rs index 34af464ae..66ee390cc 100644 --- a/crates/nokv/src/bin/nokv.rs +++ b/crates/nokv/src/bin/nokv.rs @@ -2246,6 +2246,17 @@ mod tests { ObjectStoreConfig::s3(fake_s3_options()) } + fn test_tiered_object_config(object_root: PathBuf) -> ObjectStoreConfig { + ObjectStoreConfig::tiered_local_with_options( + LocalObjectStoreOptions::new(object_root), + fake_s3_options(), + TieredObjectStoreOptions { + put_policy: nokv_object::TieredPutPolicy::HotThenBackgroundCold, + ..TieredObjectStoreOptions::default() + }, + ) + } + fn spawn_test_server() -> SocketAddr { spawn_test_server_with_object_config(fake_server_object_config()) } @@ -3156,10 +3167,7 @@ mod tests { let client_store = LocalObjectStore::new(LocalObjectStoreOptions::new(object_root.clone())).unwrap(); let client = NoKvFsClient::connect( - spawn_test_server_with_object_config(ObjectStoreConfig::tiered_local( - object_root, - fake_s3_options(), - )), + spawn_test_server_with_object_config(test_tiered_object_config(object_root)), client_store, ); run_workbench_mcp_requests_on_client(client, options, requests) @@ -3175,9 +3183,8 @@ mod tests { SERVER.get_or_init(|| { let object_dir = tempdir().unwrap(); let object_root = object_dir.path().join("objects"); - let addr = spawn_test_server_with_object_config(ObjectStoreConfig::tiered_local( + let addr = spawn_test_server_with_object_config(test_tiered_object_config( object_root.clone(), - fake_s3_options(), )); // The server thread serves for the process lifetime; keep its // object root alive with it. @@ -3298,6 +3305,7 @@ mod tests { "workbench_snapshot", "workbench_snapshot_renew", "workbench_snapshot_list", + "workbench_restore", ] ); assert!(names.iter().all(|name| name.starts_with("workbench_"))); @@ -3319,6 +3327,14 @@ mod tests { assert_eq!(tool["inputSchema"]["additionalProperties"], false); assert!(tool["inputSchema"]["properties"].is_object()); } + let restore = tools + .iter() + .find(|tool| tool["name"] == "workbench_restore") + .unwrap(); + let variants = restore["inputSchema"]["properties"]["at_snapshot"]["anyOf"] + .as_array() + .unwrap(); + assert!(variants.iter().all(|variant| variant["type"] != "null")); } #[test] @@ -3720,10 +3736,8 @@ mod tests { fn workbench_mcp_list_and_grep_tolerate_out_of_band_entries() { let object_dir = tempdir().unwrap(); let object_root = object_dir.path().join("objects"); - let addr = spawn_test_server_with_object_config(ObjectStoreConfig::tiered_local( - object_root.clone(), - fake_s3_options(), - )); + let addr = + spawn_test_server_with_object_config(test_tiered_object_config(object_root.clone())); let connect = || { let store = LocalObjectStore::new(LocalObjectStoreOptions::new(object_root.clone())).unwrap(); @@ -5112,6 +5126,121 @@ mod tests { ); } + #[test] + fn workbench_mcp_restore_creates_idempotent_fork_and_preserves_source() { + let id = "ckpt-restore-src-010"; + let destination = "ckpt-restore-dst-010"; + let responses = run_shared_workbench_mcp_requests(vec![ + workbench_tool_call(1, "workbench_create", serde_json::json!({"id": id})), + workbench_tool_call( + 2, + "workbench_put_file", + serde_json::json!({ + "id": id, "section": "outputs", "path": "state.txt", "text": "frozen\n" + }), + ), + workbench_tool_call( + 3, + "workbench_commit", + serde_json::json!({"id": id, "manifest": {"task": id}}), + ), + workbench_tool_call( + 4, + "workbench_snapshot", + serde_json::json!({"id": id, "name": "older"}), + ), + workbench_tool_call( + 5, + "workbench_snapshot", + serde_json::json!({"id": id, "name": "cp1"}), + ), + workbench_tool_call( + 6, + "workbench_put_file", + serde_json::json!({ + "id": id, "section": "outputs", "path": "state.txt", + "text": "current\n", "replace": true + }), + ), + workbench_tool_call( + 7, + "workbench_restore", + serde_json::json!({ + "id": id, "at_snapshot": "cp1", "destination_id": destination + }), + ), + workbench_tool_call( + 8, + "workbench_read", + serde_json::json!({ + "id": destination, "section": "outputs", "path": "state.txt" + }), + ), + workbench_tool_call( + 9, + "workbench_read", + serde_json::json!({ + "id": id, "section": "outputs", "path": "state.txt" + }), + ), + workbench_tool_call( + 10, + "workbench_restore", + serde_json::json!({ + "id": id, "at_snapshot": "cp1", "destination_id": destination + }), + ), + workbench_tool_call( + 11, + "workbench_snapshot_list", + serde_json::json!({"id": destination}), + ), + workbench_tool_call( + 12, + "workbench_read", + serde_json::json!({ + "id": destination, + "section": "metadata", + "path": "restore_manifest.json" + }), + ), + ]); + for (index, response) in responses.iter().enumerate() { + assert_ne!( + response["result"]["isError"], true, + "response {index}: {response}" + ); + } + let first = &responses[6]["result"]["structuredContent"]; + let retry = &responses[9]["result"]["structuredContent"]; + assert_eq!(first["operation_id"], retry["operation_id"]); + assert_eq!(first["destination_workbench_id"], destination); + assert_eq!( + responses[7]["result"]["structuredContent"]["items"][0]["value"]["text"], + "frozen" + ); + assert_eq!( + responses[8]["result"]["structuredContent"]["items"][0]["value"]["text"], + "current" + ); + assert_eq!( + responses[10]["result"]["structuredContent"]["checkpoint_count"], 0, + "restored workbench must start with an empty checkpoint registry" + ); + let manifest_items = responses[11]["result"]["structuredContent"]["items"] + .as_array() + .unwrap(); + let restored_from = manifest_items + .iter() + .find(|item| item["value"]["key"] == "restored_from") + .unwrap(); + assert_eq!(restored_from["value"]["value"]["workbench_id"], id); + assert_eq!( + restored_from["value"]["value"]["snapshot_id"], + first["snapshot_id"] + ); + } + #[test] fn workbench_mcp_snapshot_rejects_ttl_over_max() { let id = "ckpt-ttlmax-003"; diff --git a/crates/nokv/src/bin/nokv/workbench_mcp.rs b/crates/nokv/src/bin/nokv/workbench_mcp.rs index a54f41735..4c0253a31 100644 --- a/crates/nokv/src/bin/nokv/workbench_mcp.rs +++ b/crates/nokv/src/bin/nokv/workbench_mcp.rs @@ -4,10 +4,12 @@ use std::time::{SystemTime, UNIX_EPOCH}; use base64::Engine; use nokv_agent::AgentToolDefinition; use nokv_client::{ - decode_name_cursor, encode_name_cursor, is_artifact_write_conflict, is_metadata_not_found, + decode_name_cursor, encode_name_cursor, is_artifact_write_retryable, is_metadata_not_found, ArtifactMetadata, ClientError, NoKvFsClient, }; -use nokv_meta::MetadError; +use nokv_meta::{ + restore_operation_id, MetadError, RestoreInitialization, RestoreInitializationFile, +}; use nokv_object::ObjectStore; use nokv_types::{FileType, PathMetadata}; use serde_json::{json, Value}; @@ -41,28 +43,51 @@ const MS_PER_DAY: u64 = 86_400_000; /// Every mint/renew appends one JSON line here so checkpoints stay discoverable /// after the tool response is gone (Phase-1 seed for L1 named refs). const CHECKPOINT_REGISTRY_RELPATH: &str = "checkpoints.jsonl"; -/// Retries after a lost artifact-write CAS; every attempt re-reads current state. -const WRITE_CONFLICT_RETRIES: usize = 5; -/// Linear backoff step between conflict retries. Zero-interval retries make N -/// synchronized writers replay the same race until the retry budget runs out; -/// a growing pause plus per-process jitter (see [`write_conflict_backoff`]) -/// de-synchronizes them. -const WRITE_CONFLICT_BACKOFF_STEP_MS: u64 = 10; - -/// Sleep before retry number `attempt` (1-based) of a conflicted write. +/// Retries after a safe artifact-write restart, including a lost CAS or an +/// object-GC epoch change. Every attempt re-enters prepare and stages a fresh, +/// unattached generation. +const ARTIFACT_WRITE_RETRIES: usize = 8; +/// Linear backoff step between safe write retries. Zero-interval retries make +/// synchronized writers and a busy object reaper repeatedly collide; a +/// growing pause plus per-process jitter de-synchronizes them. +const ARTIFACT_WRITE_BACKOFF_STEP_MS: u64 = 10; + +/// Sleep before retry number `attempt` (1-based) of a restartable write. /// Linear `attempt * 10ms` backoff plus a 0-8ms desync offset derived from the /// process id and the current clock nanoseconds, so two workbench processes /// that lost the same race do not wake in lockstep (no rand dependency). -/// Wrap a write conflict that survived the whole retry budget into an -/// actionable error: the caller learns the write is safe to re-issue while -/// the original error text stays available for diagnosis. -fn write_conflict_exhausted(attempts: usize, err: impl fmt::Display) -> WorkbenchToolError { - WorkbenchToolError::new(format!( - "write conflicted with concurrent writers after {attempts} attempts; retry the call ({err})" - )) +/// Wrap a restartable write that survived the whole retry budget into a typed, +/// retryable error while preserving the original diagnostic. +fn artifact_write_retries_exhausted(attempts: usize, err: ClientError) -> WorkbenchToolError { + if let ClientError::Metadata(MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current, + }) = &err + { + return WorkbenchToolError::typed( + "StalePreparedArtifactObjectGcEpoch", + err.to_string(), + true, + json!({ + "expected": expected, + "current": current, + "attempts": attempts, + "requires_restage": true, + "retry_scope": "tool_operation", + }), + ); + } + WorkbenchToolError::typed( + "ArtifactWriteContended", + format!( + "artifact write remained contended after {attempts} attempts; retry the call ({err})" + ), + true, + json!({"attempts": attempts}), + ) } -fn write_conflict_backoff(attempt: usize) { +fn artifact_write_backoff(attempt: usize) { let jitter_ms = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|now| now.subsec_nanos() as u64) @@ -70,7 +95,7 @@ fn write_conflict_backoff(attempt: usize) { .wrapping_add(u64::from(std::process::id())) % 8; std::thread::sleep(std::time::Duration::from_millis( - (attempt as u64) * WRITE_CONFLICT_BACKOFF_STEP_MS + jitter_ms, + (attempt as u64) * ARTIFACT_WRITE_BACKOFF_STEP_MS + jitter_ms, )); } @@ -440,6 +465,21 @@ pub fn tool_definitions() -> Vec { "additionalProperties": false }), }, + AgentToolDefinition { + name: "workbench_restore", + description: + "Restore a live checkpoint into a new workbench using COW. The source remains unchanged, the destination must be absent, and exact retries are idempotent.", + parameters: json!({ + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string"}, + "at_snapshot": restore_at_snapshot_parameter_schema(), + "destination_id": {"type": "string"} + }, + "additionalProperties": false + }), + }, ] } @@ -457,6 +497,16 @@ fn at_snapshot_parameter_schema() -> Value { }) } +fn restore_at_snapshot_parameter_schema() -> Value { + json!({ + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1} + ], + "description": "Required checkpoint to restore: a snapshot id (integer) or checkpoint name (non-empty string)." + }) +} + // Mirrors of the agent find/aggregate sub-schemas (nokv-agent // agent_tool_definitions); keep them in sync when the agent schemas change. fn predicates_parameter_schema() -> Value { @@ -531,6 +581,7 @@ where "workbench_snapshot" => snapshot_workbench(client, options, args), "workbench_snapshot_renew" => renew_snapshot_workbench(client, options, args), "workbench_snapshot_list" => list_snapshots_workbench(client, options, args), + "workbench_restore" => restore_workbench(client, options, args), other => Err(WorkbenchToolError::new(format!( "unknown workbench tool {other}" ))), @@ -576,16 +627,27 @@ where ensure_parent_dirs(client, options, &id, section, &rel_path)?; let path = section_path(options, &id, section, Some(&rel_path)); let digest_uri = digest_uri(&bytes); - let metadata = artifact_metadata(options, &path, &digest_uri, &content_type); - let entry = if replace { - client - .put_artifact_replace(&path, bytes, metadata) - .map_err(client_error)? - .entry - } else { - client - .put_artifact(&path, bytes, metadata) - .map_err(client_error)? + let mut attempts = 0; + let entry = loop { + let metadata = artifact_metadata(options, &path, &digest_uri, &content_type); + let result = if replace { + client + .put_artifact_replace(&path, bytes.clone(), metadata) + .map(|result| result.entry) + } else { + client.put_artifact(&path, bytes.clone(), metadata) + }; + match result { + Ok(entry) => break entry, + Err(err) if is_artifact_write_retryable(&err) && attempts < ARTIFACT_WRITE_RETRIES => { + attempts += 1; + artifact_write_backoff(attempts); + } + Err(err) if is_artifact_write_retryable(&err) => { + return Err(artifact_write_retries_exhausted(attempts + 1, err)); + } + Err(err) => return Err(client_error(err)), + } }; Ok(json!({ @@ -633,12 +695,12 @@ where // lost race against a concurrent writer is safe to retry as-is. match client.append_artifact(&path, bytes.clone(), metadata, None) { Ok(outcome) => break outcome, - Err(err) if is_artifact_write_conflict(&err) && attempts < WRITE_CONFLICT_RETRIES => { + Err(err) if is_artifact_write_retryable(&err) && attempts < ARTIFACT_WRITE_RETRIES => { attempts += 1; - write_conflict_backoff(attempts); + artifact_write_backoff(attempts); } - Err(err) if is_artifact_write_conflict(&err) => { - return Err(write_conflict_exhausted(attempts + 1, err)) + Err(err) if is_artifact_write_retryable(&err) => { + return Err(artifact_write_retries_exhausted(attempts + 1, err)) } Err(err) => return Err(client_error(err)), } @@ -746,13 +808,13 @@ where match client.put_artifact_replace_if_generation(&path, new_bytes, metadata, body.generation) { Ok(result) => break (result, replacements), - Err(err) if is_artifact_write_conflict(&err) && attempts < WRITE_CONFLICT_RETRIES => { + Err(err) if is_artifact_write_retryable(&err) && attempts < ARTIFACT_WRITE_RETRIES => { // A writer landed since our read; re-read and re-validate. attempts += 1; - write_conflict_backoff(attempts); + artifact_write_backoff(attempts); } - Err(err) if is_artifact_write_conflict(&err) => { - return Err(write_conflict_exhausted(attempts + 1, err)) + Err(err) if is_artifact_write_retryable(&err) => { + return Err(artifact_write_retries_exhausted(attempts + 1, err)) } Err(err) => return Err(client_error(err)), } @@ -1474,6 +1536,89 @@ where })) } +fn restore_workbench( + client: &NoKvFsClient, + options: &WorkbenchMcpOptions, + args: &Value, +) -> Result +where + O: ObjectStore + Send + Sync + 'static, +{ + let id = required_workbench_id(args)?; + let destination_id = required_string(args, "destination_id")?.to_owned(); + validate_workbench_id(&destination_id)?; + if destination_id == id { + return Err(WorkbenchToolError::new( + "destination_id must differ from the source workbench id", + )); + } + let at_snapshot = args + .get("at_snapshot") + .ok_or_else(|| WorkbenchToolError::new("missing required argument at_snapshot"))?; + let snapshot_id = resolve_at_snapshot(client, options, &id, at_snapshot)?; + + let source_path = workbench_path(options, &id); + let destination_path = workbench_path(options, &destination_id); + let manifest_path = section_path( + options, + &destination_id, + "metadata", + Some("restore_manifest.json"), + ); + let operation_id = restore_operation_id(&source_path, snapshot_id, &destination_path) + .map_err(|err| WorkbenchToolError::new(err.to_string()))?; + let manifest = json!({ + "schema": "nokv.workbench.restore_manifest.v1", + "operation_id": operation_id, + "restored_from": { + "workbench_id": id, + "path": source_path.clone(), + "snapshot_id": snapshot_id, + }, + "source_workbench_id": id, + "source_path": source_path.clone(), + "destination_workbench_id": destination_id, + "destination_path": destination_path.clone(), + "snapshot_id": snapshot_id, + }); + let bytes = serde_json::to_vec_pretty(&manifest).map_err(|err| { + WorkbenchToolError::new(format!("failed to encode restore manifest: {err}")) + })?; + let outcome = client + .metadata() + .restore_subtree_path_to_fork_initialized( + &source_path, + snapshot_id, + &destination_path, + RestoreInitialization { + remove_relative_paths: vec![format!("metadata/{CHECKPOINT_REGISTRY_RELPATH}")], + files: vec![RestoreInitializationFile { + relative_path: "metadata/restore_manifest.json".to_owned(), + bytes, + content_type: "application/json".to_owned(), + mode: DEFAULT_MODE_FILE, + uid: options.uid, + gid: options.gid, + }], + }, + ) + .map_err(client_error)?; + + Ok(json!({ + "status": "success", + "operation_id": outcome.operation_id, + "state": "complete", + "source_workbench_id": id, + "destination_workbench_id": destination_id, + "source_root": outcome.source_root.get(), + "destination_root": outcome.destination_root.get(), + "snapshot_id": outcome.snapshot_id, + "read_version": outcome.read_version, + "cleanup_pending": outcome.cleanup_pending, + "restore_manifest": manifest_path, + })) +} + #[derive(Default)] struct CheckpointGroup { name: Value, @@ -1956,12 +2101,12 @@ where let metadata = artifact_metadata(options, &path, &digest_uri, "application/x-ndjson"); match client.append_artifact(&path, line.clone(), metadata, None) { Ok(_) => return Ok(()), - Err(err) if is_artifact_write_conflict(&err) && attempts < WRITE_CONFLICT_RETRIES => { + Err(err) if is_artifact_write_retryable(&err) && attempts < ARTIFACT_WRITE_RETRIES => { attempts += 1; - write_conflict_backoff(attempts); + artifact_write_backoff(attempts); } - Err(err) if is_artifact_write_conflict(&err) => { - return Err(write_conflict_exhausted(attempts + 1, err)) + Err(err) if is_artifact_write_retryable(&err) => { + return Err(artifact_write_retries_exhausted(attempts + 1, err)) } Err(err) => return Err(client_error(err)), } @@ -2852,6 +2997,57 @@ fn client_error(err: ClientError) -> WorkbenchToolError { true, json!({"snapshot_id": snapshot_id, "attempts": attempts}), ), + ClientError::Metadata(MetadError::StalePreparedArtifactObjectGcEpoch { + expected, + current, + }) => WorkbenchToolError::typed( + "StalePreparedArtifactObjectGcEpoch", + err.to_string(), + true, + json!({ + "expected": expected, + "current": current, + "requires_restage": true, + "retry_scope": "tool_operation", + }), + ), + ClientError::Metadata(MetadError::RestoreInProgress) => { + WorkbenchToolError::typed("RestoreInProgress", err.to_string(), true, json!({})) + } + ClientError::Metadata(MetadError::RestoreDestinationConflict { destination }) => { + WorkbenchToolError::typed( + "RestoreDestinationConflict", + err.to_string(), + false, + json!({"destination": destination}), + ) + } + ClientError::Metadata(MetadError::RestoreResourceLimit { + resource, + limit, + actual, + }) => WorkbenchToolError::typed( + "RestoreResourceLimit", + err.to_string(), + false, + json!({"resource": resource, "limit": limit, "actual": actual}), + ), + ClientError::Metadata(MetadError::RestoreHardlinkUnsupported { inode }) => { + WorkbenchToolError::typed( + "RestoreHardlinkUnsupported", + err.to_string(), + false, + json!({"inode": inode.get()}), + ) + } + ClientError::Metadata(MetadError::RestoreCrossShardUnsupported { inode }) => { + WorkbenchToolError::typed( + "RestoreCrossShardUnsupported", + err.to_string(), + false, + json!({"inode": inode.get()}), + ) + } _ => WorkbenchToolError::typed("NoKvClientError", err.to_string(), false, json!({})), } } @@ -2908,18 +3104,45 @@ mod tests { } #[test] - fn write_conflict_exhausted_keeps_inner_error_and_retry_guidance() { - let message = write_conflict_exhausted(6, "metadata predicate failed").to_string(); + fn artifact_write_exhaustion_is_typed_retryable_and_keeps_the_cause() { + let error = artifact_write_retries_exhausted( + 9, + ClientError::Metadata(MetadError::Metadata( + nokv_meta::MetadataError::PredicateFailed, + )), + ); + let value = error.as_value(); + let message = error.to_string(); + assert_eq!(value["code"], "ArtifactWriteContended"); + assert_eq!(value["retryable"], true); + assert_eq!(value["details"]["attempts"], 9); assert!( - message.contains("conflicted with concurrent writers after 6 attempts; retry the call"), + message.contains("remained contended after 9 attempts; retry the call"), "message: {message}" ); assert!( - message.contains("metadata predicate failed"), + message.contains("metadata command predicate failed"), "message: {message}" ); } + #[test] + fn stale_object_gc_epoch_is_a_typed_retryable_tool_error() { + let error = client_error(ClientError::Metadata( + MetadError::StalePreparedArtifactObjectGcEpoch { + expected: 11, + current: 17, + }, + )); + let value = error.as_value(); + assert_eq!(value["code"], "StalePreparedArtifactObjectGcEpoch"); + assert_eq!(value["retryable"], true); + assert_eq!(value["details"]["expected"], 11); + assert_eq!(value["details"]["current"], 17); + assert_eq!(value["details"]["requires_restage"], true); + assert_eq!(value["details"]["retry_scope"], "tool_operation"); + } + #[test] fn split_piped_grep_pattern_enforces_alternative_cap() { let forwarded_for = |pattern: String| { diff --git a/scripts/lingtai-workbench/CHECKPOINT_RESTORE_LIVE_E2E.md b/scripts/lingtai-workbench/CHECKPOINT_RESTORE_LIVE_E2E.md new file mode 100644 index 000000000..3f178d654 --- /dev/null +++ b/scripts/lingtai-workbench/CHECKPOINT_RESTORE_LIVE_E2E.md @@ -0,0 +1,138 @@ + + +# Checkpoint/Restore Live E2E + +`checkpoint_restore_live_e2e.py` is the checked-in acceptance path for the +NoKV to LingTai workbench checkpoint/restore contract. It starts an isolated +real stack: + +```text +RustFS Docker +-> nokv serve (100ms history/object GC workers) +-> nokv mcp --profile workbench +-> LingTai Agent registry/init registration, placeholder expansion, retry, and handler +-> three real LingTai MCPClient sessions from the resolved per-agent launch configs +``` + +The harness owns unique ports, a unique bucket and container, and a temporary +metadata directory. It always stops the server and removes its RustFS container. +Pass `--keep-state` only when preserving the metadata and logs is useful for +debugging. Every process wait, tool call, HTTP poll, and GC loop has a deadline. + +## Run + +Use the Python environment from the exact LingTai checkout under acceptance. +For each agent the harness writes real `mcp_registry.jsonl` and `init.json` +inputs, boots `Agent`, captures the registered client's resolved launch config, +closes that client, calls `Agent._retry_failed_mcps()`, and verifies the root is +unchanged. It then invokes the registered `workbench_create` handler before +opening the three independent MCP sessions used by the remaining scenarios. + +```bash +~/lingtai-kernel/.venv/bin/python \ + ./scripts/lingtai-workbench/checkpoint_restore_live_e2e.py \ + --lingtai-kernel-dir ~/lingtai-kernel \ + --profile quick +``` + +The full acceptance counts are: + +```bash +~/lingtai-kernel/.venv/bin/python \ + ./scripts/lingtai-workbench/checkpoint_restore_live_e2e.py \ + --lingtai-kernel-dir ~/lingtai-kernel \ + --profile full \ + --require-all +``` + +`full` runs 100 concurrent 1-day/90-day renew rounds per client, 200 +short-lease renew/reaper races, and 101 historical entries with `limit=7`. +`quick` keeps the same assertions with smaller local iteration counts. + +Required local commands are `docker`, `aws`, and Cargo. The RustFS image, +ports, NoKV binary, deadlines, and state directory are configurable in +`--help`. Use `--no-build` only with a binary built from the checkout under +test. + +While the restore track is being stacked on top of the lease/root-binding and +historical-index tracks, this command validates A+B without claiming restore +coverage: + +```bash +~/lingtai-kernel/.venv/bin/python \ + ./scripts/lingtai-workbench/checkpoint_restore_live_e2e.py \ + --lingtai-kernel-dir ~/lingtai-kernel \ + --profile quick \ + --allow-missing-restore +``` + +The JSON summary marks every unavailable C-only scenario `skipped`. This is a +development-only A+B command. The final integrated acceptance command is +exactly the `--profile full --require-all` command above: `--require-all` is +valid only with `full` and turns any skipped scenario into a failing exit. + +## Assertions + +The live harness verifies: + +- LingTai reads the real registry/init inputs, expands each agent root, recovers + a deliberately closed registered MCP through `_retry_failed_mcps()`, invokes + the registered handler, and preserves the resolved roots across restart; +- two concurrent LingTai MCP clients cannot shorten an acknowledged renewal; +- every manual-GC and server-health snapshot-reaper payload satisfies + `reaped + conflicted == expired_candidates`; +- 100-300ms renew/reaper races include a deterministic + `scan -> renew -> conditional delete` interleaving, require the background + reaper health counter's `conflicted` delta to be positive, and retain + snapshot readability after every successful renew; +- two expanded agent roots using the same workbench id reject foreign numeric + ids and forged checkpoint names for stat, list, read, and renew with + `SnapshotRootMismatch`, before and after restart; +- a just-acknowledged checkpoint registry update and its pinned point-read + survive immediate `SIGKILL` and reopen without relying on destructors or a + final checkpoint; +- delete, rename, and delete/recreate historical listings agree with point + reads; every page respects `limit`, truncated pages are non-empty, cursors + advance, and the full 101-entry/`limit=7` run produces exactly 15 pages with + no omission or duplicate; +- destination first visibility already has no `metadata/checkpoints.jsonl`, + already has a structured `metadata/restore_manifest.json`, and leaves the + source registry unchanged; +- restored nested, renamed, deleted, and current-only paths exactly match the + checkpoint while the source remains at its current state, and the terminal + restore response carries the complete typed outcome; +- restore is COW, source-preserving, idempotent across a second MCP client and + server restart, performs no body PUT beyond the one permitted manifest PUT, + and rejects another snapshot at the same destination with + `RestoreDestinationConflict`; +- retiring source checkpoints and deleting the source cannot reclaim + fork-shared RustFS objects, while deleting the fork eventually releases + those exact objects. + +## Deterministic Live-Test Surfaces and Coverage Boundary + +The short-lease race probes `snapshot PATH 200`. An older A+B binary reports a +skip in the development-only mode; the final breaking CLI enables the real +race, and `--require-all` rejects that skip. + +Operator in-place rollback, FUSE mount/cache coherence, FUSE writeback journal +replay, and FUSE object-GC restaging are deliberately outside this Workbench +acceptance boundary. They belong to separate operator/FUSE correctness tracks +and must not become implicit dependencies of `workbench_restore`. + +The live harness cannot directly enumerate durable fork-base references. Exact +RustFS object-key retention and reclamation provide live, indirect leak +evidence; internal base-reference invariants remain deterministic Rust +recovery/TDD acceptance. + +## Static Tests + +```bash +python3 ./scripts/lingtai-workbench/checkpoint_restore_live_e2e_test.py +ruff check ./scripts/lingtai-workbench/checkpoint_restore_live_e2e.py \ + ./scripts/lingtai-workbench/checkpoint_restore_live_e2e_test.py +git diff --check +``` diff --git a/scripts/lingtai-workbench/README.md b/scripts/lingtai-workbench/README.md index c7c6fd8d8..f39072a02 100644 --- a/scripts/lingtai-workbench/README.md +++ b/scripts/lingtai-workbench/README.md @@ -23,7 +23,8 @@ Run the full local preflight and MCP install path with: - starts or verifies RustFS and the `nokv-lingtai-workbench` bucket - verifies or starts the NoKV server at `127.0.0.1:7799` - checks that the LingTai TUI runtime can see the `nokv-workbench` skill -- checks that the workbench MCP exposes `workbench_*` tools +- checks that the workbench MCP exposes snapshot renewal, discovery, and + restore-to-fork tools - idempotently installs the MCP registration into the selected LingTai agent Defaults: @@ -181,13 +182,21 @@ The MCP server exposes workbench tools with the `workbench_` prefix: ```text workbench_create workbench_put_file +workbench_append +workbench_edit workbench_list workbench_stat workbench_read workbench_grep +workbench_search +workbench_aggregate +workbench_catalog workbench_find workbench_commit workbench_snapshot +workbench_snapshot_renew +workbench_snapshot_list +workbench_restore ``` The server registration name remains `nokv-workbench`; that is the MCP server @@ -201,3 +210,8 @@ Run the installer tests with: python3 ./scripts/lingtai-workbench/install_workbench_mcp_test.py bash -n ./scripts/lingtai-workbench/up.sh ``` + +The environment-gated checkpoint/restore acceptance test additionally requires +RustFS, a real `nokv serve` process, and the LingTai source checkout selected by +`LINGTAI_KERNEL_DIR`. It validates the MCP subprocess path rather than replacing +it with a test JSON-RPC client. diff --git a/scripts/lingtai-workbench/checkpoint_restore_live_e2e.py b/scripts/lingtai-workbench/checkpoint_restore_live_e2e.py new file mode 100755 index 000000000..03070f7f9 --- /dev/null +++ b/scripts/lingtai-workbench/checkpoint_restore_live_e2e.py @@ -0,0 +1,2260 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 + +"""Live NoKV -> LingTai workbench checkpoint/restore acceptance harness. + +The harness intentionally uses the real process boundaries: + + RustFS Docker -> nokv serve -> LingTai Agent MCP registration/retry + -> nokv mcp --profile workbench -> LingTai MCPClient + +It owns an isolated RustFS container, bucket, metadata directory, and server +port. Every process wait and network poll has a deadline. The default quick +profile is suitable for local iteration; the full profile runs the acceptance +counts from the checkpoint/restore plan. +""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import dataclasses +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Callable +from unittest.mock import MagicMock + + +BASE_WORKBENCH_TOOLS = ( + "workbench_create", + "workbench_put_file", + "workbench_append", + "workbench_edit", + "workbench_list", + "workbench_stat", + "workbench_read", + "workbench_grep", + "workbench_search", + "workbench_aggregate", + "workbench_catalog", + "workbench_find", + "workbench_commit", + "workbench_snapshot", + "workbench_snapshot_renew", + "workbench_snapshot_list", +) +RESTORE_TOOL = "workbench_restore" +WORKBENCH_ROOT_TEMPLATE = "/agents/{agent_id}/wb" + + +class AcceptanceError(RuntimeError): + """A deterministic acceptance assertion failed.""" + + +@dataclasses.dataclass(frozen=True) +class WorkloadProfile: + renew_rounds: int + reaper_rounds: int + history_entries: int + page_limit: int = 7 + + +def workload_profile(name: str) -> WorkloadProfile: + if name == "quick": + return WorkloadProfile(renew_rounds=8, reaper_rounds=20, history_entries=22) + if name == "full": + return WorkloadProfile(renew_rounds=100, reaper_rounds=200, history_entries=101) + raise AcceptanceError(f"unknown workload profile: {name}") + + +@dataclasses.dataclass(frozen=True) +class ToolError: + code: str + message: str + retryable: bool + details: dict[str, Any] + + +@dataclasses.dataclass(frozen=True) +class ResolvedMcpLaunch: + command: str + args: list[str] + env: dict[str, str] + root: str + + +def decode_tool_error(result: dict[str, Any]) -> ToolError: + """Decode LingTai MCPClient's error envelope into the public typed error.""" + if "code" in result and "message" in result: + payload: Any = result + elif result.get("status") == "error": + payload = result.get("message") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError as exc: + raise AcceptanceError( + f"MCP error is not structured JSON: {payload!r}" + ) from exc + else: + raise AcceptanceError(f"result is not an MCP tool error: {result!r}") + if not isinstance(payload, dict): + raise AcceptanceError(f"MCP error payload is not an object: {payload!r}") + code = payload.get("code") + message = payload.get("message") + retryable = payload.get("retryable") + details = payload.get("details") + if not isinstance(code, str) or not isinstance(message, str): + raise AcceptanceError(f"typed MCP error lacks code/message: {payload!r}") + if not isinstance(retryable, bool) or not isinstance(details, dict): + raise AcceptanceError(f"typed MCP error lacks retryable/details: {payload!r}") + return ToolError(code, message, retryable, details) + + +def parse_snapshot_id(output: str) -> int: + match = re.search(r"\bid=(\d+)\b", output) + if match is None: + raise AcceptanceError(f"snapshot CLI output lacks id=: {output!r}") + return int(match.group(1)) + + +def parse_snapshot_expiry(output: str) -> int: + match = re.search(r"\blease_expires_unix_ms=(\d+)\b", output) + if match is None: + raise AcceptanceError( + f"snapshot CLI output lacks lease_expires_unix_ms=: {output!r}" + ) + return int(match.group(1)) + + +class PageAccumulator: + """Verify cursor progress, uniqueness, and global dentry-name ordering.""" + + def __init__(self, page_limit: int) -> None: + self.page_limit = page_limit + self.page_count = 0 + self.page_sizes: list[int] = [] + self.names: list[str] = [] + self._seen_names: set[str] = set() + self._seen_cursors: set[str] = set() + + def add(self, names: list[str], next_cursor: str | None, truncated: bool) -> None: + self.page_count += 1 + self.page_sizes.append(len(names)) + if len(names) > self.page_limit: + raise AcceptanceError( + f"page returned {len(names)} entries above limit {self.page_limit}" + ) + if names != sorted(names): + raise AcceptanceError(f"page is not sorted by name: {names!r}") + for name in names: + if name in self._seen_names: + raise AcceptanceError(f"duplicate entry across pages: {name}") + if self.names and name <= self.names[-1]: + raise AcceptanceError( + f"pages are not globally sorted: {self.names[-1]!r} then {name!r}" + ) + self._seen_names.add(name) + self.names.append(name) + if truncated: + if not names: + raise AcceptanceError("truncated page is empty") + if not isinstance(next_cursor, str) or not next_cursor: + raise AcceptanceError("truncated page lacks next_cursor") + if next_cursor in self._seen_cursors: + raise AcceptanceError(f"pagination cursor repeated: {next_cursor}") + self._seen_cursors.add(next_cursor) + elif next_cursor is not None: + raise AcceptanceError("terminal page returned a next_cursor") + + +def validate_tool_contract(tools: list[dict[str, Any]], require_restore: bool) -> bool: + """Validate the final 17-tool workbench surface and restore schema.""" + by_name = {tool.get("name"): tool for tool in tools} + restore_available = RESTORE_TOOL in by_name + expected = set(BASE_WORKBENCH_TOOLS) + if restore_available: + expected.add(RESTORE_TOOL) + actual = set(by_name) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise AcceptanceError( + f"unexpected workbench tool surface; missing={missing}, extra={extra}" + ) + if require_restore and not restore_available: + raise AcceptanceError( + "workbench_restore is missing; integrate the restore-to-fork track " + "or pass --allow-missing-restore while validating A+B only" + ) + if not restore_available: + return False + schema = by_name[RESTORE_TOOL].get("schema") + if not isinstance(schema, dict): + raise AcceptanceError("workbench_restore lacks an input schema") + required = set(schema.get("required", [])) + properties = schema.get("properties") + expected_fields = {"id", "at_snapshot", "destination_id"} + if required != expected_fields: + raise AcceptanceError( + f"workbench_restore required fields must be exactly {sorted(expected_fields)}" + ) + if not isinstance(properties, dict) or set(properties) != expected_fields: + raise AcceptanceError( + f"workbench_restore properties must be exactly {sorted(expected_fields)}" + ) + if schema.get("additionalProperties") is not False: + raise AcceptanceError("workbench_restore must reject additional properties") + at_snapshot = properties.get("at_snapshot") + if not isinstance(at_snapshot, dict): + raise AcceptanceError("workbench_restore at_snapshot lacks a schema") + alternatives = at_snapshot.get("anyOf") + if ( + not isinstance(alternatives, list) + or len(alternatives) != 2 + or any(not isinstance(alternative, dict) for alternative in alternatives) + ): + raise AcceptanceError( + "workbench_restore at_snapshot must use exactly two object alternatives" + ) + accepted_types = { + alternative.get("type") + for alternative in alternatives + if isinstance(alternative, dict) + } + if accepted_types != {"integer", "string"}: + raise AcceptanceError( + "workbench_restore at_snapshot must accept only a non-negative " + f"snapshot id or checkpoint name, got {sorted(str(t) for t in accepted_types)}" + ) + integer_schema = next( + alternative + for alternative in alternatives + if isinstance(alternative, dict) and alternative.get("type") == "integer" + ) + if integer_schema.get("minimum") != 0: + raise AcceptanceError( + "workbench_restore numeric at_snapshot must be non-negative" + ) + string_schema = next( + alternative + for alternative in alternatives + if isinstance(alternative, dict) and alternative.get("type") == "string" + ) + if string_schema.get("minLength") != 1: + raise AcceptanceError( + "workbench_restore checkpoint names must be non-empty strings" + ) + return True + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _tail(path: Path, lines: int = 100) -> str: + if not path.exists(): + return "" + return "\n".join( + path.read_text(encoding="utf-8", errors="replace").splitlines()[-lines:] + ) + + +@dataclasses.dataclass +class LiveConfig: + repo_root: Path + cargo_bin: Path + nokv_bin: Path + lingtai_kernel_dir: Path + state_dir: Path + server_port: int + rustfs_port: int + rustfs_console_port: int + rustfs_image: str + bucket: str + container: str + profile: WorkloadProfile + command_timeout: float + tool_timeout: float + startup_timeout: float + gc_deadline: float + build: bool + keep_state: bool + allow_missing_restore: bool + require_all: bool + + +class LiveEnvironment: + def __init__(self, config: LiveConfig) -> None: + self.config = config + self.server_bind = f"127.0.0.1:{config.server_port}" + self.s3_endpoint = f"http://127.0.0.1:{config.rustfs_port}" + self.server_log = config.state_dir / "nokv-server.log" + self._server: subprocess.Popen[str] | None = None + self._server_log_handle: Any = None + self._container_started = False + self._server_env_overrides: dict[str, str] = {} + + @property + def aws_env(self) -> dict[str, str]: + env = os.environ.copy() + env.update( + { + "AWS_ACCESS_KEY_ID": "rustfsadmin", + "AWS_SECRET_ACCESS_KEY": "rustfsadmin", + "AWS_DEFAULT_REGION": "us-east-1", + "AWS_EC2_METADATA_DISABLED": "true", + } + ) + return env + + def common_nokv_args(self) -> list[str]: + return [ + "--server-bind", + self.server_bind, + "--object-backend", + "rustfs", + "--s3-endpoint", + self.s3_endpoint, + "--s3-bucket", + self.config.bucket, + "--s3-access-key-id", + "rustfsadmin", + "--s3-secret-access-key", + "rustfsadmin", + "--no-metadata-checkpoint-archive", + ] + + def mcp_args(self, root: str) -> list[str]: + return self.common_nokv_args() + [ + "mcp", + "--profile", + "workbench", + "--workbench-root", + root, + ] + + def run( + self, + args: list[str], + *, + timeout: float | None = None, + env: dict[str, str] | None = None, + cwd: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + try: + result = subprocess.run( + args, + cwd=cwd or self.config.repo_root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout or self.config.command_timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise AcceptanceError( + f"command exceeded deadline ({exc.timeout}s): {args!r}" + ) from exc + if result.returncode != 0: + raise AcceptanceError( + f"command failed ({result.returncode}): {args!r}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + def build(self) -> None: + if not self.config.build: + if not self.config.nokv_bin.is_file(): + raise AcceptanceError(f"NoKV binary not found: {self.config.nokv_bin}") + return + self.run( + [str(self.config.cargo_bin), "build", "-p", "nokv", "--bin", "nokv"], + timeout=max(self.config.startup_timeout, 1200), + ) + if not self.config.nokv_bin.is_file(): + raise AcceptanceError(f"cargo did not produce {self.config.nokv_bin}") + + def start_rustfs(self) -> None: + env = self.aws_env + env.update( + { + "LINGTAI_WORKBENCH_DATA_ROOT": str(self.config.state_dir), + "LINGTAI_WORKBENCH_RUSTFS_CONTAINER": self.config.container, + "LINGTAI_WORKBENCH_RUSTFS_IMAGE": self.config.rustfs_image, + "LINGTAI_WORKBENCH_RUSTFS_HOST": "127.0.0.1", + "LINGTAI_WORKBENCH_RUSTFS_PORT": str(self.config.rustfs_port), + "LINGTAI_WORKBENCH_RUSTFS_CONSOLE_PORT": str( + self.config.rustfs_console_port + ), + "LINGTAI_WORKBENCH_S3_ENDPOINT": self.s3_endpoint, + "LINGTAI_WORKBENCH_S3_BUCKET": self.config.bucket, + "LINGTAI_WORKBENCH_RUSTFS_DATA_DIR": str( + self.config.state_dir / "rustfs" + ), + } + ) + script = self.config.repo_root / "scripts/lingtai-workbench/start_rustfs.sh" + self.run( + [str(script)], + timeout=self.config.startup_timeout, + env=env, + ) + self._container_started = True + + def start_server(self) -> None: + if self._server is not None: + raise AcceptanceError("NoKV server is already running") + self.config.state_dir.mkdir(parents=True, exist_ok=True) + self._server_log_handle = self.server_log.open("a", encoding="utf-8") + args = ( + [str(self.config.nokv_bin)] + + self.common_nokv_args() + + [ + "--meta", + str(self.config.state_dir / "meta"), + "--object-gc-interval-ms", + "100", + "--object-gc-limit", + "4096", + "--history-gc-interval-ms", + "100", + "--history-gc-limit", + "4096", + "serve", + ] + ) + server_env = self.aws_env + server_env["NOKV_TEST_SNAPSHOT_BARRIER_DIR"] = str( + self.config.state_dir / "snapshot-barriers" + ) + server_env["NOKV_TEST_BARRIER_TIMEOUT_MS"] = str( + int(self.config.tool_timeout * 1000) + ) + server_env.update(self._server_env_overrides) + self._server = subprocess.Popen( + args, + cwd=self.config.repo_root, + env=server_env, + text=True, + stdout=self._server_log_handle, + stderr=subprocess.STDOUT, + ) + deadline = time.monotonic() + self.config.startup_timeout + last_error = "not attempted" + while time.monotonic() < deadline: + if self._server.poll() is not None: + raise AcceptanceError( + f"NoKV server exited during startup ({self._server.returncode})\n" + f"{_tail(self.server_log)}" + ) + try: + if self.http_text("/readyz", timeout=1).strip() == "ready": + return + except (AcceptanceError, urllib.error.URLError, OSError) as exc: + last_error = str(exc) + time.sleep(0.1) + raise AcceptanceError( + f"NoKV server did not become ready before deadline: {last_error}\n" + f"{_tail(self.server_log)}" + ) + + def stop_server(self) -> None: + process = self._server + self._server = None + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + if self._server_log_handle is not None: + self._server_log_handle.close() + self._server_log_handle = None + + def kill_server(self) -> None: + """Stop the server without running Rust destructors or a final checkpoint.""" + process = self._server + self._server = None + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=5) + if self._server_log_handle is not None: + self._server_log_handle.close() + self._server_log_handle = None + + def restart_server(self, env_overrides: dict[str, str] | None = None) -> None: + self.stop_server() + self._server_env_overrides = dict(env_overrides or {}) + self.start_server() + + def cleanup(self) -> None: + self.stop_server() + if self._container_started or shutil.which("docker"): + subprocess.run( + ["docker", "rm", "-f", self.config.container], + text=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + check=False, + ) + if not self.config.keep_state: + shutil.rmtree(self.config.state_dir, ignore_errors=True) + + def cli(self, *command: str) -> str: + result = self.run( + [str(self.config.nokv_bin)] + self.common_nokv_args() + list(command), + env=self.aws_env, + ) + return result.stdout + + def cli_result( + self, *command: str, timeout: float | None = None + ) -> subprocess.CompletedProcess[str]: + args = [str(self.config.nokv_bin)] + self.common_nokv_args() + list(command) + try: + return subprocess.run( + args, + cwd=self.config.repo_root, + env=self.aws_env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout or self.config.command_timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise AcceptanceError( + f"CLI probe exceeded deadline ({exc.timeout}s): {args!r}" + ) from exc + + def http_text(self, path: str, *, timeout: float = 5, method: str = "GET") -> str: + request = urllib.request.Request( + f"http://{self.server_bind}{path}", method=method + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read().decode("utf-8") + except urllib.error.URLError: + raise + except Exception as exc: + raise AcceptanceError(f"HTTP {method} {path} failed: {exc}") from exc + + def manual_gc(self) -> dict[str, Any]: + raw = self.http_text("/gc?limit=4096", timeout=30, method="POST") + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise AcceptanceError(f"manual GC returned invalid JSON: {raw!r}") from exc + if not isinstance(value, dict): + raise AcceptanceError(f"manual GC did not return an object: {value!r}") + return value + + def stats(self) -> dict[str, Any]: + raw = self.http_text("/stats", timeout=10) + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise AcceptanceError(f"stats returned invalid JSON: {raw!r}") from exc + if not isinstance(value, dict): + raise AcceptanceError(f"stats did not return an object: {value!r}") + return value + + def object_puts(self) -> int: + value = self.stats().get("object_puts") + if not isinstance(value, int) or value < 0: + raise AcceptanceError(f"stats lacks a valid object_puts counter: {value!r}") + return value + + def object_keys(self) -> set[str]: + result = self.run( + [ + "aws", + "--endpoint-url", + self.s3_endpoint, + "s3api", + "list-objects-v2", + "--bucket", + self.config.bucket, + "--output", + "json", + ], + env=self.aws_env, + ) + payload = json.loads(result.stdout) + return { + item["Key"] + for item in payload.get("Contents", []) + if isinstance(item, dict) and isinstance(item.get("Key"), str) + } + + +class WorkbenchClient: + def __init__( + self, + mcp_client_class: type, + environment: LiveEnvironment, + launch: ResolvedMcpLaunch, + ) -> None: + self.root = launch.root + self._timeout = environment.config.tool_timeout + self._client = mcp_client_class( + command=launch.command, + args=launch.args, + env=launch.env, + ) + self._client.start() + + def close(self) -> None: + self._client.close() + + def tools(self) -> list[dict[str, Any]]: + return self._client.list_tools(timeout=min(self._timeout, 30)) + + def raw_call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = self._client.call_tool(name, arguments, timeout=self._timeout) + if not isinstance(result, dict): + raise AcceptanceError(f"{name} returned a non-object: {result!r}") + return result + + def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = self.raw_call(name, arguments) + if result.get("status") == "error" or "code" in result: + error = decode_tool_error(result) + raise AcceptanceError( + f"{name} failed with {error.code}: {error.message}; " + f"retryable={error.retryable}, details={error.details}" + ) + if result.get("status") != "success": + raise AcceptanceError(f"{name} returned malformed success: {result!r}") + return result + + def expect_error( + self, + name: str, + arguments: dict[str, Any], + expected_code: str, + ) -> ToolError: + error = decode_tool_error(self.raw_call(name, arguments)) + if error.code != expected_code: + raise AcceptanceError( + f"{name} returned {error.code}, expected {expected_code}: {error.message}" + ) + return error + + +def _read_lines(result: dict[str, Any]) -> list[str]: + if result.get("bytes_encoding") == "base64": + raw = base64.b64decode(result.get("bytes", ""), validate=True) + return raw.decode("utf-8").splitlines() + items = result.get("items") + if not isinstance(items, list): + raise AcceptanceError(f"read response lacks items: {result!r}") + lines: list[str] = [] + for item in items: + value = item.get("value") if isinstance(item, dict) else None + text = value.get("text") if isinstance(value, dict) else None + if not isinstance(text, str): + raise AcceptanceError(f"read response contains a non-text item: {item!r}") + lines.append(text) + return lines + + +def _read_json_object(result: dict[str, Any]) -> dict[str, Any]: + if result.get("format") != "structured": + raise AcceptanceError(f"JSON object read has the wrong format: {result!r}") + if result.get("record_type") != "json_object": + raise AcceptanceError(f"read response is not a JSON object: {result!r}") + items = result.get("items") + if not isinstance(items, list): + raise AcceptanceError(f"JSON object read lacks items: {result!r}") + value: dict[str, Any] = {} + for item in items: + record = item.get("value") if isinstance(item, dict) else None + key = record.get("key") if isinstance(record, dict) else None + if not isinstance(key, str) or "value" not in record: + raise AcceptanceError( + f"JSON object read contains a malformed record: {item!r}" + ) + if key in value: + raise AcceptanceError(f"JSON object read repeated key {key!r}") + value[key] = record["value"] + if result.get("truncated") is True: + raise AcceptanceError( + "JSON object read was unexpectedly truncated; manifest validation " + "requires every top-level field" + ) + if result.get("next_cursor") is not None: + raise AcceptanceError( + f"terminal JSON object read returned next_cursor: {result!r}" + ) + return value + + +class AcceptanceSuite: + def __init__( + self, + environment: LiveEnvironment, + agent_class: type, + mcp_client_class: type, + ) -> None: + self.env = environment + self.agent_class = agent_class + self.mcp_client_class = mcp_client_class + self.clients: list[WorkbenchClient] = [] + self.results: dict[str, dict[str, Any]] = {} + self.restore_available = False + self.root_a = "" + self.root_b = "" + self.client_a: WorkbenchClient + self.client_a2: WorkbenchClient + self.client_b: WorkbenchClient + self.foreign_workbench_id = "shared-root-binding" + self.foreign_snapshot_id = 0 + self.foreign_snapshot_name = "forged-agent-a" + self.restore_state: dict[str, Any] = {} + self._registration_generation = 0 + + def _record( + self, + name: str, + status: str, + started: float, + details: dict[str, Any] | None = None, + ) -> None: + self.results[name] = { + "status": status, + "duration_seconds": round(time.monotonic() - started, 3), + "details": details or {}, + } + + def scenario( + self, name: str, function: Callable[[], dict[str, Any] | None] + ) -> None: + print(f"[live-e2e] START {name}", flush=True) + started = time.monotonic() + try: + details = function() or {} + except Exception as exc: + self._record(name, "failed", started, {"error": str(exc)}) + print(f"[live-e2e] FAIL {name}: {exc}", flush=True) + raise + self._record(name, "passed", started, details) + print(f"[live-e2e] PASS {name}", flush=True) + + def skip(self, name: str, reason: str) -> None: + started = time.monotonic() + self._record(name, "skipped", started, {"reason": reason}) + print(f"[live-e2e] SKIP {name}: {reason}", flush=True) + + @staticmethod + def _mock_lingtai_service() -> MagicMock: + service = MagicMock() + service.get_adapter.return_value = MagicMock() + service.provider = "gemini" + service.model = "nokv-live-acceptance" + return service + + @staticmethod + def _resolved_launch_from_client(client: Any) -> ResolvedMcpLaunch: + command = getattr(client, "_command", None) + args = getattr(client, "_args", None) + env = getattr(client, "_env", None) + if not isinstance(command, str) or not isinstance(args, list): + raise AcceptanceError("LingTai MCP client lacks its resolved launch config") + if any(not isinstance(argument, str) for argument in args): + raise AcceptanceError(f"LingTai MCP args are malformed: {args!r}") + if not isinstance(env, dict) or any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in env.items() + ): + raise AcceptanceError("LingTai MCP client lacks a string environment") + try: + root_index = args.index("--workbench-root") + 1 + root = args[root_index] + except (ValueError, IndexError) as exc: + raise AcceptanceError( + f"LingTai resolved MCP args lack --workbench-root: {args!r}" + ) from exc + if "{" in root or not root.startswith("/"): + raise AcceptanceError(f"LingTai left an unresolved root: {root!r}") + return ResolvedMcpLaunch(command, list(args), dict(env), root) + + def _registered_agent_launch(self, agent_name: str) -> ResolvedMcpLaunch: + """Exercise real Agent registration+retry, then reuse its resolved config.""" + workdir = self.env.config.state_dir / "agents" / agent_name + workdir.mkdir(parents=True, exist_ok=True) + mcp_env = { + key: value + for key, value in self.env.aws_env.items() + if key not in {"LINGTAI_AGENT_DIR", "LINGTAI_MCP_NAME"} + } + registry = { + "name": "nokv-workbench", + "summary": "NoKV live acceptance workbench.", + "transport": "stdio", + "command": str(self.env.config.nokv_bin), + "args": self.env.mcp_args(WORKBENCH_ROOT_TEMPLATE), + "source": "nokv-live-acceptance", + } + (workdir / "mcp_registry.jsonl").write_text( + json.dumps(registry, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + (workdir / "init.json").write_text( + json.dumps( + { + "mcp": { + "nokv-workbench": { + "type": "stdio", + "command": str(self.env.config.nokv_bin), + "args": self.env.mcp_args(WORKBENCH_ROOT_TEMPLATE), + "env": mcp_env, + } + } + }, + separators=(",", ":"), + ), + encoding="utf-8", + ) + + agent: Any | None = None + try: + agent = self.agent_class( + service=self._mock_lingtai_service(), + agent_name=agent_name, + working_dir=workdir, + capabilities={"mcp": {}}, + ) + specs = getattr(agent, "_mcp_init_specs", {}) + spec = specs.get("nokv-workbench") if isinstance(specs, dict) else None + initial_client = spec.get("client") if isinstance(spec, dict) else None + if initial_client is None: + raise AcceptanceError("LingTai Agent did not register the NoKV MCP") + initial_launch = self._resolved_launch_from_client(initial_client) + + initial_client.close() + retry = agent._retry_failed_mcps() + if retry.get("recovered") != ["nokv-workbench"]: + raise AcceptanceError( + f"LingTai Agent MCP retry did not recover NoKV: {retry!r}" + ) + retried_spec = agent._mcp_init_specs.get("nokv-workbench") + retried_client = ( + retried_spec.get("client") if isinstance(retried_spec, dict) else None + ) + if retried_client is None: + raise AcceptanceError("LingTai MCP retry returned no live client") + retried_launch = self._resolved_launch_from_client(retried_client) + if retried_launch.root != initial_launch.root: + raise AcceptanceError( + "LingTai resolved workbench root changed during MCP retry" + ) + + self._registration_generation += 1 + handler = getattr(agent, "_tool_handlers", {}).get("workbench_create") + if not callable(handler): + raise AcceptanceError( + "LingTai Agent did not install the registered MCP handler" + ) + handler_result = handler( + {"id": f"lingtai-registration-probe-{self._registration_generation}"} + ) + if not isinstance(handler_result, dict) or handler_result.get( + "status" + ) != "success": + raise AcceptanceError( + f"registered LingTai MCP handler failed: {handler_result!r}" + ) + return retried_launch + finally: + if agent is not None: + agent.stop(timeout=5.0) + + def connect_clients(self) -> None: + launch_a = self._registered_agent_launch("checkpoint-agent-a") + launch_b = self._registered_agent_launch("checkpoint-agent-b") + expected_a = launch_a.root + expected_b = launch_b.root + if expected_a == expected_b: + raise AcceptanceError( + "LingTai placeholder expansion did not isolate agents" + ) + if self.root_a and (self.root_a != expected_a or self.root_b != expected_b): + raise AcceptanceError( + "LingTai resolved workbench root changed after restart" + ) + self.root_a, self.root_b = expected_a, expected_b + self.client_a = WorkbenchClient(self.mcp_client_class, self.env, launch_a) + self.client_a2 = WorkbenchClient(self.mcp_client_class, self.env, launch_a) + self.client_b = WorkbenchClient(self.mcp_client_class, self.env, launch_b) + self.clients = [self.client_a, self.client_a2, self.client_b] + self.restore_available = validate_tool_contract( + self.client_a.tools(), not self.env.config.allow_missing_restore + ) + + def close_clients(self) -> None: + for client in self.clients: + try: + client.close() + except Exception: + pass + self.clients = [] + + def restart_stack(self, env_overrides: dict[str, str] | None = None) -> None: + self.close_clients() + self.env.restart_server(env_overrides) + self.connect_clients() + + def crash_restart_stack(self) -> None: + # Kill immediately after the caller-observed ACK. Closing MCP clients + # first would add enough delay to hide an async WAL acknowledgment bug. + self.env.kill_server() + self.close_clients() + self.env.start_server() + self.connect_clients() + + @staticmethod + def _put( + client: WorkbenchClient, + workbench_id: str, + path: str, + text: str, + *, + replace: bool = False, + ) -> dict[str, Any]: + return client.call( + "workbench_put_file", + { + "id": workbench_id, + "section": "outputs", + "path": path, + "text": text, + "replace": replace, + }, + ) + + @staticmethod + def _create_committed(client: WorkbenchClient, workbench_id: str) -> None: + client.call("workbench_create", {"id": workbench_id}) + client.call( + "workbench_commit", + {"id": workbench_id, "manifest": {"acceptance": workbench_id}}, + ) + + @staticmethod + def _assert_read( + client: WorkbenchClient, + workbench_id: str, + path: str, + expected_line: str, + *, + at_snapshot: int | str | None = None, + ) -> None: + args: dict[str, Any] = { + "id": workbench_id, + "section": "outputs", + "path": path, + } + if at_snapshot is not None: + args["at_snapshot"] = at_snapshot + lines = _read_lines(client.call("workbench_read", args)) + if lines != [expected_line]: + raise AcceptanceError( + f"unexpected content for {workbench_id}/{path}: {lines!r}, " + f"expected {[expected_line]!r}" + ) + + @staticmethod + def _assert_output_absent( + client: WorkbenchClient, workbench_id: str, path: str + ) -> None: + result = client.raw_call( + "workbench_stat", + {"id": workbench_id, "section": "outputs", "path": path}, + ) + error = decode_tool_error(result) + if "not found" not in error.message.lower(): + raise AcceptanceError( + f"expected {workbench_id}/{path} to be absent, got {error}" + ) + + def _list_all( + self, + client: WorkbenchClient, + workbench_id: str, + *, + at_snapshot: int | str | None = None, + ) -> PageAccumulator: + cursor: str | None = None + pages = PageAccumulator(self.env.config.profile.page_limit) + deadline = time.monotonic() + self.env.config.tool_timeout + while True: + if time.monotonic() >= deadline: + raise AcceptanceError("workbench_list pagination exceeded deadline") + args: dict[str, Any] = { + "id": workbench_id, + "section": "outputs", + "limit": self.env.config.profile.page_limit, + } + if cursor is not None: + args["cursor"] = cursor + if at_snapshot is not None: + args["at_snapshot"] = at_snapshot + result = client.call("workbench_list", args) + entries = result.get("entries") + if not isinstance(entries, list): + raise AcceptanceError(f"list response lacks entries: {result!r}") + names = [entry.get("name") for entry in entries] + if any(not isinstance(name, str) for name in names): + raise AcceptanceError( + f"list response contains malformed names: {names!r}" + ) + next_cursor = result.get("next_cursor") + truncated = result.get("truncated") + if not isinstance(truncated, bool): + raise AcceptanceError( + f"list response lacks truncated boolean: {result!r}" + ) + pages.add(names, next_cursor, truncated) + if not truncated: + return pages + cursor = next_cursor + + def scenario_placeholder_and_tools(self) -> dict[str, Any]: + return { + "root_a": self.root_a, + "root_b": self.root_b, + "tool_count": len(self.client_a.tools()), + "restore_available": self.restore_available, + "agent_registration_retry_handler": True, + } + + def scenario_linearizable_renew(self) -> dict[str, Any]: + workbench_id = "lease-linearization" + self._create_committed(self.client_a, workbench_id) + self._put(self.client_a, workbench_id, "lease.txt", "lease-base\n") + minted = self.client_a.call( + "workbench_snapshot", + {"id": workbench_id, "name": "lease-base", "ttl_days": 1}, + ) + snapshot_id = int(minted["snapshot_id"]) + barrier = threading.Barrier(2) + + def renew_worker(client: WorkbenchClient, ttl_days: int) -> list[int]: + barrier.wait(timeout=self.env.config.tool_timeout) + expiries: list[int] = [] + for _ in range(self.env.config.profile.renew_rounds): + result = client.call( + "workbench_snapshot_renew", + { + "id": workbench_id, + "snapshot_id": snapshot_id, + "ttl_days": ttl_days, + }, + ) + expiry = result.get("lease_expires_at") + if not isinstance(expiry, int): + raise AcceptanceError( + f"renew lacks authoritative expiry: {result!r}" + ) + expiries.append(expiry) + return expiries + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + short = executor.submit(renew_worker, self.client_a, 1) + long = executor.submit(renew_worker, self.client_a2, 90) + promised = short.result(timeout=self.env.config.tool_timeout * 2) + promised.extend(long.result(timeout=self.env.config.tool_timeout * 2)) + listed = self.client_a.call("workbench_snapshot_list", {"id": workbench_id}) + checkpoints = listed.get("checkpoints") + if not isinstance(checkpoints, list): + raise AcceptanceError(f"snapshot list malformed: {listed!r}") + row = next( + (row for row in checkpoints if row.get("snapshot_id") == snapshot_id), None + ) + if row is None: + raise AcceptanceError(f"renewed snapshot missing from registry: {listed!r}") + final_expiry = row.get("live_lease_expires_unix_ms") + if not isinstance(final_expiry, int) or final_expiry < max(promised): + raise AcceptanceError( + f"renew shortened a successful promise: final={final_expiry}, " + f"max_success={max(promised)}" + ) + self.env.cli( + "retire-snapshot", + f"{self.root_a}/{workbench_id}", + str(snapshot_id), + ) + return { + "snapshot_id": snapshot_id, + "calls": len(promised), + "max_promised_expiry": max(promised), + "final_expiry": final_expiry, + } + + def scenario_reaper_accounting(self) -> dict[str, Any]: + gc = self.env.manual_gc() + candidates, reaped, conflicted = self._reap_counts(gc) + return { + "expired_candidates": candidates, + "reaped": reaped, + "conflicted": conflicted, + } + + @staticmethod + def _reap_counts(payload: dict[str, Any]) -> tuple[int, int, int]: + try: + reap = payload["object_gc"]["snapshot_reap"] + candidates = int(reap["expired_candidates"]) + reaped = int(reap["reaped"]) + conflicted = int(reap["conflicted"]) + except (KeyError, TypeError, ValueError) as exc: + raise AcceptanceError( + f"GC/stats payload lacks snapshot_reap counters: {payload!r}" + ) from exc + if reaped + conflicted != candidates: + raise AcceptanceError( + "snapshot reaper accounting violated: " + f"reaped={reaped}, conflicted={conflicted}, candidates={candidates}" + ) + return candidates, reaped, conflicted + + def probe_short_lease_mint(self) -> tuple[bool, str]: + workbench_id = "renew-reaper-race" + self._create_committed(self.client_a, workbench_id) + self._put(self.client_a, workbench_id, "race.txt", "race-value\n") + root = f"{self.root_a}/{workbench_id}" + result = self.env.cli_result("snapshot", root, "200") + if result.returncode != 0: + diagnostic = f"{result.stdout}\n{result.stderr}".lower() + if "too many arguments" in diagnostic or "usage:" in diagnostic: + return ( + False, + "NoKV CLI does not yet support breaking `snapshot PATH [LEASE_MS]`; " + "the final integration capability probe will enable this scenario", + ) + raise AcceptanceError( + "short-lease snapshot capability probe failed unexpectedly: " + f"stdout={result.stdout!r}, stderr={result.stderr!r}" + ) + snapshot_id = parse_snapshot_id(result.stdout) + self.env.cli("retire-snapshot", root, str(snapshot_id)) + return True, "" + + def _wait_for_marker(self, path: Path) -> None: + deadline = time.monotonic() + self.env.config.tool_timeout + while not path.exists(): + if time.monotonic() >= deadline: + raise AcceptanceError( + f"live-test barrier marker did not appear before deadline: {path}" + ) + time.sleep(0.002) + + @staticmethod + def _release_marker(path: Path) -> None: + path.write_text("continue\n", encoding="utf-8") + + def _deterministic_reaper_conflict( + self, workbench_id: str, root: str + ) -> dict[str, int]: + _, _, health_conflicts_before = self._reap_counts(self.env.stats()) + barrier_dir = self.env.config.state_dir / "snapshot-barriers" + shutil.rmtree(barrier_dir, ignore_errors=True) + barrier_dir.mkdir(parents=True) + minted = self.env.cli_result("snapshot", root, "300") + if minted.returncode != 0: + raise AcceptanceError( + "deterministic short-lease mint failed: " + f"stdout={minted.stdout!r}, stderr={minted.stderr!r}" + ) + snapshot_id = parse_snapshot_id(minted.stdout) + lease_expiry = parse_snapshot_expiry(minted.stdout) + renew_arm = barrier_dir / f"{snapshot_id}.renew-read.arm" + renew_ready = barrier_dir / f"{snapshot_id}.renew-read.ready" + renew_continue = barrier_dir / f"{snapshot_id}.renew-read.continue" + reaper_arm = barrier_dir / f"{snapshot_id}.reaper-scan.arm" + reaper_ready = barrier_dir / f"{snapshot_id}.reaper-scan.ready" + reaper_continue = barrier_dir / f"{snapshot_id}.reaper-scan.continue" + renew_arm.write_text("armed\n", encoding="utf-8") + reaper_arm.write_text("armed\n", encoding="utf-8") + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + renew_future = executor.submit( + self.client_a.raw_call, + "workbench_snapshot_renew", + { + "id": workbench_id, + "snapshot_id": snapshot_id, + "ttl_days": 1, + }, + ) + self._wait_for_marker(renew_ready) + if time.time_ns() // 1_000_000 >= lease_expiry: + raise AcceptanceError( + "renew did not reach its pre-commit barrier before lease expiry" + ) + while True: + remaining_ms = lease_expiry - time.time_ns() // 1_000_000 + if remaining_ms <= 0: + break + time.sleep(min(remaining_ms / 1000, 0.002)) + + # The server's real background worker must own the old scan so its + # durable health counter, not only a one-off manual GC response, + # records the version conflict. + self._wait_for_marker(reaper_ready) + self._release_marker(renew_continue) + renew_result = renew_future.result(timeout=self.env.config.tool_timeout) + if renew_result.get("status") != "success": + error = decode_tool_error(renew_result) + raise AcceptanceError( + "pre-expiry renew lost the deterministic race: " + f"{error.code}: {error.message}" + ) + self._release_marker(reaper_continue) + + health_conflicts = health_conflicts_before + deadline = time.monotonic() + self.env.config.tool_timeout + while health_conflicts <= health_conflicts_before: + _, _, health_conflicts = self._reap_counts(self.env.stats()) + if time.monotonic() >= deadline: + raise AcceptanceError( + "background reaper health never recorded the forced CAS conflict" + ) + time.sleep(0.002) + self._assert_read( + self.client_a, + workbench_id, + "race.txt", + "race-value", + at_snapshot=snapshot_id, + ) + self.env.cli("retire-snapshot", root, str(snapshot_id)) + return { + "manual_conflicts": 0, + "health_conflicts": health_conflicts - health_conflicts_before, + } + + def scenario_renew_reaper_short_lease(self) -> dict[str, Any]: + workbench_id = "renew-reaper-race" + root = f"{self.root_a}/{workbench_id}" + deterministic = self._deterministic_reaper_conflict(workbench_id, root) + successes = 1 + terminal_failures = 0 + manual_conflicts = deterministic["manual_conflicts"] + health_conflicts_max = deterministic["health_conflicts"] + offsets_ms = (185, 190, 195, 198, 200) + + for round_index in range(self.env.config.profile.reaper_rounds - 1): + minted = self.env.cli_result("snapshot", root, "200") + if minted.returncode != 0: + raise AcceptanceError( + f"short-lease mint failed in round {round_index}: " + f"stdout={minted.stdout!r}, stderr={minted.stderr!r}" + ) + snapshot_id = parse_snapshot_id(minted.stdout) + target = time.monotonic() + offsets_ms[round_index % len(offsets_ms)] / 1000 + barrier = threading.Barrier(3) + + def renew() -> dict[str, Any]: + barrier.wait(timeout=self.env.config.tool_timeout) + return self.client_a.raw_call( + "workbench_snapshot_renew", + { + "id": workbench_id, + "snapshot_id": snapshot_id, + "ttl_days": 1, + }, + ) + + def reap() -> dict[str, Any]: + barrier.wait(timeout=self.env.config.tool_timeout) + return self.env.manual_gc() + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + renew_future = executor.submit(renew) + reap_future = executor.submit(reap) + while True: + remaining = target - time.monotonic() + if remaining <= 0: + break + time.sleep(min(remaining, 0.002)) + barrier.wait(timeout=self.env.config.tool_timeout) + renew_result = renew_future.result(timeout=self.env.config.tool_timeout) + gc_result = reap_future.result(timeout=self.env.config.tool_timeout) + + _, _, conflicted = self._reap_counts(gc_result) + manual_conflicts += conflicted + try: + _, _, health_conflicted = self._reap_counts(self.env.stats()) + except AcceptanceError: + health_conflicted = 0 + health_conflicts_max = max(health_conflicts_max, health_conflicted) + + if renew_result.get("status") == "success": + promised_expiry = renew_result.get("lease_expires_at") + if not isinstance(promised_expiry, int): + raise AcceptanceError( + f"successful race renew lacks authoritative expiry: {renew_result!r}" + ) + successes += 1 + # Drain newer scans before asserting the old scan cannot reap a + # successfully renewed version. Each drain is itself bounded by + # the HTTP request deadline; no probabilistic fixed sleep is used. + for _ in range(3): + drained = self.env.manual_gc() + _, _, drained_conflicts = self._reap_counts(drained) + manual_conflicts += drained_conflicts + self._assert_read( + self.client_a, + workbench_id, + "race.txt", + "race-value", + at_snapshot=snapshot_id, + ) + self.env.cli("retire-snapshot", root, str(snapshot_id)) + else: + error = decode_tool_error(renew_result) + if error.code not in {"SnapshotLeaseExpired", "SnapshotNotFound"}: + raise AcceptanceError( + f"race renew failed with unexpected typed error: {error}" + ) + terminal_failures += 1 + self.env.manual_gc() + + if successes == 0: + raise AcceptanceError( + "short-lease race produced no successful renew to validate" + ) + observed_conflicts = manual_conflicts + health_conflicts_max + if observed_conflicts == 0: + raise AcceptanceError( + "short-lease race observed no reaper CAS conflict; rerun is not an " + "acceptance substitute because conflicted > 0 is a required invariant" + ) + return { + "rounds": self.env.config.profile.reaper_rounds, + "deterministic_barrier": True, + "renew_successes": successes, + "terminal_failures": terminal_failures, + "manual_conflicts": manual_conflicts, + "health_conflicts_max": health_conflicts_max, + } + + def _assert_foreign_snapshot_rejected(self, at_snapshot: int | str) -> int: + workbench_id = self.foreign_workbench_id + checks = [ + ( + "workbench_stat", + {"id": workbench_id, "section": "outputs", "path": "owner.txt"}, + ), + ("workbench_list", {"id": workbench_id, "section": "outputs"}), + ( + "workbench_read", + {"id": workbench_id, "section": "outputs", "path": "owner.txt"}, + ), + ] + for tool, args in checks: + args["at_snapshot"] = at_snapshot + self.client_b.expect_error(tool, args, "SnapshotRootMismatch") + renew_target = ( + {"snapshot_id": at_snapshot} + if isinstance(at_snapshot, int) + else {"name": at_snapshot} + ) + self.client_b.expect_error( + "workbench_snapshot_renew", + {"id": workbench_id, "ttl_days": 30, **renew_target}, + "SnapshotRootMismatch", + ) + return len(checks) + 1 + + def scenario_root_binding(self) -> dict[str, Any]: + workbench_id = self.foreign_workbench_id + for client, content in ((self.client_a, "agent-a"), (self.client_b, "agent-b")): + self._create_committed(client, workbench_id) + self._put(client, workbench_id, "owner.txt", content + "\n") + minted = self.client_a.call( + "workbench_snapshot", + {"id": workbench_id, "name": "agent-a-owned", "ttl_days": 7}, + ) + snapshot_id = int(minted["snapshot_id"]) + self.foreign_snapshot_id = snapshot_id + typed_checks = self._assert_foreign_snapshot_rejected(snapshot_id) + + registry_row = json.dumps( + { + "name": self.foreign_snapshot_name, + "snapshot_id": snapshot_id, + "read_version": minted["read_version"], + "lease_expires_unix_ms": minted["lease_expires_at"], + "created_at": 1, + "reason": "mint", + }, + separators=(",", ":"), + ) + self.client_b.call( + "workbench_put_file", + { + "id": workbench_id, + "section": "metadata", + "path": "checkpoints.jsonl", + "text": registry_row + "\n", + }, + ) + typed_checks += self._assert_foreign_snapshot_rejected( + self.foreign_snapshot_name + ) + self._assert_read(self.client_a, workbench_id, "owner.txt", "agent-a") + self._assert_read(self.client_b, workbench_id, "owner.txt", "agent-b") + return {"snapshot_id": snapshot_id, "typed_checks": typed_checks} + + def scenario_historical_scan(self) -> dict[str, Any]: + workbench_id = "history-pagination" + self._create_committed(self.client_a, workbench_id) + expected = [ + f"entry-{index:03d}.txt" + for index in range(self.env.config.profile.history_entries) + ] + for index, name in enumerate(expected): + self._put( + self.client_a, + workbench_id, + name, + f"snapshot-{index:03d}\n", + ) + minted = self.client_a.call( + "workbench_snapshot", + {"id": workbench_id, "name": "history-base", "ttl_days": 7}, + ) + snapshot_id = int(minted["snapshot_id"]) + outputs = f"{self.root_a}/{workbench_id}/outputs" + self.env.cli("rm", f"{outputs}/{expected[0]}") + self.env.cli( + "rename", f"{outputs}/{expected[1]}", f"{outputs}/renamed-entry.txt" + ) + self.env.cli("rm", f"{outputs}/{expected[2]}") + self._put( + self.client_a, + workbench_id, + expected[2], + "current-recreated\n", + ) + + historical = self._list_all( + self.client_a, workbench_id, at_snapshot=snapshot_id + ) + if historical.names != expected: + raise AcceptanceError( + f"snapshot pagination mismatch: got={historical.names!r}, expected={expected!r}" + ) + current = self._list_all(self.client_a, workbench_id) + expected_current = sorted( + (set(expected) - {expected[0], expected[1]}) | {"renamed-entry.txt"} + ) + if current.names != expected_current: + raise AcceptanceError( + f"current pagination mismatch: got={current.names!r}, " + f"expected={expected_current!r}" + ) + page_limit = self.env.config.profile.page_limit + if len(expected) > page_limit and historical.page_count <= 1: + raise AcceptanceError("snapshot list did not exercise multiple pages") + if len(expected_current) > page_limit and current.page_count <= 1: + raise AcceptanceError("current list did not exercise multiple pages") + if len(expected) == 101 and page_limit == 7: + expected_historical_pages = (len(expected) + page_limit - 1) // page_limit + expected_current_pages = ( + len(expected_current) + page_limit - 1 + ) // page_limit + if historical.page_count != expected_historical_pages: + raise AcceptanceError( + "full snapshot pagination did not produce exactly 15 pages: " + f"{historical.page_count}" + ) + if current.page_count != expected_current_pages: + raise AcceptanceError( + "full current pagination did not produce exactly 15 pages: " + f"{current.page_count}" + ) + for index in range(3): + self._assert_read( + self.client_a, + workbench_id, + expected[index], + f"snapshot-{index:03d}", + at_snapshot=snapshot_id, + ) + self._assert_read(self.client_a, workbench_id, expected[2], "current-recreated") + self.env.cli("retire-snapshot", f"{self.root_a}/{workbench_id}", str(snapshot_id)) + return { + "snapshot_id": snapshot_id, + "entries": len(expected), + "page_limit": page_limit, + "historical_pages": historical.page_count, + "current_pages": current.page_count, + } + + def scenario_restart_root_binding(self) -> dict[str, Any]: + old_roots = (self.root_a, self.root_b) + self.restart_stack() + if old_roots != (self.root_a, self.root_b): + raise AcceptanceError("resolved roots changed across server/MCP restart") + typed_checks = self._assert_foreign_snapshot_rejected( + self.foreign_snapshot_id + ) + typed_checks += self._assert_foreign_snapshot_rejected( + self.foreign_snapshot_name + ) + return { + "root_a": self.root_a, + "root_b": self.root_b, + "typed_checks": typed_checks, + } + + def scenario_ack_durability_after_sigkill(self) -> dict[str, Any]: + workbench_id = "ack-durability" + self._create_committed(self.client_a, workbench_id) + self._put(self.client_a, workbench_id, "durable.txt", "durable-before-kill\n") + minted = self.client_a.call( + "workbench_snapshot", + {"id": workbench_id, "name": "acked-before-kill", "ttl_days": 7}, + ) + snapshot_id = int(minted["snapshot_id"]) + + self.crash_restart_stack() + + listed = self.client_a.call("workbench_snapshot_list", {"id": workbench_id}) + checkpoints = listed.get("checkpoints") + if not isinstance(checkpoints, list) or not any( + row.get("snapshot_id") == snapshot_id + and row.get("name") == "acked-before-kill" + for row in checkpoints + ): + raise AcceptanceError( + "an acknowledged snapshot/registry update disappeared after SIGKILL: " + f"{listed!r}" + ) + self._assert_read( + self.client_a, + workbench_id, + "durable.txt", + "durable-before-kill", + at_snapshot=snapshot_id, + ) + self.env.cli("retire-snapshot", f"{self.root_a}/{workbench_id}", str(snapshot_id)) + return {"snapshot_id": snapshot_id, "signal": "SIGKILL"} + + def scenario_restore_to_fork(self) -> dict[str, Any]: + source = "restore-source" + destination = "restore-destination" + self._create_committed(self.client_a, source) + keys_before_old_body = self.env.object_keys() + old_put = self._put(self.client_a, source, "result.txt", "checkpoint-value\n") + self._put( + self.client_a, + source, + "nested/kept.txt", + "checkpoint-nested\n", + ) + self._put( + self.client_a, + source, + "renamed-before.txt", + "checkpoint-renamed\n", + ) + self._put( + self.client_a, + source, + "deleted-after.txt", + "checkpoint-deleted\n", + ) + keys_after_old_body = self.env.object_keys() + old_body_keys = keys_after_old_body - keys_before_old_body + if not old_body_keys: + raise AcceptanceError("source file upload produced no RustFS object") + + older = self.client_a.call( + "workbench_snapshot", + {"id": source, "name": "older-checkpoint", "ttl_days": 7}, + ) + target = self.client_a.call( + "workbench_snapshot", + {"id": source, "name": "restore-point", "ttl_days": 7}, + ) + target_id = int(target["snapshot_id"]) + self._put( + self.client_a, + source, + "result.txt", + "source-current\n", + replace=True, + ) + self._put( + self.client_a, + source, + "nested/kept.txt", + "source-current-nested\n", + replace=True, + ) + source_outputs = f"{self.root_a}/{source}/outputs" + self.env.cli("rm", f"{source_outputs}/deleted-after.txt") + self.env.cli( + "rename", + f"{source_outputs}/renamed-before.txt", + f"{source_outputs}/renamed-after.txt", + ) + self._put( + self.client_a, + source, + "current-only.txt", + "source-current-only\n", + ) + objects_before_restore = self.env.object_keys() + object_puts_before_restore = self.env.object_puts() + restore_args = { + "id": source, + "at_snapshot": "restore-point", + "destination_id": destination, + } + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + restore_future = executor.submit( + self.client_a.call, RESTORE_TOOL, restore_args + ) + visibility_deadline = time.monotonic() + self.env.config.tool_timeout + while time.monotonic() < visibility_deadline: + if restore_future.done(): + # Surface a restore failure immediately instead of masking + # it as a destination visibility timeout. + restore_future.result() + observed = self.client_a2.raw_call( + "workbench_stat", {"id": destination} + ) + if observed.get("status") == "success": + break + error = decode_tool_error(observed) + if "not found" not in error.message.lower(): + raise AcceptanceError( + f"destination visibility probe failed unexpectedly: {error}" + ) + else: + raise AcceptanceError( + "restored destination did not become visible before deadline" + ) + + # First-visible destination contract: source-owned checkpoint aliases + # are already absent and the restore manifest is already readable. + destination_checkpoints = self.client_a2.call( + "workbench_snapshot_list", {"id": destination} + ) + if destination_checkpoints.get("checkpoint_count") != 0: + raise AcceptanceError( + "destination became visible with inherited checkpoint aliases: " + f"{destination_checkpoints!r}" + ) + inherited_registry = self.client_a2.raw_call( + "workbench_stat", + { + "id": destination, + "section": "metadata", + "path": "checkpoints.jsonl", + }, + ) + inherited_registry_error = decode_tool_error(inherited_registry) + if "not found" not in inherited_registry_error.message.lower(): + raise AcceptanceError( + "destination checkpoints.jsonl did not report absence at first " + f"visibility: {inherited_registry_error}" + ) + first_manifest = self.client_a2.call( + "workbench_read", + { + "id": destination, + "section": "metadata", + "path": "restore_manifest.json", + }, + ) + manifest = _read_json_object(first_manifest) + first_source_checkpoints = self.client_a2.call( + "workbench_snapshot_list", {"id": source} + ) + if first_source_checkpoints.get("checkpoint_count", 0) < 2: + raise AcceptanceError( + "source registry changed when destination first became visible: " + f"{first_source_checkpoints!r}" + ) + restored = restore_future.result(timeout=self.env.config.tool_timeout) + operation_id = restored.get("operation_id") + if not isinstance(operation_id, str) or not operation_id: + raise AcceptanceError(f"restore lacks operation_id: {restored!r}") + expected_manifest_path = ( + f"{self.root_a}/{destination}/metadata/restore_manifest.json" + ) + source_root = restored.get("source_root") + destination_root = restored.get("destination_root") + if ( + restored.get("state") != "complete" + or restored.get("source_workbench_id") != source + or restored.get("destination_workbench_id") != destination + or restored.get("snapshot_id") != target_id + or restored.get("read_version") != target.get("read_version") + or not isinstance(source_root, int) + or source_root <= 0 + or not isinstance(destination_root, int) + or destination_root <= 0 + or destination_root == source_root + or restored.get("cleanup_pending") is not False + or restored.get("restore_manifest") != expected_manifest_path + ): + raise AcceptanceError(f"restore response contract is malformed: {restored!r}") + + # Recheck the terminal response state as well as the first-visible state. + destination_checkpoints = self.client_a.call( + "workbench_snapshot_list", {"id": destination} + ) + if destination_checkpoints.get("checkpoint_count") != 0: + raise AcceptanceError( + "restored destination inherited source checkpoint aliases: " + f"{destination_checkpoints!r}" + ) + restored_from = manifest.get("restored_from") + if ( + manifest.get("operation_id") != operation_id + or not isinstance(restored_from, dict) + or restored_from.get("workbench_id") != source + or restored_from.get("path") != f"{self.root_a}/{source}" + or restored_from.get("snapshot_id") != target_id + ): + raise AcceptanceError( + f"restore manifest does not bind operation/source/snapshot: {manifest!r}" + ) + source_checkpoints = self.client_a.call( + "workbench_snapshot_list", {"id": source} + ) + if source_checkpoints.get("checkpoint_count", 0) < 2: + raise AcceptanceError( + f"restore modified the source checkpoint registry: {source_checkpoints!r}" + ) + self._assert_read(self.client_a, destination, "result.txt", "checkpoint-value") + self._assert_read(self.client_a, source, "result.txt", "source-current") + self._assert_read( + self.client_a, destination, "nested/kept.txt", "checkpoint-nested" + ) + self._assert_read( + self.client_a, + destination, + "renamed-before.txt", + "checkpoint-renamed", + ) + self._assert_read( + self.client_a, + destination, + "deleted-after.txt", + "checkpoint-deleted", + ) + self._assert_output_absent(self.client_a, destination, "renamed-after.txt") + self._assert_output_absent(self.client_a, destination, "current-only.txt") + self._assert_read( + self.client_a, + source, + "nested/kept.txt", + "source-current-nested", + ) + self._assert_read( + self.client_a, + source, + "renamed-after.txt", + "checkpoint-renamed", + ) + self._assert_read( + self.client_a, source, "current-only.txt", "source-current-only" + ) + self._assert_output_absent(self.client_a, source, "renamed-before.txt") + self._assert_output_absent(self.client_a, source, "deleted-after.txt") + destination_outputs = self._list_all(self.client_a, destination) + source_current_outputs = self._list_all(self.client_a, source) + if destination_outputs.names != [ + "deleted-after.txt", + "nested", + "renamed-before.txt", + "result.txt", + ]: + raise AcceptanceError( + f"restored output tree differs from checkpoint: {destination_outputs.names!r}" + ) + if source_current_outputs.names != [ + "current-only.txt", + "nested", + "renamed-after.txt", + "result.txt", + ]: + raise AcceptanceError( + f"restore changed source current tree: {source_current_outputs.names!r}" + ) + + objects_after_restore = self.env.object_keys() + object_puts_after_restore = self.env.object_puts() + uploaded_by_restore = objects_after_restore - objects_before_restore + if len(uploaded_by_restore) > 1: + raise AcceptanceError( + "COW restore uploaded more than the one allowed restore_manifest body: " + f"{sorted(uploaded_by_restore)!r}" + ) + restore_object_puts = object_puts_after_restore - object_puts_before_restore + if restore_object_puts < 0 or restore_object_puts > 1: + raise AcceptanceError( + "COW restore issued body PUTs beyond restore_manifest: " + f"delta={restore_object_puts}" + ) + object_puts_before_retry = self.env.object_puts() + retried = self.client_a2.call( + RESTORE_TOOL, + {"id": source, "at_snapshot": target_id, "destination_id": destination}, + ) + if retried.get("operation_id") != operation_id: + raise AcceptanceError( + f"restore retry changed operation id: {operation_id} -> " + f"{retried.get('operation_id')}" + ) + if self.env.object_keys() != objects_after_restore: + raise AcceptanceError("idempotent restore retry uploaded new object bodies") + if self.env.object_puts() != object_puts_before_retry: + raise AcceptanceError("idempotent restore retry repeated an object PUT") + + conflict_snapshot = self.client_a.call( + "workbench_snapshot", + {"id": source, "name": "different-point", "ttl_days": 7}, + ) + self.client_a.expect_error( + RESTORE_TOOL, + { + "id": source, + "at_snapshot": int(conflict_snapshot["snapshot_id"]), + "destination_id": destination, + }, + "RestoreDestinationConflict", + ) + + self.restore_state = { + "source": source, + "destination": destination, + "older_snapshot_id": int(older["snapshot_id"]), + "target_snapshot_id": target_id, + "conflict_snapshot_id": int(conflict_snapshot["snapshot_id"]), + "operation_id": operation_id, + "old_body_keys": old_body_keys, + "old_digest_uri": old_put.get("digest_uri"), + } + return { + "operation_id": operation_id, + "snapshot_id": target_id, + "new_objects": len(uploaded_by_restore), + "object_puts": restore_object_puts, + } + + def scenario_restore_retry_after_restart(self) -> dict[str, Any]: + state = self.restore_state + self.restart_stack() + retried = self.client_a.call( + RESTORE_TOOL, + { + "id": state["source"], + "at_snapshot": state["target_snapshot_id"], + "destination_id": state["destination"], + }, + ) + if ( + retried.get("operation_id") != state["operation_id"] + or retried.get("state") != "complete" + or retried.get("snapshot_id") != state["target_snapshot_id"] + or not isinstance(retried.get("source_root"), int) + or not isinstance(retried.get("destination_root"), int) + or retried.get("cleanup_pending") is not False + ): + raise AcceptanceError(f"restart retry response is inconsistent: {retried!r}") + self._assert_read( + self.client_a, + state["destination"], + "result.txt", + "checkpoint-value", + ) + self._assert_read( + self.client_a, state["source"], "result.txt", "source-current" + ) + self._assert_read( + self.client_a, + state["destination"], + "nested/kept.txt", + "checkpoint-nested", + ) + self._assert_read( + self.client_a, + state["source"], + "current-only.txt", + "source-current-only", + ) + self.client_b.expect_error( + "workbench_read", + { + "id": self.foreign_workbench_id, + "section": "outputs", + "path": "owner.txt", + "at_snapshot": self.foreign_snapshot_id, + }, + "SnapshotRootMismatch", + ) + self.env.cli( + "retire-snapshot", + f"{self.root_a}/{self.foreign_workbench_id}", + str(self.foreign_snapshot_id), + ) + return {"operation_id": retried["operation_id"]} + + def _delete_restored_workbench(self, destination: str) -> None: + root = f"{self.root_a}/{destination}" + for path in ( + f"{root}/outputs/result.txt", + f"{root}/outputs/nested/kept.txt", + f"{root}/outputs/renamed-before.txt", + f"{root}/outputs/deleted-after.txt", + f"{root}/metadata/run_manifest.json", + f"{root}/metadata/restore_manifest.json", + ): + self.env.cli("rm", path) + self.env.cli("rmdir", f"{root}/outputs/nested") + for section in ("input", "scripts", "outputs", "logs", "metadata"): + self.env.cli("rmdir", f"{root}/{section}") + self.env.cli("rmdir", root) + + def _delete_restore_source(self, source: str) -> None: + root = f"{self.root_a}/{source}" + for path in ( + f"{root}/outputs/result.txt", + f"{root}/outputs/nested/kept.txt", + f"{root}/outputs/renamed-after.txt", + f"{root}/outputs/current-only.txt", + f"{root}/metadata/run_manifest.json", + f"{root}/metadata/checkpoints.jsonl", + ): + self.env.cli("rm", path) + self.env.cli("rmdir", f"{root}/outputs/nested") + for section in ("input", "scripts", "outputs", "logs", "metadata"): + self.env.cli("rmdir", f"{root}/{section}") + self.env.cli("rmdir", root) + + def scenario_fork_retention_and_release(self) -> dict[str, Any]: + state = self.restore_state + source_root = f"{self.root_a}/{state['source']}" + for snapshot_id in ( + state["older_snapshot_id"], + state["target_snapshot_id"], + state["conflict_snapshot_id"], + ): + self.env.cli("retire-snapshot", source_root, str(snapshot_id)) + self._delete_restore_source(state["source"]) + old_keys = set(state["old_body_keys"]) + retention_deadline = time.monotonic() + self.env.config.gc_deadline + last_retention_gc: dict[str, Any] = {} + while time.monotonic() < retention_deadline: + last_retention_gc = self.env.manual_gc() + try: + blocked = int(last_retention_gc["object_gc"]["blocked_by_snapshots"]) + except (KeyError, TypeError, ValueError) as exc: + raise AcceptanceError( + f"manual GC lacks blocked_by_snapshots: {last_retention_gc!r}" + ) from exc + if blocked > 0: + break + if not old_keys.issubset(self.env.object_keys()): + raise AcceptanceError( + "source cleanup reclaimed a fork-owned body before reporting retention" + ) + time.sleep(0.05) + else: + raise AcceptanceError( + "GC never exercised the durable fork-base retention hold: " + f"last_gc={last_retention_gc!r}" + ) + if not old_keys.issubset(self.env.object_keys()): + raise AcceptanceError( + "source checkpoint retirement reclaimed a body still referenced by the fork" + ) + self._assert_read( + self.client_a, + state["destination"], + "result.txt", + "checkpoint-value", + ) + self._assert_read( + self.client_a, + state["destination"], + "nested/kept.txt", + "checkpoint-nested", + ) + self._delete_restored_workbench(state["destination"]) + + deadline = time.monotonic() + self.env.config.gc_deadline + last_gc: dict[str, Any] = {} + while time.monotonic() < deadline: + last_gc = self.env.manual_gc() + if old_keys.isdisjoint(self.env.object_keys()): + break + time.sleep(0.1) + else: + raise AcceptanceError( + f"fork deletion did not release shared objects before deadline; " + f"keys={sorted(old_keys)!r}, last_gc={last_gc!r}" + ) + source_stat = self.client_a.raw_call("workbench_stat", {"id": state["source"]}) + source_error = decode_tool_error(source_stat) + if "not found" not in source_error.message.lower(): + raise AcceptanceError( + f"deleted source unexpectedly remained visible: {source_error}" + ) + return { + "source_deleted": True, + "released_object_keys": sorted(old_keys), + "retention_gc": last_retention_gc, + } + + def run(self) -> dict[str, dict[str, Any]]: + self.connect_clients() + self.scenario( + "lingtai_placeholder_and_tool_contract", self.scenario_placeholder_and_tools + ) + self.scenario("linearizable_snapshot_renew", self.scenario_linearizable_renew) + self.scenario("snapshot_reaper_accounting", self.scenario_reaper_accounting) + short_lease_supported, reason = self.probe_short_lease_mint() + if short_lease_supported: + self.scenario( + "renew_reaper_short_lease_race", + self.scenario_renew_reaper_short_lease, + ) + else: + self.skip("renew_reaper_short_lease_race", reason) + self.scenario("service_root_binding", self.scenario_root_binding) + self.scenario( + "snapshot_historical_scan_and_pagination", self.scenario_historical_scan + ) + self.scenario( + "acked_metadata_survives_sigkill", + self.scenario_ack_durability_after_sigkill, + ) + self.scenario("root_binding_after_restart", self.scenario_restart_root_binding) + + if self.restore_available: + self.scenario("restore_to_fork", self.scenario_restore_to_fork) + self.scenario( + "restore_idempotency_after_restart", + self.scenario_restore_retry_after_restart, + ) + self.scenario( + "fork_base_retention_and_release", + self.scenario_fork_retention_and_release, + ) + else: + reason = "workbench_restore is not present on the A+B-only binary" + for name in ( + "restore_to_fork", + "restore_idempotency_after_restart", + "fork_base_retention_and_release", + ): + self.skip(name, reason) + return self.results + + +def _load_lingtai(lingtai_kernel_dir: Path) -> tuple[type, type]: + source = lingtai_kernel_dir / "src" + if not source.is_dir(): + raise AcceptanceError(f"LingTai source directory not found: {source}") + sys.path.insert(0, str(source)) + try: + from lingtai.agent import Agent + from lingtai.services.mcp import MCPClient + except Exception as exc: + raise AcceptanceError( + "failed to import the real LingTai Agent/MCPClient; run with the " + "LingTai environment, for example `uv run --project ~/lingtai-kernel ...`" + ) from exc + agent_source = Path(sys.modules[Agent.__module__].__file__).resolve() + try: + agent_source.relative_to(source.resolve()) + except ValueError as exc: + raise AcceptanceError( + f"imported LingTai from {agent_source}, expected checkout {source.resolve()}" + ) from exc + if not hasattr(Agent, "_expand_agent_placeholders"): + raise AcceptanceError( + "LingTai checkout lacks Agent._expand_agent_placeholders; use PR #813 or newer" + ) + return Agent, MCPClient + + +def _require_commands(names: tuple[str, ...]) -> None: + missing = [name for name in names if shutil.which(name) is None] + if missing: + raise AcceptanceError(f"required commands are missing: {', '.join(missing)}") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + script = Path(__file__).resolve() + repo_root = script.parents[2] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=("quick", "full"), default="quick") + parser.add_argument("--repo-root", type=Path, default=repo_root) + parser.add_argument( + "--cargo-bin", + type=Path, + default=Path( + os.environ.get( + "CARGO", + shutil.which("cargo") or str(Path("~/.cargo/bin/cargo").expanduser()), + ) + ), + ) + parser.add_argument("--nokv-bin", type=Path) + parser.add_argument( + "--lingtai-kernel-dir", + type=Path, + default=Path( + os.environ.get("LINGTAI_KERNEL_DIR", "~/lingtai-kernel") + ).expanduser(), + ) + parser.add_argument("--state-dir", type=Path) + parser.add_argument("--server-port", type=int, default=0) + parser.add_argument("--rustfs-port", type=int, default=0) + parser.add_argument("--rustfs-console-port", type=int, default=0) + parser.add_argument("--rustfs-image", default="rustfs/rustfs:latest") + parser.add_argument("--command-timeout", type=float, default=120) + parser.add_argument("--tool-timeout", type=float, default=120) + parser.add_argument("--startup-timeout", type=float, default=180) + parser.add_argument("--gc-deadline", type=float, default=30) + parser.add_argument("--no-build", action="store_true") + parser.add_argument("--keep-state", action="store_true") + parser.add_argument( + "--allow-missing-restore", + action="store_true", + help="Run A+B scenarios when workbench_restore has not been integrated yet.", + ) + parser.add_argument( + "--require-all", + action="store_true", + help="Fail when any acceptance scenario is skipped.", + ) + args = parser.parse_args(argv) + if args.require_all and args.profile != "full": + parser.error("--require-all requires --profile full") + return args + + +def _live_config(args: argparse.Namespace) -> LiveConfig: + repo_root = args.repo_root.expanduser().resolve() + nokv_bin = ( + args.nokv_bin.expanduser().resolve() + if args.nokv_bin + else repo_root / "target/debug/nokv" + ) + if args.state_dir: + state_dir = args.state_dir.expanduser().resolve() + if state_dir.exists() and any(state_dir.iterdir()): + raise AcceptanceError(f"state directory must be empty: {state_dir}") + state_dir.mkdir(parents=True, exist_ok=True) + else: + target = repo_root / "target" + target.mkdir(parents=True, exist_ok=True) + state_dir = Path(tempfile.mkdtemp(prefix="checkpoint-live-e2e-", dir=target)) + suffix = f"{os.getpid()}-{int(time.time())}" + server_port = args.server_port or _free_port() + rustfs_port = args.rustfs_port or _free_port() + rustfs_console_port = args.rustfs_console_port or _free_port() + if len({server_port, rustfs_port, rustfs_console_port}) != 3: + raise AcceptanceError("server and RustFS ports must be distinct") + return LiveConfig( + repo_root=repo_root, + # Preserve rustup's `cargo` symlink. Resolving it to the rustup binary + # changes argv[0] and makes rustup parse `build` as its own command. + cargo_bin=args.cargo_bin.expanduser().absolute(), + nokv_bin=nokv_bin, + lingtai_kernel_dir=args.lingtai_kernel_dir.expanduser().resolve(), + state_dir=state_dir, + server_port=server_port, + rustfs_port=rustfs_port, + rustfs_console_port=rustfs_console_port, + rustfs_image=args.rustfs_image, + bucket=f"nokv-checkpoint-e2e-{suffix}", + container=f"nokv-checkpoint-e2e-{suffix}", + profile=workload_profile(args.profile), + command_timeout=args.command_timeout, + tool_timeout=args.tool_timeout, + startup_timeout=args.startup_timeout, + gc_deadline=args.gc_deadline, + build=not args.no_build, + keep_state=args.keep_state, + allow_missing_restore=args.allow_missing_restore, + require_all=args.require_all, + ) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + config = _live_config(args) + environment = LiveEnvironment(config) + suite: AcceptanceSuite | None = None + exit_code = 0 + try: + _require_commands(("docker", "aws")) + if config.build and not config.cargo_bin.is_file(): + raise AcceptanceError(f"cargo executable not found: {config.cargo_bin}") + agent_class, mcp_client_class = _load_lingtai(config.lingtai_kernel_dir) + environment.build() + environment.start_rustfs() + environment.start_server() + suite = AcceptanceSuite(environment, agent_class, mcp_client_class) + suite.run() + if config.require_all and any( + result["status"] == "skipped" for result in suite.results.values() + ): + raise AcceptanceError( + "--require-all rejected one or more skipped scenarios" + ) + except Exception as exc: + exit_code = 1 + print(f"[live-e2e] acceptance failed: {exc}", file=sys.stderr, flush=True) + finally: + if suite is not None: + suite.close_clients() + summary = { + "status": "passed" if exit_code == 0 else "failed", + "profile": args.profile, + "server_bind": environment.server_bind, + "s3_endpoint": environment.s3_endpoint, + "bucket": config.bucket, + "state_dir": str(config.state_dir), + "results": suite.results if suite is not None else {}, + } + print(json.dumps(summary, indent=2, sort_keys=True), flush=True) + environment.cleanup() + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lingtai-workbench/checkpoint_restore_live_e2e_test.py b/scripts/lingtai-workbench/checkpoint_restore_live_e2e_test.py new file mode 100755 index 000000000..da1144722 --- /dev/null +++ b/scripts/lingtai-workbench/checkpoint_restore_live_e2e_test.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic unit tests for the live checkpoint/restore harness.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import sys +import unittest +from pathlib import Path + + +sys.dont_write_bytecode = True +MODULE_PATH = Path(__file__).with_name("checkpoint_restore_live_e2e.py") + + +def load_module(): + spec = importlib.util.spec_from_file_location( + "checkpoint_restore_live_e2e", MODULE_PATH + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class CheckpointRestoreLiveE2ETest(unittest.TestCase): + def setUp(self): + self.module = load_module() + + def test_profiles_keep_quick_and_full_workloads_explicit(self): + quick = self.module.workload_profile("quick") + full = self.module.workload_profile("full") + + self.assertEqual(quick.page_limit, 7) + self.assertEqual(full.page_limit, 7) + self.assertLess(quick.renew_rounds, full.renew_rounds) + self.assertEqual(full.renew_rounds, 100) + self.assertLess(quick.reaper_rounds, full.reaper_rounds) + self.assertEqual(full.reaper_rounds, 200) + self.assertEqual(full.history_entries, 101) + + def test_snapshot_cli_id_parser_is_stable_across_extra_fields(self): + self.assertEqual( + self.module.parse_snapshot_id( + "snapshot /agents/a/wb id=184 version=99 lease_expires_at=123\n" + ), + 184, + ) + with self.assertRaisesRegex(self.module.AcceptanceError, "lacks id"): + self.module.parse_snapshot_id("snapshot 184 version=99") + self.assertEqual( + self.module.parse_snapshot_expiry( + "snapshot /agents/a/wb id=184 version=99 " + "lease_expires_unix_ms=123\n" + ), + 123, + ) + with self.assertRaisesRegex(self.module.AcceptanceError, "lacks lease"): + self.module.parse_snapshot_expiry("snapshot /x id=1 version=2") + + def test_decode_tool_error_preserves_typed_contract(self): + error = self.module.decode_tool_error( + { + "status": "error", + "message": json.dumps( + { + "code": "SnapshotRootMismatch", + "message": "foreign snapshot", + "retryable": False, + "details": {"snapshot_id": 42}, + } + ), + } + ) + + self.assertEqual(error.code, "SnapshotRootMismatch") + self.assertEqual(error.details["snapshot_id"], 42) + self.assertFalse(error.retryable) + + def test_decode_tool_error_rejects_success(self): + with self.assertRaisesRegex( + self.module.AcceptanceError, "not an MCP tool error" + ): + self.module.decode_tool_error({"status": "success"}) + + def test_page_accumulator_rejects_duplicates_and_cursor_stalls(self): + pages = self.module.PageAccumulator(2) + pages.add(["a", "b"], "62", True) + + with self.assertRaisesRegex(self.module.AcceptanceError, "duplicate entry"): + pages.add(["b", "c"], "63", True) + + pages = self.module.PageAccumulator(2) + pages.add(["a"], "61", True) + with self.assertRaisesRegex(self.module.AcceptanceError, "cursor repeated"): + pages.add(["b"], "61", True) + + def test_page_accumulator_requires_global_order_and_terminal_cursor(self): + pages = self.module.PageAccumulator(2) + pages.add(["a", "b"], "62", True) + pages.add(["c"], None, False) + self.assertEqual(pages.names, ["a", "b", "c"]) + self.assertEqual(pages.page_count, 2) + self.assertEqual(pages.page_sizes, [2, 1]) + + with self.assertRaisesRegex(self.module.AcceptanceError, "not sorted"): + self.module.PageAccumulator(2).add(["b", "a"], None, False) + + with self.assertRaisesRegex(self.module.AcceptanceError, "next_cursor"): + self.module.PageAccumulator(2).add(["a"], "61", False) + + def test_page_accumulator_enforces_limit_and_nonempty_truncated_page(self): + with self.assertRaisesRegex(self.module.AcceptanceError, "above limit"): + self.module.PageAccumulator(2).add(["a", "b", "c"], None, False) + with self.assertRaisesRegex(self.module.AcceptanceError, "truncated page is empty"): + self.module.PageAccumulator(2).add([], "61", True) + + def test_tool_contract_requires_restore_schema(self): + base = [ + {"name": name, "schema": {"type": "object"}} + for name in self.module.BASE_WORKBENCH_TOOLS + ] + restore = { + "name": "workbench_restore", + "schema": { + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string"}, + "at_snapshot": { + "anyOf": [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1}, + ] + }, + "destination_id": {"type": "string"}, + }, + "additionalProperties": False, + }, + } + + self.assertTrue(self.module.validate_tool_contract(base + [restore], True)) + self.assertFalse(self.module.validate_tool_contract(base, False)) + with self.assertRaisesRegex(self.module.AcceptanceError, "workbench_restore"): + self.module.validate_tool_contract(base, True) + + def test_tool_contract_rejects_extra_restore_arguments(self): + tools = [ + {"name": name, "schema": {"type": "object"}} + for name in self.module.BASE_WORKBENCH_TOOLS + ] + tools.append( + { + "name": "workbench_restore", + "schema": { + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {}, + "at_snapshot": {}, + "destination_id": {}, + "in_place": {}, + }, + "additionalProperties": False, + }, + } + ) + + with self.assertRaisesRegex(self.module.AcceptanceError, "exactly"): + self.module.validate_tool_contract(tools, True) + + def test_tool_contract_rejects_loose_restore_snapshot_alternatives(self): + base = [ + {"name": name, "schema": {"type": "object"}} + for name in self.module.BASE_WORKBENCH_TOOLS + ] + + def restore_schema(alternatives): + return { + "name": "workbench_restore", + "schema": { + "type": "object", + "required": ["id", "at_snapshot", "destination_id"], + "properties": { + "id": {"type": "string"}, + "at_snapshot": {"anyOf": alternatives}, + "destination_id": {"type": "string"}, + }, + "additionalProperties": False, + }, + } + + with self.assertRaisesRegex(self.module.AcceptanceError, "exactly two"): + self.module.validate_tool_contract( + base + + [ + restore_schema( + [ + {"type": "integer", "minimum": 0}, + {"type": "string", "minLength": 1}, + {"type": "null"}, + ] + ) + ], + True, + ) + with self.assertRaisesRegex(self.module.AcceptanceError, "non-empty"): + self.module.validate_tool_contract( + base + + [ + restore_schema( + [ + {"type": "integer", "minimum": 0}, + {"type": "string"}, + ] + ) + ], + True, + ) + + def test_reaper_counts_reject_inconsistent_accounting(self): + valid = { + "object_gc": { + "snapshot_reap": { + "expired_candidates": 3, + "reaped": 2, + "conflicted": 1, + } + } + } + self.assertEqual( + self.module.AcceptanceSuite._reap_counts(valid), + (3, 2, 1), + ) + valid["object_gc"]["snapshot_reap"]["conflicted"] = 0 + with self.assertRaisesRegex(self.module.AcceptanceError, "accounting violated"): + self.module.AcceptanceSuite._reap_counts(valid) + + def test_require_all_rejects_quick_profile(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + self.module.parse_args(["--require-all"]) + self.assertEqual(raised.exception.code, 2) + args = self.module.parse_args(["--profile", "full", "--require-all"]) + self.assertEqual(args.profile, "full") + self.assertTrue(args.require_all) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/lingtai-workbench/up.sh b/scripts/lingtai-workbench/up.sh index c16703bc9..2bacaa735 100755 --- a/scripts/lingtai-workbench/up.sh +++ b/scripts/lingtai-workbench/up.sh @@ -129,10 +129,13 @@ check_mcp_tools() { python3 -c 'import json, sys; print("\n".join(tool["name"] for tool in json.loads(sys.stdin.readline())["result"]["tools"]))' )" grep -qx 'workbench_put_file' <<<"${names}" || die "workbench_put_file is missing from MCP tools" + grep -qx 'workbench_snapshot_renew' <<<"${names}" || die "workbench_snapshot_renew is missing from MCP tools" + grep -qx 'workbench_snapshot_list' <<<"${names}" || die "workbench_snapshot_list is missing from MCP tools" + grep -qx 'workbench_restore' <<<"${names}" || die "workbench_restore is missing from MCP tools" if grep -q '^nokv_workbench_' <<<"${names}"; then die "old nokv_workbench_* tools are still exposed; rebuild target/debug/nokv" fi - log "MCP tools ready" + log "MCP checkpoint/restore tools ready" } main() {