diff --git a/crates/layerfs-content/src/object/id.rs b/crates/layerfs-content/src/object/id.rs index f3755cc0c..49fb6a3b5 100644 --- a/crates/layerfs-content/src/object/id.rs +++ b/crates/layerfs-content/src/object/id.rs @@ -83,10 +83,71 @@ mod tests { use super::*; #[test] - fn ids_are_fixed_and_text_round_trips() { + fn fixed_width_hex_covers_every_byte_positions_and_seeded_ids() { + for value in u8::MIN..=u8::MAX { + for index in [0, DIGEST_BYTES - 1] { + let mut bytes = [0; DIGEST_BYTES]; + bytes[index] = value; + let text = ObjectId::from_bytes(&bytes).unwrap().to_string(); + assert_eq!(text.len(), DIGEST_BYTES * 2); + assert_eq!(&text[index * 2..index * 2 + 2], format!("{value:02x}")); + assert_eq!(text.parse::().unwrap().to_bytes(), bytes); + } + } + let mut mixed = [0; DIGEST_BYTES]; + for (index, value) in [(1, 0x01), (2, 0x10), (15, 0xab), (30, 0xfe), (31, 0xff)] { + mixed[index] = value; + } + let text = ObjectId::from_bytes(&mixed).unwrap().to_string(); + assert!(text.starts_with("000110")); + assert_eq!(&text[30..32], "ab"); + assert!(text.ends_with("feff")); + + let mut state = 0x6a09_e667_f3bc_c909_u64; + for _ in 0..128 { + let mut bytes = [0; DIGEST_BYTES]; + for byte in &mut bytes { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + *byte = state as u8; + } + let id = ObjectId::from_bytes(&bytes).unwrap(); + assert_eq!(id.to_string().parse::().unwrap(), id); + } + } + + #[test] + fn parsing_accepts_hex_case_and_displays_canonical_lowercase() { + let id = ObjectId::for_bytes(b"case-equivalence"); + let lowercase = id.to_string(); + let uppercase = lowercase.to_ascii_uppercase(); + assert_eq!(lowercase.parse::().unwrap(), id); + assert_eq!(uppercase.parse::().unwrap(), id); + assert_eq!( + uppercase.parse::().unwrap().to_string(), + lowercase + ); + } + + #[test] + fn parsing_rejects_wrong_lengths_and_non_hex_text() { + for value in [String::new(), "0".repeat(63), "0".repeat(65)] { + assert_eq!( + value.parse::(), + Err(CoreError::InvalidIdentityText) + ); + } + let mut non_hex = "0".repeat(DIGEST_BYTES * 2); + non_hex.replace_range(31..32, "g"); + assert_eq!( + non_hex.parse::(), + Err(CoreError::InvalidIdentityText) + ); let id = ObjectId::for_bytes(b"payload"); assert_eq!(id.as_bytes().len(), DIGEST_BYTES); assert_eq!(id.to_string().parse::().unwrap(), id); + assert_eq!(ObjectId::from_reader(&b"payload"[..]).unwrap(), id); assert_eq!( ObjectId::from_bytes(&[0; DIGEST_BYTES]).unwrap().to_bytes(), [0; DIGEST_BYTES] @@ -98,9 +159,5 @@ mod tests { actual: DIGEST_BYTES - 1 }) ); - assert_eq!( - "zz".parse::(), - Err(CoreError::InvalidIdentityText) - ); } } diff --git a/crates/layerfs-layerstack-store/src/query.rs b/crates/layerfs-layerstack-store/src/query.rs index b7790a120..c97b4c59e 100644 --- a/crates/layerfs-layerstack-store/src/query.rs +++ b/crates/layerfs-layerstack-store/src/query.rs @@ -393,6 +393,25 @@ impl LayerStackStore { }) } + pub fn reachable_root_storage(&self, root: ObjectId) -> Result { + let mut seen = crate::SpillableObjectSet::empty()?; + let mut active = BTreeSet::new(); + let mut objects = 0_u64; + let mut encoded_bytes = 0_u64; + traverse_root( + self, + root, + &mut seen, + &mut active, + &mut objects, + &mut encoded_bytes, + )?; + Ok(CanonicalStorage { + objects, + encoded_bytes, + }) + } + pub fn storage_snapshot(&self) -> Result { fn len(path: &std::path::Path) -> Result { match std::fs::metadata(path) { @@ -467,8 +486,17 @@ fn traverse_root( return Err(StoreError::Integrity("object cycle")); } let canonical = store.db.read_object_row(id)?; - *objects = objects.saturating_add(1); - *encoded_bytes = encoded_bytes.saturating_add(canonical.len() as u64); + *objects = objects + .checked_add(1) + .ok_or(StoreError::Integrity("reachable storage overflow"))?; + *encoded_bytes = encoded_bytes + .checked_add( + canonical + .len() + .try_into() + .map_err(|_| StoreError::Integrity("reachable storage overflow"))?, + ) + .ok_or(StoreError::Integrity("reachable storage overflow"))?; let mut children = layerfs_content::object::references::referenced_objects(&canonical)?; children.sort(); children.dedup(); diff --git a/crates/layerfs-layerstack-store/tests/v4.rs b/crates/layerfs-layerstack-store/tests/v4.rs index 1bf97cfd2..1a21fc074 100644 --- a/crates/layerfs-layerstack-store/tests/v4.rs +++ b/crates/layerfs-layerstack-store/tests/v4.rs @@ -1,7 +1,12 @@ use layerfs_content::filesystem::ContentChange; +use layerfs_content::{ + encode_object, object::references::referenced_objects, CanonicalName, DirectoryEntry, Object, + ObjectId, ObjectKind, ObjectReference, +}; use layerfs_layerstack_store::{ - apply_changes, AddLayerResult, CommitOutcome, EntityName, LayerStackInitialization, - LayerStackInitializationReceipt, LayerStackStore, LocalForkSource, ObjectSource, StoreError, + apply_changes, AddLayerResult, CanonicalStorage, CommitOutcome, EntityName, + LayerStackInitialization, LayerStackInitializationReceipt, LayerStackStore, LocalForkSource, + ObjectSource, StoreError, }; use std::collections::BTreeSet; @@ -119,6 +124,20 @@ fn one_store_initialize_fork_commit_add_and_dedup_are_atomic() { ) .unwrap(); let pinned = store.pin_branch(branch_id).unwrap(); + assert_eq!(pinned.branch.head_commit_id, None); + assert_eq!( + pinned.root, + store + .layer(pinned.branch.base_layer_id) + .unwrap() + .unwrap() + .root_id + ); + assert_eq!( + store.reachable_root_storage(pinned.root).unwrap(), + closure_oracle(&store, &[pinned.root]).unwrap() + ); + let base_root = pinned.root; let built = apply_changes( &pinned.reader, pinned.root, @@ -160,6 +179,20 @@ fn one_store_initialize_fork_commit_add_and_dedup_are_atomic() { Some(commit_id) ); assert!(store.commit(commit_id).unwrap().is_some()); + let committed = store.pin_branch(branch_id).unwrap(); + assert_eq!(committed.branch.head_commit_id, Some(commit_id)); + assert_eq!( + committed.root, + store.commit(commit_id).unwrap().unwrap().root_id + ); + assert_eq!( + store.reachable_root_storage(committed.root).unwrap(), + closure_oracle(&store, &[committed.root]).unwrap() + ); + assert_eq!( + store.reachable_storage().unwrap(), + closure_oracle(&store, &[base_root, committed.root]).unwrap() + ); let layer_id = match store.add_layer(branch_id).unwrap() { AddLayerResult::Added { layer_id } => layer_id, outcome => panic!("unexpected Add outcome: {outcome:?}"), @@ -426,6 +459,132 @@ fn visible_missing_and_same_length_corrupt_objects_are_integrity_errors() { std::fs::remove_dir_all(root).unwrap(); } +#[test] +fn reachable_root_storage_is_exact_deduplicated_and_root_scoped() { + let root = temp("reachable-root"); + let path = root.join("store.sqlite"); + drop(LayerStackStore::create(&path).unwrap()); + let shared = insert_object(&path, bytes_object(b"shared")); + let left = insert_object(&path, directory_object(&[shared])); + let right = insert_object(&path, directory_object(&[shared, shared])); + let graph_root = insert_object(&path, directory_object(&[left, right])); + let unrelated = insert_object(&path, bytes_object(b"unrelated")); + let child_bytes = bytes_object(b"child"); + let child = insert_object(&path, child_bytes.clone()); + let parent = insert_object(&path, directory_object(&[child])); + let malformed = insert_object(&path, bytes_object(b"LFS4FSR\0malformed")); + let mut deep_root = insert_object(&path, bytes_object(b"deep-leaf")); + for _ in 0..64 { + deep_root = insert_object(&path, directory_object(&[deep_root])); + } + + let store = LayerStackStore::connect(&path).unwrap(); + assert_eq!( + store.reachable_root_storage(shared).unwrap(), + closure_oracle(&store, &[shared]).unwrap() + ); + let expected = closure_oracle(&store, &[graph_root]).unwrap(); + assert_eq!(expected.objects, 4); + assert_eq!(store.reachable_root_storage(graph_root).unwrap(), expected); + let deep = store.reachable_root_storage(deep_root).unwrap(); + assert_eq!(deep.objects, 65); + assert_eq!(deep, closure_oracle(&store, &[deep_root]).unwrap()); + drop(store); + + rusqlite::Connection::open(&path) + .unwrap() + .execute("DELETE FROM objects WHERE object_id=?1", [child.as_bytes()]) + .unwrap(); + corrupt_object(&path, unrelated); + let store = LayerStackStore::connect(&path).unwrap(); + assert_eq!(store.reachable_root_storage(graph_root).unwrap(), expected); + store.read_object(parent).unwrap(); + assert!(matches!( + store.reachable_root_storage(parent), + Err(StoreError::Integrity("visible object missing")) + )); + store.read_object(malformed).unwrap(); + assert!(store.reachable_root_storage(malformed).is_err()); + drop(store); + + insert_object(&path, child_bytes); + corrupt_object(&path, child); + let store = LayerStackStore::connect(&path).unwrap(); + store.read_object(parent).unwrap(); + assert!(matches!( + store.reachable_root_storage(parent), + Err(StoreError::Integrity("object identity")) + )); + + drop(store); + std::fs::remove_dir_all(root).unwrap(); +} + +fn bytes_object(value: &[u8]) -> Vec { + encode_object(&Object::bytes(value.to_vec()).unwrap()).unwrap() +} + +fn directory_object(children: &[ObjectId]) -> Vec { + let entries = children + .iter() + .enumerate() + .map(|(index, id)| { + DirectoryEntry::new( + CanonicalName::new(&format!("entry-{index}")).unwrap(), + ObjectReference::new(ObjectKind::Bytes, *id), + ) + }) + .collect(); + encode_object(&Object::directory(entries).unwrap()).unwrap() +} + +fn insert_object(path: &std::path::Path, bytes: Vec) -> ObjectId { + let id = ObjectId::for_bytes(&bytes); + rusqlite::Connection::open(path) + .unwrap() + .execute( + "INSERT OR IGNORE INTO objects(object_id,bytes) VALUES(?1,?2)", + rusqlite::params![id.as_bytes(), bytes], + ) + .unwrap(); + id +} + +fn corrupt_object(path: &std::path::Path, id: ObjectId) { + rusqlite::Connection::open(path) + .unwrap() + .execute( + "UPDATE objects SET bytes=zeroblob(length(bytes)) WHERE object_id=?1", + [id.as_bytes()], + ) + .unwrap(); +} + +fn closure_oracle( + store: &LayerStackStore, + roots: &[ObjectId], +) -> Result { + let mut pending = roots.to_vec(); + let mut seen = BTreeSet::new(); + let mut storage = CanonicalStorage { + objects: 0, + encoded_bytes: 0, + }; + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + let bytes = store.read_object(id)?; + storage.objects = storage.objects.checked_add(1).unwrap(); + storage.encoded_bytes = storage + .encoded_bytes + .checked_add(bytes.len().try_into().unwrap()) + .unwrap(); + pending.extend(referenced_objects(&bytes)?); + } + Ok(storage) +} + fn pragma(connection: &rusqlite::Connection, name: &str) -> i64 { connection .pragma_query_value(None, name, |row| row.get(0)) diff --git a/tools/layerfs-eval/src/main.rs b/tools/layerfs-eval/src/main.rs index add4ef129..0a9f77520 100644 --- a/tools/layerfs-eval/src/main.rs +++ b/tools/layerfs-eval/src/main.rs @@ -1,4 +1,5 @@ -use layerfs_sdk::{BranchId, LayerStackStore, ObjectSource}; +use layerfs_sdk::{BranchId, LayerStackStore}; +use std::io::Write; fn main() { if let Err(error) = run() { @@ -9,14 +10,149 @@ fn main() { fn run() -> Result<(), Box> { let arguments = std::env::args().skip(1).collect::>(); - let (store, branch_id) = match arguments.as_slice() { - [mode, store, id] if mode == "check" => { - (LayerStackStore::connect(store)?, id.parse::()?) + run_with(&arguments, &mut std::io::stdout().lock()) +} + +fn run_with( + arguments: &[String], + output: &mut dyn Write, +) -> Result<(), Box> { + let (store_path, branch_id, receipt_v1) = match arguments { + [mode, store, id] if mode == "check" => (store, id.parse::()?, false), + [mode, store, id, receipt] if mode == "check" && receipt == "--receipt-v1" => { + (store, id.parse::()?, true) } - _ => return Err("usage: layerfs-eval check ".into()), + _ => return Err("usage: layerfs-eval check [--receipt-v1]".into()), }; + let store = LayerStackStore::connect(store_path)?; let pinned = store.pin_branch(branch_id)?; - pinned.reader.read_object(pinned.root)?; - println!("{:?} {}", pinned.branch.head_commit_id, pinned.root); + let reachable = store.reachable_root_storage(pinned.root)?; + let success = if receipt_v1 { + let head = pinned + .branch + .head_commit_id + .map_or_else(|| "none".to_owned(), |id| id.to_string()); + receipt( + &pinned.branch.id.to_string(), + &pinned.layer_stack.id.to_string(), + &pinned.branch.base_layer_id.to_string(), + &head, + &pinned.root.to_string(), + reachable.objects, + reachable.encoded_bytes, + ) + } else { + legacy( + &format!("{:?}", pinned.branch.head_commit_id), + &pinned.root.to_string(), + ) + }; + output.write_all(success.as_bytes())?; Ok(()) } + +fn receipt( + branch: &str, + stack: &str, + base: &str, + head: &str, + root: &str, + objects: u64, + bytes: u64, +) -> String { + format!( + "layerfs_eval_receipt_version=1\nbranch_id={branch}\nlayer_stack_id={stack}\n\ + base_layer_id={base}\nhead_commit_id={head}\nroot_object_id={root}\n\ + reachable_objects={objects}\nreachable_encoded_bytes={bytes}\nstatus=ok\n" + ) +} + +fn legacy(head: &str, root: &str) -> String { + format!("{head} {root}\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use layerfs_sdk::{CommitId, LayerId, LayerStackId}; + + #[test] + fn default_output_is_unchanged_and_receipt_is_exact_and_repeatable() { + let branch = BranchId::from_bytes([0x11; 17]).unwrap().to_string(); + let stack = LayerStackId::from_bytes([0x31; 17]).unwrap().to_string(); + let layer = LayerId::from_bytes([0x32; 33]).unwrap().to_string(); + let root = "00".repeat(32); + let expected = format!( + "layerfs_eval_receipt_version=1\nbranch_id={branch}\nlayer_stack_id={stack}\n\ + base_layer_id={layer}\nhead_commit_id=none\nroot_object_id={root}\n\ + reachable_objects=1\nreachable_encoded_bytes=13\nstatus=ok\n" + ); + assert_eq!(legacy("None", &root), format!("None {root}\n")); + assert_eq!( + receipt(&branch, &stack, &layer, "none", &root, 1, 13), + expected + ); + assert_eq!( + receipt(&branch, &stack, &layer, "none", &root, 1, 13), + expected + ); + assert!(strict_receipt(expected.as_bytes())); + let mut out_of_order = expected.lines().collect::>(); + out_of_order.swap(1, 2); + let invalid = [ + expected.replacen("base_layer_id=", "branch_id=", 1), + expected.replacen("status=ok", "result=ok", 1), + expected.replacen(&format!("base_layer_id={layer}\n"), "", 1), + format!("{}\n", out_of_order.join("\n")), + expected.replacen(&branch, &"00".repeat(17), 1), + expected.replacen(&root, &"g0".repeat(32), 1), + expected.replacen("reachable_objects=1", "reachable_objects=01", 1), + format!("{expected}trailing"), + ]; + for value in invalid { + assert!(!strict_receipt(value.as_bytes()), "accepted {value:?}"); + } + } + + fn strict_receipt(bytes: &[u8]) -> bool { + let Ok(text) = std::str::from_utf8(bytes) else { + return false; + }; + let Some(text) = text.strip_suffix('\n') else { + return false; + }; + let fields = text.split('\n').collect::>(); + let [version, branch, stack, layer, head, root, objects, encoded_bytes, status] = + fields.as_slice() + else { + return false; + }; + let decimal = |field: &str, key: &str| { + field + .strip_prefix(key) + .and_then(|value| value.parse::().ok().map(|number| (value, number))) + .is_some_and(|(value, number)| number.to_string() == value) + }; + *version == "layerfs_eval_receipt_version=1" + && canonical_id::(branch, "branch_id=") + && canonical_id::(stack, "layer_stack_id=") + && canonical_id::(layer, "base_layer_id=") + && (*head == "head_commit_id=none" || canonical_id::(head, "head_commit_id=")) + && root.strip_prefix("root_object_id=").is_some_and(|value| { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) + && decimal(objects, "reachable_objects=") + && decimal(encoded_bytes, "reachable_encoded_bytes=") + && *status == "status=ok" + } + + fn canonical_id(field: &str, key: &str) -> bool { + field + .strip_prefix(key) + .and_then(|value| value.parse::().ok().map(|id| (value, id))) + .is_some_and(|(value, id)| id.to_string() == value) + } +}