Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 62 additions & 5 deletions crates/layerfs-content/src/object/id.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,71 @@ mod tests {
use super::*;

#[test]
fn ids_are_fixed_and_text_round_trips() {
fn fixed_width_hex_covers_every_byte_positions_and_seeded_ids() {
for value in u8::MIN..=u8::MAX {
for index in [0, DIGEST_BYTES - 1] {
let mut bytes = [0; DIGEST_BYTES];
bytes[index] = value;
let text = ObjectId::from_bytes(&bytes).unwrap().to_string();
assert_eq!(text.len(), DIGEST_BYTES * 2);
assert_eq!(&text[index * 2..index * 2 + 2], format!("{value:02x}"));
assert_eq!(text.parse::<ObjectId>().unwrap().to_bytes(), bytes);
}
}
let mut mixed = [0; DIGEST_BYTES];
for (index, value) in [(1, 0x01), (2, 0x10), (15, 0xab), (30, 0xfe), (31, 0xff)] {
mixed[index] = value;
}
let text = ObjectId::from_bytes(&mixed).unwrap().to_string();
assert!(text.starts_with("000110"));
assert_eq!(&text[30..32], "ab");
assert!(text.ends_with("feff"));

let mut state = 0x6a09_e667_f3bc_c909_u64;
for _ in 0..128 {
let mut bytes = [0; DIGEST_BYTES];
for byte in &mut bytes {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
*byte = state as u8;
}
let id = ObjectId::from_bytes(&bytes).unwrap();
assert_eq!(id.to_string().parse::<ObjectId>().unwrap(), id);
}
}

#[test]
fn parsing_accepts_hex_case_and_displays_canonical_lowercase() {
let id = ObjectId::for_bytes(b"case-equivalence");
let lowercase = id.to_string();
let uppercase = lowercase.to_ascii_uppercase();
assert_eq!(lowercase.parse::<ObjectId>().unwrap(), id);
assert_eq!(uppercase.parse::<ObjectId>().unwrap(), id);
assert_eq!(
uppercase.parse::<ObjectId>().unwrap().to_string(),
lowercase
);
}

#[test]
fn parsing_rejects_wrong_lengths_and_non_hex_text() {
for value in [String::new(), "0".repeat(63), "0".repeat(65)] {
assert_eq!(
value.parse::<ObjectId>(),
Err(CoreError::InvalidIdentityText)
);
}
let mut non_hex = "0".repeat(DIGEST_BYTES * 2);
non_hex.replace_range(31..32, "g");
assert_eq!(
non_hex.parse::<ObjectId>(),
Err(CoreError::InvalidIdentityText)
);
let id = ObjectId::for_bytes(b"payload");
assert_eq!(id.as_bytes().len(), DIGEST_BYTES);
assert_eq!(id.to_string().parse::<ObjectId>().unwrap(), id);
assert_eq!(ObjectId::from_reader(&b"payload"[..]).unwrap(), id);
assert_eq!(
ObjectId::from_bytes(&[0; DIGEST_BYTES]).unwrap().to_bytes(),
[0; DIGEST_BYTES]
Expand All @@ -98,9 +159,5 @@ mod tests {
actual: DIGEST_BYTES - 1
})
);
assert_eq!(
"zz".parse::<ObjectId>(),
Err(CoreError::InvalidIdentityText)
);
}
}
32 changes: 30 additions & 2 deletions crates/layerfs-layerstack-store/src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,25 @@ impl LayerStackStore {
})
}

pub fn reachable_root_storage(&self, root: ObjectId) -> Result<CanonicalStorage> {
let mut seen = crate::SpillableObjectSet::empty()?;
let mut active = BTreeSet::new();
let mut objects = 0_u64;
let mut encoded_bytes = 0_u64;
traverse_root(
self,
root,
&mut seen,
&mut active,
&mut objects,
&mut encoded_bytes,
)?;
Ok(CanonicalStorage {
objects,
encoded_bytes,
})
}

pub fn storage_snapshot(&self) -> Result<StoreStorageSnapshot> {
fn len(path: &std::path::Path) -> Result<u64> {
match std::fs::metadata(path) {
Expand Down Expand Up @@ -467,8 +486,17 @@ fn traverse_root(
return Err(StoreError::Integrity("object cycle"));
}
let canonical = store.db.read_object_row(id)?;
*objects = objects.saturating_add(1);
*encoded_bytes = encoded_bytes.saturating_add(canonical.len() as u64);
*objects = objects
.checked_add(1)
.ok_or(StoreError::Integrity("reachable storage overflow"))?;
*encoded_bytes = encoded_bytes
.checked_add(
canonical
.len()
.try_into()
.map_err(|_| StoreError::Integrity("reachable storage overflow"))?,
)
.ok_or(StoreError::Integrity("reachable storage overflow"))?;
let mut children = layerfs_content::object::references::referenced_objects(&canonical)?;
children.sort();
children.dedup();
Expand Down
163 changes: 161 additions & 2 deletions crates/layerfs-layerstack-store/tests/v4.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
use layerfs_content::filesystem::ContentChange;
use layerfs_content::{
encode_object, object::references::referenced_objects, CanonicalName, DirectoryEntry, Object,
ObjectId, ObjectKind, ObjectReference,
};
use layerfs_layerstack_store::{
apply_changes, AddLayerResult, CommitOutcome, EntityName, LayerStackInitialization,
LayerStackInitializationReceipt, LayerStackStore, LocalForkSource, ObjectSource, StoreError,
apply_changes, AddLayerResult, CanonicalStorage, CommitOutcome, EntityName,
LayerStackInitialization, LayerStackInitializationReceipt, LayerStackStore, LocalForkSource,
ObjectSource, StoreError,
};
use std::collections::BTreeSet;

Expand Down Expand Up @@ -119,6 +124,20 @@ fn one_store_initialize_fork_commit_add_and_dedup_are_atomic() {
)
.unwrap();
let pinned = store.pin_branch(branch_id).unwrap();
assert_eq!(pinned.branch.head_commit_id, None);
assert_eq!(
pinned.root,
store
.layer(pinned.branch.base_layer_id)
.unwrap()
.unwrap()
.root_id
);
assert_eq!(
store.reachable_root_storage(pinned.root).unwrap(),
closure_oracle(&store, &[pinned.root]).unwrap()
);
let base_root = pinned.root;
let built = apply_changes(
&pinned.reader,
pinned.root,
Expand Down Expand Up @@ -160,6 +179,20 @@ fn one_store_initialize_fork_commit_add_and_dedup_are_atomic() {
Some(commit_id)
);
assert!(store.commit(commit_id).unwrap().is_some());
let committed = store.pin_branch(branch_id).unwrap();
assert_eq!(committed.branch.head_commit_id, Some(commit_id));
assert_eq!(
committed.root,
store.commit(commit_id).unwrap().unwrap().root_id
);
assert_eq!(
store.reachable_root_storage(committed.root).unwrap(),
closure_oracle(&store, &[committed.root]).unwrap()
);
assert_eq!(
store.reachable_storage().unwrap(),
closure_oracle(&store, &[base_root, committed.root]).unwrap()
);
let layer_id = match store.add_layer(branch_id).unwrap() {
AddLayerResult::Added { layer_id } => layer_id,
outcome => panic!("unexpected Add outcome: {outcome:?}"),
Expand Down Expand Up @@ -426,6 +459,132 @@ fn visible_missing_and_same_length_corrupt_objects_are_integrity_errors() {
std::fs::remove_dir_all(root).unwrap();
}

#[test]
fn reachable_root_storage_is_exact_deduplicated_and_root_scoped() {
let root = temp("reachable-root");
let path = root.join("store.sqlite");
drop(LayerStackStore::create(&path).unwrap());
let shared = insert_object(&path, bytes_object(b"shared"));
let left = insert_object(&path, directory_object(&[shared]));
let right = insert_object(&path, directory_object(&[shared, shared]));
let graph_root = insert_object(&path, directory_object(&[left, right]));
let unrelated = insert_object(&path, bytes_object(b"unrelated"));
let child_bytes = bytes_object(b"child");
let child = insert_object(&path, child_bytes.clone());
let parent = insert_object(&path, directory_object(&[child]));
let malformed = insert_object(&path, bytes_object(b"LFS4FSR\0malformed"));
let mut deep_root = insert_object(&path, bytes_object(b"deep-leaf"));
for _ in 0..64 {
deep_root = insert_object(&path, directory_object(&[deep_root]));
}

let store = LayerStackStore::connect(&path).unwrap();
assert_eq!(
store.reachable_root_storage(shared).unwrap(),
closure_oracle(&store, &[shared]).unwrap()
);
let expected = closure_oracle(&store, &[graph_root]).unwrap();
assert_eq!(expected.objects, 4);
assert_eq!(store.reachable_root_storage(graph_root).unwrap(), expected);
let deep = store.reachable_root_storage(deep_root).unwrap();
assert_eq!(deep.objects, 65);
assert_eq!(deep, closure_oracle(&store, &[deep_root]).unwrap());
drop(store);

rusqlite::Connection::open(&path)
.unwrap()
.execute("DELETE FROM objects WHERE object_id=?1", [child.as_bytes()])
.unwrap();
corrupt_object(&path, unrelated);
let store = LayerStackStore::connect(&path).unwrap();
assert_eq!(store.reachable_root_storage(graph_root).unwrap(), expected);
store.read_object(parent).unwrap();
assert!(matches!(
store.reachable_root_storage(parent),
Err(StoreError::Integrity("visible object missing"))
));
store.read_object(malformed).unwrap();
assert!(store.reachable_root_storage(malformed).is_err());
drop(store);

insert_object(&path, child_bytes);
corrupt_object(&path, child);
let store = LayerStackStore::connect(&path).unwrap();
store.read_object(parent).unwrap();
assert!(matches!(
store.reachable_root_storage(parent),
Err(StoreError::Integrity("object identity"))
));

drop(store);
std::fs::remove_dir_all(root).unwrap();
}

fn bytes_object(value: &[u8]) -> Vec<u8> {
encode_object(&Object::bytes(value.to_vec()).unwrap()).unwrap()
}

fn directory_object(children: &[ObjectId]) -> Vec<u8> {
let entries = children
.iter()
.enumerate()
.map(|(index, id)| {
DirectoryEntry::new(
CanonicalName::new(&format!("entry-{index}")).unwrap(),
ObjectReference::new(ObjectKind::Bytes, *id),
)
})
.collect();
encode_object(&Object::directory(entries).unwrap()).unwrap()
}

fn insert_object(path: &std::path::Path, bytes: Vec<u8>) -> ObjectId {
let id = ObjectId::for_bytes(&bytes);
rusqlite::Connection::open(path)
.unwrap()
.execute(
"INSERT OR IGNORE INTO objects(object_id,bytes) VALUES(?1,?2)",
rusqlite::params![id.as_bytes(), bytes],
)
.unwrap();
id
}

fn corrupt_object(path: &std::path::Path, id: ObjectId) {
rusqlite::Connection::open(path)
.unwrap()
.execute(
"UPDATE objects SET bytes=zeroblob(length(bytes)) WHERE object_id=?1",
[id.as_bytes()],
)
.unwrap();
}

fn closure_oracle(
store: &LayerStackStore,
roots: &[ObjectId],
) -> Result<CanonicalStorage, StoreError> {
let mut pending = roots.to_vec();
let mut seen = BTreeSet::new();
let mut storage = CanonicalStorage {
objects: 0,
encoded_bytes: 0,
};
while let Some(id) = pending.pop() {
if !seen.insert(id) {
continue;
}
let bytes = store.read_object(id)?;
storage.objects = storage.objects.checked_add(1).unwrap();
storage.encoded_bytes = storage
.encoded_bytes
.checked_add(bytes.len().try_into().unwrap())
.unwrap();
pending.extend(referenced_objects(&bytes)?);
}
Ok(storage)
}

fn pragma(connection: &rusqlite::Connection, name: &str) -> i64 {
connection
.pragma_query_value(None, name, |row| row.get(0))
Expand Down
Loading
Loading