From 395202fb5af8fab58a2d8e7f76665505a8e459a7 Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Mon, 27 Jul 2026 10:57:23 -0700 Subject: [PATCH 1/2] erofs: Add new erofs utility crate Add the ability to write meta-only EROFS images based on our VFS of stone layout records. --- Cargo.lock | 11 + Cargo.toml | 3 + crates/erofs/Cargo.toml | 18 + crates/erofs/src/lib.rs | 754 +++++++++++++++++++++++++++++++++++++ crates/vfs/src/tree/mod.rs | 94 +++-- 5 files changed, 847 insertions(+), 33 deletions(-) create mode 100644 crates/erofs/Cargo.toml create mode 100644 crates/erofs/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6e8447bfc..db39b8cc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -990,6 +990,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erofs" +version = "0.1.0" +dependencies = [ + "astr", + "crc", + "fs-err", + "stone", + "vfs", +] + [[package]] name = "errno" version = "0.3.14" diff --git a/Cargo.toml b/Cargo.toml index 30c542ea3..3d903af76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ rust-version = "1.94" [workspace.dependencies] astr.path = "crates/astr" +vfs.path = "crates/vfs" +stone.path = "crates/stone" blsforme = { git = "https://github.com/AerynOS/blsforme.git", rev = "55da60a60b6f883818fbd67c67557439787f2fce" } bytes = "1.6.0" camino = "1.1.10" @@ -113,6 +115,7 @@ cbindgen = "0.29.2" indexmap = { version = "2.14.0", features = ["serde"] } arc-swap = "1.9.1" indoc = "2.0.7" +crc = "3" [workspace.lints.rust] rust_2018_idioms = { level = "warn", priority = -1 } diff --git a/crates/erofs/Cargo.toml b/crates/erofs/Cargo.toml new file mode 100644 index 000000000..5bc930587 --- /dev/null +++ b/crates/erofs/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2026 AerynOS Developers +# SPDX-License-Identifier: MPL-2.0 + +[package] +name = "erofs" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true + +[dependencies] +astr.workspace = true +crc.workspace = true +fs-err.workspace = true +stone.workspace = true +vfs.workspace = true + +[lints] +workspace = true diff --git a/crates/erofs/src/lib.rs b/crates/erofs/src/lib.rs new file mode 100644 index 000000000..cf01b70e8 --- /dev/null +++ b/crates/erofs/src/lib.rs @@ -0,0 +1,754 @@ +// SPDX-FileCopyrightText: 2026 AerynOS Developers +// SPDX-License-Identifier: MPL-2.0 + +//! EROFS utilities + +use std::{ + collections::{BTreeMap, HashMap, btree_map}, + io::{self, BufWriter, Write}, + os::unix::fs::MetadataExt, + path::Path, +}; + +use astr::AStr; +use fs_err as fs; +use stone::{StonePayloadLayoutFile, StonePayloadLayoutRecord}; +use vfs::tree::Element; + +// TODO: Configurable? +const BLOCK_SIZE_BITS: u8 = 12; +const BLOCK_SIZE: u64 = 1 << (BLOCK_SIZE_BITS as u64); +const ZERO_BLOCK: [u8; BLOCK_SIZE as usize] = [0; BLOCK_SIZE as usize]; + +const SUPER_BLOCK_OFFSET: u64 = 1024; +const SUPER_BLOCK_SIZE: usize = BLOCK_SIZE as usize - SUPER_BLOCK_OFFSET as usize; +const SUPER_MAGIC_V1: u32 = 0xE0F5_E1E2; + +// https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#superblock-checksum +const FEATURE_COMPAT_SB_CHKSUM: u32 = 0x1; + +// https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#inode-data-layouts +const INODE_FLAT_INLINE: u16 = 2; +const INODE_FLAT_PLAIN: u16 = 0; +// https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#i-format-field +const INODE_VERSION_EXTENDED: u16 = 1; + +const DIRENT_SIZE: usize = 12; +const SLOT_SIZE: usize = 32; + +const ST_IFDIR: u16 = 0o040_000; +const ST_IFREG: u16 = 0o100_000; +const ST_IFLNK: u16 = 0o120_000; + +/// A writer capable of producing an EROFS meta-only image from +/// a [`vfs::Tree`] of [`StonePayloadLayoutRecord`] entries. +#[derive(Debug, Clone, Copy, Default)] +pub struct MetaImageWriter { + xattr_namespace: XattrNamespace, +} + +impl MetaImageWriter { + /// Returns a new, default [`MetaImageWriter`]. + pub fn new() -> Self { + Self::default() + } + + /// Specify which namespace the xattrs should be written to. + /// + /// Defaults to [`XattrNamespace::Trusted`]. + pub fn with_xattr_namespace(self, xattr_namespace: XattrNamespace) -> Self { + Self { xattr_namespace } + } + + /// Writes an EROFS meta-only image to the provided `writer` using + /// the provided [`vfs::Tree`] of [`StonePayloadLayoutRecord`] entries. + /// + /// `cas_dir` must be the path to the CAS backing for the provided vfstree. + pub fn write(self, tree: &vfs::Tree, cas_dir: &Path, writer: &mut W) -> io::Result<()> + where + T: AsRef, + W: Write, + { + write_meta_image(tree, cas_dir, &self.xattr_namespace, writer) + } +} + +/// The namespace that extended attributes get written to. +#[derive(Debug, Clone, Copy, Default)] +#[repr(u8)] +pub enum XattrNamespace { + /// User extended attributes (`user.*`) + User = 1, + /// Trusted extended attributes (`trusted.*`) + #[default] + Trusted = 4, +} + +fn write_meta_image( + tree: &vfs::Tree, + cas_dir: &Path, + xattr_namespace: &XattrNamespace, + writer: &mut W, +) -> io::Result<()> +where + T: AsRef, + W: Write, +{ + // Buffer by block size + let writer = &mut BufWriter::with_capacity(BLOCK_SIZE as usize, writer); + + // Get root element of VFS tree (/) + let root_element = tree + .structured() + .ok_or_else(|| io::Error::other("vfs missing root / directory"))?; + + // Build inodes from vfs + let mut inodes: Vec> = Vec::with_capacity(tree.len() as usize); + build_inodes(&root_element, &mut inodes, None); + + // Compute layout + let layout = compute_layout(cas_dir, &inodes)?; + + // Write all blocks + + // Write superblock + write_padded(writer, BLOCK_SIZE as usize, |writer| write_superblock(writer, &layout))?; + + // Write shared attrs area, block aligned + write_padded(writer, BLOCK_SIZE as usize, |writer| { + write_shared_xattrs(writer, xattr_namespace, &layout.redirects) + })?; + + // Write each meta block + let mut inode_idx = 0; + for _ in 0..layout.meta_blocks { + write_padded(writer, BLOCK_SIZE as usize, |writer| { + let mut cursor = 0usize; + + while inode_idx < layout.inodes.len() { + let inode = &layout.inodes[inode_idx]; + let aligned_size = inode.aligned_size(); + + if cursor + aligned_size > BLOCK_SIZE as usize { + break; + } + + write_padded(writer, SLOT_SIZE, |writer| write_inode(writer, inode, &layout))?; + + cursor += aligned_size; + inode_idx += 1; + } + + Ok(()) + })?; + } + debug_assert!(inode_idx == layout.inodes.len(), "All inodes should be written"); + + // Write dir blocks + for packed_dirent in layout.packed_dirents.values() { + for block in &packed_dirent.blocks { + write_padded(writer, BLOCK_SIZE as usize, |writer| { + write_dirent_block(writer, block, &layout.nid_mapping) + })?; + } + } + + writer.flush()?; + + Ok(()) +} + +struct Dirent<'a> { + name: &'a str, + child_ino: u64, + file_type: DirentFileType, +} + +// https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#file-type-values +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DirentFileType { + Unknown = 0, + RegFile = 1, + Dir = 2, + Chrdev = 3, + Blkdev = 4, + Fifo = 5, + Sock = 6, + Symlink = 7, +} + +fn dirent_file_type(file: &StonePayloadLayoutFile) -> DirentFileType { + match file { + StonePayloadLayoutFile::Directory(_) => DirentFileType::Dir, + StonePayloadLayoutFile::Regular(_, _) => DirentFileType::RegFile, + StonePayloadLayoutFile::Symlink(_, _) => DirentFileType::Symlink, + StonePayloadLayoutFile::CharacterDevice(..) => DirentFileType::Chrdev, + StonePayloadLayoutFile::BlockDevice(..) => DirentFileType::Blkdev, + StonePayloadLayoutFile::Fifo(..) => DirentFileType::Fifo, + StonePayloadLayoutFile::Socket(..) => DirentFileType::Sock, + StonePayloadLayoutFile::Unknown(..) => DirentFileType::Unknown, + } +} + +struct Inode<'a> { + ino: u64, + mode: u16, + uid: u32, + gid: u32, + kind: InodeKind<'a>, +} + +impl<'a> Inode<'a> { + fn aligned_size(&self) -> usize { + let size = match &self.kind { + InodeKind::Dir { .. } => 64, + InodeKind::Reg { .. } => { + 64 + + // Inlined xattrs + (12 + 4 * 2) + } + InodeKind::Symlink { source, .. } => { + 64 + + // Inlined symlink data + source.len() + } + }; + + // Inodes must align to slot boundary + let num_slots = size.div_ceil(SLOT_SIZE); + + num_slots * SLOT_SIZE + } +} + +enum InodeKind<'a> { + Dir(InodeDir<'a>), + Reg { cas_path: AStr }, + Symlink { source: &'a [u8] }, +} + +struct InodeDir<'a> { + children: Vec>, + num_hardlinks: u32, +} + +fn build_inodes<'a, T>(element: &'a Element<'a, T>, inodes: &mut Vec>, parent_ino: Option) -> Option +where + T: AsRef, +{ + let ino = inodes.len() as u64; + let layout = element.item().as_ref(); + + match &layout.file { + StonePayloadLayoutFile::Directory(_) => { + inodes.push(Inode { + ino, + mode: ST_IFDIR | (layout.mode & 0o7777) as u16, + uid: layout.uid, + gid: layout.gid, + kind: InodeKind::Dir(InodeDir { + // Filled in after we collect children + children: vec![ + Dirent { + name: ".", + child_ino: ino, + file_type: DirentFileType::Dir, + }, + Dirent { + name: "..", + child_ino: parent_ino.unwrap_or(ino), + file_type: DirentFileType::Dir, + }, + ], + // Filled in after adding all children + num_hardlinks: 0, + }), + }); + + for child in element.children() { + let Some(child_nid) = build_inodes(child, inodes, Some(ino)) else { + continue; + }; + + if let InodeKind::Dir(InodeDir { children, .. }) = &mut inodes[ino as usize].kind { + let layout = child.item().as_ref(); + let name = child.file_name(); + children.push(Dirent { + name, + child_ino: child_nid, + file_type: dirent_file_type(&layout.file), + }); + } + } + + let inode = &mut inodes[ino as usize]; + + if let InodeKind::Dir(InodeDir { + children, + num_hardlinks, + .. + }) = &mut inode.kind + { + // Ensure children are sorted + children.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes())); + + // ., .., and each subdirs .. + // + // Technically `..` points to the parent, but that + // means the parent also links to this so it + // contributes the same, so we can simply just + // always take the total number of subdirs + *num_hardlinks = children.iter().filter(|e| e.file_type == DirentFileType::Dir).count() as u32; + } + + Some(ino) + } + StonePayloadLayoutFile::Regular(id, _) => { + let cas_path = AStr::from(cas_path(id)); + + inodes.push(Inode { + ino, + mode: ST_IFREG | (layout.mode & 0xFFFF) as u16, + uid: layout.uid, + gid: layout.gid, + kind: InodeKind::Reg { cas_path }, + }); + + Some(ino) + } + StonePayloadLayoutFile::Symlink(source, _) => { + inodes.push(Inode { + ino, + mode: ST_IFLNK | (layout.mode & 0xFFFF) as u16, + uid: layout.uid, + gid: layout.gid, + kind: InodeKind::Symlink { + source: source.as_bytes(), + }, + }); + + Some(ino) + } + _ => None, + } +} + +struct ComputedLayout<'a> { + inodes: &'a [Inode<'a>], + redirects: BTreeMap, + nid_mapping: HashMap, + packed_dirents: BTreeMap>, + xattr_blkaddr: u32, + meta_blkaddr: u32, + meta_blocks: u32, + total_blocks: u32, +} + +#[derive(Debug, Clone, Copy)] +struct CasRedirect { + /// Offset into the shared xattr area + offset: u32, + /// Size of the underlying CAS file this redirect to + size: u64, +} + +fn compute_layout<'a>(cas_dir: &Path, inodes: &'a [Inode<'a>]) -> io::Result> { + let mut inode_dirs = vec![]; + let mut redirects = BTreeMap::new(); + let mut nid_mapping = HashMap::new(); + let mut packed_dirents = BTreeMap::new(); + let mut metadata_bytes = 0u64; + + // Compute inode placement (nid) + for inode in inodes { + // Inode metadata entries are aligned to "slots" + let aligned_size = inode.aligned_size(); + + // Skip to next block if this inode doesn't fit cleanly within a block + if metadata_bytes % BLOCK_SIZE + aligned_size as u64 > BLOCK_SIZE { + metadata_bytes += BLOCK_SIZE - (metadata_bytes % BLOCK_SIZE); + } + + // Nid is the relative slot offset into the meta block + let nid = metadata_bytes / SLOT_SIZE as u64; + // Add to mapping table so we can reference the real NID offsets + // when writing out direntry blocks + nid_mapping.insert(inode.ino, nid); + + // Track total size so we know how many metablocks will be used + metadata_bytes += aligned_size as u64; + + match &inode.kind { + InodeKind::Dir(inode_dir) => { + // Push each dir to a smaller collection which will + // be used later to compute all direntry blocks + inode_dirs.push((inode.ino, inode_dir)); + } + InodeKind::Reg { cas_path, .. } => { + // Get the unique set of cas paths & stat their actual + // size for accurate inode size. We will later + // calculate its relative offset in the shared xattr + // block & use this map as a lookup when writing + // this inodes metadata. + if let btree_map::Entry::Vacant(vacant) = redirects.entry(cas_path.clone()) { + // `cas_path` is the relative path from `cas_dir`, but made absolute to the + // root of the erofs tree. We can strip `/` & rejoin them to get the actual + // path to the file on this system. + let size = fs::metadata(cas_dir.join(cas_path.trim_start_matches('/')))?.size(); + + vacant.insert(CasRedirect { offset: 0, size }); + } + } + InodeKind::Symlink { .. } => {} + } + } + + // Compute shared xattr area size + + // We write out a metacopy entry as the first offset without + // any hash == fs-verity is disabled. All regular files will + // reference this single xattr. + let mut xattr_bytes = xattr_entry_size("overlay.metacopy", b""); + + // Each unique cas has a redirect xattr. Its offset will be + // the related inodes inlined shared pointer looked up from + // this redirect map. + for (cas_path, redirect) in redirects.iter_mut() { + redirect.offset = (xattr_bytes / 4) as u32; + xattr_bytes += xattr_entry_size("overlay.redirect", cas_path.as_bytes()); + } + + // Layout all block addresses + + // First block after superblock + let xattr_blkaddr = 1u32; + let xattr_blocks = (xattr_bytes).div_ceil(BLOCK_SIZE as usize) as u32; + + // After xattr block + let meta_blkaddr = xattr_blkaddr + xattr_blocks; + let meta_blocks = (metadata_bytes).div_ceil(BLOCK_SIZE) as u32; + + // After meta block + let dirent_blkaddr = meta_blkaddr + meta_blocks; + + // Compute each direntry blkaddr + let mut dirent_blkaddr_cursor = dirent_blkaddr; + // Each dir gets a continuous range of packed blocks + // to store its dir entries in + for (ino, dir) in inode_dirs { + // Pack dir entries into as few blocks as possible, + // ensuring each entry falls completely within a clean block + let packed_dirent = pack_dirent(dirent_blkaddr_cursor, &dir.children); + let num_blocks = packed_dirent.blocks.len() as u32; + + // Next dir lands on the next sequential block + dirent_blkaddr_cursor += num_blocks; + + // Track this dirent per inode so we can + // reference its blkaddr in the inode metadata + packed_dirents.insert(ino, packed_dirent); + } + + let total_blocks = dirent_blkaddr_cursor; + + Ok(ComputedLayout { + inodes, + redirects, + nid_mapping, + packed_dirents, + xattr_blkaddr, + meta_blkaddr, + meta_blocks, + total_blocks, + }) +} + +fn cas_path(id: &u128) -> String { + let hash = format!("{id:02x}"); + + if hash.len() >= 10 { + format!("/{}/{}/{}/{hash}", &hash[..2], &hash[2..4], &hash[4..6]) + } else { + format!("/{hash}") + } +} + +/// Header + xattr suffix string + value string, aligned to 4 bytes +fn xattr_entry_size(suffix: &str, value: &[u8]) -> usize { + let size = 4 + suffix.len() + value.len(); + + if size.is_multiple_of(4) { + size + } else { + size + (4 - size % 4) + } +} + +struct PaddedAdapter<'a, T: Write> { + inner: &'a mut T, + written: usize, +} + +impl<'a, T: Write> Write for PaddedAdapter<'a, T> { + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.inner.write(buf)?; + self.written += written; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + let written = self.inner.write_vectored(bufs)?; + self.written += written; + Ok(written) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.inner.write_all(buf)?; + self.written += buf.len(); + Ok(()) + } +} + +fn write_padded( + writer: &mut W, + alignment: usize, + mut f: impl FnMut(&mut PaddedAdapter<'_, W>) -> io::Result<()>, +) -> io::Result<()> { + let mut writer = PaddedAdapter { + inner: writer, + written: 0, + }; + + f(&mut writer)?; + + let written = writer.written; + + if !written.is_multiple_of(alignment) { + let pad = alignment - written % alignment; + + writer.write_all(&ZERO_BLOCK[..pad])?; + } + + Ok(()) +} + +fn write_shared_xattrs( + writer: &mut W, + namespace: &XattrNamespace, + redirects: &BTreeMap, +) -> io::Result<()> { + write_xattr_entry(writer, namespace, "overlay.metacopy", b"")?; + + for path in redirects.keys() { + write_xattr_entry(writer, namespace, "overlay.redirect", path.as_bytes())?; + } + + Ok(()) +} + +fn write_xattr_entry( + writer: &mut W, + namespace: &XattrNamespace, + suffix: &str, + value: &[u8], +) -> io::Result<()> { + write_padded(writer, 4, |writer| { + // https://erofs.docs.kernel.org/en/latest/ondisk/xattrs.html#xattr-entry-record + writer.write_all(&[suffix.len() as u8, *namespace as u8])?; + writer.write_all(&(value.len() as u16).to_le_bytes())?; + writer.write_all(suffix.as_bytes())?; + writer.write_all(value) + }) +} + +fn write_inode(writer: &mut W, inode: &Inode<'_>, layout: &ComputedLayout<'_>) -> io::Result<()> { + let (data_layout, i_u_startblk, i_xattr_shared_count, i_size, i_nlink, redirect) = match &inode.kind { + InodeKind::Dir(InodeDir { num_hardlinks, .. }) => { + let dirent = layout + .packed_dirents + .get(&inode.ino) + .expect("inode dir must have packed dirent"); + + ( + INODE_FLAT_PLAIN, + dirent.blkaddr, + 0u8, + dirent.blocks.len() as u64 * BLOCK_SIZE, + *num_hardlinks, + None, + ) + } + InodeKind::Reg { cas_path } => { + let redirect = layout + .redirects + .get(cas_path) + .copied() + .expect("cas path must have redirect"); + + ( + INODE_FLAT_PLAIN, + 0, + // 2 shared refs + 2, + redirect.size, + 1, + Some(redirect), + ) + } + InodeKind::Symlink { source, .. } => (INODE_FLAT_INLINE, 0, 0, source.len() as u64, 1, None), + }; + + let i_format: u16 = (data_layout << 1) | INODE_VERSION_EXTENDED; + + let i_xattr_icount = if i_xattr_shared_count > 0 { + i_xattr_shared_count + 1 + } else { + 0 + }; + + // Extended 64 byte inode + // https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#extended-inode-64-bytes + writer.write_all(&i_format.to_le_bytes())?; + writer.write_all(&(i_xattr_icount as u16).to_le_bytes())?; + writer.write_all(&inode.mode.to_le_bytes())?; + writer.write_all(&0u16.to_le_bytes())?; + writer.write_all(&i_size.to_le_bytes())?; + writer.write_all(&i_u_startblk.to_le_bytes())?; + writer.write_all(&(inode.ino as u32).to_le_bytes())?; + writer.write_all(&inode.uid.to_le_bytes())?; + writer.write_all(&inode.gid.to_le_bytes())?; + writer.write_all(&0u64.to_le_bytes())?; + writer.write_all(&0u32.to_le_bytes())?; + writer.write_all(&i_nlink.to_le_bytes())?; + writer.write_all(&[0u8; 16])?; + + // Inline data + if let InodeKind::Symlink { source, .. } = &inode.kind { + writer.write_all(source)?; + } + + // Inline xattrs + if let InodeKind::Reg { .. } = &inode.kind { + // Header + // https://erofs.docs.kernel.org/en/latest/ondisk/xattrs.html#inline-xattr-body-header + writer.write_all(&0u32.to_le_bytes())?; + writer.write_all(&[i_xattr_shared_count])?; + writer.write_all(&[0u8; 7])?; + + // Shared xattr index values + // Metacopy + writer.write_all(&0u32.to_le_bytes())?; + // Redirect + let redirect_id = redirect.expect("regular file always has redirect").offset; + writer.write_all(&redirect_id.to_le_bytes())?; + } + + Ok(()) +} + +struct PackedDirent<'a> { + blkaddr: u32, + blocks: Vec<&'a [Dirent<'a>]>, +} + +fn pack_dirent<'a>(blkaddr: u32, children: &'a [Dirent<'a>]) -> PackedDirent<'a> { + let mut start = 0; + let mut running_size = 0; + let mut blocks = vec![]; + + for (child_i, child) in children.iter().enumerate() { + running_size += DIRENT_SIZE + child.name.len(); + + // Write if this is the last entry that will fit on a full block + let is_full = child_i == children.len() - 1 + || running_size + DIRENT_SIZE + children[child_i + 1].name.len() > BLOCK_SIZE as usize; + + if is_full { + blocks.push(&children[start..=child_i]); + + start = child_i + 1; + running_size = 0; + } + } + + PackedDirent { blkaddr, blocks } +} + +fn write_dirent_block( + writer: &mut W, + children: &[Dirent<'_>], + nid_mapping: &HashMap, +) -> io::Result<()> { + let num_entries = children.len(); + + // Record the offset each filename will be at + let mut name_offsets: Vec = Vec::with_capacity(num_entries); + let mut cursor = num_entries * DIRENT_SIZE; + for entry in children { + name_offsets.push(cursor as u16); + cursor += entry.name.len(); + } + + // Write each record, using the offset recorded above + for (i, entry) in children.iter().enumerate() { + let nid = nid_mapping + .get(&entry.child_ino) + .copied() + .expect("inode must have nid mapping"); + + // https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#directory-entry-record + writer.write_all(&nid.to_le_bytes())?; + writer.write_all(&name_offsets[i].to_le_bytes())?; + writer.write_all(&[entry.file_type as u8, 0])?; + } + + // Write each filename + // + // https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#filename-encoding + for entry in children { + writer.write_all(entry.name.as_bytes())?; + } + Ok(()) +} + +#[rustfmt::skip] +fn write_superblock( + writer: &mut W, + layout: &ComputedLayout<'_>, +) -> io::Result<()> { + let mut buf = [0u8; SUPER_BLOCK_SIZE]; + + // Populate required / used fields + // https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#field-definitions + buf[0..4].copy_from_slice(&SUPER_MAGIC_V1.to_le_bytes()); // magic + buf[8..12].copy_from_slice(&FEATURE_COMPAT_SB_CHKSUM.to_le_bytes()); // feature_compat + buf[12] = BLOCK_SIZE_BITS; // blkszbits + buf[16..24].copy_from_slice(&(layout.inodes.len() as u64).to_le_bytes()); // inos + buf[36..40].copy_from_slice(&layout.total_blocks.to_le_bytes()); // blocks + buf[40..44].copy_from_slice(&layout.meta_blkaddr.to_le_bytes()); // meta_blkaddr + buf[44..48].copy_from_slice(&layout.xattr_blkaddr.to_le_bytes()); // xattr_blkaddr + + // Add checksum + let checksum = crc32c(buf.as_slice()); + buf[4..8].copy_from_slice(&checksum.to_le_bytes()); + + // Ensure entire block is zeroized + writer.write_all(&ZERO_BLOCK[..SUPER_BLOCK_OFFSET as usize])?; + writer.write_all(buf.as_slice()) +} + +fn crc32c(bytes: &[u8]) -> u32 { + use crc::{CRC_32_ISCSI, Crc}; + + const CRC32C: Crc = Crc::::new(&CRC_32_ISCSI); + + CRC32C.checksum(bytes) + // Undo XOR per: + // https://erofs.docs.kernel.org/en/latest/ondisk/core_ondisk.html#superblock-checksum + ^ CRC32C.algorithm.xorout +} diff --git a/crates/vfs/src/tree/mod.rs b/crates/vfs/src/tree/mod.rs index dbf0b7c10..e996dd7d6 100644 --- a/crates/vfs/src/tree/mod.rs +++ b/crates/vfs/src/tree/mod.rs @@ -55,8 +55,11 @@ struct File { inner: T, } -impl File { - pub fn new(inner: T) -> Self { +impl File { + pub fn new(inner: T) -> Self + where + T: BlitFile, + { let path = VfsPath::new(inner.path()); Self { @@ -80,13 +83,13 @@ impl File { /// Actual tree implementation, encapsulating indextree #[derive(Debug)] -pub struct Tree { +pub struct Tree { arena: Arena>, map: HashMap, length: u64, } -impl Tree { +impl Tree { /// Construct a new Tree with specified capacity fn with_capacity(capacity: usize) -> Self { Tree { @@ -159,6 +162,38 @@ impl Tree { Ok(()) } + /// Iterate using a TreeIterator, starting at the `/` node + pub fn iter(&self) -> TreeIterator<'_, T> { + TreeIterator { + parent: self, + enume: self.resolve_node("/").map(|n| n.descendants(&self.arena)), + } + } + + /// Return structured view beginning at `/` + pub fn structured(&self) -> Option> { + self.resolve_node("/").map(|root| self.structured_children(root)) + } + + /// For the given node, recursively convert to Element::Directory of Child + fn structured_children(&self, start: &NodeId) -> Element<'_, T> { + let node = &self.arena[*start]; + let item = node.get(); + let partial = item.file_name(); + + if item.kind.is_directory() { + let children = start + .children(&self.arena) + .map(|c| self.structured_children(&c)) + .collect::>(); + Element::Directory(partial, &item.inner, children) + } else { + Element::Child(partial, &item.inner) + } + } +} + +impl Tree { pub fn print(&self) { let root = self.resolve_node("/").unwrap(); eprintln!("{:#?}", root.debug_pretty_print(&self.arena)); @@ -203,50 +238,43 @@ impl Tree { Ok(()) } +} - /// Iterate using a TreeIterator, starting at the `/` node - pub fn iter(&self) -> TreeIterator<'_, T> { - TreeIterator { - parent: self, - enume: self.resolve_node("/").map(|n| n.descendants(&self.arena)), +pub enum Element<'a, T> { + Directory(&'a str, &'a T, Vec>), + Child(&'a str, &'a T), +} + +impl<'a, T> Element<'a, T> { + pub fn file_name(&self) -> &str { + match self { + Element::Directory(name, _, _) => name, + Element::Child(name, _) => name, } } - /// Return structured view beginning at `/` - pub fn structured(&self) -> Option> { - self.resolve_node("/").map(|root| self.structured_children(root)) + pub fn item(&self) -> &T { + match self { + Element::Directory(_, file, _) => file, + Element::Child(_, file) => file, + } } - /// For the given node, recursively convert to Element::Directory of Child - fn structured_children(&self, start: &NodeId) -> Element<'_, T> { - let node = &self.arena[*start]; - let item = node.get(); - let partial = item.file_name(); - - if item.kind.is_directory() { - let children = start - .children(&self.arena) - .map(|c| self.structured_children(&c)) - .collect::>(); - Element::Directory(partial, &item.inner, children) - } else { - Element::Child(partial, &item.inner) + pub fn children(&self) -> &[Element<'a, T>] { + match self { + Element::Directory(_, _, elements) => elements, + Element::Child(_, _) => &[], } } } -pub enum Element<'a, T: BlitFile> { - Directory(&'a str, &'a T, Vec>), - Child(&'a str, &'a T), -} - /// Simple DFS iterator for a Tree -pub struct TreeIterator<'a, T: BlitFile> { +pub struct TreeIterator<'a, T> { parent: &'a Tree, enume: Option>>, } -impl<'a, T: BlitFile> Iterator for TreeIterator<'a, T> { +impl<'a, T> Iterator for TreeIterator<'a, T> { type Item = &'a T; fn next(&mut self) -> Option { From 85e2a8d9c7423792e02b15abc79efd81daad96df Mon Sep 17 00:00:00 2001 From: Cory Forsstrom Date: Tue, 28 Jul 2026 14:33:36 -0700 Subject: [PATCH 2/2] moss: Add state erofs command --- Cargo.lock | 1 + moss/Cargo.toml | 1 + moss/src/cli/state.rs | 62 ++++++++++++++++++++++++++++++++++++++++++- moss/src/fstree.rs | 6 +++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index db39b8cc6..afbc56f54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1993,6 +1993,7 @@ dependencies = [ "container", "dag", "derive_more", + "erofs", "fnmatch", "fs-err", "futures-util", diff --git a/moss/Cargo.toml b/moss/Cargo.toml index 8e06d801d..f31f5418f 100644 --- a/moss/Cargo.toml +++ b/moss/Cargo.toml @@ -18,6 +18,7 @@ testing = [] config = { path = "../crates/config" } container = { path = "../crates/container" } dag = { path = "../crates/dag" } +erofs = { path = "../crates/erofs" } tools_buildinfo = { path = "../crates/tools_buildinfo" } stone = { path = "../crates/stone" } tracing_common = { path = "../crates/tracing_common" } diff --git a/moss/src/cli/state.rs b/moss/src/cli/state.rs index 0e854de0a..123208bcc 100644 --- a/moss/src/cli/state.rs +++ b/moss/src/cli/state.rs @@ -8,7 +8,7 @@ use std::{ use chrono::Local; use clap::{ArgAction, ArgMatches, Command, CommandFactory, FromArgMatches, Parser, arg}; -use fs_err as fs; +use fs_err::{self as fs, File}; use moss::{ Installation, State, client::{self, Client, prune}, @@ -67,6 +67,18 @@ pub fn command() -> Command { .about("Verify and fix system states and assets") .arg(arg!(--verbose "Vebose output").action(ArgAction::SetTrue)), ) + .subcommand( + Command::new("format-erofs") + .about("Format state as an erofs meta-only image") + .arg(arg!( "State id to be formatted").value_parser(clap::value_parser!(u64))) + .arg(arg!( "Path to write the image to").value_parser(clap::value_parser!(PathBuf))) + .arg( + arg!(--namespace "Namespace to write xattrs under") + .default_value("trusted") + .value_parser(clap::value_parser!(XattrNamespace)) + .action(ArgAction::Set), + ), + ) .subcommand(Export::command()) // For profiling only, hence hidden. // @@ -99,6 +111,7 @@ pub fn handle(args: &ArgMatches, installation: Installation) -> Result<(), Error Some(("remove", args)) => remove(args, installation), Some(("verify", args)) => verify(args, installation), Some(("export", args)) => export(args, installation), + Some(("format-erofs", args)) => format_erofs(args, installation), _ => unreachable!(), } } @@ -325,6 +338,53 @@ fn print_state_selections(state: State, client: &Client) -> Result<(), Error> { Ok(()) } +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +enum XattrNamespace { + Trusted, + User, +} + +impl From for erofs::XattrNamespace { + fn from(value: XattrNamespace) -> Self { + match value { + XattrNamespace::Trusted => Self::Trusted, + XattrNamespace::User => Self::User, + } + } +} + +fn format_erofs(args: &ArgMatches, installation: Installation) -> Result<(), Error> { + let id = *args.get_one::("ID").unwrap() as i32; + let image_path = args.get_one::("IMAGE").unwrap(); + let xattr_namespace = args.get_one::("namespace").copied().unwrap().into(); + + let assets_path = installation.assets_path("v2"); + let client = Client::new(environment::NAME, installation)?; + + let state = client.get_state(id.into())?; + let vfs = client.vfs(state.selections.iter().map(|s| &s.package))?; + + let image_writer = erofs::MetaImageWriter::new().with_xattr_namespace(xattr_namespace); + + let mut out = File::create(image_path)?; + + image_writer.write(&vfs, &assets_path, &mut out)?; + + println!("State {id} EROFS meta-only imaged saved to {image_path:?}"); + println!(); + println!("Usage example:"); + println!(); + println!("````"); + println!("sudo mount -t erofs {} ./erofs", image_path.display()); + println!( + "sudo mount -t overlay overlay -o lowerdir=./erofs::{} ./overlay", + assets_path.display() + ); + println!("````"); + + Ok(()) +} + #[derive(Clone, Debug)] struct Format { name: String, diff --git a/moss/src/fstree.rs b/moss/src/fstree.rs index 3e79f8332..7385acc12 100644 --- a/moss/src/fstree.rs +++ b/moss/src/fstree.rs @@ -92,6 +92,12 @@ impl fmt::Display for PendingFile { } } +impl AsRef for PendingFile { + fn as_ref(&self) -> &StonePayloadLayoutRecord { + &self.layout + } +} + /// Build a [`vfs::Tree`] for the specified layouts. /// /// Returns a newly built [`vfs::Tree`] that can be used in