From bf71811521598cd0521c5479cc8cdddae458aac1 Mon Sep 17 00:00:00 2001 From: Hunter Beast Date: Thu, 9 Jul 2026 11:00:21 -0600 Subject: [PATCH 1/6] lean init --- .gitignore | 9 +- .gitmodules | 24 + AGENTS.md | 34 +- Carbonado.lean | 26 + Carbonado/Adamantine.lean | 274 ++++ Carbonado/Bao.lean | 8 + Carbonado/Bao/Blake3.lean | 372 +++++ Carbonado/Bao/Product.lean | 97 ++ Carbonado/Bao/Tree.lean | 335 ++++ Carbonado/Cli.lean | 543 +++++++ Carbonado/Compress.lean | 170 ++ Carbonado/Constants.lean | 184 +++ Carbonado/Crypto.lean | 10 + Carbonado/Crypto/AESCTR.lean | 168 ++ Carbonado/Crypto/EtM.lean | 264 ++++ Carbonado/Crypto/HMAC.lean | 35 + Carbonado/Crypto/SHA512.lean | 134 ++ Carbonado/Crypto/Util.lean | 189 +++ Carbonado/Directory.lean | 651 ++++++++ Carbonado/Fec.lean | 9 + Carbonado/Fec/Galois.lean | 133 ++ Carbonado/Fec/Inboard.lean | 234 +++ Carbonado/Fec/Matrix.lean | 147 ++ Carbonado/Fec/RS.lean | 286 ++++ Carbonado/Ffi.lean | 102 ++ Carbonado/Filepack.lean | 569 +++++++ Carbonado/Header.lean | 197 +++ Carbonado/Main.lean | 1365 +++++++++++++++++ Carbonado/Outboard.lean | 204 +++ Carbonado/Pipeline.lean | 450 ++++++ Carbonado/Scrub.lean | 194 +++ Carbonado/Shard.lean | 196 +++ Carbonado/Slh.lean | 276 ++++ Carbonado/Stream.lean | 107 ++ CarbonadoTest.lean | 27 + CarbonadoTest/Bao.lean | 235 +++ CarbonadoTest/Compress.lean | 98 ++ CarbonadoTest/Directory.lean | 327 ++++ CarbonadoTest/EtM.lean | 305 ++++ CarbonadoTest/Fec.lean | 307 ++++ CarbonadoTest/Pipeline.lean | 410 +++++ CarbonadoTest/Scaffold.lean | 28 + CarbonadoTest/Slh.lean | 99 ++ Cargo.toml | 9 +- carbonado-sys/Cargo.toml | 11 + carbonado-sys/build.rs | 36 + carbonado-sys/src/lib.rs | 94 ++ docs/ABI.md | 196 +++ docs/GAPS.md | 54 + docs/LIMITS.md | 137 ++ docs/PARITY.md | 136 ++ docs/PROOFS.md | 44 + docs/SPEC-MATRIX.md | 22 + docs/TEST_CONTRACT.md | 140 ++ docs/VISION.md | 31 + flake.lock | 117 ++ flake.nix | 332 ++++ include/carbonado.h | 88 ++ lake-manifest.json | 6 + lakefile.toml | 13 + lean-toolchain | 1 + nix/native/carbonado_abi.c | 73 + nix/native/carbonado_zstd.c | 158 ++ nix/native/default.nix | 100 ++ nix/tooling-purity.nix | 96 ++ ref/README.md | 29 + ref/bao-tree | 1 + ref/bitcoinpqc | 1 + ref/blake3 | 1 + ref/crates/README.md | 11 + ref/crates/ctr-0.9.2/.cargo_vcs_info.json | 6 + ref/crates/ctr-0.9.2/CHANGELOG.md | 79 + ref/crates/ctr-0.9.2/Cargo.toml | 68 + ref/crates/ctr-0.9.2/Cargo.toml.orig | 33 + ref/crates/ctr-0.9.2/LICENSE-APACHE | 201 +++ ref/crates/ctr-0.9.2/LICENSE-MIT | 26 + ref/crates/ctr-0.9.2/README.md | 59 + ref/crates/ctr-0.9.2/benches/aes128.rs | 26 + ref/crates/ctr-0.9.2/src/backend.rs | 83 + ref/crates/ctr-0.9.2/src/ctr_core.rs | 156 ++ ref/crates/ctr-0.9.2/src/flavors.rs | 44 + ref/crates/ctr-0.9.2/src/flavors/ctr128.rs | 158 ++ ref/crates/ctr-0.9.2/src/flavors/ctr32.rs | 158 ++ ref/crates/ctr-0.9.2/src/flavors/ctr64.rs | 158 ++ ref/crates/ctr-0.9.2/src/lib.rs | 90 ++ .../tests/ctr128/data/aes128-ctr.blb | Bin 0 -> 2039 bytes .../tests/ctr128/data/aes256-ctr.blb | Bin 0 -> 2055 bytes ref/crates/ctr-0.9.2/tests/ctr128/mod.rs | 12 + ref/crates/ctr-0.9.2/tests/ctr32/be.rs | 86 ++ ref/crates/ctr-0.9.2/tests/ctr32/le.rs | 96 ++ ref/crates/ctr-0.9.2/tests/ctr32/mod.rs | 9 + ref/crates/ctr-0.9.2/tests/gost/mod.rs | 55 + ref/crates/ctr-0.9.2/tests/mod.rs | 5 + ref/parity-harness/README.md | 33 + .../drivers/bao-vectors/Cargo.lock | 340 ++++ .../drivers/bao-vectors/Cargo.toml | 10 + .../drivers/bao-vectors/src/main.rs | 199 +++ .../drivers/etm-vectors/Cargo.lock | 163 ++ .../drivers/etm-vectors/Cargo.toml | 16 + .../drivers/etm-vectors/src/main.rs | 132 ++ .../drivers/rs-vectors/Cargo.lock | 148 ++ .../drivers/rs-vectors/Cargo.toml | 13 + .../drivers/rs-vectors/src/main.rs | 100 ++ ref/reed-solomon-erasure | 1 + ref/rustcrypto-block-ciphers | 1 + ref/rustcrypto-hashes | 1 + ref/rustcrypto-macs | 1 + ref/zstd | 1 + src/backend/mod.rs | 124 ++ src/lib.rs | 3 + 110 files changed, 14659 insertions(+), 4 deletions(-) create mode 100644 .gitmodules create mode 100644 Carbonado.lean create mode 100644 Carbonado/Adamantine.lean create mode 100644 Carbonado/Bao.lean create mode 100644 Carbonado/Bao/Blake3.lean create mode 100644 Carbonado/Bao/Product.lean create mode 100644 Carbonado/Bao/Tree.lean create mode 100644 Carbonado/Cli.lean create mode 100644 Carbonado/Compress.lean create mode 100644 Carbonado/Constants.lean create mode 100644 Carbonado/Crypto.lean create mode 100644 Carbonado/Crypto/AESCTR.lean create mode 100644 Carbonado/Crypto/EtM.lean create mode 100644 Carbonado/Crypto/HMAC.lean create mode 100644 Carbonado/Crypto/SHA512.lean create mode 100644 Carbonado/Crypto/Util.lean create mode 100644 Carbonado/Directory.lean create mode 100644 Carbonado/Fec.lean create mode 100644 Carbonado/Fec/Galois.lean create mode 100644 Carbonado/Fec/Inboard.lean create mode 100644 Carbonado/Fec/Matrix.lean create mode 100644 Carbonado/Fec/RS.lean create mode 100644 Carbonado/Ffi.lean create mode 100644 Carbonado/Filepack.lean create mode 100644 Carbonado/Header.lean create mode 100644 Carbonado/Main.lean create mode 100644 Carbonado/Outboard.lean create mode 100644 Carbonado/Pipeline.lean create mode 100644 Carbonado/Scrub.lean create mode 100644 Carbonado/Shard.lean create mode 100644 Carbonado/Slh.lean create mode 100644 Carbonado/Stream.lean create mode 100644 CarbonadoTest.lean create mode 100644 CarbonadoTest/Bao.lean create mode 100644 CarbonadoTest/Compress.lean create mode 100644 CarbonadoTest/Directory.lean create mode 100644 CarbonadoTest/EtM.lean create mode 100644 CarbonadoTest/Fec.lean create mode 100644 CarbonadoTest/Pipeline.lean create mode 100644 CarbonadoTest/Scaffold.lean create mode 100644 CarbonadoTest/Slh.lean create mode 100644 carbonado-sys/Cargo.toml create mode 100644 carbonado-sys/build.rs create mode 100644 carbonado-sys/src/lib.rs create mode 100644 docs/ABI.md create mode 100644 docs/GAPS.md create mode 100644 docs/LIMITS.md create mode 100644 docs/PARITY.md create mode 100644 docs/PROOFS.md create mode 100644 docs/SPEC-MATRIX.md create mode 100644 docs/TEST_CONTRACT.md create mode 100644 docs/VISION.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 include/carbonado.h create mode 100644 lake-manifest.json create mode 100644 lakefile.toml create mode 100644 lean-toolchain create mode 100644 nix/native/carbonado_abi.c create mode 100644 nix/native/carbonado_zstd.c create mode 100644 nix/native/default.nix create mode 100644 nix/tooling-purity.nix create mode 100644 ref/README.md create mode 160000 ref/bao-tree create mode 160000 ref/bitcoinpqc create mode 160000 ref/blake3 create mode 100644 ref/crates/README.md create mode 100644 ref/crates/ctr-0.9.2/.cargo_vcs_info.json create mode 100644 ref/crates/ctr-0.9.2/CHANGELOG.md create mode 100644 ref/crates/ctr-0.9.2/Cargo.toml create mode 100644 ref/crates/ctr-0.9.2/Cargo.toml.orig create mode 100644 ref/crates/ctr-0.9.2/LICENSE-APACHE create mode 100644 ref/crates/ctr-0.9.2/LICENSE-MIT create mode 100644 ref/crates/ctr-0.9.2/README.md create mode 100644 ref/crates/ctr-0.9.2/benches/aes128.rs create mode 100644 ref/crates/ctr-0.9.2/src/backend.rs create mode 100644 ref/crates/ctr-0.9.2/src/ctr_core.rs create mode 100644 ref/crates/ctr-0.9.2/src/flavors.rs create mode 100644 ref/crates/ctr-0.9.2/src/flavors/ctr128.rs create mode 100644 ref/crates/ctr-0.9.2/src/flavors/ctr32.rs create mode 100644 ref/crates/ctr-0.9.2/src/flavors/ctr64.rs create mode 100644 ref/crates/ctr-0.9.2/src/lib.rs create mode 100644 ref/crates/ctr-0.9.2/tests/ctr128/data/aes128-ctr.blb create mode 100644 ref/crates/ctr-0.9.2/tests/ctr128/data/aes256-ctr.blb create mode 100644 ref/crates/ctr-0.9.2/tests/ctr128/mod.rs create mode 100644 ref/crates/ctr-0.9.2/tests/ctr32/be.rs create mode 100644 ref/crates/ctr-0.9.2/tests/ctr32/le.rs create mode 100644 ref/crates/ctr-0.9.2/tests/ctr32/mod.rs create mode 100644 ref/crates/ctr-0.9.2/tests/gost/mod.rs create mode 100644 ref/crates/ctr-0.9.2/tests/mod.rs create mode 100644 ref/parity-harness/README.md create mode 100644 ref/parity-harness/drivers/bao-vectors/Cargo.lock create mode 100644 ref/parity-harness/drivers/bao-vectors/Cargo.toml create mode 100644 ref/parity-harness/drivers/bao-vectors/src/main.rs create mode 100644 ref/parity-harness/drivers/etm-vectors/Cargo.lock create mode 100644 ref/parity-harness/drivers/etm-vectors/Cargo.toml create mode 100644 ref/parity-harness/drivers/etm-vectors/src/main.rs create mode 100644 ref/parity-harness/drivers/rs-vectors/Cargo.lock create mode 100644 ref/parity-harness/drivers/rs-vectors/Cargo.toml create mode 100644 ref/parity-harness/drivers/rs-vectors/src/main.rs create mode 160000 ref/reed-solomon-erasure create mode 160000 ref/rustcrypto-block-ciphers create mode 160000 ref/rustcrypto-hashes create mode 160000 ref/rustcrypto-macs create mode 160000 ref/zstd create mode 100644 src/backend/mod.rs diff --git a/.gitignore b/.gitignore index 5b43a2e..93f1fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,11 @@ recovered_dir/ /*.adam.c[0-9][0-9].out /*.adam.c[0-9][0-9].par /*.adam.c[0-9].ots -/*.adam.c[0-9][0-9].ots \ No newline at end of file +/*.adam.c[0-9][0-9].ots + +# Nix / Lean build artifacts +/result +/result-* +/.lake +/**/.lake +ref/parity-harness/**/target/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2837049 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,24 @@ +[submodule "ref/bao-tree"] + path = ref/bao-tree + url = https://github.com/SurmountSystems/bao-tree.git +[submodule "ref/reed-solomon-erasure"] + path = ref/reed-solomon-erasure + url = https://github.com/darrenldl/reed-solomon-erasure.git +[submodule "ref/bitcoinpqc"] + path = ref/bitcoinpqc + url = https://github.com/cryptoquick/libbitcoinpqc-bindings.git +[submodule "ref/blake3"] + path = ref/blake3 + url = https://github.com/BLAKE3-team/BLAKE3.git +[submodule "ref/zstd"] + path = ref/zstd + url = https://github.com/facebook/zstd.git +[submodule "ref/rustcrypto-block-ciphers"] + path = ref/rustcrypto-block-ciphers + url = https://github.com/RustCrypto/block-ciphers.git +[submodule "ref/rustcrypto-macs"] + path = ref/rustcrypto-macs + url = https://github.com/RustCrypto/MACs.git +[submodule "ref/rustcrypto-hashes"] + path = ref/rustcrypto-hashes + url = https://github.com/RustCrypto/hashes.git diff --git a/AGENTS.md b/AGENTS.md index 1761fca..fafe029 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,38 @@ # AGENTS.md — Carbonado Development Guidelines **Project:** Carbonado (bitmask-stack/carbonado) -**Mission:** Apocalypse-resistant archival format for consensus-critical data, with a focus on Bitcoin quantum resistance. -**Current Status (as of 2026-07):** Symmetric v2 stack (`CARBONADO20\n`, AES-256-CTR + HMAC-SHA512 EtM) stable. **P1:** `SLICE_LEN=4096`, keyed 4 KiB Bao groups, seekable slice verify. **P2:** streaming-first encode/decode. **P3:** segment sharding. **P4:** Adamantine 1.0 directory archives (see §7.1). FEC: reed-solomon-erasure RS 4/8. Outboard + scrub complete. +**Mission:** Apocalypse-resistant archival format for consensus-critical data, with a focus on Bitcoin quantum resistance. + +## Dual-backend product model (SSOT for parity) + +| Concern | Role | +|---------|------| +| **Rust** (`src/`, default `backend-rust`) | First-class engine; production library + CLI | +| **Rust tests** (`tests/`) | **Normative behavioral contract** — both backends must pass | +| **Lean 4** (`Carbonado/`, AOT `libcarbonado`) | Second engine: proofs + wire/C ABI compatible implementation | +| **Nix flakes** | Build Lean AOT, purity/no-sorry, package `libcarbonado` | +| **`ref/`** | Pinned oracles (bao-tree, RustCrypto, zstd, …) | + +**Parity bar (G8):** `cargo test` with `backend-rust` (default) and `backend-lean` (links Lean AOT C). Same tests; not separate Lean-only demos. See [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md), [docs/ABI.md](docs/ABI.md), [docs/PARITY.md](docs/PARITY.md), [docs/GAPS.md](docs/GAPS.md). + +| Concern | Allowed | +|---------|---------| +| Lean product logic, proofs, AOT | `Carbonado/`, `CarbonadoTest/` (not `Tests/` — collides with Rust `tests/` on Darwin) | +| Rust product + contract tests | `src/`, `tests/` (stay; do not delete for Lean purity) | +| Build Lean / packaging | Nix flakes (`flake.nix`, `nix/`) | +| Oracles / pins | `ref/` | + +**Prove everything:** machine-checked Lean theorems **and** bit-match via the Rust suite on `backend-lean`. + +**Agents:** implement and verify (`nix build`, `nix flake check`, `cargo test`). Do **not** create commits or push; agents do not own git history. **Never regress `backend-rust` `cargo test`.** Do **not** implement Phase 1+ C ABI exports unless that phase is the assigned task. + +**Gates:** `nix flake check` (Lean); `cargo test` (Rust default); dual-backend CI as G8 lands (Phase 0 docs closed; Phase 1+ engineering open — see GAPS P0–P5). + +**C ABI:** normative surface in `include/carbonado.h` + [docs/ABI.md](docs/ABI.md). Phase 0–1 honesty: encode/decode may still return `NOT_IMPLEMENTED` until Phase 1 wiring. + +--- + +**Current Status (as of 2026-07):** Lean 4 + Nix product scaffold started (`Carbonado/Constants`, flake). Historical Rust: Symmetric v2 stack (`CARBONADO20\n`, AES-256-CTR + HMAC-SHA512 EtM) stable. **P1:** `SLICE_LEN=4096`, keyed 4 KiB Bao groups, seekable slice verify. **P2:** streaming-first encode/decode. **P3:** segment sharding. **P4:** Adamantine 1.0 directory archives (see §7.1). FEC: reed-solomon-erasure RS 4/8. Outboard + scrub complete. **Unified streaming stack (three independent axes — do not conflate):** | Axis | Status | diff --git a/Carbonado.lean b/Carbonado.lean new file mode 100644 index 0000000..1dcab5f --- /dev/null +++ b/Carbonado.lean @@ -0,0 +1,26 @@ +/- + Carbonado — apocalypse-resistant archival format. + + Product implementation and proofs: Lean 4. + Build and packaging: Nix flakes. +-/ +import Carbonado.Constants +import Carbonado.Crypto +import Carbonado.Fec +import Carbonado.Bao +import Carbonado.Header +import Carbonado.Compress +import Carbonado.Slh +import Carbonado.Pipeline +import Carbonado.Stream +import Carbonado.Scrub +import Carbonado.Shard +import Carbonado.Adamantine +import Carbonado.Filepack +import Carbonado.Outboard +import Carbonado.Directory +import Carbonado.Cli +import Carbonado.Ffi + +/-- Library root namespace. -/ +def Carbonado.versionString : String := "lean-dual-backend-0" diff --git a/Carbonado/Adamantine.lean b/Carbonado/Adamantine.lean new file mode 100644 index 0000000..a6c260c --- /dev/null +++ b/Carbonado/Adamantine.lean @@ -0,0 +1,274 @@ +/- + Adamantine 1.0 directory catalog wire envelope (Program G). + + Normative layout (AGENTS §7.1 / Rust `src/adamantine.rs`): + ``` + Offset Size Field + 0 13 magic ADAMANTINE10\n + 13 1 carbonado_fmt 0x0E | 0x0F + 14 1 flags u8 (bit0 REQUIRE_OTS; bits 1–7 must be 0) + 15 4 payload_len u32 LE + 19 N payload manifest + Bao bundle (see Filepack / payload) + ``` + + Payload framing (Rust `adamantine_payload`): + ``` + [u32 LE manifest_len][manifest bytes][u32 LE bundle_len][bundle bytes] + ``` + + Dev magics `ADAMANTINE1\n` / `ADAMANTINE2\n` are rejected with distinct errors. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util + +namespace Carbonado.Adamantine + +open Carbonado.Constants +open Carbonado.Crypto.Util + +/-- Adamantine 1.0 magic: `ADAMANTINE10\n` (13 bytes). -/ +def adamantineMagic : List UInt8 := + [0x41, 0x44, 0x41, 0x4d, 0x41, 0x4e, 0x54, 0x49, 0x4e, 0x45, 0x31, 0x30, 0x0a] + +theorem adamantineMagic_length : adamantineMagic.length = 13 := by native_decide + +theorem adamantineMagic_eq_literal : + adamantineMagic = + [0x41, 0x44, 0x41, 0x4d, 0x41, 0x4e, 0x54, 0x49, 0x4e, 0x45, 0x31, 0x30, 0x0a] := by + rfl + +/-- Total Adamantine header length. -/ +def adamantineHeaderLen : Nat := 19 + +theorem adamantineHeaderLen_eq : adamantineHeaderLen = 19 := by native_decide + +/-- Catalog carbonado_fmt public (c14). -/ +def adamantineFmtPublic : UInt8 := 0x0E + +/-- Catalog carbonado_fmt encrypted (c15). -/ +def adamantineFmtEncrypted : UInt8 := 0x0F + +/-- Flag bit 0: REQUIRE_OTS (per-entry proofs required at decode). -/ +def adamantineFlagRequireOts : UInt8 := 1 + +/-- Allowed flags mask (only bit 0). -/ +def adamantineFlagsMask : UInt8 := adamantineFlagRequireOts + +/-- Max rkyv / Lean-native manifest payload (16 MiB). -/ +def maxManifestPayloadLen : Nat := 16 * 1024 * 1024 + +/-- Max Bao bundle (256 MiB). -/ +def maxBaoBundleLen : Nat := 256 * 1024 * 1024 + +/-- Max total Adamantine payload. -/ +def maxAdamantinePayloadLen : Nat := + maxManifestPayloadLen + 4 + maxBaoBundleLen + +/-- Strict Adamantine error taxonomy (exact-match in tests). -/ +inductive AdamantineError where + /-- Input shorter than 19-byte header, or length fields inconsistent. -/ + | invalidHeader + /-- Magic is not a recognized Adamantine form. -/ + | invalidMagic + /-- Legacy/dev magic `ADAMANTINE1\n` or unsupported major.minor. -/ + | unsupportedVersion (major minor : UInt8) + /-- carbonado_fmt not c14/c15. -/ + | invalidCarbonadoFormat (fmt : UInt8) + /-- Reserved flag bits set. -/ + | invalidFlags (flags : UInt8) + /-- Declared payload larger than DoS cap. -/ + | payloadTooLarge (declared max : Nat) + /-- Declared payload length exceeds available bytes. -/ + | payloadLengthMismatch (expected available : Nat) + deriving DecidableEq, Repr + +/-- Parsed Adamantine 1.0 header (excluding payload). -/ +structure AdamantineHeader where + carbonadoFmt : UInt8 + flags : UInt8 + deriving DecidableEq, Repr, Inhabited + +/-- Magic as ByteArray. -/ +def adamantineMagicBA : ByteArray := ofList adamantineMagic + +/-- Legacy v1 magic `ADAMANTINE1\n` (12 bytes). -/ +def adamantineMagicV1 : ByteArray := + ofList [0x41, 0x44, 0x41, 0x4d, 0x41, 0x4e, 0x54, 0x49, 0x4e, 0x45, 0x31, 0x0a] + +/-- Dev v2 magic `ADAMANTINE2\n` (12 bytes). -/ +def adamantineMagicDevV2 : ByteArray := + ofList [0x41, 0x44, 0x41, 0x4d, 0x41, 0x4e, 0x54, 0x49, 0x4e, 0x45, 0x32, 0x0a] + +/-- Validate catalog carbonado_fmt is c14 or c15. -/ +def validateCarbonadoFmt (fmt : UInt8) : Except AdamantineError Unit := + if fmt == adamantineFmtPublic || fmt == adamantineFmtEncrypted then + .ok () + else + .error (.invalidCarbonadoFormat fmt) + +/-- Validate flags: only bit 0 allowed. -/ +def validateFlags (flags : UInt8) : Except AdamantineError Unit := + if flags &&& (~~~adamantineFlagsMask) != 0 then + .error (.invalidFlags flags) + else + .ok () + +/-- + Parse unsupported `ADAMANTINE{d}{d?}\n` version digits. + Returns `none` when the prefix is not Adamantine-shaped. +-/ +def parseUnsupportedMagicVersion (magic : ByteArray) : Option (UInt8 × UInt8) := + if magic.size < 12 then + none + else if !ctEq (magic.extract 0 10) + (ofList [0x41, 0x44, 0x41, 0x4d, 0x41, 0x4e, 0x54, 0x49, 0x4e, 0x45]) then + none + else + let b10 := magic.get! 10 + let b11 := if magic.size > 11 then magic.get! 11 else 0x0a + let isDigit (b : UInt8) : Bool := b ≥ 0x30 && b ≤ 0x39 + if b11 == 0x0a then + if isDigit b10 then some (b10 - 0x30, 0) else none + else if isDigit b11 && magic.size > 12 && magic.get! 12 == 0x0a && isDigit b10 then + some (b10 - 0x30, b11 - 0x30) + else + none + +/-- Prepend Adamantine 1.0 header to a payload (no size cap check — use `buildPayload` first). -/ +def encodeAdamantine (payload : ByteArray) (carbonadoFmt flags : UInt8) : ByteArray := + Id.run do + let mut out := ByteArray.empty + out := appendBA out adamantineMagicBA + out := out.push carbonadoFmt + out := out.push flags + out := appendBA out (putUInt32LE (UInt32.ofNat payload.size)) + out := appendBA out payload + pure out + +/-- Strip and validate Adamantine 1.0 header; exact payload length required (no trailer). -/ +def decodeAdamantine (bytes : ByteArray) : + Except AdamantineError (ByteArray × AdamantineHeader) := + if bytes.size < adamantineHeaderLen then + .error .invalidHeader + else + let magic13 := bytes.extract 0 13 + if ctEq magic13 adamantineMagicBA then + let carbonadoFmt := bytes.get! 13 + let flags := bytes.get! 14 + match validateCarbonadoFmt carbonadoFmt, validateFlags flags with + | .error e, _ => .error e + | _, .error e => .error e + | .ok (), .ok () => + let payloadLen := UInt32.toNat (getUInt32LE bytes 15) + if payloadLen > maxAdamantinePayloadLen then + .error (.payloadTooLarge payloadLen maxAdamantinePayloadLen) + else + let payloadEnd := adamantineHeaderLen + payloadLen + if bytes.size < payloadEnd then + .error (.payloadLengthMismatch payloadLen (bytes.size - adamantineHeaderLen)) + else if bytes.size != payloadEnd then + .error .invalidHeader + else + .ok (bytes.extract adamantineHeaderLen payloadEnd, + { carbonadoFmt := carbonadoFmt, flags := flags }) + else if bytes.size ≥ 12 && ctEq (bytes.extract 0 12) adamantineMagicV1 then + .error (.unsupportedVersion 1 0) + else if bytes.size ≥ 12 && ctEq (bytes.extract 0 12) adamantineMagicDevV2 then + .error (.unsupportedVersion 2 0) + else + match parseUnsupportedMagicVersion magic13 with + | some (maj, min) => .error (.unsupportedVersion maj min) + | none => .error .invalidMagic + +/-- Build payload: `[u32 LE man_len][man][u32 LE bun_len][bun]`. -/ +def buildPayload (manifest bundle : ByteArray) : Except AdamantineError ByteArray := + if manifest.size > maxManifestPayloadLen then + .error (.payloadTooLarge manifest.size maxManifestPayloadLen) + else if bundle.size > maxBaoBundleLen then + .error (.payloadTooLarge bundle.size maxBaoBundleLen) + else + let total := 4 + manifest.size + 4 + bundle.size + if total > maxAdamantinePayloadLen then + .error (.payloadTooLarge total maxAdamantinePayloadLen) + else + Id.run do + let mut out := ByteArray.empty + out := appendBA out (putUInt32LE (UInt32.ofNat manifest.size)) + out := appendBA out manifest + out := appendBA out (putUInt32LE (UInt32.ofNat bundle.size)) + out := appendBA out bundle + pure (.ok out) + +/-- Split payload into (manifest, bundle); exact length required. -/ +def splitPayload (payload : ByteArray) : Except AdamantineError (ByteArray × ByteArray) := + if payload.size > maxAdamantinePayloadLen then + .error (.payloadTooLarge payload.size maxAdamantinePayloadLen) + else if payload.size < 8 then + .error (.payloadLengthMismatch 8 payload.size) + else + let manLen := UInt32.toNat (getUInt32LE payload 0) + if manLen > maxManifestPayloadLen then + .error (.payloadTooLarge manLen maxManifestPayloadLen) + else + let bundleLenOff := 4 + manLen + if payload.size < bundleLenOff + 4 then + .error (.payloadLengthMismatch (bundleLenOff + 4) payload.size) + else + let bunLen := UInt32.toNat (getUInt32LE payload bundleLenOff) + if bunLen > maxBaoBundleLen then + .error (.payloadTooLarge bunLen maxBaoBundleLen) + else + let bunStart := bundleLenOff + 4 + let bunEnd := bunStart + bunLen + if bunEnd > payload.size then + .error (.payloadLengthMismatch bunEnd payload.size) + else if payload.size != bunEnd then + .error (.payloadLengthMismatch bunEnd payload.size) + else + .ok (payload.extract 4 bundleLenOff, payload.extract bunStart bunEnd) + +/-- Slice bundle at offset/len (fail-closed on OOB). -/ +def bundleSlice (bundle : ByteArray) (offset len : Nat) : Except AdamantineError ByteArray := + if len == 0 then + .ok ByteArray.empty + else if offset + len > bundle.size then + .error (.payloadLengthMismatch (offset + len) bundle.size) + else + .ok (bundle.extract offset (offset + len)) + +/-- Header encode/decode roundtrip identity (empty payload). -/ +theorem encode_decode_empty_public : + (match decodeAdamantine (encodeAdamantine ByteArray.empty adamantineFmtPublic 0) with + | .ok (p, h) => p.size == 0 && h.carbonadoFmt == adamantineFmtPublic && h.flags == 0 + | .error _ => false) = true := by + native_decide + +/-- Invalid flags (reserved bits) rejected. -/ +theorem invalid_flags_bit1 : + (match decodeAdamantine (encodeAdamantine ByteArray.empty adamantineFmtPublic 2) with + | .error (.invalidFlags 2) => true + | _ => false) = true := by + native_decide + +/-- Bad carbonado_fmt rejected. -/ +theorem invalid_fmt_c0 : + (match decodeAdamantine (encodeAdamantine ByteArray.empty 0 0) with + | .error (.invalidCarbonadoFormat 0) => true + | _ => false) = true := by + native_decide + +/-- Short buffer → invalidHeader. -/ +theorem short_header : + (match decodeAdamantine (ofList [1, 2, 3]) with + | .error .invalidHeader => true + | _ => false) = true := by + native_decide + +/-- Dev v2 magic → unsupportedVersion 2 0. -/ +theorem dev_v2_rejected : + (match decodeAdamantine (appendBA adamantineMagicDevV2 (replicate 7 0)) with + | .error (.unsupportedVersion 2 0) => true + | _ => false) = true := by + native_decide + +end Carbonado.Adamantine diff --git a/Carbonado/Bao.lean b/Carbonado/Bao.lean new file mode 100644 index 0000000..364f3d4 --- /dev/null +++ b/Carbonado/Bao.lean @@ -0,0 +1,8 @@ +/- + Carbonado keyed Bao surface (Program D). + + Modules: Blake3, Tree (geometry + encode/decode), Product (format keying). +-/ +import Carbonado.Bao.Blake3 +import Carbonado.Bao.Tree +import Carbonado.Bao.Product diff --git a/Carbonado/Bao/Blake3.lean b/Carbonado/Bao/Blake3.lean new file mode 100644 index 0000000..42ac192 --- /dev/null +++ b/Carbonado/Bao/Blake3.lean @@ -0,0 +1,372 @@ +/- + BLAKE3 (reference algorithm) — pure Lean. + + Parity target: `ref/blake3` 1.8.5 (portable / reference semantics). + Used for keyed Bao: `hash`, `keyed_hash`, `derive_key`, hazmat subtree/parent CVs. +-/ +import Carbonado.Crypto.Util + +namespace Carbonado.Bao.Blake3 + +open Carbonado.Crypto.Util + +def outLen : Nat := 32 +def keyLen : Nat := 32 +def blockLen : Nat := 64 +def chunkLen : Nat := 1024 + +private def CHUNK_START : UInt32 := (1 : UInt32) <<< 0 +private def CHUNK_END : UInt32 := (1 : UInt32) <<< 1 +private def PARENT : UInt32 := (1 : UInt32) <<< 2 +private def ROOT : UInt32 := (1 : UInt32) <<< 3 +private def KEYED_HASH : UInt32 := (1 : UInt32) <<< 4 +private def DERIVE_KEY_CONTEXT : UInt32 := (1 : UInt32) <<< 5 +private def DERIVE_KEY_MATERIAL : UInt32 := (1 : UInt32) <<< 6 + +private def IV : Array UInt32 := #[ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 +] + +private def MSG_PERM : Array Nat := #[ + 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8 +] + +/-- 8-word chaining value. -/ +abbrev CV := Array UInt32 + +private def rotr32 (x : UInt32) (n : Nat) : UInt32 := + let n32 : UInt32 := UInt32.ofNat n + let m32 : UInt32 := UInt32.ofNat (32 - n) + (x >>> n32) ||| (x <<< m32) + +private def g (state : Array UInt32) (a b c d : Nat) (mx my : UInt32) : Array UInt32 := + Id.run do + let mut s := state + s := s.set! a (s[a]! + s[b]! + mx) + s := s.set! d (rotr32 (s[d]! ^^^ s[a]!) 16) + s := s.set! c (s[c]! + s[d]!) + s := s.set! b (rotr32 (s[b]! ^^^ s[c]!) 12) + s := s.set! a (s[a]! + s[b]! + my) + s := s.set! d (rotr32 (s[d]! ^^^ s[a]!) 8) + s := s.set! c (s[c]! + s[d]!) + s := s.set! b (rotr32 (s[b]! ^^^ s[c]!) 7) + pure s + +private def round (state : Array UInt32) (m : Array UInt32) : Array UInt32 := + Id.run do + let mut s := state + s := g s 0 4 8 12 m[0]! m[1]! + s := g s 1 5 9 13 m[2]! m[3]! + s := g s 2 6 10 14 m[4]! m[5]! + s := g s 3 7 11 15 m[6]! m[7]! + s := g s 0 5 10 15 m[8]! m[9]! + s := g s 1 6 11 12 m[10]! m[11]! + s := g s 2 7 8 13 m[12]! m[13]! + s := g s 3 4 9 14 m[14]! m[15]! + pure s + +private def permute (m : Array UInt32) : Array UInt32 := + Id.run do + let mut out : Array UInt32 := Array.replicate 16 0 + for i in [:16] do + out := out.set! i m[MSG_PERM[i]!]! + pure out + +private def getUInt32LE (bs : ByteArray) (off : Nat) : UInt32 := + let b0 := (bs.get! off).toUInt32 + let b1 := (bs.get! (off + 1)).toUInt32 + let b2 := (bs.get! (off + 2)).toUInt32 + let b3 := (bs.get! (off + 3)).toUInt32 + b0 ||| (b1 <<< 8) ||| (b2 <<< 16) ||| (b3 <<< 24) + +private def putUInt32LE (x : UInt32) : ByteArray := + Id.run do + let mut out := ByteArray.empty + out := out.push (UInt8.ofNat (UInt32.toNat x % 256)) + out := out.push (UInt8.ofNat (UInt32.toNat (x >>> 8) % 256)) + out := out.push (UInt8.ofNat (UInt32.toNat (x >>> 16) % 256)) + out := out.push (UInt8.ofNat (UInt32.toNat (x >>> 24) % 256)) + pure out + +private def wordsFromLE (bs : ByteArray) (nWords : Nat) : Array UInt32 := + Id.run do + let mut w : Array UInt32 := Array.replicate nWords 0 + for i in [:nWords] do + w := w.set! i (getUInt32LE bs (i * 4)) + pure w + +private def compress (cv : CV) (blockWords : Array UInt32) (counter : UInt64) + (blockLen' : UInt32) (flags : UInt32) : Array UInt32 := + Id.run do + let counterLow : UInt32 := UInt32.ofNat (UInt64.toNat (counter &&& 0xffffffff)) + let counterHigh : UInt32 := UInt32.ofNat (UInt64.toNat (counter >>> 32)) + let mut state : Array UInt32 := #[ + cv[0]!, cv[1]!, cv[2]!, cv[3]!, + cv[4]!, cv[5]!, cv[6]!, cv[7]!, + IV[0]!, IV[1]!, IV[2]!, IV[3]!, + counterLow, counterHigh, blockLen', flags + ] + let mut block := blockWords + for _ in [:7] do + state := round state block + block := permute block + for i in [:8] do + state := state.set! i (state[i]! ^^^ state[i + 8]!) + state := state.set! (i + 8) (state[i + 8]! ^^^ cv[i]!) + pure state + +private def first8 (full : Array UInt32) : CV := + #[full[0]!, full[1]!, full[2]!, full[3]!, full[4]!, full[5]!, full[6]!, full[7]!] + +/-- Output just prior to choosing CV vs root bytes. -/ +private structure Output where + inputCV : CV + blockWords : Array UInt32 + counter : UInt64 + blockLen' : UInt32 + flags : UInt32 + deriving Repr + +private def Output.chainingValue (o : Output) : CV := + first8 (compress o.inputCV o.blockWords o.counter o.blockLen' o.flags) + +private def Output.rootOutputBytes (o : Output) (outLen' : Nat) : ByteArray := + Id.run do + let mut out := ByteArray.empty + let mut blockCounter : UInt64 := 0 + while out.size < outLen' do + let words := compress o.inputCV o.blockWords blockCounter o.blockLen' (o.flags ||| ROOT) + for wi in [:16] do + if out.size ≥ outLen' then break + let wordBytes := putUInt32LE words[wi]! + for bi in [:4] do + if out.size < outLen' then + out := out.push (wordBytes.get! bi) + blockCounter := blockCounter + 1 + pure out + +private structure ChunkState where + chainingValue : CV + chunkCounter : UInt64 + block : ByteArray + blockLen' : Nat + blocksCompressed : Nat + flags : UInt32 + +private def ChunkState.new (keyWords : CV) (chunkCounter : UInt64) (flags : UInt32) : ChunkState := + { chainingValue := keyWords + chunkCounter := chunkCounter + block := replicate blockLen 0 + blockLen' := 0 + blocksCompressed := 0 + flags := flags } + +private def ChunkState.len (cs : ChunkState) : Nat := + blockLen * cs.blocksCompressed + cs.blockLen' + +private def ChunkState.startFlag (cs : ChunkState) : UInt32 := + if cs.blocksCompressed == 0 then CHUNK_START else 0 + +private def ChunkState.update (cs : ChunkState) (input : ByteArray) : ChunkState := + Id.run do + let mut cs := cs + let mut off : Nat := 0 + while off < input.size do + if cs.blockLen' == blockLen then + let blockWords := wordsFromLE cs.block 16 + cs := { cs with + chainingValue := first8 (compress cs.chainingValue blockWords cs.chunkCounter + (UInt32.ofNat blockLen) (cs.flags ||| cs.startFlag)) + blocksCompressed := cs.blocksCompressed + 1 + block := replicate blockLen 0 + blockLen' := 0 } + let want := blockLen - cs.blockLen' + let take := min want (input.size - off) + for i in [:take] do + cs := { cs with block := cs.block.set! (cs.blockLen' + i) (input.get! (off + i)) } + cs := { cs with blockLen' := cs.blockLen' + take } + off := off + take + pure cs + +private def ChunkState.output (cs : ChunkState) : Output := + let blockWords := wordsFromLE cs.block 16 + { inputCV := cs.chainingValue + blockWords := blockWords + counter := cs.chunkCounter + blockLen' := UInt32.ofNat cs.blockLen' + flags := cs.flags ||| cs.startFlag ||| CHUNK_END } + +private def parentOutput (left right keyWords : CV) (flags : UInt32) : Output := + Id.run do + let mut blockWords : Array UInt32 := Array.replicate 16 0 + for i in [:8] do + blockWords := blockWords.set! i left[i]! + blockWords := blockWords.set! (i + 8) right[i]! + pure { + inputCV := keyWords + blockWords := blockWords + counter := 0 + blockLen' := UInt32.ofNat blockLen + flags := PARENT ||| flags + } + +private def parentCV (left right keyWords : CV) (flags : UInt32) : CV := + (parentOutput left right keyWords flags).chainingValue + +/-- Incremental hasher (reference §5.1). -/ +structure Hasher where + chunkState : ChunkState + keyWords : CV + cvStack : Array CV + cvStackLen : Nat + flags : UInt32 + +private def Hasher.newInternal (keyWords : CV) (flags : UInt32) : Hasher := + { chunkState := ChunkState.new keyWords 0 flags + keyWords := keyWords + cvStack := Array.replicate 54 (Array.replicate 8 (0 : UInt32)) + cvStackLen := 0 + flags := flags } + +def Hasher.new : Hasher := Hasher.newInternal IV 0 + +def Hasher.newKeyed (key : ByteArray) : Hasher := + let keyWords := wordsFromLE (resize key keyLen) 8 + Hasher.newInternal keyWords KEYED_HASH + +private def Hasher.pushStack (h : Hasher) (cv : CV) : Hasher := + { h with + cvStack := h.cvStack.set! h.cvStackLen cv + cvStackLen := h.cvStackLen + 1 } + +private def Hasher.popStack (h : Hasher) : Hasher × CV := + let len := h.cvStackLen - 1 + ({ h with cvStackLen := len }, h.cvStack[len]!) + +private def Hasher.addChunkCV (h : Hasher) (newCV : CV) (totalChunks : UInt64) : Hasher := + Id.run do + let mut h := h + let mut newCV := newCV + let mut total := totalChunks + while (total &&& 1) == 0 do + let (h', left) := h.popStack + h := h' + newCV := parentCV left newCV h.keyWords h.flags + total := total >>> 1 + pure (h.pushStack newCV) + +/-- Bytes accepted so far by this hasher. -/ +def Hasher.count (h : Hasher) : Nat := + (UInt64.toNat h.chunkState.chunkCounter) * chunkLen + h.chunkState.len + +/-- Set input byte offset (must be multiple of 1024; hasher must be empty). -/ +def Hasher.setInputOffset (h : Hasher) (offset : UInt64) : Hasher := + let counter := offset / UInt64.ofNat chunkLen + { h with chunkState := ChunkState.new h.keyWords counter h.flags } + +def Hasher.update (h : Hasher) (input : ByteArray) : Hasher := + Id.run do + let mut h := h + let mut off : Nat := 0 + while off < input.size do + if h.chunkState.len == chunkLen then + let chunkCV := h.chunkState.output.chainingValue + let totalChunks := h.chunkState.chunkCounter + 1 + h := h.addChunkCV chunkCV totalChunks + h := { h with chunkState := ChunkState.new h.keyWords totalChunks h.flags } + let want := chunkLen - h.chunkState.len + let take := min want (input.size - off) + h := { h with chunkState := h.chunkState.update (input.extract off (off + take)) } + off := off + take + pure h + +def Hasher.finalize (h : Hasher) : ByteArray := + Id.run do + let mut output := h.chunkState.output + let mut remaining := h.cvStackLen + while remaining > 0 do + remaining := remaining - 1 + output := parentOutput h.cvStack[remaining]! output.chainingValue h.keyWords h.flags + pure (output.rootOutputBytes outLen) + +/-- Non-root chaining value of the current subtree (hazmat; for Bao leaves). -/ +def Hasher.finalizeNonRoot (h : Hasher) : CV := + Id.run do + let mut output := h.chunkState.output + let mut remaining := h.cvStackLen + while remaining > 0 do + remaining := remaining - 1 + output := parentOutput h.cvStack[remaining]! output.chainingValue h.keyWords h.flags + pure output.chainingValue + +def Hasher.newDeriveKey (context : String) : Hasher := + Id.run do + let mut ctxH := Hasher.newInternal IV DERIVE_KEY_CONTEXT + ctxH := ctxH.update (utf8 context) + let contextKey := ctxH.finalize + let contextKeyWords := wordsFromLE contextKey 8 + pure (Hasher.newInternal contextKeyWords DERIVE_KEY_MATERIAL) + +/-- CV → 32 little-endian bytes (as blake3::Hash). -/ +def cvToBytes (cv : CV) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:8] do + out := appendBA out (putUInt32LE cv[i]!) + pure out + +def bytesToCV (bs : ByteArray) : CV := + wordsFromLE (resize bs outLen) 8 + +/-- Standard BLAKE3 hash. -/ +def hash (data : ByteArray) : ByteArray := + (Hasher.new.update data).finalize + +/-- BLAKE3 keyed hash (32-byte key). -/ +def keyedHash (key data : ByteArray) : ByteArray := + ((Hasher.newKeyed key).update data).finalize + +/-- BLAKE3 `derive_key(context, key_material)` → 32 bytes. -/ +def deriveKey (context : String) (keyMaterial : ByteArray) : ByteArray := + ((Hasher.newDeriveKey context).update keyMaterial).finalize + +/-- Hash a Bao/BLAKE3 subtree (keyed mode). + + * `isRoot = true` → `keyed_hash(key, data)` (must be start_chunk 0). + * else → non-root CV at `start_chunk * 1024` input offset. +-/ +def keyedHashSubtree (startChunk : Nat) (data : ByteArray) (isRoot : Bool) + (key : ByteArray) : ByteArray := + if isRoot then + keyedHash key data + else + let h0 := Hasher.newKeyed key + let h1 := h0.setInputOffset (UInt64.ofNat (startChunk * chunkLen)) + let h2 := h1.update data + cvToBytes h2.finalizeNonRoot + +/-- Merge two child CVs (keyed mode). -/ +def keyedParentCV (left right : ByteArray) (isRoot : Bool) (key : ByteArray) : ByteArray := + let keyWords := wordsFromLE (resize key keyLen) 8 + let leftCV := bytesToCV left + let rightCV := bytesToCV right + let o := parentOutput leftCV rightCV keyWords KEYED_HASH + if isRoot then + o.rootOutputBytes outLen + else + cvToBytes o.chainingValue + +/-- Empty-string hash golden (official). -/ +theorem hash_empty : + toHex (hash ByteArray.empty) = + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" := by + native_decide + +/-- Official `abc` hash golden. -/ +theorem hash_abc : + toHex (hash (utf8 "abc")) = + "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85" := by + native_decide + +end Carbonado.Bao.Blake3 diff --git a/Carbonado/Bao/Product.lean b/Carbonado/Bao/Product.lean new file mode 100644 index 0000000..8c1b6a5 --- /dev/null +++ b/Carbonado/Bao/Product.lean @@ -0,0 +1,97 @@ +/- + Carbonado product-facing keyed Bao API. + + Format-byte verification key: + blake3::derive_key("carbonado-v2/verification", &[format]) + (see AGENTS.md / Rust `crypto::carbonado_verification_key`). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Bao.Blake3 +import Carbonado.Bao.Tree + +namespace Carbonado.Bao.Product + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Bao.Blake3 +open Carbonado.Bao.Tree + +/-- Derive the 32-byte keyed-Bao BLAKE3 key for format level `format` (c0–c15). -/ +def carbonadoVerificationKey (format : UInt8) : ByteArray := + deriveKey verificationContext (ofList [format]) + +/-- Keyed Bao root for logical `data` under Carbonado format byte. -/ +def rootForFormat (format : UInt8) (data : ByteArray) : ByteArray := + keyedRoot (carbonadoVerificationKey format) data + +/-- Encode inboard `[u64le|response]` for format; returns `(root, artifact)`. -/ +def encodeInboardForFormat (format : UInt8) (data : ByteArray) : ByteArray × ByteArray := + encodeInboard (carbonadoVerificationKey format) data + +/-- Decode/verify inboard under format + expected root. -/ +def decodeInboardForFormat (format : UInt8) (root input : ByteArray) : + Except BaoError ByteArray := + decodeInboard (carbonadoVerificationKey format) root input + +/-- Verify inboard only. -/ +def verifyInboardForFormat (format : UInt8) (root input : ByteArray) : + Except BaoError Unit := + verifyInboard (carbonadoVerificationKey format) root input + +/-- Post-order outboard `(root, sidecar)` for format. -/ +def encodeOutboardForFormat (format : UInt8) (data : ByteArray) : ByteArray × ByteArray := + createOutboard (carbonadoVerificationKey format) data + +/-- Verify bare main + outboard sidecar under format. -/ +def verifyOutboardForFormat (format : UInt8) (root bare outboard : ByteArray) : + Except BaoError Unit := + verifyOutboard (carbonadoVerificationKey format) root bare outboard + +/-- Encode slice response for format. -/ +def encodeSliceForFormat (format : UInt8) (data : ByteArray) (index count : Nat) : + ByteArray × ByteArray := + encodeSliceResponse (carbonadoVerificationKey format) data index count + +/-- Stream-authenticate slice response under format (no full plaintext required). + + `contentLen` is the logical file size (from header / out-of-band), not derived from + the untrusted response body. +-/ +def decodeSliceForFormat (format : UInt8) (root : ByteArray) (contentLen index count : Nat) + (response : ByteArray) : Except BaoError ByteArray := + decodeSliceResponse (carbonadoVerificationKey format) root contentLen index count response + +/-- Extract/verify slice from full inboard under format (auth-first). -/ +def verifySliceInboardForFormat (format : UInt8) (root input : ByteArray) + (index count : Nat) : Except BaoError ByteArray := + verifySliceInboard (carbonadoVerificationKey format) root input index count + +/-- Different format bytes yield different verification keys. -/ +theorem verification_key_format_domain : + toHex (carbonadoVerificationKey 4) ≠ toHex (carbonadoVerificationKey 6) := by + native_decide + +/-- Root commits to format: same data, different format → different root. -/ +theorem root_commits_to_format : + let data := ofList [0, 1, 2, 3, 4] + toHex (rootForFormat 4 data) ≠ toHex (rootForFormat 6 data) := by + native_decide + +/-- Empty inboard roundtrip under c4. -/ +theorem encode_decode_empty_c4 : + (match encodeInboardForFormat 4 ByteArray.empty with + | (root, art) => + match decodeInboardForFormat 4 root art with + | .ok d => d.size == 0 + | .error _ => false) = true := by + native_decide + +/-- Hello inboard root matches keyed_hash under c4. -/ +theorem hello_root_eq_keyed_hash : + let data := utf8 "hello" + let key := carbonadoVerificationKey 4 + toHex (rootForFormat 4 data) = toHex (keyedHash key data) := by + native_decide + +end Carbonado.Bao.Product diff --git a/Carbonado/Bao/Tree.lean b/Carbonado/Bao/Tree.lean new file mode 100644 index 0000000..25d03c7 --- /dev/null +++ b/Carbonado/Bao/Tree.lean @@ -0,0 +1,335 @@ +/- + Keyed Bao tree (bao-tree 76-keyed-bao semantics) for Carbonado product paths. + + Geometry: BLAKE3 chunk = 1024 B; Carbonado leaf group = 4 chunks = 4096 B + (`BlockSize::from_chunk_log(2)` / `sliceLen`). + + Tree recursion is over **leaf groups** (4 KiB), matching `BaoTree` + `keyed_encode_ranges_*` + with `BAO_BLOCK_SIZE`. Leaf hashes use BLAKE3 keyed subtree hashing over the group bytes. + + Slice verification authenticates untrusted response bytes against `(key, root, contentLen)` + via `decodeRec` (bao-tree `keyed_decode_ranges` analogue) — not re-encode oracles. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Bao.Blake3 + +namespace Carbonado.Bao.Tree + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Bao.Blake3 + +/-- Strict Bao / verification errors (distinct failure modes). -/ +inductive BaoError where + /-- Expected root / leaf / parent hash did not match under the given key. -/ + | authenticationFailed + /-- Response stream ended before required parent pair or leaf bytes. -/ + | truncatedResponse + /-- Response longer than geometry consumed (trailing garbage after a valid decode). -/ + | trailingData + /-- Inboard prefix missing or shorter than 8 bytes. -/ + | invalidPrefix + /-- Root hash argument not 32 bytes. -/ + | invalidRootLength + /-- Slice index past end of content. -/ + | invalidSliceIndex + /-- `count = 0` on verify/decode-slice APIs (empty success would skip auth). -/ + | invalidSliceCount + deriving DecidableEq, Repr + +/-- Carbonado Bao chunk-group log: 2 → 4 KiB leaves. -/ +def blockChunkLog : Nat := baoChunkLog + +/-- Bytes per leaf group (= `sliceLen` = 4096). -/ +def leafBytes : Nat := chunkLen * (2 ^ blockChunkLog) + +theorem leafBytes_eq_sliceLen : leafBytes = sliceLen := by native_decide + +/-- Blake3 chunks covered by one Carbonado slice / leaf group. -/ +def chunksPerSlice : Nat := 2 ^ blockChunkLog + +/-- Smallest power of two ≥ `n` (n ≥ 1). -/ +def nextPow2 (n : Nat) : Nat := + if n ≤ 1 then 1 + else 2 ^ (Nat.log2 (n - 1) + 1) + +/-- Query over leaf-group indices (not BLAKE3 chunks). -/ +inductive LeafQuery where + | all + | range (start end_ : Nat) + deriving DecidableEq, Repr + +def LeafQuery.isEmpty : LeafQuery → Bool + | .all => false + | .range s e => decide (e ≤ s) + +def LeafQuery.isAll : LeafQuery → Bool + | .all => true + | .range _ _ => false + +def LeafQuery.split (q : LeafQuery) (startLeaf midLeaf : Nat) : LeafQuery × LeafQuery := + match q with + | .all => (.all, .all) + | .range s e => + let leftS := max s startLeaf + let leftE := min e midLeaf + let rightS := max s midLeaf + let rightE := e + let left : LeafQuery := + if leftE ≤ leftS then .range 0 0 else .range leftS leftE + let right : LeafQuery := + if rightE ≤ rightS then .range 0 0 else .range rightS rightE + (left, right) + +/-- Number of leaf groups covering `dataLen` bytes. -/ +def leafGroupCount (dataLen : Nat) : Nat := + if dataLen == 0 then 0 + else (dataLen + leafBytes - 1) / leafBytes + +/-- Hash one leaf group (up to 4 KiB) at blake3 start chunk `startChunk`. -/ +def hashLeafGroup (startChunk : Nat) (group : ByteArray) (isRoot : Bool) + (key : ByteArray) : ByteArray := + keyedHashSubtree startChunk group isRoot key + +/-- Recursive keyed encode over 4 KiB leaf groups. + + Returns `(hash, emitted_bytes)`. Parent pairs are pre-order (before children), + matching bao-tree full-range / range responses at `BlockSize` log=2. +-/ +partial def encodeRec (startLeaf : Nat) (data : ByteArray) (isRoot : Bool) + (query : LeafQuery) (emitData : Bool) (key : ByteArray) : ByteArray × ByteArray := + let nLeaves := leafGroupCount data.size + if nLeaves ≤ 1 then + let startChunk := startLeaf * chunksPerSlice + let h := hashLeafGroup startChunk data isRoot key + let emitted := + if emitData && !query.isEmpty then data else ByteArray.empty + (h, emitted) + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let midLeaf := startLeaf + mid + let (lQ, rQ) := query.split startLeaf midLeaf + let leftData := data.extract 0 (min midBytes data.size) + let rightData := data.extract (min midBytes data.size) data.size + let (leftH, leftEm) := encodeRec startLeaf leftData false lQ emitData key + let (rightH, rightEm) := encodeRec midLeaf rightData false rQ emitData key + let parentH := keyedParentCV leftH rightH isRoot key + let emitParent := !query.isEmpty + let emitted := + if emitParent then + appendBA (appendBA leftH rightH) (appendBA leftEm rightEm) + else + appendBA leftEm rightEm + (parentH, emitted) + +/-- Keyed Bao root over full data (≡ `blake3::keyed_hash` / `create_keyed` root). -/ +def keyedRoot (key data : ByteArray) : ByteArray := + keyedHash key data + +/-- Full-range inboard response bytes (no length prefix). -/ +def encodeResponseAll (key data : ByteArray) : ByteArray × ByteArray := + encodeRec 0 data true .all true key + +/-- Post-order outboard sidecar (parent pairs only). -/ +partial def outboardRec (startLeaf : Nat) (data : ByteArray) (isRoot : Bool) + (key : ByteArray) : ByteArray × ByteArray := + let nLeaves := leafGroupCount data.size + if nLeaves ≤ 1 then + let startChunk := startLeaf * chunksPerSlice + (hashLeafGroup startChunk data isRoot key, ByteArray.empty) + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let midLeaf := startLeaf + mid + let leftData := data.extract 0 (min midBytes data.size) + let rightData := data.extract (min midBytes data.size) data.size + let (leftH, leftOb) := outboardRec startLeaf leftData false key + let (rightH, rightOb) := outboardRec midLeaf rightData false key + let parentH := keyedParentCV leftH rightH isRoot key + let pair := appendBA leftH rightH + (parentH, appendBA (appendBA leftOb rightOb) pair) + +/-- Post-order outboard for full data under `key`. -/ +def createOutboard (key data : ByteArray) : ByteArray × ByteArray := + outboardRec 0 data true key + +/-- Inboard artifact: `[u64le content_len | response]`. -/ +def encodeInboard (key data : ByteArray) : ByteArray × ByteArray := + let (root, resp) := encodeResponseAll key data + let lenPrefix := putUInt64LE (UInt64.ofNat data.size) + (root, appendBA lenPrefix resp) + +/-- Parse inboard content-length prefix. -/ +def contentLenPrefix (input : ByteArray) : Except BaoError Nat := + if input.size < 8 then + .error .invalidPrefix + else + .ok (UInt64.toNat (getUInt64LE input 0)) + +/-- Decode full-range / partial response over leaf groups. + + Authenticates included leaves and parent pairs against `expected` (root or CV). + Sibling hashes for unqueried sides are still read from the stream and bound into + the parent CV (bao-tree range decode semantics). +-/ +partial def decodeRec (startLeaf : Nat) (contentLen : Nat) (isRoot : Bool) + (query : LeafQuery) (key : ByteArray) (expected : ByteArray) + (input : ByteArray) (pos : Nat) : Except BaoError (ByteArray × Nat) := do + if query.isEmpty then + -- Empty sub-query: no bytes consumed; caller already authenticated via parent pair. + pure (ByteArray.empty, pos) + else + let nLeaves := leafGroupCount contentLen + if nLeaves ≤ 1 then + if pos + contentLen > input.size then + throw .truncatedResponse + let data := input.extract pos (pos + contentLen) + let startChunk := startLeaf * chunksPerSlice + let h := hashLeafGroup startChunk data isRoot key + if !ctEq h expected then + throw .authenticationFailed + pure (data, pos + contentLen) + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let midLeaf := startLeaf + mid + let leftLen := min midBytes contentLen + let rightLen := contentLen - leftLen + let (lQ, rQ) := query.split startLeaf midLeaf + if pos + 64 > input.size then + throw .truncatedResponse + let leftH := input.extract pos (pos + 32) + let rightH := input.extract (pos + 32) (pos + 64) + let parentH := keyedParentCV leftH rightH isRoot key + if !ctEq parentH expected then + throw .authenticationFailed + let pos := pos + 64 + let (leftData, pos) ← + decodeRec startLeaf leftLen false lQ key leftH input pos + let (rightData, pos) ← + decodeRec midLeaf rightLen false rQ key rightH input pos + pure (appendBA leftData rightData, pos) + +/-- Verify and decode full inboard `[u64le|response]` under `key` and expected root. -/ +def decodeInboard (key root input : ByteArray) : Except BaoError ByteArray := do + if root.size != outLen then + throw .invalidRootLength + let contentLen ← contentLenPrefix input + let response := input.extract 8 input.size + if contentLen == 0 then + let expect := keyedRoot key ByteArray.empty + if !ctEq expect root then + throw .authenticationFailed + if response.size != 0 then + throw .trailingData + pure ByteArray.empty + else + let (data, endPos) ← + decodeRec 0 contentLen true .all key root response 0 + if endPos < response.size then + throw .trailingData + if endPos > response.size then + throw .truncatedResponse + if data.size != contentLen then + throw .authenticationFailed + if !ctEq (keyedRoot key data) root then + throw .authenticationFailed + pure data + +/-- Verify inboard without retaining body. -/ +def verifyInboard (key root input : ByteArray) : Except BaoError Unit := do + let _ ← decodeInboard key root input + pure () + +/-- Verify bare main + post-order outboard against root. -/ +def verifyOutboard (key root bare outboard : ByteArray) : Except BaoError Unit := do + if root.size != outLen then + throw .invalidRootLength + let (gotRoot, gotOb) := createOutboard key bare + if !ctEq gotRoot root then + throw .authenticationFailed + if !ctEq gotOb outboard then + throw .authenticationFailed + pure () + +/-- Slice index/count → leaf-group query. -/ +def sliceLeafQuery (index count : Nat) : LeafQuery := + .range index (index + count) + +/-- Encode keyed slice response for `count` slices starting at `index`. -/ +def encodeSliceResponse (key data : ByteArray) (index count : Nat) : + ByteArray × ByteArray := + encodeRec 0 data true (sliceLeafQuery index count) true key + +/-- Authenticate and decode a standalone slice response (stream verify). + + Does **not** require full plaintext. Uses `decodeRec` against `(key, root, contentLen)`. + Returns only the authenticated slice bytes recovered from `response`. + + * `count = 0` → `invalidSliceCount` (empty success would skip authentication) + * short stream → `truncatedResponse` + * trailing garbage after geometry → `trailingData` + * hash mismatch / wrong key → `authenticationFailed` +-/ +def decodeSliceResponse (key root : ByteArray) (contentLen index count : Nat) + (response : ByteArray) : Except BaoError ByteArray := do + if root.size != outLen then + throw .invalidRootLength + if count == 0 then + throw .invalidSliceCount + if contentLen == 0 then + throw .invalidSliceIndex + let sliceStart := index * leafBytes + if sliceStart ≥ contentLen then + throw .invalidSliceIndex + let (data, endPos) ← + decodeRec 0 contentLen true (sliceLeafQuery index count) key root response 0 + if endPos < response.size then + throw .trailingData + if endPos > response.size then + throw .truncatedResponse + pure data + +/-- Test-oracle helper: re-encode slice and compare (not a product verify path). + + Prefer `decodeSliceResponse` for authentication of untrusted responses. +-/ +def sliceResponseMatchesEncode (key data : ByteArray) (index count : Nat) + (response : ByteArray) : Bool := + let (_root, enc) := encodeSliceResponse key data index count + ctEq enc response + +/-- Decode full inboard (always authenticates), then extract `count` slices at `index`. + + Integrity runs **before** the `count = 0` empty return — corrupt inboard never succeeds. + `count = 0` after successful decode returns empty (extract semantics, not skip-auth). +-/ +def extractSliceFromInboard (key root input : ByteArray) (index count : Nat) : + Except BaoError ByteArray := do + let data ← decodeInboard key root input + if count == 0 then + return ByteArray.empty + let sliceStart := index * leafBytes + if sliceStart ≥ data.size then + throw .invalidSliceIndex + let sliceEnd := min data.size (sliceStart + count * leafBytes) + pure (data.extract sliceStart sliceEnd) + +/-- Strict verify of slice inside full inboard: full decode then extract. + + Same auth-first contract as `extractSliceFromInboard`. +-/ +def verifySliceInboard (key root input : ByteArray) (index count : Nat) : + Except BaoError ByteArray := + extractSliceFromInboard key root input index count + +/-- Root equals keyed_hash (determinism / multi-dimensional naming basis). -/ +theorem root_eq_keyed_hash (key data : ByteArray) : + keyedRoot key data = keyedHash key data := rfl + +end Carbonado.Bao.Tree diff --git a/Carbonado/Cli.lean b/Carbonado/Cli.lean new file mode 100644 index 0000000..433ce4c --- /dev/null +++ b/Carbonado/Cli.lean @@ -0,0 +1,543 @@ +/- + AOT product CLI (Program G). + + Subcommands: + * `demo` / (no args) — full self-test suite (Programs A–G) + * `version` / `help` + * `encode ` — single-file (headered) or directory → archive + * `decode ` — headered file or `.adam.c14`/`.adam.c15` catalog + * `slh parse|verify` — SLH1 wire helpers + + Directory default outdir: `{input}-archive/` (never `.`). + `--encrypted` for c15; `--master <64 hex>`; `--format <0-15>` single-file. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Header +import Carbonado.Pipeline +import Carbonado.Outboard +import Carbonado.Adamantine +import Carbonado.Filepack +import Carbonado.Directory +import Carbonado.Slh +import Carbonado.Bao.Blake3 + +namespace Carbonado.Cli + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Header +open Carbonado.Pipeline +open Carbonado.Outboard +open Carbonado.Adamantine +open Carbonado.Filepack +open Carbonado.Directory +open Carbonado.Slh +open Carbonado.Bao.Blake3 + +/-- CLI error taxonomy (exact; maps lower errors when needed). -/ +inductive CliError where + | usage (msg : String) + | io (msg : String) + | badMasterHex + | zeroMasterEncrypted + | invalidFormat + | directory (e : DirectoryError) + | pipeline (e : PipelineError) + | slh (e : SlhError) + | notFound (path : String) + deriving Repr + +def versionString : String := "lean-program-g-0" + +def helpText : String := + "carbonado — apocalypse-resistant archival format (Lean 4 AOT)\n" ++ + "\n" ++ + "Usage:\n" ++ + " carbonado Run built-in self-test demo\n" ++ + " carbonado demo Same as no-args demo\n" ++ + " carbonado version Print version\n" ++ + " carbonado help Show this help\n" ++ + " carbonado encode [opts]\n" ++ + " Single file → headered {bao_root_hex}.c{fmt:02x}; directory → Adamantine archive\n" ++ + " --format <0-15> single-file format (default 0 public raw)\n" ++ + " --encrypted directory catalog c15 (requires --master)\n" ++ + " --master 32-byte master key as 64 hex chars\n" ++ + " -o output file or directory (default for dir: {input}-archive/)\n" ++ + " carbonado decode [opts]\n" ++ + " Headered archive or {root}.adam.c14/.adam.c15 catalog\n" ++ + " --master master key (required for encrypted)\n" ++ + " -o output file or directory\n" ++ + " carbonado slh parse Validate SLH1 sidecar wire (7860 B)\n" ++ + " carbonado slh verify --root --pk \n" ++ + " Cryptographic bind-to-root (fails closed exit 1 until SLH-DSA FFI — LIMITS)\n" + +/-- Parse 64 hex chars → 32 bytes. -/ +def parseMaster (s : String) : Except CliError ByteArray := + match fromHex? s with + | none => .error .badMasterHex + | some b => + if b.size != 32 then .error .badMasterHex + else .ok b + +/-- All-zero master. -/ +def zeroMaster : ByteArray := replicate 32 0 + +/-- Reject all-zero master on encrypted paths. -/ +def rejectZeroEncrypted (master : ByteArray) (encrypted : Bool) : Except CliError Unit := + if !encrypted then .ok () + else + let allZero := + Id.run do + let mut z := true + for i in [:master.size] do + if master.get! i != 0 then z := false + pure z + if allZero then .error .zeroMasterEncrypted else .ok () + +/-- Read OS CSPRNG bytes (Linux/macOS `/dev/urandom`). -/ +partial def readUrandom (n : Nat) : IO ByteArray := do + if n == 0 then + pure ByteArray.empty + else + let h ← IO.FS.Handle.mk "/dev/urandom" .read + let mut acc := ByteArray.empty + let mut left := n + while left > 0 do + let chunk ← h.read (USize.ofNat (min left 4096)) + if chunk.size == 0 then + throw (IO.userError "urandom: unexpected EOF") + acc := acc.append chunk + left := left - chunk.size + pure acc + +/-- CLI options bag. -/ +structure CliOpts where + format : Option Nat := none + encrypted : Bool := false + master : Option ByteArray := none + output : Option String := none + rootHex : Option String := none + pkHex : Option String := none + deriving Inhabited + +/-- Sequential parse allowing interleaved flags and positionals. -/ +def parseArgsSimple (args : List String) : Except CliError (CliOpts × List String) := + Id.run do + let mut opts : CliOpts := {} + let mut pos : List String := [] + let mut rest := args + let mut err : Option CliError := none + while !rest.isEmpty && err.isNone do + match rest with + | [] => pure () + | "--encrypted" :: r => + opts := { opts with encrypted := true } + rest := r + | "--format" :: v :: r => + match v.toNat? with + | none => err := some (.usage s!"bad --format: {v}") + | some n => + if n > 15 then err := some .invalidFormat + else + opts := { opts with format := some n } + rest := r + | "--master" :: v :: r => + match parseMaster v with + | .error e => err := some e + | .ok m => + opts := { opts with master := some m } + rest := r + | "-o" :: v :: r => + opts := { opts with output := some v } + rest := r + | "--root" :: v :: r => + opts := { opts with rootHex := some v } + rest := r + | "--pk" :: v :: r => + opts := { opts with pkHex := some v } + rest := r + | flag :: r => + if flag.startsWith "-" then + err := some (.usage s!"unknown flag: {flag}") + else + pos := pos ++ [flag] + rest := r + match err with + | some e => pure (.error e) + | none => pure (.ok (opts, pos)) + +/-- Default master from opts (zero if omitted). -/ +def masterOrZero (opts : CliOpts) : ByteArray := + match opts.master with + | some m => m + | none => zeroMaster + +/-- Two-digit lowercase hex for format byte (AGENTS single-file: `.c{fmt:02x}`). -/ +def formatHex2 (fmt : Nat) : String := + toHex (ofList [UInt8.ofNat (fmt % 256)]) + +/-- Default single-file output: `{bao_root_hex}.c{fmt:02x}` (AGENTS §7.1). -/ +def defaultFileOut (baoRoot : ByteArray) (fmt : Nat) : System.FilePath := + System.FilePath.mk s!"{toHex baoRoot}.c{formatHex2 fmt}" + +/-- True if path is a symlink (`test -L`; fail-closed encode/decode policy). -/ +def pathIsSymlink (path : System.FilePath) : IO Bool := do + let r ← IO.Process.output { + cmd := "test" + args := #["-L", path.toString] + } + pure (r.exitCode == 0) + +/-- Encode single file; when `outputPath` is none, write `{hash}.c{fmt:02x}`. -/ +def encodeFile (inputPath : System.FilePath) (outputPath : Option System.FilePath) + (master : ByteArray) (format : FormatBits) : IO (Except CliError Unit) := do + try + let data ← IO.FS.readBinFile inputPath + let enc := format.encrypted + match rejectZeroEncrypted master enc with + | .error e => pure (.error e) + | .ok () => + let nonce ← if enc then readUrandom 16 else pure (replicate 16 0) + match encodeHeadered master nonce data format 0 + (replicate slhPublicKeyLen 0) (replicate 8 0) with + | .error e => pure (.error (.pipeline e)) + | .ok (hdr, archive) => + let out := + match outputPath with + | some p => p + | none => defaultFileOut hdr.hash format.toUInt8.toNat + IO.FS.writeBinFile out archive + IO.println s!"encoded {inputPath} → {out} (format=0x{formatHex2 format.toUInt8.toNat}, root={toHex hdr.hash})" + pure (.ok ()) + catch e => + pure (.error (.io (toString e))) + +/-- Single-file decode (headered). -/ +def decodeFile (inputPath outputPath : System.FilePath) (master : ByteArray) : + IO (Except CliError Unit) := do + try + let data ← IO.FS.readBinFile inputPath + match decodeHeadered master data with + | .error e => pure (.error (.pipeline e)) + | .ok pt => + IO.FS.writeBinFile outputPath pt + IO.println s!"decoded {inputPath} → {outputPath} ({pt.size} bytes)" + pure (.ok ()) + catch e => + pure (.error (.io (toString e))) + +/-- Recursively collect files under `dir` with relative paths (POSIX). Fail-closed. -/ +partial def collectFiles (base : System.FilePath) (rel : String) : + IO (Except DirectoryError (Array DirFile)) := do + let entries ← base.readDir + let mut files : Array DirFile := #[] + for ent in entries do + let name := ent.fileName + if name == "." || name == ".." then + pure () + else if name.any (fun c => c == '/' || c == '\\') then + return .error .pathBackslash + else if name.any (fun c => c == Char.ofNat 0) then + return .error .pathNullByte + else + let path := ent.path + if ← pathIsSymlink path then + return .error .symlinkNotAllowed + let childRel := if rel.isEmpty then name else s!"{rel}/{name}" + let isDir ← path.isDir + if isDir then + match ← collectFiles path childRel with + | .error e => return .error e + | .ok sub => files := files ++ sub + else + match validateRelPath childRel with + | .error e => return .error (ofFilepackError e) + | .ok () => + let data ← IO.FS.readBinFile path + files := files.push { relPath := childRel, content := data } + pure (.ok files) + +/-- Count encrypted segments for nonce preallocation. -/ +def countEncryptedSegments (files : Array DirFile) (opts : DirectoryEncodeOptions) : Nat := + Id.run do + let mut n := 0 + for i in [:files.size] do + let f := files[i]! + match opts.segmentPolicy.resolve opts.catalogEncrypted f.content with + | .error _ => pure () + | .ok fmt => + let bits := FormatBits.ofUInt8 fmt + if bits.encrypted then + let chunks := splitContent f.content opts.segmentPlaintextBudget + n := n + chunks.size + if opts.catalogEncrypted then n := n + 1 + pure n + +/-- Encode directory to outdir. -/ +def encodeDir (inputDir outputDir : System.FilePath) (master : ByteArray) + (encrypted : Bool) : IO (Except CliError Unit) := do + try + let isDir ← inputDir.isDir + if !isDir then + pure (.error (.directory .notADirectory)) + else + match checkMasterPolicy master encrypted with + | .error e => pure (.error (.directory e)) + | .ok () => + match ← collectFiles inputDir "" with + | .error e => pure (.error (.directory e)) + | .ok files => + let opts : DirectoryEncodeOptions := { + catalogEncrypted := encrypted + segmentPolicy := .auto + } + let need := countEncryptedSegments files opts + let mut nonces : Array ByteArray := #[] + for _ in [:need] do + let n ← readUrandom 16 + nonces := nonces.push n + match encodeDirectory master files opts nonces with + | .error e => pure (.error (.directory e)) + | .ok arch => + IO.FS.createDirAll outputDir + IO.FS.writeBinFile (outputDir / arch.catalogFilename) arch.catalogBytes + for i in [:arch.segments.size] do + let s := arch.segments[i]! + IO.FS.writeBinFile (outputDir / s.filename) s.main + IO.println s!"encoded directory {inputDir} → {outputDir}/" + IO.println s!" catalog {arch.catalogFilename} ({arch.entryCount} entries, {arch.segments.size} segments)" + pure (.ok ()) + catch e => + pure (.error (.io (toString e))) + +/-- Decode directory catalog into outdir (reads segment mains from catalog parent). -/ +def decodeDir (catalogPath outputDir : System.FilePath) (master : ByteArray) : + IO (Except CliError Unit) := do + try + let name := catalogPath.fileName.getD "" + match parseCatalogName name with + | .error e => pure (.error (.directory e)) + | .ok (expectedRoot, catalogFmt) => + match checkMasterPolicy master (catalogFmt &&& 1 != 0) with + | .error e => pure (.error (.directory e)) + | .ok () => + let catalogBytes ← IO.FS.readBinFile catalogPath + let parent := catalogPath.parent.getD (System.FilePath.mk ".") + match decodeHeaderedWithHeader master catalogBytes with + | .error e => pure (.error (.pipeline e)) + | .ok (hdr, adamBody) => + if !ctEq hdr.hash expectedRoot then + pure (.error (.directory .catalogBaoRootMismatch)) + else + match decodeAdamantine adamBody with + | .error ae => pure (.error (.directory (ofAdamantineError ae))) + | .ok (payload, adamHdr) => + -- Same format consistency as pure `decodeDirectory`. + if adamHdr.carbonadoFmt != catalogFmt then + pure (.error (.directory (.invalidAdamantineCarbonadoFormat adamHdr.carbonadoFmt))) + else if adamHdr.flags &&& adamantineFlagRequireOts != 0 then + pure (.error (.directory .otsFeatureRequired)) + else + match splitPayload payload with + | .error ae => pure (.error (.directory (ofAdamantineError ae))) + | .ok (manBytes, baoBundle) => + match FilepackManifest.fromWireBytes manBytes expectedRoot with + | .error pe => pure (.error (.directory (ofFilepackError pe))) + | .ok manifest => + if manifest.formatLevel != catalogFmt then + pure (.error (.directory (.invalidFormatLevel manifest.formatLevel))) + else + IO.FS.createDirAll outputDir + for ei in [:manifest.entries.size] do + let entry := manifest.entries[ei]! + let mut recovered := ByteArray.empty + for si in [:entry.segments.size] do + let sref := entry.segments[si]! + match validateSegmentBundleSemantics entry.segmentFormat sref with + | .error e => return .error (.directory e) + | .ok () => pure () + match segmentFilename sref.segmentBaoRoot entry.segmentFormat with + | .error e => return .error (.directory e) + | .ok sname => + let spath := parent / sname + if ← pathIsSymlink spath then + return .error (.directory .symlinkNotAllowed) + let main ← IO.FS.readBinFile spath + if main.size != UInt64.toNat sref.mainLen then + return .error (.directory .segmentMainLenMismatch) + match bundleSlice baoBundle + (UInt32.toNat sref.verificationOutboardOffset) + (UInt32.toNat sref.verificationOutboardLen) with + | .error ae => return .error (.directory (ofAdamantineError ae)) + | .ok verOb => + let fmtBits := FormatBits.ofUInt8 entry.segmentFormat + let fecPar ← + if fmtBits.fec then + match bundleSlice baoBundle + (UInt32.toNat sref.fecParityOffset) + (UInt32.toNat sref.fecParityLen) with + | .error ae => return .error (.directory (ofAdamantineError ae)) + | .ok p => pure p + else pure ByteArray.empty + let pad := paddingForMainLen main.size fmtBits.fec + match decodeOutboardBody master sref.segmentBaoRoot main verOb + fecPar pad fmtBits with + | .error pe => return .error (.pipeline pe) + | .ok part => recovered := appendBA recovered part + match checkContentBlake3 recovered entry.contentBlake3 with + | .error e => return .error (.directory e) + | .ok () => pure () + match validateRelPath entry.relPath with + | .error pe => return .error (.directory (ofFilepackError pe)) + | .ok () => + let outPath := outputDir / entry.relPath + if ← pathIsSymlink outPath then + return .error (.directory .symlinkNotAllowed) + if let some p := outPath.parent then + IO.FS.createDirAll p + IO.FS.writeBinFile outPath recovered + IO.println s!"decoded catalog {name} → {outputDir}/ ({manifest.entries.size} files)" + pure (.ok ()) + catch e => + pure (.error (.io (toString e))) + +/-- Detect if path is directory catalog by name. -/ +def isAdamCatalogName (name : String) : Bool := + name.endsWith ".adam.c14" || name.endsWith ".adam.c15" + +/-- Default output for encode directory: `{input}-archive`. -/ +def defaultArchiveDir (input : System.FilePath) : System.FilePath := + System.FilePath.mk s!"{input}-archive" + +/-- Format CLI error for stderr. -/ +def formatCliError : CliError → String + | .usage m => s!"usage: {m}" + | .io m => s!"io: {m}" + | .badMasterHex => "invalid --master (need 64 hex chars)" + | .zeroMasterEncrypted => "zero master key not allowed for encrypted formats" + | .invalidFormat => "invalid --format (0-15)" + | .directory e => s!"directory: {repr e}" + | .pipeline e => s!"pipeline: {repr e}" + | .slh e => s!"slh: {repr e}" + | .notFound p => s!"not found: {p}" + +/-- Dispatch encode/decode/slh (not demo). -/ +def runCommand (cmd : String) (args : List String) : IO UInt32 := do + match parseArgsSimple args with + | .error e => + IO.eprintln (formatCliError e) + pure 2 + | .ok (opts, pos) => + match cmd with + | "version" => + IO.println versionString + pure 0 + | "help" | "--help" | "-h" => + IO.println helpText + pure 0 + | "encode" => + match pos with + | [input] => + let master := masterOrZero opts + let inPath := System.FilePath.mk input + let isDir ← inPath.isDir + if isDir then + let out := + match opts.output with + | some o => System.FilePath.mk o + | none => defaultArchiveDir inPath + match ← encodeDir inPath out master opts.encrypted with + | .error e => IO.eprintln (formatCliError e); pure 1 + | .ok () => pure 0 + else + let fmtN := opts.format.getD 0 + if fmtN > 15 then + IO.eprintln (formatCliError .invalidFormat); pure 2 + else + let format := + let base := FormatBits.ofUInt8 (UInt8.ofNat fmtN) + if opts.encrypted && !base.encrypted then + FormatBits.ofUInt8 (UInt8.ofNat (fmtN ||| 1)) + else base + let outOpt := opts.output.map System.FilePath.mk + match ← encodeFile inPath outOpt master format with + | .error e => IO.eprintln (formatCliError e); pure 1 + | .ok () => pure 0 + | _ => + IO.eprintln (formatCliError (.usage "encode [-o out] [--format N] [--encrypted] [--master HEX]")) + pure 2 + | "decode" => + match pos with + | [input] => + let master := masterOrZero opts + let inPath := System.FilePath.mk input + let name := inPath.fileName.getD input + if isAdamCatalogName name then + let out := + match opts.output with + | some o => System.FilePath.mk o + | none => System.FilePath.mk s!"{input}-decoded" + match ← decodeDir inPath out master with + | .error e => IO.eprintln (formatCliError e); pure 1 + | .ok () => pure 0 + else + let out := + match opts.output with + | some o => System.FilePath.mk o + | none => System.FilePath.mk s!"{input}.out" + match ← decodeFile inPath out master with + | .error e => IO.eprintln (formatCliError e); pure 1 + | .ok () => pure 0 + | _ => + IO.eprintln (formatCliError (.usage "decode [-o out] [--master HEX]")) + pure 2 + | "slh" => + match pos with + | "parse" :: [file] => + try + let bytes ← IO.FS.readBinFile file + match parseSidecar bytes with + | .error e => IO.eprintln (formatCliError (.slh e)); pure 1 + | .ok sig => + IO.println s!"SLH1 sidecar ok (signature {sig.size} bytes)" + pure 0 + catch e => + IO.eprintln (formatCliError (.io (toString e))); pure 1 + | "verify" :: [file] => + match opts.rootHex, opts.pkHex with + | some rh, some ph => + match fromHex? rh, fromHex? ph with + | some root, some pk => + try + let bytes ← IO.FS.readBinFile file + match parseSidecar bytes with + | .error e => IO.eprintln (formatCliError (.slh e)); pure 1 + | .ok sig => + -- Fail-closed: never exit 0 without real SLH-DSA verify. + -- Until FFI is linked, always-false oracle → verificationFailed → exit 1. + match verifyBound (fun _ _ _ => false) pk root sig with + | .error .verificationFailed + | .error .signatureUnavailable => + IO.eprintln "slh verify: cryptographic verification unavailable (LIMITS: no SLH-DSA FFI)" + IO.eprintln s!" wire parse ok (sigLen={sig.size}); NOT verified — exit 1" + pure 1 + | .error e => IO.eprintln (formatCliError (.slh e)); pure 1 + | .ok () => + -- Only reachable once real oracle is linked and accepts. + IO.println s!"slh verify ok root={toHex root}" + pure 0 + catch e => + IO.eprintln (formatCliError (.io (toString e))); pure 1 + | _, _ => IO.eprintln "slh verify: bad --root/--pk hex"; pure 2 + | _, _ => + IO.eprintln (formatCliError (.usage "slh verify --root HEX --pk HEX")) + pure 2 + | _ => + IO.eprintln (formatCliError (.usage "slh parse | slh verify --root HEX --pk HEX")) + pure 2 + | other => + IO.eprintln (formatCliError (.usage s!"unknown command: {other}")) + pure 2 + +end Carbonado.Cli diff --git a/Carbonado/Compress.lean b/Carbonado/Compress.lean new file mode 100644 index 0000000..8bb4721 --- /dev/null +++ b/Carbonado/Compress.lean @@ -0,0 +1,170 @@ +/- + Zstd compression for the Carbonado pipeline (Program F). + + Product AOT statically embeds libzstd (level 20) from the pinned `ref/zstd` + tree into `libcarbonado_native.a` via `nix/native` (`staticLibDeps`; **no** + shared `-lzstd`). + + **Evaluation model (LIMITS):** + * `@[extern]` symbols are used by the **compiled** AOT binary. + * Lean bodies are identity fallbacks for the elaborator when C is not linked. + * Do **not** `native_decide` over `compressRaw` / `decompressRaw` (extern needs + the native symbol at decide-time). Pure tests cover status decoding + bit-clear + pipeline paths; AOT `Main` / `demo` gate real zstd goldens and c2/c6 roundtrips. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util + +namespace Carbonado.Compress + +open Carbonado.Constants +open Carbonado.Crypto.Util + +/-- Strict zstd error taxonomy (exact-match in tests; no lumped diagnostics). -/ +inductive ZstdError where + | compressionFailed + | decompressionFailed + | outputTooLarge + | invalidInput + deriving DecidableEq, Repr + +/-- Normative compression level (AGENTS: zstd-20). -/ +def zstdLevel : UInt32 := 20 + +/-- DoS cap on decompressed output (Rust `MAX_SEGMENT_MAIN_LEN` = 256 MiB). -/ +def maxDecompressedLen : UInt64 := 256 * 1024 * 1024 + +/-- Zstd frame magic (little-endian frame descriptor prefix). -/ +def zstdMagic : List UInt8 := [0x28, 0xb5, 0x2f, 0xfd] + +theorem zstdMagic_length : zstdMagic.length = 4 := by native_decide + +/-- Pure status-prefix helper (identity payload). Used by extern Lean bodies. -/ +def statusOkPayload (payload : ByteArray) : ByteArray := + Id.run do + let mut out := ByteArray.empty + out := out.push 0 + pure (appendBA out payload) + +/-- + Raw compress: status-prefixed blob. + Compiled AOT: real `ZSTD_compress` at `level`. + Lean body: identity payload with status 0 (elaborator only; not for native_decide). +-/ +@[extern "carbonado_zstd_compress"] +def compressRaw (input : @& ByteArray) (_level : UInt32) : ByteArray := + statusOkPayload input + +/-- + Raw decompress: status-prefixed blob. + `maxOut = 0` → C uses `maxDecompressedLen`. + Lean body: identity with status 0. +-/ +@[extern "carbonado_zstd_decompress"] +def decompressRaw (input : @& ByteArray) (_maxOut : UInt64) : ByteArray := + statusOkPayload input + +/-- Map C status byte to `ZstdError` (distinct codes; no collapse). -/ +def ofStatus (code : UInt8) : ZstdError := + match code with + | 1 => .compressionFailed + | 2 => .decompressionFailed + | 3 => .outputTooLarge + | _ => .invalidInput + +/-- Decode status-prefixed blob into Except. Empty raw → invalidInput. -/ +def decodeStatusPayload (raw : ByteArray) : Except ZstdError ByteArray := + if raw.size == 0 then + .error .invalidInput + else + let code := raw.get! 0 + let payload := raw.extract 1 raw.size + if code == 0 then + .ok payload + else + .error (ofStatus code) + +/-- Compress at normative level 20 (AOT: real zstd). -/ +def compressLevel20 (input : ByteArray) : Except ZstdError ByteArray := + decodeStatusPayload (compressRaw input zstdLevel) + +/-- Decompress with 256 MiB output cap. -/ +def decompress (input : ByteArray) : Except ZstdError ByteArray := + decodeStatusPayload (decompressRaw input maxDecompressedLen) + +/-- Decompress with an explicit max output size (for tests / tight caps). -/ +def decompressWithMax (input : ByteArray) (maxOut : UInt64) : Except ZstdError ByteArray := + decodeStatusPayload (decompressRaw input maxOut) + +/-- True if `bs` begins with zstd magic (product AOT frames always do). -/ +def hasZstdMagic (bs : ByteArray) : Bool := + bs.size ≥ 4 && + bs.get! 0 == 0x28 && + bs.get! 1 == 0xb5 && + bs.get! 2 == 0x2f && + bs.get! 3 == 0xfd + +/-- ofStatus maps 1 → compressionFailed. -/ +theorem ofStatus_compress : ofStatus 1 = .compressionFailed := rfl + +/-- ofStatus maps 2 → decompressionFailed. -/ +theorem ofStatus_decompress : ofStatus 2 = .decompressionFailed := rfl + +/-- ofStatus maps 3 → outputTooLarge. -/ +theorem ofStatus_too_large : ofStatus 3 = .outputTooLarge := rfl + +/-- ofStatus maps other non-zero → invalidInput (incl. 4 and unknown). -/ +theorem ofStatus_invalid_4 : ofStatus 4 = .invalidInput := rfl + +theorem ofStatus_invalid_99 : ofStatus 99 = .invalidInput := rfl + +/-- Empty status blob → invalidInput (Bool form for Decidable). -/ +theorem decode_empty_raw : + (match decodeStatusPayload ByteArray.empty with + | .error .invalidInput => true + | _ => false) = true := by + native_decide + +/-- Status 1 with empty payload → compressionFailed. -/ +theorem decode_status_1 : + (match decodeStatusPayload (ofList [1]) with + | .error .compressionFailed => true + | _ => false) = true := by + native_decide + +/-- Status 2 → decompressionFailed. -/ +theorem decode_status_2 : + (match decodeStatusPayload (ofList [2]) with + | .error .decompressionFailed => true + | _ => false) = true := by + native_decide + +/-- Status 3 → outputTooLarge. -/ +theorem decode_status_3 : + (match decodeStatusPayload (ofList [3]) with + | .error .outputTooLarge => true + | _ => false) = true := by + native_decide + +/-- Status 4 → invalidInput. -/ +theorem decode_status_4 : + (match decodeStatusPayload (ofList [4]) with + | .error .invalidInput => true + | _ => false) = true := by + native_decide + +/-- Status 0 with payload returns the payload. -/ +theorem decode_status_ok_hello : + (match decodeStatusPayload (ofList [0, 0x68, 0x69]) with + | .ok b => ctEq b (ofList [0x68, 0x69]) + | .error _ => false) = true := by + native_decide + +/-- Pure `statusOkPayload` is status 0 + payload (identity framing). -/ +theorem statusOk_payload_identity : + (match decodeStatusPayload (statusOkPayload (ofList [1, 2, 3])) with + | .ok b => ctEq b (ofList [1, 2, 3]) + | .error _ => false) = true := by + native_decide + +end Carbonado.Compress diff --git a/Carbonado/Constants.lean b/Carbonado/Constants.lean new file mode 100644 index 0000000..4c01b94 --- /dev/null +++ b/Carbonado/Constants.lean @@ -0,0 +1,184 @@ +/- + Normative constants for the Carbonado v2 wire format and pipeline geometry. + + These match the Rust implementation (AGENTS.md / `src/constants.rs`) so Lean + models and parity oracles share a single source of sizes and labels. +-/ + +namespace Carbonado.Constants + +/-- v2 container magic: `CARBONADO20\n` (12 bytes). -/ +def magicBytes : List UInt8 := + [0x43, 0x41, 0x52, 0x42, 0x4f, 0x4e, 0x41, 0x44, 0x4f, 0x32, 0x30, 0x0a] + +theorem magicBytes_length : magicBytes.length = 12 := by native_decide + +/-- Magic is exactly ASCII `CARBONADO20` + LF (no trailing NUL). -/ +theorem magicBytes_eq_literal : + magicBytes = + [0x43, 0x41, 0x52, 0x42, 0x4f, 0x4e, 0x41, 0x44, 0x4f, 0x32, 0x30, 0x0a] := by + rfl + +/-- Header wire length (MAGIC + nonce + header_mac + hash + slh_pk + format + fields). -/ +def headerLen : Nat := 177 + +/-- AES-CTR / payload nonce size. -/ +def nonceLen : Nat := 16 + +/-- Full HMAC-SHA512 tag size (never truncated). -/ +def hmacTagLen : Nat := 64 + +/-- Bao / BLAKE3 root size. -/ +def hashLen : Nat := 32 + +/-- SLH-DSA public key slot in the header. -/ +def slhPublicKeyLen : Nat := 32 + +/-- One Bao leaf / slice length (4 KiB). -/ +def sliceLen : Nat := 4096 + +/-- Bao-tree `BlockSize::from_chunk_log` argument (2 → 4 × 1 KiB chunks = 4 KiB groups). -/ +def baoChunkLog : Nat := 2 + +theorem sliceLen_eq_bao_group : sliceLen = 1024 * 2 ^ baoChunkLog := by native_decide + +/-- RS data shards. -/ +def fecK : Nat := 4 + +/-- RS total shards (data + parity). -/ +def fecM : Nat := 8 + +/-- Stripe alignment: `sliceLen * fecK` = 16 KiB logical stripe unit for padding. -/ +def stripeUnit : Nat := sliceLen * fecK + +theorem stripeUnit_eq : stripeUnit = 16384 := by native_decide + +theorem fecM_eq_twice_fecK : fecM = 2 * fecK := by native_decide + +/-- SLH1 sidecar magic. -/ +def slh1Magic : List UInt8 := [0x53, 0x4c, 0x48, 0x31] -- "SLH1" + +theorem slh1Magic_length : slh1Magic.length = 4 := by native_decide + +theorem slh1Magic_eq_literal : + slh1Magic = [0x53, 0x4c, 0x48, 0x31] := by + rfl + +/-- SLH-DSA-SHA2-128s raw signature length. -/ +def slh1SignatureLen : Nat := 7856 + +/-- Total SLH1 sidecar file length: magic + signature. -/ +def slh1SidecarLen : Nat := 4 + slh1SignatureLen + +theorem slh1SidecarLen_eq : slh1SidecarLen = 7860 := by native_decide + +/-- Subkey domain prefix for HMAC-SHA512 derivation. -/ +def subkeyDomainPrefix : String := "carbonado-v2/" + +/-- Payload EtM MAC domain string. -/ +def etmDomain : String := "carbonado-v2-etm" + +/-- Keyed Bao KDF context (BLAKE3 derive_key). -/ +def verificationContext : String := "carbonado-v2/verification" + +/-! + Full Subkey Label Registry (AGENTS.md). Wire strings match the Rust product. + + * Master-key derived: `aes-ctr`, `etm-hmac`, `header-auth` + * Hybrid only: `ecc-chacha-poly` — PRF input is ECDH shared secret, **not** master + * SLH convenience wrappers only: `slh-dsa-seed`, `slh-dsa-seed-2` (not container security) + + Program B implements master-derived EtM subkeys (`aes-ctr`, `etm-hmac`, `header-auth`). + Hybrid/SLH seed labels remain registry-only until those programs. +-/ +inductive SubkeyLabel where + | aesCtr + | etmHmac + | headerAuth + | eccChaChaPoly + | slhDsaSeed + | slhDsaSeed2 + deriving DecidableEq, Repr + +def SubkeyLabel.toString : SubkeyLabel → String + | .aesCtr => "aes-ctr" + | .etmHmac => "etm-hmac" + | .headerAuth => "header-auth" + | .eccChaChaPoly => "ecc-chacha-poly" + | .slhDsaSeed => "slh-dsa-seed" + | .slhDsaSeed2 => "slh-dsa-seed-2" + +/-- Labels derived from the archive master key (container security). -/ +def SubkeyLabel.isMasterDerived : SubkeyLabel → Bool + | .aesCtr | .etmHmac | .headerAuth => true + | .eccChaChaPoly | .slhDsaSeed | .slhDsaSeed2 => false + +/-- Format bitmask bit values (lowest bit = Encrypted → unencrypted formats are even). -/ +def formatBitEncrypted : UInt8 := 1 +def formatBitCompression : UInt8 := 2 +def formatBitVerification : UInt8 := 4 +def formatBitFec : UInt8 := 8 + +/-- Format bitmask bits (lowest bit = Encrypted, so unencrypted formats are even). -/ +structure FormatBits where + encrypted : Bool + compression : Bool + verification : Bool + fec : Bool + deriving DecidableEq, Repr + +def FormatBits.toUInt8 (f : FormatBits) : UInt8 := + let e : UInt8 := if f.encrypted then formatBitEncrypted else 0 + let c : UInt8 := if f.compression then formatBitCompression else 0 + let v : UInt8 := if f.verification then formatBitVerification else 0 + let z : UInt8 := if f.fec then formatBitFec else 0 + e + c + v + z + +def FormatBits.ofUInt8 (b : UInt8) : FormatBits := + { encrypted := b &&& formatBitEncrypted != 0 + compression := b &&& formatBitCompression != 0 + verification := b &&& formatBitVerification != 0 + fec := b &&& formatBitFec != 0 } + +theorem formatBits_roundtrip (f : FormatBits) : + FormatBits.ofUInt8 f.toUInt8 = f := by + cases f with + | mk e c v z => + cases e <;> cases c <;> cases v <;> cases z <;> native_decide + +/-- Unencrypted formats have even numeric codes. -/ +theorem unencrypted_format_even (f : FormatBits) (h : f.encrypted = false) : + f.toUInt8 % 2 = 0 := by + cases f with + | mk e c v z => + simp [FormatBits.toUInt8, formatBitEncrypted, formatBitCompression, + formatBitVerification, formatBitFec] at h ⊢ + subst h + cases c <;> cases v <;> cases z <;> native_decide + +/-- Public catalog format c14 = Compression | Verification | Fec = 14. -/ +def formatC14 : FormatBits := + { encrypted := false, compression := true, verification := true, fec := true } + +theorem formatC14_byte : formatC14.toUInt8 = 14 := by native_decide + +/-- Encrypted catalog format c15 = Encrypted | Compression | Verification | Fec = 15. -/ +def formatC15 : FormatBits := + { encrypted := true, compression := true, verification := true, fec := true } + +theorem formatC15_byte : formatC15.toUInt8 = 15 := by native_decide + +/-- All 16 format codes round-trip through `FormatBits`. -/ +theorem format_codes_roundtrip : + (List.range 16).all (fun n => + let b := UInt8.ofNat n + (FormatBits.ofUInt8 b).toUInt8 = b) = true := by + native_decide + +/-- Header layout field sizes sum to `headerLen`. -/ +theorem headerLen_sum : + magicBytes.length + nonceLen + hmacTagLen + hashLen + slhPublicKeyLen + + 1 + 4 + 4 + 4 + 8 = headerLen := by + native_decide + +end Carbonado.Constants diff --git a/Carbonado/Crypto.lean b/Carbonado/Crypto.lean new file mode 100644 index 0000000..7ed06cd --- /dev/null +++ b/Carbonado/Crypto.lean @@ -0,0 +1,10 @@ +/- + Carbonado v2 symmetric crypto surface (Program B). + + Modules: Util, SHA512, HMAC, AESCTR, EtM. +-/ +import Carbonado.Crypto.Util +import Carbonado.Crypto.SHA512 +import Carbonado.Crypto.HMAC +import Carbonado.Crypto.AESCTR +import Carbonado.Crypto.EtM diff --git a/Carbonado/Crypto/AESCTR.lean b/Carbonado/Crypto/AESCTR.lean new file mode 100644 index 0000000..e2a3fdf --- /dev/null +++ b/Carbonado/Crypto/AESCTR.lean @@ -0,0 +1,168 @@ +/- + AES-256 block encrypt + AES-256-CTR (Ctr128BE) matching RustCrypto + `aes` 0.8.4 + `ctr` 0.9.2 (`Ctr128BE`). + + Counter: 16-byte big-endian 128-bit integer; keystream block i is + AES-256_K(BE128(nonce) + i). + + ## Unchecked primitives (caller contract) + + `expandKey256` and `ctrXor` are **low-level unchecked** helpers: + * `expandKey256` requires a key of at least `keySize` (32) bytes + * `ctrXor` requires a nonce of at least `blockSize` (16) bytes and a 32-byte key + + Short inputs panic via `ByteArray.get!` rather than returning a typed error. + Product EtM APIs (`Carbonado.Crypto.EtM`) validate key/nonce lengths first and + never call these with undersized buffers. Direct callers must supply full sizes. +-/ +import Carbonado.Crypto.Util + +namespace Carbonado.Crypto.AESCTR + +open Carbonado.Crypto.Util + +/-- AES block size. -/ +def blockSize : Nat := 16 + +/-- AES-256 key size. -/ +def keySize : Nat := 32 + +/-- Forward S-box. -/ +private def sbox : Array UInt8 := #[ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 +] + +private def rcon : Array UInt8 := #[ + 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 +] + +private def xtime (a : UInt8) : UInt8 := + let s := a <<< 1 + if (a &&& 0x80) != 0 then s ^^^ 0x1b else s + +private def mul2 (a : UInt8) : UInt8 := xtime a +private def mul3 (a : UInt8) : UInt8 := xtime a ^^^ a + +/-- AES-256 expanded key: 15 round keys × 16 bytes = 240 bytes. -/ +def expandKey256 (key : ByteArray) : ByteArray := + Id.run do + let mut w := ByteArray.empty + for i in [:32] do + w := w.push (key.get! i) + let mut i : Nat := 8 + while i < 60 do + let mut temp0 := w.get! ((i - 1) * 4) + let mut temp1 := w.get! ((i - 1) * 4 + 1) + let mut temp2 := w.get! ((i - 1) * 4 + 2) + let mut temp3 := w.get! ((i - 1) * 4 + 3) + if i % 8 == 0 then + let t0 := temp0 + temp0 := sbox[temp1.toNat]! ^^^ rcon[i / 8]! + temp1 := sbox[temp2.toNat]! + temp2 := sbox[temp3.toNat]! + temp3 := sbox[t0.toNat]! + else if i % 8 == 4 then + temp0 := sbox[temp0.toNat]! + temp1 := sbox[temp1.toNat]! + temp2 := sbox[temp2.toNat]! + temp3 := sbox[temp3.toNat]! + w := w.push (w.get! ((i - 8) * 4) ^^^ temp0) + w := w.push (w.get! ((i - 8) * 4 + 1) ^^^ temp1) + w := w.push (w.get! ((i - 8) * 4 + 2) ^^^ temp2) + w := w.push (w.get! ((i - 8) * 4 + 3) ^^^ temp3) + i := i + 1 + pure w + +private def addRoundKey (state rk : ByteArray) (round : Nat) : ByteArray := + Id.run do + let mut out := ByteArray.empty + let base := round * 16 + for i in [:16] do + out := out.push (state.get! i ^^^ rk.get! (base + i)) + pure out + +private def subBytes (state : ByteArray) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:16] do + out := out.push sbox[(state.get! i).toNat]! + pure out + +/-- ShiftRows on column-major AES state (index = row + 4*col). -/ +private def shiftRows (state : ByteArray) : ByteArray := + let get (r c : Nat) := state.get! (r + 4 * c) + Id.run do + let mut arr : Array UInt8 := Array.replicate 16 0 + for c in [:4] do + arr := arr.set! (0 + 4 * c) (get 0 c) + arr := arr.set! (1 + 4 * c) (get 1 ((c + 1) % 4)) + arr := arr.set! (2 + 4 * c) (get 2 ((c + 2) % 4)) + arr := arr.set! (3 + 4 * c) (get 3 ((c + 3) % 4)) + let mut out := ByteArray.empty + for i in [:16] do + out := out.push arr[i]! + pure out + +private def mixColumns (state : ByteArray) : ByteArray := + Id.run do + let mut arr : Array UInt8 := Array.replicate 16 0 + for c in [:4] do + let s0 := state.get! (0 + 4 * c) + let s1 := state.get! (1 + 4 * c) + let s2 := state.get! (2 + 4 * c) + let s3 := state.get! (3 + 4 * c) + arr := arr.set! (0 + 4 * c) (mul2 s0 ^^^ mul3 s1 ^^^ s2 ^^^ s3) + arr := arr.set! (1 + 4 * c) (s0 ^^^ mul2 s1 ^^^ mul3 s2 ^^^ s3) + arr := arr.set! (2 + 4 * c) (s0 ^^^ s1 ^^^ mul2 s2 ^^^ mul3 s3) + arr := arr.set! (3 + 4 * c) (mul3 s0 ^^^ s1 ^^^ s2 ^^^ mul2 s3) + let mut out := ByteArray.empty + for i in [:16] do + out := out.push arr[i]! + pure out + +/-- Encrypt one 16-byte block with AES-256 (expanded key 240 bytes). -/ +def encryptBlock (roundKeys block : ByteArray) : ByteArray := + Id.run do + let mut state := addRoundKey block roundKeys 0 + for round in [1:14] do + state := subBytes state + state := shiftRows state + state := mixColumns state + state := addRoundKey state roundKeys round + state := subBytes state + state := shiftRows state + state := addRoundKey state roundKeys 14 + pure state + +/-- AES-256-CTR keystream XOR (encrypt ≡ decrypt). Nonce is 16-byte Ctr128BE IV. -/ +def ctrXor (key nonce data : ByteArray) : ByteArray := + let rk := expandKey256 key + Id.run do + let mut counter := nonce.extract 0 16 + let mut out := ByteArray.empty + let mut off : Nat := 0 + while off < data.size do + let ks := encryptBlock rk counter + let n := min 16 (data.size - off) + for i in [:n] do + out := out.push (data.get! (off + i) ^^^ ks.get! i) + counter := incCtr128BE counter + off := off + n + pure out + +end Carbonado.Crypto.AESCTR diff --git a/Carbonado/Crypto/EtM.lean b/Carbonado/Crypto/EtM.lean new file mode 100644 index 0000000..df924a9 --- /dev/null +++ b/Carbonado/Crypto/EtM.lean @@ -0,0 +1,264 @@ +/- + Carbonado v2 symmetric Encrypt-then-MAC stack. + + Normative (AGENTS.md §2.1): + * Subkeys: `HMAC-SHA512(master, "carbonado-v2/" || label)` → 64 bytes + * AES-256-CTR (`Ctr128BE`) with 16-byte nonce + * Payload EtM: `tag = HMAC-SHA512(etm-hmac, "carbonado-v2-etm" || nonce || ct)` + * Header-path layout: `[tag(64) | ct]` + * Low-level layout: `[nonce(16) | tag(64) | ct]` + * Header MAC: `HMAC-SHA512(header-auth, auth_data)` (MAGIC is domain in auth_data) + * **MAC-before-decrypt**: tag verified before any keystream is applied + + Error check order on decrypt (parity with Rust `symmetric_decrypt_with_nonce`): + 1. ciphertext length (`invalidCiphertextLength`) + 2. master key length (`invalidKeyLength`) + 3. nonce length (`invalidNonceLength`) + 4. MAC verify (`authenticationFailed`) then keystream +-/ +import Carbonado.Constants +import Carbonado.Crypto.AESCTR +import Carbonado.Crypto.HMAC +import Carbonado.Crypto.Util + +namespace Carbonado.Crypto.EtM + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Crypto.HMAC +open Carbonado.Crypto.AESCTR + +/-- Crypto error taxonomy (strict matches in tests; distinct failure modes). -/ +inductive CryptoError where + | invalidKeyLength + | invalidCiphertextLength + | invalidNonceLength + | authenticationFailed + deriving DecidableEq, Repr + +/-- Decrypt result: plaintext is only present on success. -/ +inductive DecryptResult where + | ok (plaintext : ByteArray) + | error (e : CryptoError) + -- No `Repr` (ByteArray has no Repr instance in core Lean 4.30). + +/-- True iff the result carries plaintext. -/ +def DecryptResult.isOk : DecryptResult → Bool + | .ok _ => true + | .error _ => false + +/-- Extract plaintext if present. -/ +def DecryptResult.plaintext? : DecryptResult → Option ByteArray + | .ok pt => some pt + | .error _ => none + +/-- Minimum master key length (high-entropy 32 bytes; 64 also accepted). -/ +def minMasterLen : Nat := 32 + +/-- Derive a 64-byte subkey: `HMAC-SHA512(master, "carbonado-v2/" || label)`. -/ +def deriveSubkey (master : ByteArray) (label : String) : Except CryptoError ByteArray := + if master.size == 0 then + .error .invalidKeyLength + else + let msg := appendBA (utf8 subkeyDomainPrefix) (utf8 label) + .ok (hmacSHA512 master msg) + +/-- Derive using the registered `SubkeyLabel` enum. -/ +def deriveSubkeyLabel (master : ByteArray) (label : SubkeyLabel) : Except CryptoError ByteArray := + deriveSubkey master label.toString + +/-- First 32 bytes of `aes-ctr` subkey → AES-256 key. -/ +def aesKeyOfMaster (master : ByteArray) : Except CryptoError ByteArray := + match deriveSubkey master "aes-ctr" with + | .error e => .error e + | .ok material => + if material.size < 32 then .error .invalidKeyLength + else .ok (material.extract 0 32) + +/-- Full 64-byte `etm-hmac` subkey. -/ +def etmKeyOfMaster (master : ByteArray) : Except CryptoError ByteArray := + deriveSubkey master "etm-hmac" + +/-- Full 64-byte `header-auth` subkey. -/ +def headerAuthKeyOfMaster (master : ByteArray) : Except CryptoError ByteArray := + deriveSubkey master "header-auth" + +/-- Payload EtM MAC input: `"carbonado-v2-etm" || nonce || ciphertext`. -/ +def etmMacInput (nonce ct : ByteArray) : ByteArray := + appendBA (appendBA (utf8 etmDomain) nonce) ct + +/-- Compute the 64-byte payload EtM tag. -/ +def computePayloadTag (macKey nonce ct : ByteArray) : ByteArray := + hmacSHA512 macKey (etmMacInput nonce ct) + +/-- Header-path encrypt: output `[tag(64) | ct]`. Nonce stored out-of-band (Header). -/ +def encryptWithNonce (master nonce plaintext : ByteArray) : Except CryptoError ByteArray := + if master.size < minMasterLen then + .error .invalidKeyLength + else if nonce.size != nonceLen then + .error .invalidNonceLength + else + match aesKeyOfMaster master, etmKeyOfMaster master with + | .error e, _ => .error e + | _, .error e => .error e + | .ok aesKey, .ok macKey => + let ct := ctrXor aesKey nonce plaintext + let tag := computePayloadTag macKey nonce ct + .ok (appendBA tag ct) + +/-- + Core MAC-then-decrypt step (keys already derived). + + Keystream (`ctrXor`) is applied **only** in the success branch after `ctEq`. + The tag check is the condition of the `if` — plaintext is not built on failure. +-/ +def decryptAfterMacCheck (aesKey macKey nonce tag ct : ByteArray) : DecryptResult := + if ctEq tag (computePayloadTag macKey nonce ct) then + .ok (ctrXor aesKey nonce ct) + else + .error .authenticationFailed + +/-- + Header-path decrypt: input `[tag(64) | ct]`. + + **MAC-before-decrypt:** `decryptAfterMacCheck` verifies the full 64-byte tag + before constructing any `.ok` plaintext. + + Guard order matches Rust `symmetric_decrypt_with_nonce`: + ciphertext length → master length → nonce length → MAC. +-/ +def decryptWithNonce (master nonce input : ByteArray) : DecryptResult := + if input.size < hmacTagLen then + .error .invalidCiphertextLength + else if master.size < minMasterLen then + .error .invalidKeyLength + else if nonce.size != nonceLen then + .error .invalidNonceLength + else + let tag := input.extract 0 hmacTagLen + let ct := input.extract hmacTagLen input.size + match aesKeyOfMaster master, etmKeyOfMaster master with + | .error e, _ => .error e + | _, .error e => .error e + | .ok aesKey, .ok macKey => + decryptAfterMacCheck aesKey macKey nonce tag ct + +/-- Low-level encrypt: `[nonce(16) | tag(64) | ct]` (caller supplies nonce). -/ +def encryptEmbeddedNonce (master nonce plaintext : ByteArray) : Except CryptoError ByteArray := + match encryptWithNonce master nonce plaintext with + | .error e => .error e + | .ok inner => + -- `encryptWithNonce` already requires `nonce.size = nonceLen`. + .ok (appendBA nonce inner) + +/-- + Low-level decrypt for `[nonce(16) | tag(64) | ct]`. + Same MAC-before-decrypt discipline as `decryptWithNonce`. +-/ +def decryptEmbeddedNonce (master input : ByteArray) : DecryptResult := + if input.size < nonceLen + hmacTagLen then + .error .invalidCiphertextLength + else + let nonce := input.extract 0 nonceLen + let rest := input.extract nonceLen input.size + decryptWithNonce master nonce rest + +/-- Header MAC: `HMAC-SHA512(header-auth subkey, auth_data)` — no extra domain string. -/ +def computeHeaderMac (master authData : ByteArray) : Except CryptoError ByteArray := + if master.size < minMasterLen then + .error .invalidKeyLength + else + match headerAuthKeyOfMaster master with + | .error e => .error e + | .ok key => .ok (hmacSHA512 key authData) + +/-- Verify a header MAC; true only on exact 64-byte match. -/ +def verifyHeaderMac (master authData tag : ByteArray) : Except CryptoError Bool := + match computeHeaderMac master authData with + | .error e => .error e + | .ok expected => .ok (ctEq tag expected) + +/-! + ## MAC-before-decrypt theorems + + Control-flow contract: `.ok plaintext` is only constructed after `ctEq` succeeds. + No constant-time claim (see docs/LIMITS.md). +-/ + +/-- Tag mismatch ⇒ authentication failure (no plaintext). -/ +theorem decryptAfterMacCheck_tag_fail + (aesKey macKey nonce tag ct : ByteArray) + (h : ctEq tag (computePayloadTag macKey nonce ct) = false) : + decryptAfterMacCheck aesKey macKey nonce tag ct = .error .authenticationFailed := by + unfold decryptAfterMacCheck + rw [if_neg (by simp [h])] + +/-- Successful core decrypt ⇒ tag matched (MAC verified before keystream use). -/ +theorem decryptAfterMacCheck_ok_implies_mac + (aesKey macKey nonce tag ct pt : ByteArray) + (h : decryptAfterMacCheck aesKey macKey nonce tag ct = .ok pt) : + ctEq tag (computePayloadTag macKey nonce ct) = true := by + unfold decryptAfterMacCheck at h + by_cases hct : ctEq tag (computePayloadTag macKey nonce ct) = true + · exact hct + · have hctf : ctEq tag (computePayloadTag macKey nonce ct) = false := by + cases hc : ctEq tag (computePayloadTag macKey nonce ct) <;> simp_all + simp [hctf] at h + +/-- Authentication failure carries no plaintext. -/ +theorem decryptAfterMacCheck_auth_fail_no_plaintext + (aesKey macKey nonce tag ct : ByteArray) + (h : ctEq tag (computePayloadTag macKey nonce ct) = false) : + (decryptAfterMacCheck aesKey macKey nonce tag ct).plaintext? = none := by + rw [decryptAfterMacCheck_tag_fail aesKey macKey nonce tag ct h] + rfl + +/-- `.ok` is the only constructor that embeds plaintext bytes. -/ +theorem decryptResult_ok_iff_plaintext + (r : DecryptResult) (pt : ByteArray) : + r = .ok pt ↔ r.plaintext? = some pt := by + cases r with + | ok p => + constructor + · intro h; cases h; rfl + · intro h + simp only [DecryptResult.plaintext?] at h + injection h with h' + subst h' + rfl + | error e => + constructor + · intro h; cases h + · intro h; cases h + +/-- Short ciphertext is rejected first (Rust parity: length before master). -/ +theorem decryptWithNonce_short_input + (master nonce input : ByteArray) + (hs : (input.size < hmacTagLen) = true) : + decryptWithNonce master nonce input = .error .invalidCiphertextLength := by + unfold decryptWithNonce + simp [hs] + +/-- + Short master key is rejected when the ciphertext is long enough to pass the + length gate (Rust order: CT length first, then master). +-/ +theorem decryptWithNonce_short_master + (master nonce input : ByteArray) + (hs : (input.size < hmacTagLen) = false) + (hm : (master.size < minMasterLen) = true) : + decryptWithNonce master nonce input = .error .invalidKeyLength := by + unfold decryptWithNonce + simp [hs, hm] + +/-- Bad nonce length is a distinct error (not `invalidCiphertextLength`). -/ +theorem decryptWithNonce_bad_nonce + (master nonce input : ByteArray) + (hs : (input.size < hmacTagLen) = false) + (hm : (master.size < minMasterLen) = false) + (hn : (nonce.size != nonceLen) = true) : + decryptWithNonce master nonce input = .error .invalidNonceLength := by + unfold decryptWithNonce + simp [hs, hm, hn] + +end Carbonado.Crypto.EtM diff --git a/Carbonado/Crypto/HMAC.lean b/Carbonado/Crypto/HMAC.lean new file mode 100644 index 0000000..83af0fa --- /dev/null +++ b/Carbonado/Crypto/HMAC.lean @@ -0,0 +1,35 @@ +/- + HMAC-SHA512 (RFC 2104 / FIPS 198-1) — full 64-byte tags, never truncated. + + Parity target: RustCrypto `hmac` 0.12.1 (`ref/rustcrypto-macs`). +-/ +import Carbonado.Crypto.SHA512 +import Carbonado.Crypto.Util + +namespace Carbonado.Crypto.HMAC + +open Carbonado.Crypto.Util +open Carbonado.Crypto.SHA512 + +/-- HMAC-SHA512 with arbitrary-length key and message. Output is 64 bytes. -/ +def hmacSHA512 (key msg : ByteArray) : ByteArray := + let block := blockSize -- 128 for SHA-512 + let keyBlock : ByteArray := + if key.size > block then + resize (SHA512.hash key) block + else + resize key block + let ipad := Id.run do + let mut out := ByteArray.empty + for i in [:block] do + out := out.push (keyBlock.get! i ^^^ 0x36) + pure out + let opad := Id.run do + let mut out := ByteArray.empty + for i in [:block] do + out := out.push (keyBlock.get! i ^^^ 0x5c) + pure out + let inner := SHA512.hash (appendBA ipad msg) + SHA512.hash (appendBA opad inner) + +end Carbonado.Crypto.HMAC diff --git a/Carbonado/Crypto/SHA512.lean b/Carbonado/Crypto/SHA512.lean new file mode 100644 index 0000000..b4d96db --- /dev/null +++ b/Carbonado/Crypto/SHA512.lean @@ -0,0 +1,134 @@ +/- + SHA-512 (FIPS 180-4) — pure Lean, full 64-byte digests. + + Parity target: RustCrypto `sha2` 0.10.9 (`ref/rustcrypto-hashes`). +-/ +import Carbonado.Crypto.Util + +namespace Carbonado.Crypto.SHA512 + +open Carbonado.Crypto.Util + +/-- SHA-512 block size in bytes. -/ +def blockSize : Nat := 128 + +/-- SHA-512 digest size in bytes. -/ +def digestSize : Nat := 64 + +/-- Round constants K₀‥K₇₉ (FIPS 180-4). -/ +private def K : Array UInt64 := #[ + 0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5, + 0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70, + 0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df, + 0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b, + 0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30, + 0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec, + 0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b, + 0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b, + 0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 +] + +/-- Initial hash value H⁽⁰⁾. -/ +private def H0 : Array UInt64 := #[ + 0x6a09e667f3bcc908, 0xbb67ae8584caa73b, 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1, + 0x510e527fade682d1, 0x9b05688c2b3e6c1f, 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179 +] + +/-- Right-rotate a 64-bit word by `n` bits (0 < n < 64). -/ +private def rotr (x : UInt64) (n : Nat) : UInt64 := + let n64 : UInt64 := UInt64.ofNat n + let m64 : UInt64 := UInt64.ofNat (64 - n) + (x >>> n64) ||| (x <<< m64) + +private def Ch (x y z : UInt64) : UInt64 := (x &&& y) ^^^ ((~~~x) &&& z) +private def Maj (x y z : UInt64) : UInt64 := (x &&& y) ^^^ (x &&& z) ^^^ (y &&& z) +private def Sigma0 (x : UInt64) : UInt64 := rotr x 28 ^^^ rotr x 34 ^^^ rotr x 39 +private def Sigma1 (x : UInt64) : UInt64 := rotr x 14 ^^^ rotr x 18 ^^^ rotr x 41 +private def sigma0 (x : UInt64) : UInt64 := rotr x 1 ^^^ rotr x 8 ^^^ (x >>> (7 : UInt64)) +private def sigma1 (x : UInt64) : UInt64 := rotr x 19 ^^^ rotr x 61 ^^^ (x >>> (6 : UInt64)) + +/-- Process one 128-byte block; returns updated 8-word state. -/ +private def compress (state : Array UInt64) (block : ByteArray) : Array UInt64 := + Id.run do + let mut W : Array UInt64 := Array.replicate 80 0 + for t in [:16] do + W := W.set! t (getUInt64BE block (t * 8)) + for t in [16:80] do + let v := sigma1 W[t - 2]! + W[t - 7]! + sigma0 W[t - 15]! + W[t - 16]! + W := W.set! t v + let mut a := state[0]! + let mut b := state[1]! + let mut c := state[2]! + let mut d := state[3]! + let mut e := state[4]! + let mut f := state[5]! + let mut g := state[6]! + let mut h := state[7]! + for t in [:80] do + let T1 := h + Sigma1 e + Ch e f g + K[t]! + W[t]! + let T2 := Sigma0 a + Maj a b c + h := g + g := f + f := e + e := d + T1 + d := c + c := b + b := a + a := T1 + T2 + pure #[ + state[0]! + a, state[1]! + b, state[2]! + c, state[3]! + d, + state[4]! + e, state[5]! + f, state[6]! + g, state[7]! + h + ] + +private def appendU64BE (out : ByteArray) (x : UInt64) : ByteArray := + appendBA out (putUInt64BE x) + +/-- Pad message per FIPS 180-4 (length in bits as 128-bit BE). -/ +private def pad (msg : ByteArray) : ByteArray := + let bitLen : UInt64 := UInt64.ofNat (msg.size * 8) + let lenHigh : UInt64 := 0 + let lenLow := bitLen + let lenAfterBit := msg.size + 1 + let rem := lenAfterBit % 128 + let zeroCount := if rem ≤ 112 then 112 - rem else 128 + 112 - rem + Id.run do + let mut out := ByteArray.empty + for i in [:msg.size] do + out := out.push (msg.get! i) + out := out.push 0x80 + for _ in [:zeroCount] do + out := out.push 0 + out := appendU64BE out lenHigh + out := appendU64BE out lenLow + pure out + +/-- SHA-512 hash of an arbitrary-length message. -/ +def hash (msg : ByteArray) : ByteArray := + let padded := pad msg + Id.run do + let mut state := H0 + let mut off : Nat := 0 + while off + 128 ≤ padded.size do + let block := padded.extract off (off + 128) + state := compress state block + off := off + 128 + let mut out := ByteArray.empty + for i in [:8] do + out := appendU64BE out state[i]! + pure out + +/-- Convenience: hash a UTF-8 string. -/ +def hashString (s : String) : ByteArray := hash (utf8 s) + +end Carbonado.Crypto.SHA512 diff --git a/Carbonado/Crypto/Util.lean b/Carbonado/Crypto/Util.lean new file mode 100644 index 0000000..1023006 --- /dev/null +++ b/Carbonado/Crypto/Util.lean @@ -0,0 +1,189 @@ +/- + Byte utilities for Carbonado crypto (SHA-512, AES, EtM). + + Product code uses `ByteArray` for wire material. No secret zeroization is + claimed here (see docs/LIMITS.md). Equality helpers are logical, not CT proofs. +-/ + +namespace Carbonado.Crypto.Util + +/-- Equality for authentication tags (length-checked; logical, not a CT proof). -/ +def ctEq (a b : ByteArray) : Bool := + if a.size != b.size then + false + else + Id.run do + let mut acc : UInt8 := 0 + for i in [:a.size] do + acc := acc ||| (a.get! i ^^^ b.get! i) + pure (acc == 0) + +/-- Append `b` to `a`. -/ +def appendBA (a b : ByteArray) : ByteArray := + a.append b + +/-- XOR two byte arrays; result length is `min a.size b.size`. -/ +def xorBytes (a b : ByteArray) : ByteArray := + let n := min a.size b.size + Id.run do + let mut out := ByteArray.empty + for i in [:n] do + out := out.push (a.get! i ^^^ b.get! i) + pure out + +/-- Repeat a byte `n` times. -/ +def replicate (n : Nat) (b : UInt8) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for _ in [:n] do + out := out.push b + pure out + +/-- Pad or truncate to exactly `n` bytes (zero-pad on the right). -/ +def resize (bs : ByteArray) (n : Nat) : ByteArray := + if bs.size >= n then + bs.extract 0 n + else + appendBA bs (replicate (n - bs.size) 0) + +/-- Big-endian increment of a 16-byte counter (Ctr128BE step). -/ +def incCtr128BE (ctr : ByteArray) : ByteArray := + Id.run do + let mut out := ctr + let mut i : Int := 15 + let mut carry := true + while carry && i ≥ 0 do + let idx := i.toNat + let v := out.get! idx + if v == 0xff then + out := out.set! idx 0 + i := i - 1 + else + out := out.set! idx (v + 1) + carry := false + pure out + +/-- Read big-endian `UInt64` from `bs` starting at `off`. -/ +def getUInt64BE (bs : ByteArray) (off : Nat) : UInt64 := + let b0 := (bs.get! off).toUInt64 + let b1 := (bs.get! (off + 1)).toUInt64 + let b2 := (bs.get! (off + 2)).toUInt64 + let b3 := (bs.get! (off + 3)).toUInt64 + let b4 := (bs.get! (off + 4)).toUInt64 + let b5 := (bs.get! (off + 5)).toUInt64 + let b6 := (bs.get! (off + 6)).toUInt64 + let b7 := (bs.get! (off + 7)).toUInt64 + (b0 <<< 56) ||| (b1 <<< 48) ||| (b2 <<< 40) ||| (b3 <<< 32) ||| + (b4 <<< 24) ||| (b5 <<< 16) ||| (b6 <<< 8) ||| b7 + +/-- Write big-endian `UInt64` into a fresh 8-byte array. -/ +def putUInt64BE (x : UInt64) : ByteArray := + Id.run do + let mut out := ByteArray.empty + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 56) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 48) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 40) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 32) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 24) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 16) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 8) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat x % 256)) + pure out + +/-- Read little-endian `UInt32` from `bs` starting at `off`. -/ +def getUInt32LE (bs : ByteArray) (off : Nat) : UInt32 := + let b0 := (bs.get! off).toUInt32 + let b1 := (bs.get! (off + 1)).toUInt32 + let b2 := (bs.get! (off + 2)).toUInt32 + let b3 := (bs.get! (off + 3)).toUInt32 + b0 ||| (b1 <<< 8) ||| (b2 <<< 16) ||| (b3 <<< 24) + +/-- Write little-endian `UInt32` into a fresh 4-byte array. -/ +def putUInt32LE (x : UInt32) : ByteArray := + Id.run do + let mut out := ByteArray.empty + out := out.push (UInt8.ofNat (UInt32.toNat x % 256)) + out := out.push (UInt8.ofNat (UInt32.toNat (x >>> 8) % 256)) + out := out.push (UInt8.ofNat (UInt32.toNat (x >>> 16) % 256)) + out := out.push (UInt8.ofNat (UInt32.toNat (x >>> 24) % 256)) + pure out + +/-- Read little-endian `UInt64` from `bs` starting at `off`. -/ +def getUInt64LE (bs : ByteArray) (off : Nat) : UInt64 := + let b0 := (bs.get! off).toUInt64 + let b1 := (bs.get! (off + 1)).toUInt64 + let b2 := (bs.get! (off + 2)).toUInt64 + let b3 := (bs.get! (off + 3)).toUInt64 + let b4 := (bs.get! (off + 4)).toUInt64 + let b5 := (bs.get! (off + 5)).toUInt64 + let b6 := (bs.get! (off + 6)).toUInt64 + let b7 := (bs.get! (off + 7)).toUInt64 + b0 ||| (b1 <<< 8) ||| (b2 <<< 16) ||| (b3 <<< 24) ||| + (b4 <<< 32) ||| (b5 <<< 40) ||| (b6 <<< 48) ||| (b7 <<< 56) + +/-- Write little-endian `UInt64` into a fresh 8-byte array. -/ +def putUInt64LE (x : UInt64) : ByteArray := + Id.run do + let mut out := ByteArray.empty + out := out.push (UInt8.ofNat (UInt64.toNat x % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 8) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 16) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 24) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 32) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 40) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 48) % 256)) + out := out.push (UInt8.ofNat (UInt64.toNat (x >>> 56) % 256)) + pure out + +private def hexDigit (n : Nat) : Char := + if n < 10 then Char.ofNat ('0'.toNat + n) + else Char.ofNat ('a'.toNat + (n - 10)) + +/-- Lowercase hex encoding (for demos / golden diagnostics). -/ +def toHex (bs : ByteArray) : String := + Id.run do + let mut s : String := "" + for i in [:bs.size] do + let b := (bs.get! i).toNat + s := s.push (hexDigit (b / 16)) |>.push (hexDigit (b % 16)) + pure s + +private def hexVal (c : Char) : Option Nat := + if '0' ≤ c && c ≤ '9' then some (c.toNat - '0'.toNat) + else if 'a' ≤ c && c ≤ 'f' then some (c.toNat - 'a'.toNat + 10) + else if 'A' ≤ c && c ≤ 'F' then some (c.toNat - 'A'.toNat + 10) + else none + +/-- Parse hex string to bytes (even length). Returns `none` on error. -/ +def fromHex? (s : String) : Option ByteArray := + let chars := s.toList + if chars.length % 2 != 0 then + none + else + Id.run do + let mut out := ByteArray.empty + let mut rest := chars + let mut ok := true + while rest.length ≥ 2 && ok do + match rest with + | c0 :: c1 :: tail => + match hexVal c0, hexVal c1 with + | some hi, some lo => + out := out.push (UInt8.ofNat (hi * 16 + lo)) + rest := tail + | _, _ => ok := false + | _ => ok := false + if ok then some out else none + +/-- UTF-8 encode a String. -/ +def utf8 (s : String) : ByteArray := s.toUTF8 + +/-- Build ByteArray from a list of bytes. -/ +def ofList (bs : List UInt8) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for b in bs do + out := out.push b + pure out + +end Carbonado.Crypto.Util diff --git a/Carbonado/Directory.lean b/Carbonado/Directory.lean new file mode 100644 index 0000000..e5ec38f --- /dev/null +++ b/Carbonado/Directory.lean @@ -0,0 +1,651 @@ +/- + Adamantine directory archive pure model (Program G). + + Layout (AGENTS §7.1): + * Catalog: inboard headered `{catalog_root}.adam.c14` / `.adam.c15` + body = Adamantine10 envelope (CFP2 FilepackManifest + centralized Bao bundle) + * Segments: bare mains `{seg_root}.c12|c13|c14|c15` (no .out/.par on disk) + * Bundle: per-segment [verification_outboard][fec_parity] indexed by SegmentRef + + Path rules fail-closed via Filepack.validateRelPath. + Content integrity: BLAKE3 of recovered plaintext vs entry.content_blake3. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Header +import Carbonado.Pipeline +import Carbonado.Outboard +import Carbonado.Adamantine +import Carbonado.Filepack +import Carbonado.Bao.Blake3 +import Carbonado.Fec.Inboard + +namespace Carbonado.Directory + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Header +open Carbonado.Pipeline +open Carbonado.Outboard +open Carbonado.Adamantine +open Carbonado.Filepack +open Carbonado.Bao.Blake3 +open Carbonado.Fec.Inboard + +/-- Strict directory error taxonomy (exact-match in tests; no lumped diagnostics). -/ +inductive DirectoryError where + -- Path / layout + | pathTraversal + | pathAbsolute + | pathBackslash + | pathEmpty + | pathEmptyComponent + | pathTooLong + | pathNullByte + | notADirectory + | symlinkNotAllowed + | directoryLayoutMismatch + | missingSegment + | segmentMainLenMismatch + | catalogBaoRootMismatch + | contentBlake3Mismatch + | invalidCatalogPath + | zeroMasterKeyNotAllowed + | encryptedDirectoryNotRequested + -- Adamantine + | invalidAdamantineHeader + | invalidAdamantineMagic + | unsupportedAdamantineVersion (major minor : UInt8) + | invalidAdamantineCarbonadoFormat (fmt : UInt8) + | invalidAdamantineFlags (flags : UInt8) + | adamantinePayloadTooLarge + | adamantinePayloadLengthMismatch + -- Filepack + | invalidFilepackManifest + | unsupportedFilepackVersion (v : Nat) + | invalidFormatLevel (fmt : UInt8) + | segmentFormatMismatch (fmt : UInt8) + | legacySegmentFormat (fmt : UInt8) + | emptySegments + | invalidChunkSequence + | mainLenTooLarge + | tooManyEntries + | tooManySegments + | entriesUnsorted + | invalidWire + | missingVerificationOutboard + | missingFecParity + /-- FEC parity length does not match encode geometry for main_len. -/ + | fecParityLenMismatch + /-- FEC parity present when segment format lacks FEC (or zero main). -/ + | unexpectedFecParity + | otsFeatureRequired + | otsProofTooLarge + -- Pipeline / lower + | pipeline (e : PipelineError) + | invalidHashLength + | insufficientNonces + deriving DecidableEq, Repr + +def ofFilepackError : FilepackError → DirectoryError + | .emptyRelPath => .pathEmpty + | .relPathTooLong => .pathTooLong + | .relPathBackslash => .pathBackslash + | .relPathAbsolute => .pathAbsolute + | .relPathTraversal => .pathTraversal + | .relPathEmptyComponent => .pathEmptyComponent + | .relPathNullByte => .pathNullByte + | .unsupportedVersion v => .unsupportedFilepackVersion v + | .invalidFormatLevel f => .invalidFormatLevel f + | .segmentFormatMismatch f => .segmentFormatMismatch f + | .legacySegmentFormat f => .legacySegmentFormat f + | .emptySegments => .emptySegments + | .invalidChunkSequence => .invalidChunkSequence + | .mainLenTooLarge => .mainLenTooLarge + | .tooManyEntries => .tooManyEntries + | .tooManySegments => .tooManySegments + | .invalidWire => .invalidWire + | .invalidHashLength => .invalidHashLength + | .otsProofTooLarge => .otsProofTooLarge + | .entriesUnsorted => .entriesUnsorted + +def ofAdamantineError : AdamantineError → DirectoryError + | .invalidHeader => .invalidAdamantineHeader + | .invalidMagic => .invalidAdamantineMagic + | .unsupportedVersion maj min => .unsupportedAdamantineVersion maj min + | .invalidCarbonadoFormat f => .invalidAdamantineCarbonadoFormat f + | .invalidFlags f => .invalidAdamantineFlags f + | .payloadTooLarge _ _ => .adamantinePayloadTooLarge + | .payloadLengthMismatch _ _ => .adamantinePayloadLengthMismatch + +def ofPipelineError (e : PipelineError) : DirectoryError := .pipeline e + +/-- One logical file input for pure directory encode. -/ +structure DirFile where + relPath : String + content : ByteArray + deriving DecidableEq, Inhabited + +/-- One encoded bare segment artifact. -/ +structure SegmentArtifact where + filename : String + main : ByteArray + baoRoot : ByteArray + segmentFormat : UInt8 + chunkIndex : UInt32 + deriving DecidableEq, Inhabited + +/-- Full encoded directory archive (in-memory). -/ +structure DirectoryArchive where + catalogFilename : String + catalogBytes : ByteArray + catalogBaoRoot : ByteArray + catalogFormat : UInt8 + segments : Array SegmentArtifact + entryCount : Nat + deriving DecidableEq, Inhabited + +/-- Default segment budget = u32::MAX. -/ +def defaultSegmentPlaintextBudget : Nat := 4294967295 + +/-- Encode options (pure). -/ +structure DirectoryEncodeOptions where + catalogEncrypted : Bool := false + segmentPolicy : SegmentFormatPolicy := .auto + segmentPlaintextBudget : Nat := defaultSegmentPlaintextBudget + /-- + Require OTS flag in Adamantine header. + **Rejected at encode** until OTS stamps exist (`otsFeatureRequired`) — never mints + archives that always fail decode. + -/ + requireOts : Bool := false + deriving DecidableEq, Repr + +/-- Expected outboard FEC parity byte length for bare `main_len` (Rust `expected_fec_parity_len`). -/ +def expectedFecParityLen (mainLen : Nat) : Nat := + if mainLen == 0 then 0 + else + let chunk := (calcPaddingLen mainLen).chunkLen + (fecM - fecK) * chunk + +/-- + Validate SegmentRef bundle semantics for a directory segment (Rust + `validate_segment_bundle_semantics` subset: FEC length geometry + non-FEC hygiene). +-/ +def validateSegmentBundleSemantics (segmentFormat : UInt8) (sref : SegmentRef) : + Except DirectoryError Unit := + let fmtBits := FormatBits.ofUInt8 segmentFormat + let mainLen := UInt64.toNat sref.mainLen + let fecLen := UInt32.toNat sref.fecParityLen + if mainLen > maxSegmentMainLen then + .error .mainLenTooLarge + else if fmtBits.fec then + if mainLen > 0 && fecLen == 0 then + .error .missingFecParity + else if mainLen == 0 && fecLen != 0 then + .error .unexpectedFecParity + else if mainLen > 0 && fecLen != expectedFecParityLen mainLen then + .error .fecParityLenMismatch + else + -- Contiguous: fec_parity_offset should follow verification outboard when both present. + let expectedFecOff := + UInt32.toNat sref.verificationOutboardOffset + UInt32.toNat sref.verificationOutboardLen + if fecLen > 0 && UInt32.toNat sref.fecParityOffset != expectedFecOff then + .error .invalidWire + else + .ok () + else if fecLen != 0 then + .error .unexpectedFecParity + else + .ok () + +/-- Hex lowercase of a 32-byte root for filenames. -/ +def rootHex (root : ByteArray) : Except DirectoryError String := + if root.size != hashLen then .error .invalidHashLength + else .ok (toHex root) + +/-- Decimal segment filename `{root}.c{fmt}`. -/ +def segmentFilename (root : ByteArray) (fmt : UInt8) : Except DirectoryError String := + match rootHex root with + | .error e => .error e + | .ok h => .ok s!"{h}.c{fmt.toNat}" + +/-- Catalog filename `{root}.adam.c{14|15}`. -/ +def catalogFilename (root : ByteArray) (fmt : UInt8) : Except DirectoryError String := + match rootHex root with + | .error e => .error e + | .ok h => .ok s!"{h}.adam.c{fmt.toNat}" + +/-- Parse `{64hex}.adam.c14` / `.adam.c15` → (root, format). -/ +def parseCatalogName (name : String) : Except DirectoryError (ByteArray × UInt8) := + -- Expect at least 64 + ".adam.c" + digits + if !name.endsWith ".adam.c14" && !name.endsWith ".adam.c15" then + .error .invalidCatalogPath + else + let fmt : UInt8 := if name.endsWith ".adam.c15" then 15 else 14 + let suffixLen := if fmt == 15 then ".adam.c15".length else ".adam.c14".length + if name.length < 64 + suffixLen then + .error .invalidCatalogPath + else + let hexPart := name.dropRight suffixLen + if hexPart.length != 64 then + .error .invalidCatalogPath + else + match fromHex? hexPart with + | none => .error .invalidCatalogPath + | some root => + if root.size != hashLen then .error .invalidCatalogPath + else .ok (root, fmt) + +/-- Master key policy for catalog format. -/ +def checkMasterPolicy (master : ByteArray) (catalogEncrypted : Bool) : + Except DirectoryError Unit := + let allZero := + Id.run do + let mut z := true + for i in [:master.size] do + if master.get! i != 0 then z := false + pure z + if master.size < 32 then + .error (.pipeline .invalidKeyLength) + else if catalogEncrypted && allZero then + .error .zeroMasterKeyNotAllowed + else if !catalogEncrypted && !allZero then + .error .encryptedDirectoryNotRequested + else + .ok () + +/-- Split content by budget (last may be short; empty → one empty chunk). -/ +def splitContent (content : ByteArray) (budget : Nat) : Array ByteArray := + let b := if budget == 0 then 1 else budget + if content.size == 0 then + #[ByteArray.empty] + else + Id.run do + let mut out : Array ByteArray := #[] + let mut off : Nat := 0 + while off < content.size do + let end_ := min (off + b) content.size + out := out.push (content.extract off end_) + off := end_ + pure out + +/-- Bundle builder (concat verification + fec blobs). -/ +structure BundleBuilder where + bytes : ByteArray + deriving Inhabited + +def BundleBuilder.empty : BundleBuilder := { bytes := ByteArray.empty } + +def BundleBuilder.append (b : BundleBuilder) (blob : ByteArray) : + Except DirectoryError (BundleBuilder × UInt32 × UInt32) := + let offset := b.bytes.size + let len := blob.size + let end_ := offset + len + if end_ > maxBaoBundleLen then + .error .adamantinePayloadTooLarge + else if offset > u32Max || len > u32Max then + .error .adamantinePayloadTooLarge + else + .ok ({ bytes := appendBA b.bytes blob }, UInt32.ofNat offset, UInt32.ofNat len) + +/-- + Pure directory encode. + + `nonces` supplies one embedded-path nonce per **segment** when any segment is + encrypted (c13/c15). Public archives may pass an empty nonce array. +-/ +def encodeDirectory (master : ByteArray) (files : Array DirFile) + (opts : DirectoryEncodeOptions) (nonces : Array ByteArray) : + Except DirectoryError DirectoryArchive := + let catalogFmt : UInt8 := + if opts.catalogEncrypted then adamantineFmtEncrypted else adamantineFmtPublic + -- OTS stamps not implemented: refuse to mint REQUIRE_OTS archives that always fail decode. + if opts.requireOts then + .error .otsFeatureRequired + else + match checkMasterPolicy master opts.catalogEncrypted with + | .error e => .error e + | .ok () => + Id.run do + -- Sort files by rel_path for determinism + let mut sorted := files + -- simple insertion sort by relPath + for i in [:sorted.size] do + let mut j := i + while j > 0 && (sorted[j]!).relPath < (sorted[j - 1]!).relPath do + let tmp := sorted[j]! + sorted := sorted.set! j (sorted[j - 1]!) + sorted := sorted.set! (j - 1) tmp + j := j - 1 + + let mut entries : Array FilepackEntry := #[] + let mut segments : Array SegmentArtifact := #[] + let mut bundle := BundleBuilder.empty + let mut nonceIdx : Nat := 0 + let mut err : Option DirectoryError := none + + for fi in [:sorted.size] do + if err.isNone then + let f := sorted[fi]! + match validateRelPath f.relPath with + | .error pe => err := some (ofFilepackError pe) + | .ok () => + let contentHash := Carbonado.Bao.Blake3.hash f.content + match opts.segmentPolicy.resolve opts.catalogEncrypted f.content with + | .error pe => err := some (ofFilepackError pe) + | .ok segFmt => + let fmtBits := FormatBits.ofUInt8 segFmt + let chunks := splitContent f.content opts.segmentPlaintextBudget + let mut segs : Array SegmentRef := #[] + for ci in [:chunks.size] do + if err.isNone then + let chunk := chunks[ci]! + let nonce : ByteArray := + if fmtBits.encrypted then + if nonceIdx < nonces.size then nonces[nonceIdx]! + else ByteArray.empty + else + replicate nonceLen 0 + if fmtBits.encrypted && (nonceIdx ≥ nonces.size || nonce.size != nonceLen) then + err := some .insufficientNonces + else + if fmtBits.encrypted then nonceIdx := nonceIdx + 1 + match encodeOutboardBody master nonce chunk fmtBits with + | .error pe => err := some (ofPipelineError pe) + | .ok oenc => + -- Single-leaf trees may have empty post-order outboard (no parent pairs). + if fmtBits.fec && oenc.main.size > 0 && oenc.fecParity.size == 0 then + err := some .missingFecParity + if err.isNone then + match bundle.append oenc.verificationOutboard with + | .error e => err := some e + | .ok (b1, vo, vl) => + bundle := b1 + if fmtBits.fec then + match bundle.append oenc.fecParity with + | .error e => err := some e + | .ok (b2, fo, fl) => + bundle := b2 + match segmentFilename oenc.baoHash segFmt with + | .error e => err := some e + | .ok sname => + segments := segments.push { + filename := sname + main := oenc.main + baoRoot := oenc.baoHash + segmentFormat := segFmt + chunkIndex := UInt32.ofNat ci + } + segs := segs.push { + segmentBaoRoot := oenc.baoHash + chunkIndex := UInt32.ofNat ci + mainLen := UInt64.ofNat oenc.main.size + verificationOutboardOffset := vo + verificationOutboardLen := vl + fecParityOffset := fo + fecParityLen := fl + } + else + match segmentFilename oenc.baoHash segFmt with + | .error e => err := some e + | .ok sname => + segments := segments.push { + filename := sname + main := oenc.main + baoRoot := oenc.baoHash + segmentFormat := segFmt + chunkIndex := UInt32.ofNat ci + } + segs := segs.push { + segmentBaoRoot := oenc.baoHash + chunkIndex := UInt32.ofNat ci + mainLen := UInt64.ofNat oenc.main.size + verificationOutboardOffset := vo + verificationOutboardLen := vl + fecParityOffset := 0 + fecParityLen := 0 + } + if err.isNone then + entries := entries.push { + relPath := f.relPath + contentBlake3 := contentHash + segmentFormat := segFmt + segments := segs + otsProof := none + } + + match err with + | some e => pure (.error e) + | none => + -- Build CFP2 manifest (catalog root placeholder zeros until headered encode). + let placeholderRoot := replicate hashLen 0 + let manifest : FilepackManifest := { + version := filepackManifestVersion + formatLevel := catalogFmt + catalogBaoRoot := placeholderRoot + entries := entries + } + match manifest.toWireBytes with + | .error pe => pure (.error (ofFilepackError pe)) + | .ok manBytes => + match buildPayload manBytes bundle.bytes with + | .error ae => pure (.error (ofAdamantineError ae)) + | .ok payload => + let flags : UInt8 := if opts.requireOts then adamantineFlagRequireOts else 0 + let adamBody := encodeAdamantine payload catalogFmt flags + -- Catalog nonce: public uses zeros; encrypted needs one more nonce. + let catNonce : ByteArray := + if opts.catalogEncrypted then + if nonceIdx < nonces.size then nonces[nonceIdx]! + else ByteArray.empty + else + replicate nonceLen 0 + if opts.catalogEncrypted && + (nonceIdx ≥ nonces.size || catNonce.size != nonceLen) then + pure (.error .insufficientNonces) + else + let catFmtBits := FormatBits.ofUInt8 catalogFmt + match encodeHeadered master catNonce adamBody catFmtBits 0 + (replicate slhPublicKeyLen 0) (replicate 8 0) with + | .error pe => pure (.error (ofPipelineError pe)) + | .ok (hdr, catalogBytes) => + -- Rebind catalog root into manifest is not on wire; filename binds root. + let root := hdr.hash + match catalogFilename root catalogFmt with + | .error e => pure (.error e) + | .ok cname => + pure (.ok { + catalogFilename := cname + catalogBytes := catalogBytes + catalogBaoRoot := root + catalogFormat := catalogFmt + segments := segments + entryCount := entries.size + }) + +/-- Content integrity: BLAKE3 of recovered plaintext vs entry slot (exact error). -/ +def checkContentBlake3 (recovered expectedHash : ByteArray) : Except DirectoryError Unit := + if expectedHash.size != hashLen then + .error .invalidHashLength + else if !ctEq (Carbonado.Bao.Blake3.hash recovered) expectedHash then + .error .contentBlake3Mismatch + else + .ok () + +/-- Look up segment main by root hex + format among artifacts. -/ +def findSegment (segments : Array SegmentArtifact) (root : ByteArray) (fmt : UInt8) : + Option SegmentArtifact := + Id.run do + let mut found : Option SegmentArtifact := none + for i in [:segments.size] do + let s := segments[i]! + if ctEq s.baoRoot root && s.segmentFormat == fmt then + found := some s + pure found + +/-- + Pure directory decode from in-memory archive. + + Returns recovered files sorted by rel_path. Checks content BLAKE3. +-/ +def decodeDirectory (master : ByteArray) (archive : DirectoryArchive) : + Except DirectoryError (Array DirFile) := + match parseCatalogName archive.catalogFilename with + | .error e => .error e + | .ok (expectedRoot, catalogFmt) => + if catalogFmt != archive.catalogFormat then + .error .invalidCatalogPath + else if !ctEq expectedRoot archive.catalogBaoRoot then + .error .catalogBaoRootMismatch + else + match checkMasterPolicy master (catalogFmt &&& 1 != 0) with + | .error e => .error e + | .ok () => + -- Headered decode of catalog + match decodeHeaderedWithHeader master archive.catalogBytes with + | .error pe => .error (ofPipelineError pe) + | .ok (hdr, adamBody) => + if !ctEq hdr.hash expectedRoot then + .error .catalogBaoRootMismatch + else + match decodeAdamantine adamBody with + | .error ae => .error (ofAdamantineError ae) + | .ok (payload, adamHdr) => + if adamHdr.carbonadoFmt != catalogFmt then + .error (.invalidAdamantineCarbonadoFormat adamHdr.carbonadoFmt) + else if adamHdr.flags &&& adamantineFlagRequireOts != 0 then + -- OTS not implemented in Lean product path. + .error .otsFeatureRequired + else + match splitPayload payload with + | .error ae => .error (ofAdamantineError ae) + | .ok (manBytes, baoBundle) => + match FilepackManifest.fromWireBytes manBytes expectedRoot with + | .error pe => .error (ofFilepackError pe) + | .ok manifest => + if manifest.formatLevel != catalogFmt then + .error (.invalidFormatLevel manifest.formatLevel) + else + Id.run do + let mut out : Array DirFile := #[] + let mut err : Option DirectoryError := none + for ei in [:manifest.entries.size] do + if err.isNone then + let entry := manifest.entries[ei]! + let mut recovered := ByteArray.empty + for si in [:entry.segments.size] do + if err.isNone then + let sref := entry.segments[si]! + match findSegment archive.segments sref.segmentBaoRoot + entry.segmentFormat with + | none => err := some .missingSegment + | some art => + if art.main.size != UInt64.toNat sref.mainLen then + err := some .segmentMainLenMismatch + else + match validateSegmentBundleSemantics entry.segmentFormat sref with + | .error e => err := some e + | .ok () => + match bundleSlice baoBundle + (UInt32.toNat sref.verificationOutboardOffset) + (UInt32.toNat sref.verificationOutboardLen) with + | .error ae => err := some (ofAdamantineError ae) + | .ok verOb => + let fmtBits := FormatBits.ofUInt8 entry.segmentFormat + let fecParRes : Except DirectoryError ByteArray := + if fmtBits.fec then + match bundleSlice baoBundle + (UInt32.toNat sref.fecParityOffset) + (UInt32.toNat sref.fecParityLen) with + | .error ae => .error (ofAdamantineError ae) + | .ok p => .ok p + else + .ok ByteArray.empty + match fecParRes with + | .error e => err := some e + | .ok fecPar => + let pad := paddingForMainLen art.main.size fmtBits.fec + match decodeOutboardBody master sref.segmentBaoRoot + art.main verOb fecPar pad fmtBits with + | .error pe => err := some (ofPipelineError pe) + | .ok part => + recovered := appendBA recovered part + if err.isNone then + match checkContentBlake3 recovered entry.contentBlake3 with + | .error e => err := some e + | .ok () => + out := out.push { relPath := entry.relPath, content := recovered } + match err with + | some e => pure (.error e) + | none => pure (.ok out) + +/-- Round-trip pure directory archive. -/ +def roundtripDirectory (master : ByteArray) (files : Array DirFile) + (opts : DirectoryEncodeOptions) (nonces : Array ByteArray) : + Except DirectoryError Bool := + match encodeDirectory master files opts nonces with + | .error e => .error e + | .ok arch => + match decodeDirectory master arch with + | .error e => .error e + | .ok got => + if got.size != files.size then + .ok false + else + -- Compare as multisets by path+content (both sorted by encode). + Id.run do + let mut ok := true + for i in [:got.size] do + let g := got[i]! + -- find matching original + let mut found := false + for j in [:files.size] do + let f := files[j]! + if f.relPath == g.relPath && ctEq f.content g.content then + found := true + if !found then ok := false + pure (.ok ok) + +/-- Map Filepack path errors are distinct (theorems use ofFilepackError). -/ +theorem ofFilepack_traversal : + ofFilepackError .relPathTraversal = DirectoryError.pathTraversal := by + rfl + +theorem ofFilepack_absolute : + ofFilepackError .relPathAbsolute = DirectoryError.pathAbsolute := by + rfl + +theorem ofFilepack_backslash : + ofFilepackError .relPathBackslash = DirectoryError.pathBackslash := by + rfl + +theorem ofFilepack_null : + ofFilepackError .relPathNullByte = DirectoryError.pathNullByte := by + rfl + +theorem ofFilepack_empty_component : + ofFilepackError .relPathEmptyComponent = DirectoryError.pathEmptyComponent := by + rfl + +theorem ofFilepack_too_many_segments : + ofFilepackError .tooManySegments = DirectoryError.tooManySegments := by + rfl + +theorem ofFilepack_ots_too_large : + ofFilepackError .otsProofTooLarge = DirectoryError.otsProofTooLarge := by + rfl + +theorem ofAdamantine_flags : + ofAdamantineError (.invalidFlags 2) = DirectoryError.invalidAdamantineFlags 2 := by + rfl + +theorem ofAdamantine_magic : + ofAdamantineError .invalidMagic = DirectoryError.invalidAdamantineMagic := by + rfl + +theorem expected_fec_parity_empty : expectedFecParityLen 0 = 0 := by native_decide + +theorem expected_fec_parity_one : expectedFecParityLen 1 = 16384 := by native_decide + +end Carbonado.Directory diff --git a/Carbonado/Fec.lean b/Carbonado/Fec.lean new file mode 100644 index 0000000..8bd3b74 --- /dev/null +++ b/Carbonado/Fec.lean @@ -0,0 +1,9 @@ +/- + Carbonado Reed–Solomon 4/8 FEC surface (Program C). + + Modules: Galois (GF(2^8)), Matrix, RS codec, Inboard geometry. +-/ +import Carbonado.Fec.Galois +import Carbonado.Fec.Matrix +import Carbonado.Fec.RS +import Carbonado.Fec.Inboard diff --git a/Carbonado/Fec/Galois.lean b/Carbonado/Fec/Galois.lean new file mode 100644 index 0000000..dfebf0f --- /dev/null +++ b/Carbonado/Fec/Galois.lean @@ -0,0 +1,133 @@ +/- + GF(2^8) arithmetic matching `reed-solomon-erasure` 5.0.3 (`galois_8`). + + Generating polynomial = 29 (0x1d), same as the crate's `build.rs`. + Log/exp tables are built with the same algorithm so mul/div/exp bit-match. +-/ + +namespace Carbonado.Fec.Galois + +/-- Field order. -/ +def order : Nat := 256 + +/-- Generating polynomial used by reed-solomon-erasure (AES poly 0x1d). -/ +def generatingPolynomial : Nat := 29 + +/-- Build LOG_TABLE[b] = discrete log of `b` (log of 0 is left 0). -/ +def genLogTable : Array UInt8 := + Id.run do + let mut result : Array UInt8 := Array.replicate 256 (0 : UInt8) + let mut b : Nat := 1 + for log in [:255] do + result := result.set! b (UInt8.ofNat log) + b := b <<< 1 + if b ≥ 256 then + b := (b - 256).xor generatingPolynomial + pure result + +/-- Build EXP_TABLE of length 510: dual copy so mul can index `log_a + log_b` without mod. -/ +def genExpTable (logTable : Array UInt8) : Array UInt8 := + Id.run do + let mut result : Array UInt8 := Array.replicate 510 (0 : UInt8) + for i in [1:256] do + let log := (logTable[i]!).toNat + let v := UInt8.ofNat i + result := result.set! log v + result := result.set! (log + 255) v + pure result + +/-- Discrete-log table (index 0 unused / zero). -/ +def logTable : Array UInt8 := genLogTable + +/-- Antilog table (size 510). -/ +def expTable : Array UInt8 := genExpTable logTable + +/-- Addition = XOR. -/ +@[inline] def add (a b : UInt8) : UInt8 := a ^^^ b + +/-- Subtraction = XOR (characteristic 2). -/ +@[inline] def sub (a b : UInt8) : UInt8 := a ^^^ b + +/-- Multiplication via log/exp tables. -/ +def mul (a b : UInt8) : UInt8 := + if a == 0 || b == 0 then + 0 + else + let la := (logTable[a.toNat]!).toNat + let lb := (logTable[b.toNat]!).toNat + expTable[la + lb]! + +/-- Division via log/exp. Divisor must be non-zero (caller invariant). -/ +def div (a b : UInt8) : UInt8 := + if a == 0 then + 0 + else + let la := (logTable[a.toNat]!).toNat + let lb := (logTable[b.toNat]!).toNat + let logResult : Int := (Int.ofNat la) - (Int.ofNat lb) + let logResult := if logResult < 0 then logResult + 255 else logResult + expTable[logResult.toNat]! + +/-- `a^n` in GF(2^8). -/ +def exp (a : UInt8) (n : Nat) : UInt8 := + if n == 0 then + 1 + else if a == 0 then + 0 + else + let logA := (logTable[a.toNat]!).toNat + let logResult := (logA * n) % 255 + expTable[logResult]! + +/-- Field element index `n` (nth_internal = identity for GF(2^8) in the crate). -/ +@[inline] def nth (n : Nat) : UInt8 := UInt8.ofNat (n % 256) + +/-- Multiply every element of `input` by `c` into a fresh array. -/ +def mulSlice (c : UInt8) (input : Array UInt8) : Array UInt8 := + Id.run do + let mut out : Array UInt8 := Array.mkEmpty input.size + for i in [:input.size] do + out := out.push (mul c (input[i]!)) + pure out + +/-- `out[i] ^= mul(c, input[i])` (lengths must match). -/ +def mulSliceAdd (c : UInt8) (input : Array UInt8) (out : Array UInt8) : Array UInt8 := + Id.run do + let mut result := out + for i in [:input.size] do + let v := add (result[i]!) (mul c (input[i]!)) + result := result.set! i v + pure result + +/-- Multiply every byte of a `ByteArray` by `c`. -/ +def mulSliceBA (c : UInt8) (input : ByteArray) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:input.size] do + out := out.push (mul c (input.get! i)) + pure out + +/-- XOR-accumulate `mul(c, input)` into `out` (ByteArray form). -/ +def mulSliceAddBA (c : UInt8) (input out : ByteArray) : ByteArray := + Id.run do + let mut result := out + for i in [:input.size] do + let v := add (result.get! i) (mul c (input.get! i)) + result := result.set! i v + pure result + +/-! ## Parity samples vs reed-solomon-erasure galois_8 -/ + +theorem mul_1_1 : mul 1 1 = 1 := by native_decide +theorem mul_2_3 : mul 2 3 = 6 := by native_decide +theorem mul_0x53_0xca : mul 0x53 0xca = 0x8f := by native_decide +theorem mul_0xff_1 : mul 0xff 1 = 0xff := by native_decide +theorem mul_7_11 : mul 7 11 = 0x31 := by native_decide + +theorem div_2_3 : div 2 3 = 0xf5 := by native_decide +theorem exp_2_3 : exp 2 3 = 8 := by native_decide +theorem exp_0x53_3 : exp 0x53 3 = 0xd0 := by native_decide + +theorem mul_comm_7_11 : mul 7 11 = mul 11 7 := by native_decide + +end Carbonado.Fec.Galois diff --git a/Carbonado/Fec/Inboard.lean b/Carbonado/Fec/Inboard.lean new file mode 100644 index 0000000..68665f1 --- /dev/null +++ b/Carbonado/Fec/Inboard.lean @@ -0,0 +1,234 @@ +/- + Carbonado FEC geometry and inboard encode/decode. + + Normative (matches `src/utils.rs` + `src/stream/fec.rs`): + * `stripeUnit = sliceLen * fecK = 16384` + * `calc_padding_len` pads logical length up to a multiple of `stripeUnit` + * chunk_len = padded_len / fecK + * Inboard body = 8 concatenated shards of length `chunk_len` + * Decode: split → reconstruct any 4/8 → concat data → strip padding + + Memory (LIMITS): encode/decode materialize O(stripe) = O(padded logical × 2) + for a single segment-wide stripe (same residual as Rust FEC path). +-/ +import Carbonado.Constants +import Carbonado.Fec.RS + +namespace Carbonado.Fec.Inboard + +open Carbonado.Constants +open Carbonado.Fec.RS + +/-- Result of padding geometry: `(padding_len, chunk_len)`. -/ +structure PaddingInfo where + paddingLen : Nat + chunkLen : Nat + deriving DecidableEq, Repr + +/-- + `calc_padding_len` — pad to a multiple of `stripeUnit` (`sliceLen * fecK`). + + Empty input → `(0, 0)`. Otherwise: + `target = ceil(input / stripeUnit) * stripeUnit`, + `padding = target - input`, + `chunk = target / fecK`. +-/ +def calcPaddingLen (inputLen : Nat) : PaddingInfo := + if inputLen == 0 then + { paddingLen := 0, chunkLen := 0 } + else + let stripe := stripeUnit + let target := ((inputLen + stripe - 1) / stripe) * stripe + let paddingLen := target - inputLen + let chunkLen := target / fecK + { paddingLen := paddingLen, chunkLen := chunkLen } + +/-- Padded length for a logical payload. -/ +def paddedLen (inputLen : Nat) : Nat := + let p := calcPaddingLen inputLen + if inputLen == 0 then 0 else inputLen + p.paddingLen + +/-- Zero-extend `input` to `targetLen`. -/ +def padWithZeros (input : ByteArray) (targetLen : Nat) : ByteArray := + if input.size ≥ targetLen then + input.extract 0 targetLen + else + Id.run do + let mut out := input + for _ in [:targetLen - input.size] do + out := out.push 0 + pure out + +/-- Split a padded buffer into `fecK` data shards of `chunkLen`. -/ +def splitDataShards (padded : ByteArray) (chunkLen : Nat) : Except FecError (Array ByteArray) := + if chunkLen == 0 then + .error .badGeometry + else if padded.size != chunkLen * fecK then + .error .unevenShards + else + Id.run do + let mut shards : Array ByteArray := Array.mkEmpty fecK + for i in [:fecK] do + shards := shards.push (padded.extract (i * chunkLen) ((i + 1) * chunkLen)) + pure (.ok shards) + +/-- Concatenate shards in order. -/ +def concatShards (shards : Array ByteArray) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:shards.size] do + out := out.append (shards[i]!) + pure out + +/-- Split inboard body into `fecM` equal shards. -/ +def splitInboard (body : ByteArray) : Except FecError (Array ByteArray) := + if body.size == 0 then + .ok #[] + else if body.size % fecM != 0 then + .error .unevenShards + else + -- body.size > 0 ∧ divisible by fecM ⇒ chunkLen ≥ 1 + let chunkLen := body.size / fecM + Id.run do + let mut shards : Array ByteArray := Array.mkEmpty fecM + for i in [:fecM] do + shards := shards.push (body.extract (i * chunkLen) ((i + 1) * chunkLen)) + pure (.ok shards) + +/-- + Encode logical bytes to inboard FEC body. + + Returns `(inboard_body, padding_len, chunk_len)`. + Empty input → empty body, zero geometry. +-/ +def encodeInboard (input : ByteArray) : Except FecError (ByteArray × Nat × Nat) := + if input.size == 0 then + .ok (ByteArray.empty, 0, 0) + else + let geo := calcPaddingLen input.size + let target := input.size + geo.paddingLen + let padded := padWithZeros input target + match splitDataShards padded geo.chunkLen with + | .error e => .error e + | .ok dataShards => + Id.run do + let mut shards : Array ByteArray := dataShards + for _ in [:fecM - fecK] do + shards := shards.push (padWithZeros ByteArray.empty geo.chunkLen) + match carbonadoRS.encode shards with + | .error e => pure (.error e) + | .ok encoded => + pure (.ok (concatShards encoded, geo.paddingLen, geo.chunkLen)) + +/-- Concatenate first `fecK` data shards and strip `padding` trailing zeros. -/ +def stripPadding (dataShards : Array ByteArray) (padding : Nat) : Except FecError ByteArray := + let cat := concatShards dataShards + if padding > cat.size then + .error .paddingTooLarge + else + .ok (cat.extract 0 (cat.size - padding)) + +/-- + Decode full inboard body (all 8 shards present) with encode-time padding length. +-/ +def decodeInboard (body : ByteArray) (padding : Nat) : Except FecError ByteArray := + if body.size == 0 then + if padding == 0 then .ok ByteArray.empty + else .error .paddingTooLarge + else + match splitInboard body with + | .error e => .error e + | .ok shards => + Id.run do + let mut opts : Array (Option ByteArray) := Array.mkEmpty fecM + for i in [:shards.size] do + opts := opts.push (some (shards[i]!)) + match carbonadoRS.reconstruct opts with + | .error e => pure (.error e) + | .ok full => + let mut data : Array ByteArray := Array.mkEmpty fecK + for i in [:fecK] do + data := data.push (full[i]!) + pure (stripPadding data padding) + +/-- + Reconstruct logical payload from optional shards (scrub / chaos path). + + Requires at least `fecK` present shards of equal non-zero length. + `padding` is the encode-time padding to strip from reconstructed data. +-/ +def reconstructLogical (shards : Array (Option ByteArray)) (padding : Nat) : + Except FecError ByteArray := + if shards.size != fecM then + .error .badGeometry + else + match carbonadoRS.reconstruct shards with + | .error e => .error e + | .ok full => + Id.run do + let mut data : Array ByteArray := Array.mkEmpty fecK + for i in [:fecK] do + data := data.push (full[i]!) + pure (stripPadding data padding) + +/-- + Knock out (erase) shards at the given indices and reconstruct. + + Pure scrub helper without Bao: validates that any 4 of 8 suffice when + the remaining shards are intact. + + Every index in `missing` must be `< fecM`; out-of-range indices → `badGeometry` + (no silent ignore). +-/ +def reconstructAfterKnockout (encoded : Array ByteArray) (missing : List Nat) (padding : Nat) : + Except FecError ByteArray := + if encoded.size != fecM then + .error .badGeometry + else if missing.any (fun i => decide (i ≥ fecM)) then + .error .badGeometry + else + Id.run do + let mut opts : Array (Option ByteArray) := Array.mkEmpty fecM + for i in [:fecM] do + if missing.contains i then + opts := opts.push none + else + opts := opts.push (some (encoded[i]!)) + pure (reconstructLogical opts padding) + +/-- Split encoded inboard body into an array of 8 shards (for knockout tests). -/ +def inboardToShards (body : ByteArray) : Except FecError (Array ByteArray) := + splitInboard body + +/-- Geometry theorems: padding identity. -/ +theorem calcPaddingLen_zero : calcPaddingLen 0 = { paddingLen := 0, chunkLen := 0 } := by + native_decide + +theorem calcPaddingLen_one : + calcPaddingLen 1 = { paddingLen := 16383, chunkLen := 4096 } := by + native_decide + +theorem calcPaddingLen_stripe : + calcPaddingLen 16384 = { paddingLen := 0, chunkLen := 4096 } := by + native_decide + +theorem calcPaddingLen_stripe_plus_one : + calcPaddingLen 16385 = { paddingLen := 16383, chunkLen := 8192 } := by + native_decide + +theorem calcPaddingLen_100 : + calcPaddingLen 100 = { paddingLen := 16284, chunkLen := 4096 } := by + native_decide + +/-- One slice (4096): pad three remaining slices of the stripe. -/ +theorem calcPaddingLen_4096 : + calcPaddingLen 4096 = { paddingLen := 12288, chunkLen := 4096 } := by + native_decide + +/-- Concrete padded lengths align to `stripeUnit` (product-relevant sizes). -/ +theorem paddedLen_aligns_samples : + (List.map paddedLen [1, 100, 4096, 16384, 16385]).all + (fun p => p % stripeUnit == 0) = true := by + native_decide + +end Carbonado.Fec.Inboard diff --git a/Carbonado/Fec/Matrix.lean b/Carbonado/Fec/Matrix.lean new file mode 100644 index 0000000..1f058b8 --- /dev/null +++ b/Carbonado/Fec/Matrix.lean @@ -0,0 +1,147 @@ +/- + Dense matrices over GF(2^8) for Reed–Solomon encoding matrices. + + Matches `reed-solomon-erasure` `matrix.rs`: Vandermonde, Gaussian elimination + invert, multiply, augment, sub_matrix. +-/ +import Carbonado.Fec.Galois + +namespace Carbonado.Fec.Matrix + +open Carbonado.Fec.Galois + +/-- Row-major matrix of GF elements. -/ +structure Matrix where + rows : Nat + cols : Nat + /-- Flattened `rows * cols` elements, row-major. -/ + data : Array UInt8 + deriving Repr + +def Matrix.get (m : Matrix) (r c : Nat) : UInt8 := + m.data[r * m.cols + c]! + +def Matrix.set (m : Matrix) (r c : Nat) (v : UInt8) : Matrix := + { m with data := m.data.set! (r * m.cols + c) v } + +def Matrix.zeros (rows cols : Nat) : Matrix := + { rows := rows, cols := cols, data := Array.replicate (rows * cols) 0 } + +def Matrix.identity (size : Nat) : Matrix := + Id.run do + let mut m := zeros size size + for i in [:size] do + m := m.set i i 1 + pure m + +def Matrix.getRow (m : Matrix) (row : Nat) : Array UInt8 := + Id.run do + let mut out : Array UInt8 := Array.mkEmpty m.cols + for c in [:m.cols] do + out := out.push (m.get row c) + pure out + +def Matrix.swapRows (m : Matrix) (r1 r2 : Nat) : Matrix := + if r1 == r2 then m + else + Id.run do + let mut result := m + for c in [:m.cols] do + let a := result.get r1 c + let b := result.get r2 c + result := result.set r1 c b + result := result.set r2 c a + pure result + +def Matrix.multiply (lhs rhs : Matrix) : Matrix := + Id.run do + let mut result := zeros lhs.rows rhs.cols + for r in [:lhs.rows] do + for c in [:rhs.cols] do + let mut val : UInt8 := 0 + for i in [:lhs.cols] do + val := add val (mul (lhs.get r i) (rhs.get i c)) + result := result.set r c val + pure result + +def Matrix.augment (lhs rhs : Matrix) : Matrix := + Id.run do + let mut result := zeros lhs.rows (lhs.cols + rhs.cols) + for r in [:lhs.rows] do + for c in [:lhs.cols] do + result := result.set r c (lhs.get r c) + for c in [:rhs.cols] do + result := result.set r (lhs.cols + c) (rhs.get r c) + pure result + +def Matrix.subMatrix (m : Matrix) (rmin cmin rmax cmax : Nat) : Matrix := + Id.run do + let mut result := zeros (rmax - rmin) (cmax - cmin) + for r in [rmin:rmax] do + for c in [cmin:cmax] do + result := result.set (r - rmin) (c - cmin) (m.get r c) + pure result + +/-- Gaussian elimination to RREF (in-place style). Returns `none` if singular. -/ +def Matrix.gaussianElim (m : Matrix) : Option Matrix := + Id.run do + let mut work := m + for r in [:work.rows] do + if work.get r r == 0 then + let mut found := false + for rBelow in [r+1:work.rows] do + if !found && work.get rBelow r != 0 then + work := work.swapRows r rBelow + found := true + if work.get r r == 0 then + return none + if work.get r r != 1 then + let scale := div 1 (work.get r r) + for c in [:work.cols] do + work := work.set r c (mul scale (work.get r c)) + for rBelow in [r+1:work.rows] do + if work.get rBelow r != 0 then + let scale := work.get rBelow r + for c in [:work.cols] do + let v := add (work.get rBelow c) (mul scale (work.get r c)) + work := work.set rBelow c v + -- Clear above diagonal + for d in [:work.rows] do + for rAbove in [:d] do + if work.get rAbove d != 0 then + let scale := work.get rAbove d + for c in [:work.cols] do + let v := add (work.get rAbove c) (mul scale (work.get d c)) + work := work.set rAbove c v + pure (some work) + +/-- Invert a square matrix; `none` if singular. -/ +def Matrix.invert (m : Matrix) : Option Matrix := + if m.rows != m.cols then + none + else + let n := m.rows + let work := m.augment (identity n) + match work.gaussianElim with + | none => none + | some reduced => some (reduced.subMatrix 0 n n (n * 2)) + +/-- Vandermonde: `M[r,c] = nth(r)^c`. -/ +def Matrix.vandermonde (rows cols : Nat) : Matrix := + Id.run do + let mut result := zeros rows cols + for r in [:rows] do + let rA := nth r + for c in [:cols] do + result := result.set r c (exp rA c) + pure result + +/-- Systematic RS generator: `V * inv(top)`. -/ +def Matrix.buildRSMatrix (dataShards totalShards : Nat) : Option Matrix := + let vandermonde := Matrix.vandermonde totalShards dataShards + let top := vandermonde.subMatrix 0 0 dataShards dataShards + match top.invert with + | none => none + | some invTop => some (vandermonde.multiply invTop) + +end Carbonado.Fec.Matrix diff --git a/Carbonado/Fec/RS.lean b/Carbonado/Fec/RS.lean new file mode 100644 index 0000000..58e42c7 --- /dev/null +++ b/Carbonado/Fec/RS.lean @@ -0,0 +1,286 @@ +/- + Reed–Solomon erasure codec over GF(2^8), matching `reed-solomon-erasure` 5.0.3. + + Carbonado uses k=4 data + 4 parity (n=8). Encode fills parity from data; + reconstruct recovers missing shards from any k present shards. +-/ +import Carbonado.Constants +import Carbonado.Fec.Galois +import Carbonado.Fec.Matrix + +namespace Carbonado.Fec.RS + +open Carbonado.Constants +open Carbonado.Fec.Galois +open Carbonado.Fec.Matrix + +/-- Strict FEC error taxonomy (distinct failure modes; match exactly in tests). -/ +inductive FecError where + /-- Input length does not divide evenly over shard count / stripe geometry. -/ + | unevenShards + /-- Fewer than `dataShards` present shards for reconstruct. -/ + | tooFewShards + /-- A present shard has zero length. -/ + | emptyShard + /-- Present shards disagree on length. -/ + | incorrectShardSize + /-- Wrong number of shards for the codec (not k+parity), or invalid knockout indices. -/ + | badGeometry + /-- Padding length exceeds reconstructed data length. -/ + | paddingTooLarge + /-- RS matrix inversion failed (singular submatrix). Unreachable for valid RS(4,4) + with any k distinct generator rows; tested via `invertOrSingular` on singular matrices. -/ + | singularMatrix + deriving DecidableEq, Repr + +/-- Reed–Solomon codec parameters + precomputed systematic generator matrix. -/ +structure ReedSolomon where + dataShards : Nat + parityShards : Nat + /-- `totalShards × dataShards` systematic generator (top is identity). -/ + matrix : Matrix + deriving Repr + +def ReedSolomon.totalShards (rs : ReedSolomon) : Nat := + rs.dataShards + rs.parityShards + +/-- Invert a matrix, mapping singular → `FecError.singularMatrix` (strict taxonomy surface). -/ +def invertOrSingular (m : Matrix) : Except FecError Matrix := + match m.invert with + | none => .error .singularMatrix + | some inv => .ok inv + +/-- Construct a codec. Fails on zero data/parity or total > 256. -/ +def ReedSolomon.new (dataShards parityShards : Nat) : Except FecError ReedSolomon := + if dataShards == 0 || parityShards == 0 then + .error .badGeometry + else if dataShards + parityShards > order then + .error .badGeometry + else + let total := dataShards + parityShards + match Matrix.buildRSMatrix dataShards total with + | none => .error .singularMatrix + | some m => + .ok { + dataShards := dataShards + parityShards := parityShards + matrix := m + } + +/-- Product codec construction (uses Constants geometry; no silent zero-matrix fallback). -/ +def carbonadoRSExcept : Except FecError ReedSolomon := + ReedSolomon.new fecK (fecM - fecK) + +/-- Product RS(4,4) constructs successfully (native). -/ +theorem carbonadoRS_constructs : + (match carbonadoRSExcept with | .ok _ => true | .error _ => false) = true := by + native_decide + +/-- + Carbonado product codec: RS(`fecK`, `fecM - fecK`) → 8 total shards. + + Error branch is eliminated by `carbonadoRS_constructs` — never substitutes a + zero generator matrix (review issue #1). +-/ +def carbonadoRS : ReedSolomon := + match h : carbonadoRSExcept with + | .ok rs => rs + | .error _ => + False.elim <| by + have ht : (match carbonadoRSExcept with | .ok _ => true | .error _ => false) = true := + carbonadoRS_constructs + simp only [h] at ht + exact Bool.noConfusion ht + +/-- Codec geometry matches `Constants.fecK` / `fecM`. -/ +theorem carbonadoRS_geometry : + carbonadoRS.dataShards = fecK ∧ carbonadoRS.parityShards = fecM - fecK := by + native_decide + +/-- Zero-filled ByteArray of length `n`. -/ +private def zerosBA (n : Nat) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for _ in [:n] do + out := out.push 0 + pure out + +/-- Encode parity into the last `parityShards` of `shards` (data in first k). -/ +def ReedSolomon.encode (rs : ReedSolomon) (shards : Array ByteArray) : + Except FecError (Array ByteArray) := + -- `ReedSolomon.new` requires dataShards ≥ 1 and parityShards ≥ 1 ⇒ totalShards ≥ 2. + if shards.size != rs.totalShards then + .error .badGeometry + else + let shardLen := (shards[0]!).size + if shardLen == 0 then + .error .emptyShard + else + Id.run do + let mut ok := true + for i in [:shards.size] do + if (shards[i]!).size != shardLen then + ok := false + if !ok then + pure (.error .incorrectShardSize) + else + let mut out := shards + for p in [:rs.parityShards] do + let row := rs.dataShards + p + let mut parity := zerosBA shardLen + for iData in [:rs.dataShards] do + let coeff := rs.matrix.get row iData + let data := out[iData]! + if iData == 0 then + parity := mulSliceBA coeff data + else + parity := mulSliceAddBA coeff data parity + out := out.set! (rs.dataShards + p) parity + pure (.ok out) + +/-- Collect present-shard length; enforce non-empty + uniform. -/ +private def presentShardLen (shards : Array (Option ByteArray)) : + Except FecError Nat := + Id.run do + let mut found : Option Nat := none + for i in [:shards.size] do + match shards[i]! with + | none => pure () + | some s => + if s.size == 0 then + return .error .emptyShard + match found with + | none => found := some s.size + | some n => + if s.size != n then + return .error .incorrectShardSize + match found with + | none => pure (.error .tooFewShards) + | some n => pure (.ok n) + +/-- Reconstruct all shards (data + parity) from any `dataShards` present. -/ +def ReedSolomon.reconstruct (rs : ReedSolomon) (shards : Array (Option ByteArray)) : + Except FecError (Array ByteArray) := + if shards.size != rs.totalShards then + .error .badGeometry + else + match presentShardLen shards with + | .error e => .error e + | .ok shardLen => + Id.run do + let mut numberPresent := 0 + for i in [:shards.size] do + if (shards[i]!).isSome then + numberPresent := numberPresent + 1 + if numberPresent < rs.dataShards then + pure (.error .tooFewShards) + else if numberPresent == rs.totalShards then + let mut out : Array ByteArray := Array.mkEmpty rs.totalShards + for i in [:shards.size] do + out := out.push (shards[i]!).get! + pure (.ok out) + else + let mut validIndices : Array Nat := Array.mkEmpty rs.dataShards + let mut invalidIndices : Array Nat := Array.mkEmpty rs.dataShards + let mut subShards : Array ByteArray := Array.mkEmpty rs.dataShards + for matrixRow in [:rs.totalShards] do + match shards[matrixRow]! with + | some s => + if validIndices.size < rs.dataShards then + validIndices := validIndices.push matrixRow + subShards := subShards.push s + | none => + invalidIndices := invalidIndices.push matrixRow + let mut sub := Matrix.zeros rs.dataShards rs.dataShards + for subRow in [:rs.dataShards] do + let validIndex := validIndices[subRow]! + for c in [:rs.dataShards] do + sub := sub.set subRow c (rs.matrix.get validIndex c) + match invertOrSingular sub with + | .error e => pure (.error e) + | .ok dataDecode => + let mut out : Array ByteArray := Array.mkEmpty rs.totalShards + for i in [:rs.totalShards] do + match shards[i]! with + | some s => out := out.push s + | none => out := out.push (zerosBA shardLen) + for invIdx in [:invalidIndices.size] do + let iSlice := invalidIndices[invIdx]! + if iSlice < rs.dataShards then + let row := dataDecode.getRow iSlice + let mut decoded := zerosBA shardLen + for iData in [:rs.dataShards] do + let coeff := row[iData]! + let src := subShards[iData]! + if iData == 0 then + decoded := mulSliceBA coeff src + else + decoded := mulSliceAddBA coeff src decoded + out := out.set! iSlice decoded + let mut dataOnly : Array ByteArray := Array.mkEmpty rs.dataShards + for i in [:rs.dataShards] do + dataOnly := dataOnly.push (out[i]!) + for invIdx in [:invalidIndices.size] do + let iSlice := invalidIndices[invIdx]! + if iSlice ≥ rs.dataShards then + let p := iSlice - rs.dataShards + let mut parity := zerosBA shardLen + for iData in [:rs.dataShards] do + let coeff := rs.matrix.get (rs.dataShards + p) iData + let src := dataOnly[iData]! + if iData == 0 then + parity := mulSliceBA coeff src + else + parity := mulSliceAddBA coeff src parity + out := out.set! iSlice parity + pure (.ok out) + +/-- Reconstruct data shards only. -/ +def ReedSolomon.reconstructData (rs : ReedSolomon) (shards : Array (Option ByteArray)) : + Except FecError (Array ByteArray) := + match rs.reconstruct shards with + | .error e => .error e + | .ok full => + Id.run do + let mut data : Array ByteArray := Array.mkEmpty rs.dataShards + for i in [:rs.dataShards] do + data := data.push (full[i]!) + pure (.ok data) + +/-- Verify parity matches data under this codec. -/ +def ReedSolomon.verify (rs : ReedSolomon) (shards : Array ByteArray) : Except FecError Bool := + if shards.size != rs.totalShards then + .error .badGeometry + else + match rs.encode shards with + | .error e => .error e + | .ok encoded => + Id.run do + let mut ok := true + for p in [:rs.parityShards] do + let i := rs.dataShards + p + let a := encoded[i]! + let b := shards[i]! + if a.size != b.size then + ok := false + else + for j in [:a.size] do + if a.get! j != b.get! j then + ok := false + pure (.ok ok) + +/-- Singular 2×2 zero matrix maps to `singularMatrix` (taxonomy surface for invert). -/ +theorem invertOrSingular_zeros : + (match invertOrSingular (Matrix.zeros 2 2) with + | .error .singularMatrix => true + | _ => false) = true := by + native_decide + +/-- Identity inverts cleanly. -/ +theorem invertOrSingular_identity : + (match invertOrSingular (Matrix.identity 2) with + | .ok m => m.get 0 0 == 1 && m.get 1 1 == 1 + | .error _ => false) = true := by + native_decide + +end Carbonado.Fec.RS diff --git a/Carbonado/Ffi.lean b/Carbonado/Ffi.lean new file mode 100644 index 0000000..3ec6b87 --- /dev/null +++ b/Carbonado/Ffi.lean @@ -0,0 +1,102 @@ +/- + C ABI surface for dual-backend parity (docs/ABI.md). + + Pure helpers map Pipeline results to ABI error codes. `@[export]` entry points + are thin wrappers for the Lean AOT static library (`libcarbonado`). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Bao.Product +import Carbonado.Header +import Carbonado.Pipeline + +namespace Carbonado.Ffi + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Bao.Product +open Carbonado.Header +open Carbonado.Pipeline + +/-- ABI version (must match `include/carbonado.h` / docs/ABI.md). -/ +def abiVersion : UInt32 := 1 + +@[export carbonado_abi_version] +def carbonado_abi_version : UInt32 := abiVersion + +/-- Stable C error codes (docs/ABI.md). -/ +def ok : UInt32 := 0 +def errInvalidArgument : UInt32 := 1 +def errInvalidKeyLength : UInt32 := 2 +def errAuthentication : UInt32 := 3 +def errInvalidMagic : UInt32 := 4 +def errInvalidHeader : UInt32 := 5 +def errFec : UInt32 := 6 +def errBao : UInt32 := 7 +def errZstd : UInt32 := 8 +def errScrubUnnecessary : UInt32 := 9 +def errScrubFailed : UInt32 := 10 +def errNotImplemented : UInt32 := 11 +def errInternal : UInt32 := 12 + +/-- Collapse `PipelineError` into ABI codes (exhaustive; docs/ABI.md). -/ +def ofPipelineError : PipelineError → UInt32 + | .invalidKeyLength => errInvalidKeyLength + | .payloadAuthenticationFailed | .headerAuthenticationFailed => errAuthentication + | .badMagic => errInvalidMagic + | .invalidHeaderLength | .truncatedBody | .invalidFieldLength => errInvalidHeader + | .unevenShards | .tooFewShards | .emptyShard | .incorrectShardSize + | .badGeometry | .paddingTooLarge | .singularMatrix => errFec + | .baoAuthenticationFailed | .truncatedResponse | .trailingData + | .invalidPrefix | .invalidRootLength | .invalidSliceIndex | .invalidSliceCount => errBao + | .compressionFailed | .decompressionFailed | .decompressOutputTooLarge + | .zstdInvalidInput => errZstd + | .unnecessaryScrub => errScrubUnnecessary + | .scrubRequiresVerification | .invalidScrubbedHash => errScrubFailed + | .invalidCiphertextLength | .invalidNonceLength | .insufficientNonces => errInvalidArgument + | .invalidChunkSequence | .emptySegment => errInvalidArgument + +def masterOk (master : ByteArray) : Bool := + master.size == 32 || master.size == 64 + +/-- Pure headered encode for FFI (explicit nonce; zero SLH/meta). -/ +def encodeHeaderedBytes (master nonce plaintext : ByteArray) (format : UInt8) : + Except UInt32 ByteArray := + if !masterOk master then .error errInvalidKeyLength + else if (FormatBits.ofUInt8 format).encrypted && nonce.size != nonceLen then + .error errInvalidArgument + else + let fmt := FormatBits.ofUInt8 format + let n := if fmt.encrypted then nonce else ByteArray.mkEmpty 0 + match encodeHeadered master n plaintext fmt 0 (ByteArray.mkEmpty 32) (ByteArray.mkEmpty 8) with + | .error e => .error (ofPipelineError e) + | .ok (_hdr, archive) => .ok archive + +/-- Pure headered decode for FFI. -/ +def decodeHeaderedBytes (master archive : ByteArray) : Except UInt32 ByteArray := + if !masterOk master then .error errInvalidKeyLength + else + match decodeHeadered master archive with + | .error e => .error (ofPipelineError e) + | .ok pt => .ok pt + +/-- Format verification key (32 bytes). -/ +def verificationKeyBytes (format : UInt8) : ByteArray := + carbonadoVerificationKey format + +/-- Round-trip self-check (public or encrypted with given nonce). -/ +def roundtripHeaderedOk (master nonce plaintext : ByteArray) (format : UInt8) : Bool := + match encodeHeaderedBytes master nonce plaintext format with + | .error _ => false + | .ok arch => + match decodeHeaderedBytes master arch with + | .error _ => false + | .ok pt => ctEq pt plaintext + +theorem abiVersion_eq : abiVersion = 1 := by native_decide + +theorem masterOk_32 : masterOk (ByteArray.mkArray 32 0) = true := by native_decide + +theorem masterOk_31 : masterOk (ByteArray.mkArray 31 0) = false := by native_decide + +end Carbonado.Ffi diff --git a/Carbonado/Filepack.lean b/Carbonado/Filepack.lean new file mode 100644 index 0000000..16a8ae2 --- /dev/null +++ b/Carbonado/Filepack.lean @@ -0,0 +1,569 @@ +/- + FilepackManifest v2 for Adamantine catalogs (Program G). + + **Wire note (LIMITS):** Rust uses rkyv `FilepackManifestWire`. Lean ships a + deterministic **CFP2** native codec with the same *logical* fields (version, + format_level, entries with SegmentRef + content_blake3). Adamantine envelope + framing matches Rust (`manifest_len` + body + `bundle_len` + bundle); the + *manifest body* is Lean-native CFP2, not rkyv — interop with Rust-produced + catalogs requires a converter (tracked LIMITS). Product CLI uses CFP2 end-to-end. + + Path rules: fail-closed (no `..`, no absolute, no backslash, length caps). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Adamantine + +namespace Carbonado.Filepack + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Adamantine + +/-- FilepackManifest wire schema version (v2). -/ +def filepackManifestVersion : Nat := 2 + +/-- Max entries (DoS). -/ +def maxFilepackEntries : Nat := 100000 + +/-- Max rel_path bytes. -/ +def maxRelPathLen : Nat := 4096 + +/-- Max OTS proof blob. -/ +def maxOtsProofLen : Nat := 65536 + +/-- Max segments per entry. -/ +def maxSegmentsPerEntry : Nat := 10000 + +/-- Max total segment refs. -/ +def maxTotalSegmentRefs : Nat := 1000000 + +/-- Max bare segment main bytes. -/ +def maxSegmentMainLen : Nat := 256 * 1024 * 1024 + +/-- CFP2 magic (`CFP2`). -/ +def cfp2Magic : List UInt8 := [0x43, 0x46, 0x50, 0x32] + +theorem cfp2Magic_length : cfp2Magic.length = 4 := by native_decide + +/-- Segment format constants (directory). -/ +def segmentFormatPublicRaw : UInt8 := 0x0C -- c12 +def segmentFormatPublicCompressed : UInt8 := 0x0E -- c14 +def segmentFormatEncryptedRaw : UInt8 := 0x0D -- c13 +def segmentFormatEncryptedCompressed : UInt8 := 0x0F -- c15 + +/-- Strict Filepack / path error taxonomy. -/ +inductive FilepackError where + /-- Empty relative path. -/ + | emptyRelPath + /-- rel_path exceeds max length. -/ + | relPathTooLong + /-- Backslash present (must use `/`). -/ + | relPathBackslash + /-- Absolute path (leading `/`). -/ + | relPathAbsolute + /-- `..` component present. -/ + | relPathTraversal + /-- Empty path component (e.g. `//` or trailing `/` with empty). -/ + | relPathEmptyComponent + /-- Null byte in path. -/ + | relPathNullByte + /-- Manifest version ≠ 2. -/ + | unsupportedVersion (v : Nat) + /-- format_level not c14/c15. -/ + | invalidFormatLevel (fmt : UInt8) + /-- Segment format not c12–c15 or encryption mismatch. -/ + | segmentFormatMismatch (fmt : UInt8) + /-- Legacy c4–c7 segment format rejected. -/ + | legacySegmentFormat (fmt : UInt8) + /-- Entry has zero segments. -/ + | emptySegments + /-- chunk_index not contiguous 0..n-1. -/ + | invalidChunkSequence + /-- main_len exceeds DoS cap. -/ + | mainLenTooLarge + /-- Entry count / total refs / wire length cap. -/ + | tooManyEntries + /-- Too many segments on one entry. -/ + | tooManySegments + /-- Wire parse failure (truncated / bad magic / length). -/ + | invalidWire + /-- Root / hash field wrong size. -/ + | invalidHashLength + /-- OTS proof too large. -/ + | otsProofTooLarge + /-- Entries not sorted by rel_path. -/ + | entriesUnsorted + deriving DecidableEq, Repr + +/-- One bare segment main reference (bundle offsets into Adamantine Bao bundle). -/ +structure SegmentRef where + segmentBaoRoot : ByteArray + chunkIndex : UInt32 + mainLen : UInt64 + verificationOutboardOffset : UInt32 + verificationOutboardLen : UInt32 + fecParityOffset : UInt32 + fecParityLen : UInt32 + deriving DecidableEq, Inhabited + +/-- One file entry in the catalog. -/ +structure FilepackEntry where + relPath : String + contentBlake3 : ByteArray + segmentFormat : UInt8 + segments : Array SegmentRef + /-- Optional OTS proof (wire-encoded when present). -/ + otsProof : Option ByteArray + deriving DecidableEq, Inhabited + +/-- Directory catalog manifest (API view; catalog_bao_root bound from filename). -/ +structure FilepackManifest where + version : Nat + formatLevel : UInt8 + catalogBaoRoot : ByteArray + entries : Array FilepackEntry + deriving DecidableEq, Inhabited + +/-- + Fail-closed relative path validation (AGENTS / Rust `validate_rel_path` + extras). + + Rejects: empty, too long, `\`, absolute `/`, `..`, empty components, NUL. +-/ +def validateRelPath (rel : String) : Except FilepackError Unit := + if rel.isEmpty then + .error .emptyRelPath + else if rel.length > maxRelPathLen then + .error .relPathTooLong + else if rel.contains '\\' then + .error .relPathBackslash + else if rel.startsWith "/" then + .error .relPathAbsolute + else if rel.contains (Char.ofNat 0) then + .error .relPathNullByte + else + Id.run do + let parts := rel.splitOn "/" + let mut err : Option FilepackError := none + for p in parts do + if err.isNone then + if p == ".." then + err := some .relPathTraversal + else if p.isEmpty then + err := some .relPathEmptyComponent + match err with + | some e => pure (.error e) + | none => pure (.ok ()) + +/-- Segment format is one of c12–c15. -/ +def isDirectorySegmentFormat (fmt : UInt8) : Bool := + fmt == segmentFormatPublicRaw || fmt == segmentFormatPublicCompressed || + fmt == segmentFormatEncryptedRaw || fmt == segmentFormatEncryptedCompressed + +/-- Legacy c4–c7. -/ +def isLegacySegmentFormat (fmt : UInt8) : Bool := + fmt ≥ 0x04 && fmt ≤ 0x07 + +/-- Segment encryption bit must match catalog encryption. -/ +def segmentMatchesCatalogEncryption (segmentFmt : UInt8) (catalogEncrypted : Bool) : Bool := + let segEnc := segmentFmt &&& 1 != 0 + segEnc == catalogEncrypted + +/-- Validate segment format against catalog. -/ +def validateSegmentFormat (segmentFmt : UInt8) (catalogEncrypted : Bool) : + Except FilepackError Unit := + if isLegacySegmentFormat segmentFmt then + .error (.legacySegmentFormat segmentFmt) + else if !isDirectorySegmentFormat segmentFmt then + .error (.segmentFormatMismatch segmentFmt) + else if !segmentMatchesCatalogEncryption segmentFmt catalogEncrypted then + .error (.segmentFormatMismatch segmentFmt) + else + -- c12–c15 all include Verification|Fec bits by definition of the four codes + .ok () + +/-- Validate segments: non-empty, contiguous chunk_index 0..n-1, root size, main_len cap. -/ +def validateSegments (segments : Array SegmentRef) : Except FilepackError Unit := + if segments.size == 0 then + .error .emptySegments + else if segments.size > maxSegmentsPerEntry then + .error .tooManySegments + else + Id.run do + let n := segments.size + let mut err : Option FilepackError := none + for i in [:n] do + if err.isNone then + let s := segments[i]! + if s.segmentBaoRoot.size != hashLen then + err := some .invalidHashLength + else if UInt64.toNat s.mainLen > maxSegmentMainLen then + err := some .mainLenTooLarge + else if UInt32.toNat s.chunkIndex != i then + err := some .invalidChunkSequence + match err with + | some e => pure (.error e) + | none => pure (.ok ()) + +/-- Validate full manifest semantics (caller supplies expected catalog root). -/ +def FilepackManifest.validate (m : FilepackManifest) : Except FilepackError Unit := + if m.version != filepackManifestVersion then + .error (.unsupportedVersion m.version) + else if m.formatLevel != adamantineFmtPublic && m.formatLevel != adamantineFmtEncrypted then + .error (.invalidFormatLevel m.formatLevel) + else if m.catalogBaoRoot.size != hashLen then + .error .invalidHashLength + else if m.entries.size > maxFilepackEntries then + .error .tooManyEntries + else + let catalogEnc := m.formatLevel &&& 1 != 0 + Id.run do + let mut err : Option FilepackError := none + let mut totalSegs : Nat := 0 + let mut prevPath : Option String := none + for i in [:m.entries.size] do + if err.isNone then + let e := m.entries[i]! + match validateRelPath e.relPath with + | .error pe => err := some pe + | .ok () => + match prevPath with + | some p => + if e.relPath ≤ p then + err := some .entriesUnsorted + | none => pure () + prevPath := some e.relPath + if e.contentBlake3.size != hashLen then + err := some .invalidHashLength + else + match validateSegmentFormat e.segmentFormat catalogEnc with + | .error se => err := some se + | .ok () => + match validateSegments e.segments with + | .error se => err := some se + | .ok () => + totalSegs := totalSegs + e.segments.size + if totalSegs > maxTotalSegmentRefs then + err := some .tooManyEntries + match e.otsProof with + | some p => + if p.size > maxOtsProofLen then + err := some .otsProofTooLarge + | none => pure () + match err with + | some e => pure (.error e) + | none => pure (.ok ()) + +/-- Serialize SegmentRef (fixed 32+4+8+4+4+4+4 = 60 bytes). -/ +def SegmentRef.toBytes (s : SegmentRef) : Except FilepackError ByteArray := + if s.segmentBaoRoot.size != hashLen then + .error .invalidHashLength + else + Id.run do + let mut out := s.segmentBaoRoot + out := appendBA out (putUInt32LE s.chunkIndex) + out := appendBA out (putUInt64LE s.mainLen) + out := appendBA out (putUInt32LE s.verificationOutboardOffset) + out := appendBA out (putUInt32LE s.verificationOutboardLen) + out := appendBA out (putUInt32LE s.fecParityOffset) + out := appendBA out (putUInt32LE s.fecParityLen) + pure (.ok out) + +/-- Parse SegmentRef from bytes at offset; returns (ref, next_offset). -/ +def parseSegmentRef (bs : ByteArray) (off : Nat) : + Except FilepackError (SegmentRef × Nat) := + if off + 60 > bs.size then + .error .invalidWire + else + let root := bs.extract off (off + 32) + let chunkIndex := getUInt32LE bs (off + 32) + let mainLen := getUInt64LE bs (off + 36) + let vo := getUInt32LE bs (off + 44) + let vl := getUInt32LE bs (off + 48) + let fo := getUInt32LE bs (off + 52) + let fl := getUInt32LE bs (off + 56) + .ok ({ + segmentBaoRoot := root + chunkIndex := chunkIndex + mainLen := mainLen + verificationOutboardOffset := vo + verificationOutboardLen := vl + fecParityOffset := fo + fecParityLen := fl + }, off + 60) + +/-- Serialize one entry. -/ +def FilepackEntry.toBytes (e : FilepackEntry) : Except FilepackError ByteArray := + match validateRelPath e.relPath with + | .error err => .error err + | .ok () => + if e.contentBlake3.size != hashLen then + .error .invalidHashLength + else + let pathBytes := utf8 e.relPath + if pathBytes.size > maxRelPathLen then + .error .relPathTooLong + else if pathBytes.size > 65535 then + .error .relPathTooLong + else + match validateSegments e.segments with + | .error err => .error err + | .ok () => + Id.run do + let mut out := ByteArray.empty + out := appendBA out (putUInt32LE (UInt32.ofNat pathBytes.size)) + out := appendBA out pathBytes + out := appendBA out e.contentBlake3 + out := out.push e.segmentFormat + out := appendBA out (putUInt32LE (UInt32.ofNat e.segments.size)) + let mut err : Option FilepackError := none + for i in [:e.segments.size] do + if err.isNone then + match (e.segments[i]!).toBytes with + | .error se => err := some se + | .ok sb => out := appendBA out sb + match err with + | some se => pure (.error se) + | none => + match e.otsProof with + | none => + out := out.push 0 + pure (.ok out) + | some proof => + if proof.size > maxOtsProofLen then + pure (.error .otsProofTooLarge) + else + out := out.push 1 + out := appendBA out (putUInt32LE (UInt32.ofNat proof.size)) + out := appendBA out proof + pure (.ok out) + +/-- Parse entry at offset. -/ +def parseEntry (bs : ByteArray) (off : Nat) : Except FilepackError (FilepackEntry × Nat) := + if off + 4 > bs.size then + .error .invalidWire + else + let pathLen := UInt32.toNat (getUInt32LE bs off) + let pathStart := off + 4 + let pathEnd := pathStart + pathLen + if pathEnd + 32 + 1 + 4 > bs.size then + .error .invalidWire + else + let pathBytes := bs.extract pathStart pathEnd + match String.fromUTF8? pathBytes with + | none => .error .invalidWire + | some relPath => + match validateRelPath relPath with + | .error e => .error e + | .ok () => + let contentBlake3 := bs.extract pathEnd (pathEnd + 32) + let segmentFormat := bs.get! (pathEnd + 32) + let segCount := UInt32.toNat (getUInt32LE bs (pathEnd + 33)) + if segCount > maxSegmentsPerEntry then + .error .tooManySegments + else + Id.run do + let mut segs : Array SegmentRef := Array.mkEmpty segCount + let mut cur := pathEnd + 37 + let mut err : Option FilepackError := none + for _ in [:segCount] do + if err.isNone then + match parseSegmentRef bs cur with + | .error e => err := some e + | .ok (s, next) => + segs := segs.push s + cur := next + match err with + | some e => pure (.error e) + | none => + if cur ≥ bs.size then + pure (.error .invalidWire) + else + let hasOts := bs.get! cur + cur := cur + 1 + if hasOts == 0 then + pure (.ok ({ + relPath := relPath + contentBlake3 := contentBlake3 + segmentFormat := segmentFormat + segments := segs + otsProof := none + }, cur)) + else if hasOts == 1 then + if cur + 4 > bs.size then + pure (.error .invalidWire) + else + let otsLen := UInt32.toNat (getUInt32LE bs cur) + cur := cur + 4 + if otsLen > maxOtsProofLen then + pure (.error .otsProofTooLarge) + else if cur + otsLen > bs.size then + pure (.error .invalidWire) + else + let proof := bs.extract cur (cur + otsLen) + pure (.ok ({ + relPath := relPath + contentBlake3 := contentBlake3 + segmentFormat := segmentFormat + segments := segs + otsProof := some proof + }, cur + otsLen)) + else + pure (.error .invalidWire) + +/-- Serialize wire body (no catalog root — bound out-of-band). -/ +def FilepackManifest.toWireBytes (m : FilepackManifest) : Except FilepackError ByteArray := + match m.validate with + | .error e => .error e + | .ok () => + Id.run do + let mut out := ofList cfp2Magic + out := appendBA out (putUInt32LE (UInt32.ofNat m.version)) + out := out.push m.formatLevel + out := appendBA out (putUInt32LE (UInt32.ofNat m.entries.size)) + let mut err : Option FilepackError := none + for i in [:m.entries.size] do + if err.isNone then + match (m.entries[i]!).toBytes with + | .error e => err := some e + | .ok eb => out := appendBA out eb + match err with + | some e => pure (.error e) + | none => pure (.ok out) + +/-- Parse wire body; bind catalog root from filename / caller. -/ +def FilepackManifest.fromWireBytes (bytes : ByteArray) (catalogBaoRoot : ByteArray) : + Except FilepackError FilepackManifest := + if catalogBaoRoot.size != hashLen then + .error .invalidHashLength + else if bytes.size < 4 + 4 + 1 + 4 then + .error .invalidWire + else if !ctEq (bytes.extract 0 4) (ofList cfp2Magic) then + .error .invalidWire + else + let version := UInt32.toNat (getUInt32LE bytes 4) + if version != filepackManifestVersion then + .error (.unsupportedVersion version) + else + let formatLevel := bytes.get! 8 + let entryCount := UInt32.toNat (getUInt32LE bytes 9) + if entryCount > maxFilepackEntries then + .error .tooManyEntries + else + Id.run do + let mut entries : Array FilepackEntry := Array.mkEmpty entryCount + let mut cur : Nat := 13 + let mut err : Option FilepackError := none + for _ in [:entryCount] do + if err.isNone then + match parseEntry bytes cur with + | .error e => err := some e + | .ok (e, next) => + entries := entries.push e + cur := next + match err with + | some e => pure (.error e) + | none => + if cur != bytes.size then + pure (.error .invalidWire) + else + let m : FilepackManifest := { + version := version + formatLevel := formatLevel + catalogBaoRoot := catalogBaoRoot + entries := entries + } + match m.validate with + | .error e => pure (.error e) + | .ok () => pure (.ok m) + +/-- Segment format policy (subset of Rust). -/ +inductive SegmentFormatPolicy where + | auto + | forceRaw + | forceCompressed + | forceC12 + | forceC14 + | forceC13 + | forceC15 + deriving DecidableEq, Repr + +/-- Simple incompressible heuristic: empty or high-entropy-looking magic prefixes. -/ +def isLikelyIncompressible (data : ByteArray) : Bool := + if data.size == 0 then + true + else if data.size ≥ 4 then + -- gzip, zip, png, jpeg, zstd, 7z, pdf, webp-ish + let b0 := data.get! 0 + let b1 := data.get! 1 + let b2 := data.get! 2 + let b3 := data.get! 3 + (b0 == 0x1f && b1 == 0x8b) || -- gzip + (b0 == 0x50 && b1 == 0x4b) || -- zip/pk + (b0 == 0x89 && b1 == 0x50 && b2 == 0x4e && b3 == 0x47) || -- png + (b0 == 0xff && b1 == 0xd8) || -- jpeg + (b0 == 0x28 && b1 == 0xb5 && b2 == 0x2f && b3 == 0xfd) || -- zstd + (b0 == 0x37 && b1 == 0x7a) || -- 7z + (b0 == 0x25 && b1 == 0x50 && b2 == 0x44 && b3 == 0x46) -- %PDF + else + false + +/-- Resolve segment format for one file. -/ +def SegmentFormatPolicy.resolve (self : SegmentFormatPolicy) (catalogEncrypted : Bool) + (data : ByteArray) : Except FilepackError UInt8 := + let fmt : UInt8 := + match self with + | .auto => + if catalogEncrypted then + if isLikelyIncompressible data then segmentFormatEncryptedRaw + else segmentFormatEncryptedCompressed + else if isLikelyIncompressible data then segmentFormatPublicRaw + else segmentFormatPublicCompressed + | .forceRaw => + if catalogEncrypted then segmentFormatEncryptedRaw else segmentFormatPublicRaw + | .forceCompressed => + if catalogEncrypted then segmentFormatEncryptedCompressed + else segmentFormatPublicCompressed + | .forceC12 => segmentFormatPublicRaw + | .forceC14 => segmentFormatPublicCompressed + | .forceC13 => segmentFormatEncryptedRaw + | .forceC15 => segmentFormatEncryptedCompressed + validateSegmentFormat fmt catalogEncrypted |>.map (fun _ => fmt) + +/-- Path validation unit tests as theorems. -/ +theorem rel_empty : + (match validateRelPath "" with | .error .emptyRelPath => true | _ => false) = true := by + native_decide + +theorem rel_traversal : + (match validateRelPath "a/../b" with | .error .relPathTraversal => true | _ => false) = + true := by + native_decide + +theorem rel_absolute : + (match validateRelPath "/etc/passwd" with | .error .relPathAbsolute => true | _ => false) = + true := by + native_decide + +theorem rel_backslash : + (match validateRelPath "a\\b" with | .error .relPathBackslash => true | _ => false) = + true := by + native_decide + +theorem rel_ok : + (match validateRelPath "src/main.lean" with | .ok () => true | _ => false) = true := by + native_decide + +theorem rel_empty_component : + (match validateRelPath "a//b" with | .error .relPathEmptyComponent => true | _ => false) = + true := by + native_decide + +/-- Path with embedded NUL is rejected (`relPathNullByte`). -/ +theorem rel_null : + (match validateRelPath ("a" ++ String.singleton (Char.ofNat 0) ++ "b") with + | .error .relPathNullByte => true + | _ => false) = true := by + native_decide + +end Carbonado.Filepack diff --git a/Carbonado/Header.lean b/Carbonado/Header.lean new file mode 100644 index 0000000..77d31cf --- /dev/null +++ b/Carbonado/Header.lean @@ -0,0 +1,197 @@ +/- + Carbonado v2 Header wire codec (177 bytes). + + Normative layout (AGENTS.md / Rust `file::Header`): + MAGIC(12) + payload_nonce(16) + header_mac(64) + hash(32) + slh_public_key(32) + + format(1) + chunk_index(u32 LE) + encoded_len(u32 LE) + padding_len(u32 LE) + + metadata(8) = 177 + + Header is **never encrypted**. Integrity via `header_mac` under `header-auth` subkey. + Verification must happen before trusting any metadata (header-MAC-before-body). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Crypto.EtM + +namespace Carbonado.Header + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Crypto.EtM + +/-- Header parse / build errors (distinct; not folded into payload EtM failures). -/ +inductive HeaderError where + /-- Input shorter than `headerLen` (177). -/ + | invalidHeaderLength + /-- Magic prefix is not `CARBONADO20\n`. -/ + | badMagic + /-- `header_mac` did not verify under the master key. -/ + | headerAuthenticationFailed + /-- Master key shorter than 32 bytes when computing/verifying MAC. -/ + | invalidKeyLength + /-- Hash / slh / nonce field wrong size when constructing. -/ + | invalidFieldLength + deriving DecidableEq, Repr + +/-- Parsed / constructed v2 Header (public metadata only). -/ +structure Header where + payloadNonce : ByteArray + headerMac : ByteArray + hash : ByteArray + slhPublicKey : ByteArray + format : UInt8 + chunkIndex : UInt32 + encodedLen : UInt32 + paddingLen : UInt32 + /-- Always 8 bytes on the wire; all-zero when absent. -/ + metadata : ByteArray + deriving DecidableEq, Inhabited + +/-- Build `auth_data` for header MAC (113 bytes; no separate domain string). -/ +def buildAuthData + (payloadNonce hash slhPublicKey : ByteArray) + (format : UInt8) + (chunkIndex encodedLen paddingLen : UInt32) + (metadata : ByteArray) : Except HeaderError ByteArray := + if payloadNonce.size != nonceLen then + .error .invalidFieldLength + else if hash.size != hashLen then + .error .invalidFieldLength + else if slhPublicKey.size != slhPublicKeyLen then + .error .invalidFieldLength + else if metadata.size != 8 then + .error .invalidFieldLength + else + Id.run do + let mut out := ByteArray.empty + for b in magicBytes do + out := out.push b + out := appendBA out payloadNonce + out := appendBA out hash + out := appendBA out slhPublicKey + out := out.push format + out := appendBA out (putUInt32LE chunkIndex) + out := appendBA out (putUInt32LE encodedLen) + out := appendBA out (putUInt32LE paddingLen) + out := appendBA out metadata + pure (.ok out) + +/-- auth_data length is fixed at 113 (= headerLen − header_mac). -/ +theorem authData_len_formula : + magicBytes.length + nonceLen + hashLen + slhPublicKeyLen + 1 + 4 + 4 + 4 + 8 = 113 := by + native_decide + +/-- Construct Header, computing `header_mac` under `master`. -/ +def Header.new + (master payloadNonce hash slhPublicKey : ByteArray) + (format : UInt8) + (chunkIndex encodedLen paddingLen : UInt32) + (metadata : ByteArray) : Except HeaderError Header := + match buildAuthData payloadNonce hash slhPublicKey format chunkIndex encodedLen paddingLen + metadata with + | .error e => .error e + | .ok auth => + -- Exhaustive CryptoError match (no catch-all → invalidKeyLength). + match computeHeaderMac master auth with + | .error .invalidKeyLength => .error .invalidKeyLength + | .error .invalidCiphertextLength => .error .invalidKeyLength + | .error .invalidNonceLength => .error .invalidKeyLength + | .error .authenticationFailed => .error .invalidKeyLength + | .ok mac => + .ok { + payloadNonce := payloadNonce + headerMac := mac + hash := hash + slhPublicKey := slhPublicKey + format := format + chunkIndex := chunkIndex + encodedLen := encodedLen + paddingLen := paddingLen + metadata := metadata + } + +/-- Serialize Header to exactly 177 wire bytes. -/ +def Header.toBytes (h : Header) : Except HeaderError ByteArray := + if h.payloadNonce.size != nonceLen then .error .invalidFieldLength + else if h.headerMac.size != hmacTagLen then .error .invalidFieldLength + else if h.hash.size != hashLen then .error .invalidFieldLength + else if h.slhPublicKey.size != slhPublicKeyLen then .error .invalidFieldLength + else if h.metadata.size != 8 then .error .invalidFieldLength + else + Id.run do + let mut out := ByteArray.empty + for b in magicBytes do + out := out.push b + out := appendBA out h.payloadNonce + out := appendBA out h.headerMac + out := appendBA out h.hash + out := appendBA out h.slhPublicKey + out := out.push h.format + out := appendBA out (putUInt32LE h.chunkIndex) + out := appendBA out (putUInt32LE h.encodedLen) + out := appendBA out (putUInt32LE h.paddingLen) + out := appendBA out h.metadata + pure (.ok out) + +/-- Parse Header from wire bytes (no MAC verify — call `verify` next). -/ +def parse (bytes : ByteArray) : Except HeaderError Header := + if bytes.size < headerLen then + .error .invalidHeaderLength + else + Id.run do + let mut magicOk := true + for i in [:magicBytes.length] do + if bytes.get! i != magicBytes[i]! then + magicOk := false + if !magicOk then + pure (.error .badMagic) + else + let payloadNonce := bytes.extract 12 28 + let headerMac := bytes.extract 28 92 + let hash := bytes.extract 92 124 + let slhPublicKey := bytes.extract 124 156 + let format := bytes.get! 156 + let chunkIndex := getUInt32LE bytes 157 + let encodedLen := getUInt32LE bytes 161 + let paddingLen := getUInt32LE bytes 165 + let metadata := bytes.extract 169 177 + pure (.ok { + payloadNonce := payloadNonce + headerMac := headerMac + hash := hash + slhPublicKey := slhPublicKey + format := format + chunkIndex := chunkIndex + encodedLen := encodedLen + paddingLen := paddingLen + metadata := metadata + }) + +/-- Verify `header_mac` under master; does not return plaintext body. -/ +def verify (master : ByteArray) (h : Header) : Except HeaderError Unit := + match buildAuthData h.payloadNonce h.hash h.slhPublicKey h.format + h.chunkIndex h.encodedLen h.paddingLen h.metadata with + | .error e => .error e + | .ok auth => + -- Exhaustive CryptoError match (computeHeaderMac only yields invalidKeyLength today). + match verifyHeaderMac master auth h.headerMac with + | .error .invalidKeyLength => .error .invalidKeyLength + | .error .invalidCiphertextLength => .error .invalidKeyLength + | .error .invalidNonceLength => .error .invalidKeyLength + | .error .authenticationFailed => .error .invalidKeyLength + | .ok true => .ok () + | .ok false => .error .headerAuthenticationFailed + +/-- Parse + verify in one step (header MAC before trusting metadata). -/ +def parseAndVerify (master bytes : ByteArray) : Except HeaderError Header := + match parse bytes with + | .error e => .error e + | .ok h => + match verify master h with + | .error e => .error e + | .ok () => .ok h + +/-- Wire length of a successful `toBytes` is `headerLen`. -/ +theorem headerLen_eq_177 : headerLen = 177 := by native_decide + +end Carbonado.Header diff --git a/Carbonado/Main.lean b/Carbonado/Main.lean new file mode 100644 index 0000000..df3374b --- /dev/null +++ b/Carbonado/Main.lean @@ -0,0 +1,1365 @@ +import Carbonado.Constants +import Carbonado.Crypto +import Carbonado.Fec +import Carbonado.Bao +import Carbonado.Header +import Carbonado.Compress +import Carbonado.Slh +import Carbonado.Pipeline +import Carbonado.Stream +import Carbonado.Scrub +import Carbonado.Shard +import Carbonado.Adamantine +import Carbonado.Filepack +import Carbonado.Outboard +import Carbonado.Directory +import Carbonado.Cli + +-- Large patterned Bao vectors + partial tree recursion need a deeper elaborator budget. +set_option maxRecDepth 2048 + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Crypto.SHA512 +open Carbonado.Crypto.HMAC +open Carbonado.Crypto.AESCTR +open Carbonado.Crypto.EtM +open Carbonado.Fec.Galois +open Carbonado.Fec.Matrix +open Carbonado.Fec.RS +open Carbonado.Fec.Inboard +open Carbonado.Bao.Blake3 +open Carbonado.Bao.Product +open Carbonado.Header +open Carbonado.Compress +open Carbonado.Slh +open Carbonado.Pipeline +open Carbonado.Stream +open Carbonado.Scrub +open Carbonado.Shard +open Carbonado.Adamantine +open Carbonado.Filepack +open Carbonado.Outboard +open Carbonado.Directory +open Carbonado.Cli +-- Note: do not open Carbonado.Bao.Tree — name clash with Fec.Inboard.encodeInboard/decodeInboard. + +/-- Expected magic bytes: ASCII `CARBONADO20\n`. -/ +private def expectedMagic : List UInt8 := + [0x43, 0x41, 0x52, 0x42, 0x4f, 0x4e, 0x41, 0x44, 0x4f, 0x32, 0x30, 0x0a] + +private def master42 : ByteArray := replicate 32 0x42 +private def nonce11 : ByteArray := replicate 16 0x11 + +private def fail (msg : String) : IO Unit := do + IO.eprintln s!"FAIL: {msg}" + IO.Process.exit 1 + +private def expectHex (label : String) (got : ByteArray) (wantHex : String) : IO Unit := do + let g := toHex got + if g != wantHex then + fail s!"{label}: got {g} want {wantHex}" + +private def expectTrue (label : String) (b : Bool) : IO Unit := do + if !b then fail label + +/-- Patterned bytes `i % 251` for Bao parity vectors (name avoids shadowing FEC locals). -/ +private def baoPattern (n : Nat) : ByteArray := Id.run do + let mut out := ByteArray.empty + for i in [:n] do + out := out.push (UInt8.ofNat (i % 251)) + pure out + +/-- Scaffold + Program B–G demo (EtM + FEC + Bao + pipeline + zstd/SLH + Adamantine/CLI). -/ +def runDemo : IO Unit := do + IO.println "carbonado (Lean 4 AOT — Program G Adamantine + CLI)" + IO.println s!"magic length = {magicBytes.length} (expect 12)" + IO.println s!"headerLen = {headerLen}" + IO.println s!"sliceLen = {sliceLen}" + IO.println s!"baoChunkLog = {baoChunkLog}" + IO.println s!"leafBytes = {Carbonado.Bao.Tree.leafBytes}" + IO.println s!"verificationContext = {verificationContext}" + IO.println s!"fecK = {fecK} fecM = {fecM} stripeUnit = {stripeUnit}" + IO.println s!"hmacTagLen = {hmacTagLen} (full HMAC-SHA512)" + IO.println s!"slh1SidecarLen = {slh1SidecarLen}" + IO.println s!"sample public c14 format byte = {formatC14.toUInt8}" + IO.println s!"sample encrypted c15 format byte = {formatC15.toUInt8}" + if magicBytes != expectedMagic then + fail "magic bytes (expect CARBONADO20\\n)" + if magicBytes.length != 12 then fail "magic length" + if headerLen != 177 then fail "headerLen" + if sliceLen != 4096 then fail "sliceLen" + if baoChunkLog != 2 then fail "baoChunkLog" + if nonceLen != 16 then fail "nonceLen" + if hashLen != 32 then fail "hashLen" + if slh1SignatureLen != 7856 then fail "slh1SignatureLen" + if stripeUnit != 16384 then fail "stripeUnit" + if fecK != 4 || fecM != 8 then fail "FEC geometry" + if hmacTagLen != 64 then fail "hmacTagLen" + if slh1SidecarLen != 7860 then fail "slh1SidecarLen" + if formatC14.toUInt8 != 14 then fail "formatC14" + if formatC15.toUInt8 != 15 then fail "formatC15" + IO.println "scaffold constants ok" + + -- SHA-512 goldens (FIPS / common) + expectHex "sha512(empty)" (Carbonado.Crypto.SHA512.hash ByteArray.empty) + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" + expectHex "sha512(abc)" (hashString "abc") + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + expectHex "sha512(fox)" (hashString "The quick brown fox jumps over the lazy dog") + "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6" + IO.println "sha512 goldens ok" + + -- HMAC-SHA512 RFC 4231 test case 1 + let hmacKey := replicate 20 0x0b + expectHex "hmac-rfc4231-1" (hmacSHA512 hmacKey (utf8 "Hi There")) + "87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854" + IO.println "hmac goldens ok" + + -- AES-256-CTR NIST SP 800-38A F.5.5 + let nistKey := ofList [ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4] + let nistCtr := ofList [ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff] + let nistPt := ofList [ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10] + expectHex "aes-ctr-nist" (ctrXor nistKey nistCtr nistPt) + "601ec313775789a5b7a7f504bbf3d228f443e3ca4d62b59aca84e990cacaf5c52b0930daa23de94ce87017ba2d84988ddfc9c58db67aada613c2dd08457941a6" + IO.println "aes-ctr nist golden ok" + + -- Subkeys (master = 0x42×32) + match deriveSubkey master42 "aes-ctr" with + | .error e => fail s!"derive aes-ctr: {repr e}" + | .ok skAes => + expectHex "subkey aes-ctr" skAes + "6f15fb9936ca3e4d2ecc5bd80bcc06c12d67361b72dcf5e8edc8312092f42a28494d106e3340595717f67ab0ec91b0b8d0ea653853ca129a4515ea6df74a5ca7" + match deriveSubkey master42 "etm-hmac" with + | .error e => fail s!"derive etm-hmac: {repr e}" + | .ok skEtm => + expectHex "subkey etm-hmac" skEtm + "d9795accc69d8966b12f051575d3efc53725697a14d8117ffcef149eea20bb859e025d2e76f5b47bb1bc73af3c34aceb317ba0d5d00e5b3f36e7f21accae3609" + match deriveSubkey master42 "header-auth" with + | .error e => fail s!"derive header-auth: {repr e}" + | .ok skHdr => + expectHex "subkey header-auth" skHdr + "af1f7d5c23422538fab14c8343eaef42918230ba04fef171176b3c01a89e9bab0ae60b90586d9863f4a4231d91eb984516f0be04c20d4c7e784f0fe459de2b19" + IO.println "subkey goldens ok" + + -- EtM header-path goldens + roundtrips + let cases : List (String × ByteArray × String) := [ + ("empty", ByteArray.empty, + "74e137726bc9f0a9e55add833d1ac0c187bb366f22f0a2be1189536828d77dfc2d021e8aad99fab802c664db3b8ec1e0c46198f44dd3f3e9321bded8263a6aa2"), + ("hello", utf8 "hello", + "1d05aa600755696228225aeada6672b65266554ef6a2e2b5f4a083870ad00f534747fbd18d98f4c6e449a4b64e954b77f65bd63eba1a9a0e080cca3c296760b08f0a0e143c"), + ("multi", utf8 "The quick brown fox jumps over the lazy dog", + "38cddad202dc0f7daacd53b35573b43a3c79bbcfa44aee039d661c92d5be8f16701b996b7474d66610627fb714376524cfcbb4aa30d8e7062ca9b61cf32eccd9b307075822eda7e16b52f04354c83f70cd6b5a9ab6817560693ae25a0cdd1883752eeccaff8bb0228d84c6") + ] + for (name, pt, want) in cases do + match encryptWithNonce master42 nonce11 pt with + | .error e => fail s!"encrypt {name}: {repr e}" + | .ok blob => + expectHex s!"etm {name}" blob want + match decryptWithNonce master42 nonce11 blob with + | .ok pt' => + expectTrue s!"roundtrip {name}" (toHex pt' == toHex pt) + | .error e => fail s!"decrypt {name}: {repr e}" + IO.println "etm header-path goldens + roundtrip ok" + + -- Low-level embedded nonce layout + match encryptEmbeddedNonce master42 nonce11 (utf8 "hello") with + | .error e => fail s!"encryptEmbedded: {repr e}" + | .ok low => + expectHex "low_level_hello" low + "111111111111111111111111111111111d05aa600755696228225aeada6672b65266554ef6a2e2b5f4a083870ad00f534747fbd18d98f4c6e449a4b64e954b77f65bd63eba1a9a0e080cca3c296760b08f0a0e143c" + match decryptEmbeddedNonce master42 low with + | .ok pt => expectTrue "low roundtrip" (toHex pt == toHex (utf8 "hello")) + | .error e => fail s!"decryptEmbedded: {repr e}" + IO.println "etm low-level layout ok" + + -- Tampered tag → authenticationFailed (strict match) + match encryptWithNonce master42 nonce11 (utf8 "hello") with + | .error e => fail s!"encrypt for tamper: {repr e}" + | .ok blob => + let mut bad := blob + -- flip first tag byte + bad := bad.set! 0 (bad.get! 0 ^^^ 0x01) + match decryptWithNonce master42 nonce11 bad with + | .error .authenticationFailed => pure () + | .error e => fail s!"tamper: expected authenticationFailed, got {repr e}" + | .ok _ => fail "tamper: decrypt succeeded on bad tag" + IO.println "tampered tag → authenticationFailed ok" + + -- Ciphertext-body tamper (byte past the 64-byte tag) → authenticationFailed + match encryptWithNonce master42 nonce11 (utf8 "hello") with + | .error e => fail s!"encrypt for ct tamper: {repr e}" + | .ok blob => + if blob.size ≤ hmacTagLen then + fail "ct tamper: blob has no ciphertext body" + else + let mut bad := blob + bad := bad.set! hmacTagLen (bad.get! hmacTagLen ^^^ 0x01) + match decryptWithNonce master42 nonce11 bad with + | .error .authenticationFailed => pure () + | .error e => fail s!"ct body tamper: expected authenticationFailed, got {repr e}" + | .ok _ => fail "ct body tamper: decrypt succeeded" + IO.println "ct body tamper → authenticationFailed ok" + + -- Wrong master → authenticationFailed + let wrongMaster := replicate 32 0x43 + match encryptWithNonce master42 nonce11 (utf8 "hello") with + | .error e => fail s!"encrypt for wrong key: {repr e}" + | .ok blob => + match decryptWithNonce wrongMaster nonce11 blob with + | .error .authenticationFailed => pure () + | .error e => fail s!"wrong key: expected authenticationFailed, got {repr e}" + | .ok _ => fail "wrong key: decrypt succeeded" + IO.println "wrong key → authenticationFailed ok" + + -- Short ciphertext → invalidCiphertextLength + match decryptWithNonce master42 nonce11 (replicate 10 0) with + | .error .invalidCiphertextLength => pure () + | .error e => fail s!"short ct: expected invalidCiphertextLength, got {repr e}" + | .ok _ => fail "short ct: ok" + IO.println "short ciphertext → invalidCiphertextLength ok" + + -- Dual-invalid (short master + short CT): Rust/Lean both report CT length first + match decryptWithNonce (replicate 16 0x42) nonce11 (replicate 10 0) with + | .error .invalidCiphertextLength => pure () + | .error e => fail s!"dual short: expected invalidCiphertextLength, got {repr e}" + | .ok _ => fail "dual short: ok" + IO.println "dual short master+ct → invalidCiphertextLength ok" + + -- Short master (long enough CT) → invalidKeyLength + match decryptWithNonce (replicate 16 0x42) nonce11 (replicate 64 0) with + | .error .invalidKeyLength => pure () + | .error e => fail s!"short master: expected invalidKeyLength, got {repr e}" + | .ok _ => fail "short master: ok" + IO.println "short master → invalidKeyLength ok" + + -- Bad nonce length → invalidNonceLength (encrypt + decrypt) + match encryptWithNonce master42 (replicate 8 0x11) (utf8 "hi") with + | .error .invalidNonceLength => pure () + | .error e => fail s!"encrypt bad nonce: expected invalidNonceLength, got {repr e}" + | .ok _ => fail "encrypt bad nonce: ok" + match decryptWithNonce master42 (replicate 8 0x11) (replicate 64 0) with + | .error .invalidNonceLength => pure () + | .error e => fail s!"decrypt bad nonce: expected invalidNonceLength, got {repr e}" + | .ok _ => fail "decrypt bad nonce: ok" + IO.println "bad nonce → invalidNonceLength ok" + + -- Embedded short input → invalidCiphertextLength + match decryptEmbeddedNonce master42 (replicate 20 0) with + | .error .invalidCiphertextLength => pure () + | .error e => fail s!"embedded short: expected invalidCiphertextLength, got {repr e}" + | .ok _ => fail "embedded short: ok" + IO.println "embedded short → invalidCiphertextLength ok" + + -- Header MAC goldens + match computeHeaderMac master42 (utf8 "CARBONADO20\n") with + | .error e => fail s!"header mac magic: {repr e}" + | .ok tag => + expectHex "header_mac(MAGIC)" tag + "c02b40016162e5abf37a007183f2117a46fb74175529188dc98786c9cab370691c7903dcf552765f7764ec2c392af0863b618ea295ed026e8a47b304f0127937" + -- sample full auth_data (113 bytes) + let mut auth := ByteArray.empty + for b in magicBytes do auth := auth.push b + for _ in [:16] do auth := auth.push 0x11 + for _ in [:32] do auth := auth.push 0xcd + for _ in [:32] do auth := auth.push 0x00 + auth := auth.push 0x05 + for _ in [:4] do auth := auth.push 0x00 -- chunk 0 LE + auth := auth.push 100; auth := auth.push 0; auth := auth.push 0; auth := auth.push 0 -- encoded_len 100 LE + for _ in [:4] do auth := auth.push 0x00 -- padding + for _ in [:8] do auth := auth.push 0x00 -- metadata + expectTrue "auth_data len" (auth.size == 113) + match computeHeaderMac master42 auth with + | .error e => fail s!"header mac sample: {repr e}" + | .ok tag => + expectHex "header_mac(sample_auth)" tag + "72b887d72cf53ae234f4802ac1984405a14dfdc9a0494ceb2067d24af0f4063f2dc94e7cd754289722d17a318845398dcc929190c3c39aae0b415af8867c0699" + match verifyHeaderMac master42 auth tag with + | .ok true => pure () + | .ok false => fail "verifyHeaderMac: expected true on good tag" + | .error e => fail s!"verifyHeaderMac good: {repr e}" + let badTag := tag.set! 0 (tag.get! 0 ^^^ 1) + match verifyHeaderMac master42 auth badTag with + | .ok false => pure () + | .ok true => fail "verifyHeaderMac: expected false on bad tag" + | .error e => fail s!"verifyHeaderMac bad: {repr e}" + IO.println "header mac goldens ok" + IO.println "header mac verify false path ok" + + IO.println "etm stack ok" + + -- Program C: GF + RS 4/8 + inboard geometry + expectTrue "gf mul 0x53*0xca" (mul 0x53 0xca == 0x8f) + expectTrue "gf div 2/3" (div 2 3 == 0xf5) + expectTrue "gf exp 2^3" (exp 2 3 == 8) + IO.println "gf goldens ok" + + let p0 := calcPaddingLen 0 + let p1 := calcPaddingLen 1 + let p4k := calcPaddingLen 4096 + let pStripe := calcPaddingLen 16384 + let pPlus := calcPaddingLen 16385 + expectTrue "pad0" (p0.paddingLen == 0 && p0.chunkLen == 0) + expectTrue "pad1" (p1.paddingLen == 16383 && p1.chunkLen == 4096) + expectTrue "pad4096" (p4k.paddingLen == 12288 && p4k.chunkLen == 4096) + expectTrue "pad16384" (pStripe.paddingLen == 0 && pStripe.chunkLen == 4096) + expectTrue "pad16385" (pPlus.paddingLen == 16383 && pPlus.chunkLen == 8192) + expectTrue "rs geometry" (carbonadoRS.dataShards == fecK && carbonadoRS.parityShards == fecM - fecK) + IO.println "padding geometry ok" + + -- 1-byte shard encode golden + let s1 : Array ByteArray := #[ + ofList [1], ofList [2], ofList [3], ofList [4], + ofList [0], ofList [0], ofList [0], ofList [0]] + match carbonadoRS.encode s1 with + | .error e => fail s!"encode len1: {repr e}" + | .ok enc => + expectTrue "parity0" ((enc[4]!).get! 0 == 0x45) + expectTrue "parity1" ((enc[5]!).get! 0 == 0x5e) + expectTrue "parity2" ((enc[6]!).get! 0 == 0x67) + expectTrue "parity3" ((enc[7]!).get! 0 == 0x78) + -- reconstruct parity-only + let opts : Array (Option ByteArray) := #[ + none, none, none, none, + some (enc[4]!), some (enc[5]!), some (enc[6]!), some (enc[7]!)] + match carbonadoRS.reconstruct opts with + | .error e => fail s!"reconstruct parity-only: {repr e}" + | .ok full => + expectTrue "recon0" ((full[0]!).get! 0 == 1) + expectTrue "recon1" ((full[1]!).get! 0 == 2) + expectTrue "recon2" ((full[2]!).get! 0 == 3) + expectTrue "recon3" ((full[3]!).get! 0 == 4) + IO.println "rs encode/reconstruct goldens ok" + + -- Inboard hello roundtrip (16 KiB stripe; O(stripe) memory) + match encodeInboard (utf8 "hello") with + | .error e => fail s!"encodeInboard hello: {repr e}" + | .ok (body, pad, chunk) => + expectTrue "hello pad" (pad == 16379) + expectTrue "hello chunk" (chunk == 4096) + expectTrue "hello body len" (body.size == 32768) + expectTrue "hello head ascii" (body.get! 0 == 'h'.toNat.toUInt8 && body.get! 4 == 'o'.toNat.toUInt8) + match decodeInboard body pad with + | .error e => fail s!"decodeInboard hello: {repr e}" + | .ok pt => + expectTrue "hello roundtrip" (toHex pt == toHex (utf8 "hello")) + -- Knock out all data shards; reconstruct from parity only + match inboardToShards body with + | .error e => fail s!"split hello: {repr e}" + | .ok shards => + match reconstructAfterKnockout shards [0, 1, 2, 3] pad with + | .error e => fail s!"knockout data: {repr e}" + | .ok pt => + expectTrue "hello knockout data" (toHex pt == toHex (utf8 "hello")) + match reconstructAfterKnockout shards [4, 5, 6, 7] pad with + | .error e => fail s!"knockout parity: {repr e}" + | .ok pt => + expectTrue "hello knockout parity" (toHex pt == toHex (utf8 "hello")) + match reconstructAfterKnockout shards [0, 2, 5, 7] pad with + | .error e => fail s!"knockout mixed: {repr e}" + | .ok pt => + expectTrue "hello knockout mixed" (toHex pt == toHex (utf8 "hello")) + IO.println "inboard hello roundtrip + knockout ok" + + -- Pattern i%251 length 100 + let mut pat := ByteArray.empty + for i in [:100] do + pat := pat.push (UInt8.ofNat (i % 251)) + match encodeInboard pat with + | .error e => fail s!"encode pat100: {repr e}" + | .ok (body, pad, _) => + expectTrue "pat100 pad" (pad == 16284) + match decodeInboard body pad with + | .error e => fail s!"decode pat100: {repr e}" + | .ok pt => + expectTrue "pat100 roundtrip" (toHex pt == toHex pat) + match inboardToShards body with + | .error e => fail s!"split pat100: {repr e}" + | .ok shards => + expectHex "pat100 parity0 head" ((shards[4]!).extract 0 8) + "001b362d6c775a41" + IO.println "inboard pattern roundtrip ok" + + -- Strict error paths + match decodeInboard (ofList [1, 2, 3]) 0 with + | .error .unevenShards => pure () + | .error e => fail s!"uneven: expected unevenShards, got {repr e}" + | .ok _ => fail "uneven: ok" + IO.println "unevenShards ok" + + match carbonadoRS.reconstruct #[ + some (ofList [1]), some (ofList [2]), some (ofList [3]), + none, none, none, none, none] with + | .error .tooFewShards => pure () + | .error e => fail s!"tooFew: expected tooFewShards, got {repr e}" + | .ok _ => fail "tooFew: ok" + IO.println "tooFewShards ok" + + match carbonadoRS.reconstruct #[ + some ByteArray.empty, some (ofList [1]), some (ofList [2]), some (ofList [3]), + none, none, none, none] with + | .error .emptyShard => pure () + | .error e => fail s!"emptyShard: expected emptyShard, got {repr e}" + | .ok _ => fail "emptyShard: ok" + IO.println "emptyShard ok" + + match carbonadoRS.reconstruct #[ + some (ofList [1]), some (ofList [1, 2]), some (ofList [3]), some (ofList [4]), + none, none, none, none] with + | .error .incorrectShardSize => pure () + | .error e => fail s!"incorrectSize: expected incorrectShardSize, got {repr e}" + | .ok _ => fail "incorrectSize: ok" + IO.println "incorrectShardSize ok" + + match carbonadoRS.reconstruct #[some (ofList [1]), some (ofList [2])] with + | .error .badGeometry => pure () + | .error e => fail s!"badGeometry: expected badGeometry, got {repr e}" + | .ok _ => fail "badGeometry: ok" + IO.println "badGeometry ok" + + match stripPadding #[ofList [1], ofList [2], ofList [3], ofList [4]] 5 with + | .error .paddingTooLarge => pure () + | .error e => fail s!"paddingTooLarge: expected paddingTooLarge, got {repr e}" + | .ok _ => fail "paddingTooLarge: ok" + IO.println "paddingTooLarge ok" + + match invertOrSingular (Matrix.zeros 2 2) with + | .error .singularMatrix => pure () + | .error e => fail s!"singular: expected singularMatrix, got {repr e}" + | .ok _ => fail "singular: ok" + IO.println "singularMatrix ok" + + match ReedSolomon.new 0 4 with + | .error .badGeometry => pure () + | .error e => fail s!"new0data: expected badGeometry, got {repr e}" + | .ok _ => fail "new0data: ok" + match carbonadoRS.encode #[ofList [1], ofList [2]] with + | .error .badGeometry => pure () + | .error e => fail s!"encode bad geo: expected badGeometry, got {repr e}" + | .ok _ => fail "encode bad geo: ok" + match carbonadoRS.encode #[ + ByteArray.empty, ofList [1], ofList [2], ofList [3], + ofList [0], ofList [0], ofList [0], ofList [0]] with + | .error .emptyShard => pure () + | .error e => fail s!"encode empty: expected emptyShard, got {repr e}" + | .ok _ => fail "encode empty: ok" + IO.println "encode/new guards ok" + + match carbonadoRS.encode #[ + ofList [1], ofList [2], ofList [3], ofList [4], + ofList [0], ofList [0], ofList [0], ofList [0]] with + | .error e => fail s!"verify setup: {repr e}" + | .ok enc => + match carbonadoRS.verify enc with + | .ok true => pure () + | .ok false => fail "verify good: expected true" + | .error e => fail s!"verify good: {repr e}" + let mut bad := enc + bad := bad.set! 4 (ofList [(enc[4]!).get! 0 ^^^ 0x01]) + match carbonadoRS.verify bad with + | .ok false => pure () + | .ok true => fail "verify bad: expected false" + | .error e => fail s!"verify bad: {repr e}" + IO.println "verify good/bad ok" + + match reconstructAfterKnockout + #[ofList [1], ofList [2], ofList [3], ofList [4], + ofList [0], ofList [0], ofList [0], ofList [0]] [0, 99] 0 with + | .error .badGeometry => pure () + | .error e => fail s!"knockout oob: expected badGeometry, got {repr e}" + | .ok _ => fail "knockout oob: ok" + IO.println "knockout oob badGeometry ok" + + IO.println "fec stack ok" + + -- Program D: BLAKE3 + keyed Bao (4 KiB leaves, format-byte key) + expectHex "blake3(empty)" (Carbonado.Bao.Blake3.hash ByteArray.empty) + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + expectHex "blake3(abc)" (Carbonado.Bao.Blake3.hash (utf8 "abc")) + "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85" + IO.println "blake3 goldens ok" + + expectHex "vkey c4" (carbonadoVerificationKey 4) + "6f1b6d31098f44f98e31231fe4244d532d1263556a45fe370d74cae1a447ffbf" + expectHex "vkey c6" (carbonadoVerificationKey 6) + "3923848b90f499febc394b34cbac1f3c2d9a98a89b93beb4bb6361d1db5d4615" + expectHex "vkey c14" (carbonadoVerificationKey 14) + "4f1be70ef9d46fe13ad7c55230a7703d90ee6601d0e24b041c5db041a8a26d44" + expectTrue "vkey domain" (toHex (carbonadoVerificationKey 4) != toHex (carbonadoVerificationKey 6)) + IO.println "verification key goldens ok" + + expectHex "keyed root hello c4" (rootForFormat 4 (utf8 "hello")) + "f8a7892045a78f933cca82f9ef17046c453ad166e5463e5e93c88cf614443d86" + expectHex "keyed root pat100 c4" (rootForFormat 4 (baoPattern 100)) + "27e8845ee8cfeed082734d4409991f9db4e9e4d0476de3ed196d89f4a2f37077" + expectHex "keyed root pat100 c6" (rootForFormat 6 (baoPattern 100)) + "1e478e01260caf8df6ad29f09c754a5e8423ee5ceceee4209db543828416147f" + expectTrue "root commits to format" + (toHex (rootForFormat 4 (baoPattern 100)) != toHex (rootForFormat 6 (baoPattern 100))) + IO.println "keyed root goldens ok" + + -- Inboard empty / hello / pat100 + let (r0, a0) := encodeInboardForFormat 4 ByteArray.empty + expectHex "inboard empty" a0 "0000000000000000" + expectHex "inboard empty root" r0 + "51ceb9e65f98d4ae586f2345d792b4d5af14f4222bfaa95dac58313034498294" + match decodeInboardForFormat 4 r0 a0 with + | .ok d => expectTrue "empty roundtrip" (d.size == 0) + | .error e => fail s!"empty decode: {repr e}" + + let (rHello, aHello) := encodeInboardForFormat 4 (utf8 "hello") + expectHex "inboard hello" aHello "050000000000000068656c6c6f" + match decodeInboardForFormat 4 rHello aHello with + | .ok d => expectTrue "hello roundtrip" (toHex d == toHex (utf8 "hello")) + | .error e => fail s!"hello decode: {repr e}" + + let bao100 := baoPattern 100 + let (r100, a100) := encodeInboardForFormat 4 bao100 + expectTrue "inboard pat100 len" (a100.size == 108) + match decodeInboardForFormat 4 r100 a100 with + | .ok d => expectTrue "pat100 roundtrip" (toHex d == toHex bao100) + | .error e => fail s!"pat100 decode: {repr e}" + IO.println "inboard encode/decode ok" + + -- Multi-leaf (5000 B) inboard + outboard + let bao5k := baoPattern 5000 + let (r5k, a5k) := encodeInboardForFormat 4 bao5k + expectHex "root pat5000" r5k + "d1168cb56536e2e0ae67934258019b239b875a07d812505df43155db53ae53ad" + expectTrue "inboard 5000 total" (a5k.size == 5072) + match decodeInboardForFormat 4 r5k a5k with + | .ok d => expectTrue "pat5000 roundtrip" (toHex d == toHex bao5k) + | .error e => fail s!"pat5000 decode: {repr e}" + let (rOb, ob) := encodeOutboardForFormat 4 bao5k + expectTrue "outboard root match" (toHex rOb == toHex r5k) + expectTrue "outboard len" (ob.size == 64) + expectHex "outboard 5000" ob + "d5b0f4c38a9c1ddc9d00230cb53225677b7b41f8fa15f7de4750aa32a7882cfdd0b117458bfb503d3f195a969fdc9d8f2b94984022a2a940b81b238f505e3b80" + match verifyOutboardForFormat 4 rOb bao5k ob with + | .ok _ => pure () + | .error e => fail s!"outboard verify: {repr e}" + IO.println "outboard encode/verify ok" + + -- Slice first group of 5000: stream decode (no plaintext oracle) + let (rSlice, sliceEnc) := encodeSliceForFormat 4 bao5k 0 1 + expectTrue "slice root" (toHex rSlice == toHex r5k) + expectTrue "slice enc len" (sliceEnc.size == 4160) + match decodeSliceForFormat 4 rSlice 5000 0 1 sliceEnc with + | .ok s => + expectTrue "slice size" (s.size == 4096) + expectTrue "slice bytes" (toHex s == toHex (bao5k.extract 0 4096)) + | .error e => fail s!"slice decode: {repr e}" + match verifySliceInboardForFormat 4 r5k a5k 0 1 with + | .ok s => expectTrue "slice inboard size" (s.size == 4096) + | .error e => fail s!"slice inboard: {repr e}" + -- count=0 after full inboard auth → empty ok + match verifySliceInboardForFormat 4 r5k a5k 0 0 with + | .ok s => expectTrue "count0 after auth" (s.size == 0) + | .error e => fail s!"count0 inboard: {repr e}" + -- corrupt inboard + count=0 must fail (auth-first) + let mut bad5k := a5k + bad5k := bad5k.set! 20 (bad5k.get! 20 ^^^ 0x01) + match verifySliceInboardForFormat 4 r5k bad5k 0 0 with + | Except.error .authenticationFailed => pure () + | Except.error e => fail s!"count0 corrupt: expected authenticationFailed, got {repr e}" + | Except.ok _ => fail "count0 corrupt: ok" + IO.println "slice encode/stream-decode ok" + + -- Three-leaf tree (12288 B): deeper nesting + let bao12k := baoPattern 12288 + let (r12, a12) := encodeInboardForFormat 4 bao12k + expectHex "root pat12288" r12 + "7390719e0ff132dd988f246240f60bd70e8b6cb52836f9978340676a5b442c9d" + expectTrue "inboard 12288 total" (a12.size == 12424) + match decodeInboardForFormat 4 r12 a12 with + | .ok d => expectTrue "pat12288 roundtrip" (toHex d == toHex bao12k) + | .error e => fail s!"pat12288 decode: {repr e}" + let (rOb12, ob12) := encodeOutboardForFormat 4 bao12k + expectTrue "outboard 12288 root" (toHex rOb12 == toHex r12) + expectTrue "outboard 12288 len" (ob12.size == 128) + -- middle slice (leaf index 1) stream decode + let (_rs, midSlice) := encodeSliceForFormat 4 bao12k 1 1 + expectTrue "mid slice enc len" (midSlice.size == 4224) + match decodeSliceForFormat 4 r12 12288 1 1 midSlice with + | .ok s => + expectTrue "mid slice size" (s.size == 4096) + expectTrue "mid slice bytes" (toHex s == toHex (bao12k.extract 4096 8192)) + | .error e => fail s!"mid slice decode: {repr e}" + IO.println "three-leaf tree ok" + + -- Wrong format key → authenticationFailed + match decodeInboardForFormat 6 r100 a100 with + | Except.error .authenticationFailed => pure () + | Except.error e => fail s!"wrong format: expected authenticationFailed, got {repr e}" + | Except.ok _ => fail "wrong format: ok" + IO.println "wrong format key → authenticationFailed ok" + + -- Slice wrong key → authenticationFailed (stream path) + match decodeSliceForFormat 6 rSlice 5000 0 1 sliceEnc with + | Except.error .authenticationFailed => pure () + | Except.error e => fail s!"slice wrong key: expected authenticationFailed, got {repr e}" + | Except.ok _ => fail "slice wrong key: ok" + IO.println "slice wrong key → authenticationFailed ok" + + -- Truncated response → truncatedResponse + match decodeInboardForFormat 4 r5k (a5k.extract 0 20) with + | Except.error .truncatedResponse => pure () + | Except.error e => fail s!"trunc: expected truncatedResponse, got {repr e}" + | Except.ok _ => fail "trunc: ok" + IO.println "truncated response → truncatedResponse ok" + + -- Truncated slice response → truncatedResponse + match decodeSliceForFormat 4 rSlice 5000 0 1 (sliceEnc.extract 0 20) with + | Except.error .truncatedResponse => pure () + | Except.error e => fail s!"slice trunc: expected truncatedResponse, got {repr e}" + | Except.ok _ => fail "slice trunc: ok" + IO.println "truncated slice → truncatedResponse ok" + + -- Trailing garbage after valid inboard response → trailingData + let mut trail := a100 + trail := trail.push 0xaa + match decodeInboardForFormat 4 r100 trail with + | Except.error .trailingData => pure () + | Except.error e => fail s!"trail: expected trailingData, got {repr e}" + | Except.ok _ => fail "trail: ok" + IO.println "trailing data → trailingData ok" + + -- Trailing garbage on slice response → trailingData + let mut sliceTrail := sliceEnc + sliceTrail := sliceTrail.push 0xbb + match decodeSliceForFormat 4 rSlice 5000 0 1 sliceTrail with + | Except.error .trailingData => pure () + | Except.error e => fail s!"slice trail: expected trailingData, got {repr e}" + | Except.ok _ => fail "slice trail: ok" + IO.println "slice trailing data → trailingData ok" + + -- Invalid prefix → invalidPrefix + match Carbonado.Bao.Tree.contentLenPrefix (ofList [1, 2, 3]) with + | Except.error .invalidPrefix => pure () + | Except.error e => fail s!"prefix: expected invalidPrefix, got {repr e}" + | Except.ok _ => fail "prefix: ok" + IO.println "short prefix → invalidPrefix ok" + + -- Invalid root length → invalidRootLength + match decodeInboardForFormat 4 (ofList [0]) a100 with + | Except.error .invalidRootLength => pure () + | Except.error e => fail s!"rootlen: expected invalidRootLength, got {repr e}" + | Except.ok _ => fail "rootlen: ok" + IO.println "bad root length → invalidRootLength ok" + + -- Invalid slice index → invalidSliceIndex + match verifySliceInboardForFormat 4 r100 a100 5 1 with + | Except.error .invalidSliceIndex => pure () + | Except.error e => fail s!"slice idx: expected invalidSliceIndex, got {repr e}" + | Except.ok _ => fail "slice idx: ok" + IO.println "bad slice index → invalidSliceIndex ok" + + -- count=0 on stream decode → invalidSliceCount + match decodeSliceForFormat 4 rSlice 5000 0 0 sliceEnc with + | Except.error .invalidSliceCount => pure () + | Except.error e => fail s!"slice count0: expected invalidSliceCount, got {repr e}" + | Except.ok _ => fail "slice count0: ok" + IO.println "slice count 0 → invalidSliceCount ok" + + -- Tampered inboard body → authenticationFailed + let mut badArt := a100 + if badArt.size > 10 then + badArt := badArt.set! 10 (badArt.get! 10 ^^^ 0x01) + match decodeInboardForFormat 4 r100 badArt with + | Except.error .authenticationFailed => pure () + | Except.error e => fail s!"tamper body: expected authenticationFailed, got {repr e}" + | Except.ok _ => fail "tamper body: ok" + IO.println "tampered body → authenticationFailed ok" + + -- Tampered slice response → authenticationFailed + let mut badSlice := sliceEnc + badSlice := badSlice.set! 70 (badSlice.get! 70 ^^^ 0x01) + match decodeSliceForFormat 4 rSlice 5000 0 1 badSlice with + | Except.error .authenticationFailed => pure () + | Except.error e => fail s!"slice tamper: expected authenticationFailed, got {repr e}" + | Except.ok _ => fail "slice tamper: ok" + IO.println "tampered slice → authenticationFailed ok" + + IO.println "bao stack ok" + + -- Program E: Header + pipeline c0–c15 + stream bounds + scrub + shard + match Header.new master42 nonce11 (replicate 32 0xcd) (replicate 32 0) 5 0 100 0 + (replicate 8 0) with + | .error e => fail s!"Header.new: {repr e}" + | .ok h => + match h.toBytes with + | .error e => fail s!"Header.toBytes: {repr e}" + | .ok wire => + expectTrue "header wire 177" (wire.size == 177) + match parseAndVerify master42 wire with + | .ok h2 => + expectTrue "header roundtrip format" (h2.format == 5) + expectTrue "header roundtrip nonce" (toHex h2.payloadNonce == toHex nonce11) + | .error e => fail s!"parseAndVerify: {repr e}" + -- Tamper header_mac → headerAuthenticationFailed + let mut badWire := wire + badWire := badWire.set! 28 (badWire.get! 28 ^^^ 0x01) + match parseAndVerify master42 badWire with + | .error .headerAuthenticationFailed => pure () + | .error e => fail s!"tamper hdr: expected headerAuthenticationFailed, got {repr e}" + | .ok _ => fail "tamper hdr: ok" + IO.println "header wire + verify ok" + + -- badMagic (HeaderError, mapped to PipelineError on decodeHeadered) + match parseAndVerify master42 (replicate 177 0) with + | .error .badMagic => pure () + | .error e => fail s!"badMagic: expected HeaderError.badMagic, got {repr e}" + | .ok _ => fail "badMagic: ok" + -- Full archive with bad magic after length pad → pipeline badMagic + match decodeHeadered master42 (replicate 200 0) with + | .error .badMagic => pure () + | .error e => fail s!"pipe badMagic: expected badMagic, got {repr e}" + | .ok _ => fail "pipe badMagic: ok" + IO.println "badMagic → HeaderError/PipelineError.badMagic ok" + + -- invalidHeaderLength + match decodeHeadered master42 (ofList [1, 2, 3]) with + | .error .invalidHeaderLength => pure () + | .error e => fail s!"short hdr: expected invalidHeaderLength, got {repr e}" + | .ok _ => fail "short hdr: ok" + IO.println "short header → invalidHeaderLength ok" + + -- invalidFieldLength + match Header.new master42 nonce11 (ofList [1]) (replicate 32 0) 0 0 0 0 (replicate 8 0) with + | .error .invalidFieldLength => pure () + | .error e => fail s!"bad field: expected invalidFieldLength, got {repr e}" + | .ok _ => fail "bad field: ok" + IO.println "invalidFieldLength ok" + + -- Format matrix body roundtrips (all 16 formats; AOT uses real zstd-20 when Compression bit set) + match formatMatrixRoundtrip master42 nonce11 (utf8 "hi") with + | .ok true => pure () + | .ok false => fail "format matrix: mismatch" + | .error e => fail s!"format matrix: {repr e}" + IO.println "format matrix c0–c15 roundtrip ok" + + -- Headered encrypted path (c5) + public bao (c4) + match roundtripHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 5) with + | .ok true => pure () + | .ok false => fail "headered c5 mismatch" + | .error e => fail s!"headered c5: {repr e}" + match roundtripHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) with + | .ok true => pure () + | .ok false => fail "headered c4 mismatch" + | .error e => fail s!"headered c4: {repr e}" + -- c12 and c15 (FEC + Bao) + match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 12) with + | .ok true => pure () + | .ok false => fail "c12 mismatch" + | .error e => fail s!"c12: {repr e}" + match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 15) with + | .ok true => pure () + | .ok false => fail "c15 mismatch" + | .error e => fail s!"c15: {repr e}" + IO.println "headered + c12/c15 roundtrip ok" + + -- encoded_len bound: short body → truncatedBody; trailer ignored (c0 + c5) + match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) 0 + zeroSlhPk zeroMeta with + | .error e => fail s!"enc c0 for len: {repr e}" + | .ok (_h, arch) => + -- trailer after body still recovers + let withTrailer := appendBA arch (ofList [0xaa, 0xbb, 0xcc]) + match decodeHeadered master42 withTrailer with + | .ok pt => expectTrue "c0 trailer ignore" (toHex pt == toHex (utf8 "hello")) + | .error e => fail s!"c0 trailer: {repr e}" + -- short body + if arch.size > headerLen + 1 then + let short := arch.extract 0 (headerLen + 1) + match decodeHeadered master42 short with + | .error .truncatedBody => pure () + | .error e => fail s!"short body: expected truncatedBody, got {repr e}" + | .ok _ => fail "short body: ok" + match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 5) 0 + zeroSlhPk zeroMeta with + | .error e => fail s!"enc c5 for trailer: {repr e}" + | .ok (_h, arch) => + let withTrailer := appendBA arch (ofList [0xde, 0xad]) + match decodeHeadered master42 withTrailer with + | .ok pt => expectTrue "c5 trailer ignore" (toHex pt == toHex (utf8 "hello")) + | .error e => fail s!"c5 trailer: {repr e}" + IO.println "encoded_len truncatedBody + trailer ignore ok" + + -- Payload auth failure via pipeline decrypt (embedded, tampered after encode) + match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 1) false with + | .error e => fail s!"enc c1: {repr e}" + | .ok enc => + let mut bad := enc.body + if bad.size > 20 then + bad := bad.set! 20 (bad.get! 20 ^^^ 0x01) + match decodeBody master42 nonce11 enc.baoHash bad enc.info.paddingLen + (FormatBits.ofUInt8 1) false with + | .error .payloadAuthenticationFailed => pure () + | .error e => fail s!"payload tamper: expected payloadAuthenticationFailed, got {repr e}" + | .ok _ => fail "payload tamper: ok" + IO.println "payload tamper → payloadAuthenticationFailed ok" + + -- Composition: short ciphertext → invalidCiphertextLength (c1, no Bao) + match decodeBody master42 nonce11 zeroHash (replicate 10 0) 0 + (FormatBits.ofUInt8 1) false with + | .error .invalidCiphertextLength => pure () + | .error e => fail s!"short ct pipe: expected invalidCiphertextLength, got {repr e}" + | .ok _ => fail "short ct pipe: ok" + -- Composition: FEC padding too large on empty body with pad>0 + match decodeBody master42 nonce11 zeroHash ByteArray.empty 5 + (FormatBits.ofUInt8 8) false with + | .error .paddingTooLarge => pure () + | .error e => fail s!"pad large: expected paddingTooLarge, got {repr e}" + | .ok _ => fail "pad large: ok" + -- Composition: truncated Bao response via pipeline c4 (short prefix → invalidPrefix) + match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error e => fail s!"enc c4 trunc: {repr e}" + | .ok enc => + match decodeBody master42 nonce11 enc.baoHash (enc.body.extract 0 4) + enc.info.paddingLen (FormatBits.ofUInt8 4) false with + | .error .invalidPrefix => pure () + | .error e => fail s!"trunc bao: expected invalidPrefix, got {repr e}" + | .ok _ => fail "trunc bao: ok" + -- Composition: trailing garbage after valid Bao inboard → trailingData + match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error e => fail s!"enc c4 trail: {repr e}" + | .ok enc => + let trailed := appendBA enc.body (ofList [0xaa]) + match decodeBody master42 nonce11 enc.baoHash trailed + enc.info.paddingLen (FormatBits.ofUInt8 4) false with + | .error .trailingData => pure () + | .error e => fail s!"trail bao: expected trailingData, got {repr e}" + | .ok _ => fail "trail bao: ok" + -- Composition: invalid root length + match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error e => fail s!"enc c4 root: {repr e}" + | .ok enc => + match decodeBody master42 nonce11 (ofList [0]) enc.body + enc.info.paddingLen (FormatBits.ofUInt8 4) false with + | .error .invalidRootLength => pure () + | .error e => fail s!"root len: expected invalidRootLength, got {repr e}" + | .ok _ => fail "root len: ok" + IO.println "composition invalidCiphertextLength + paddingTooLarge + bao trunc ok" + + -- invalidNonceLength / invalidKeyLength via pipeline + match encodeBody master42 (replicate 8 0) (utf8 "hi") (FormatBits.ofUInt8 1) false with + | .error .invalidNonceLength => pure () + | .error e => fail s!"bad nonce pipe: expected invalidNonceLength, got {repr e}" + | .ok _ => fail "bad nonce pipe: ok" + match encodeBody (replicate 16 0) nonce11 (utf8 "hi") (FormatBits.ofUInt8 1) false with + | .error .invalidKeyLength => pure () + | .error e => fail s!"short master pipe: expected invalidKeyLength, got {repr e}" + | .ok _ => fail "short master pipe: ok" + IO.println "pipeline invalidNonceLength + invalidKeyLength ok" + + -- Bao auth fail via pipeline (wrong root on c4) + match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error e => fail s!"enc c4: {repr e}" + | .ok enc => + let badRoot := replicate 32 0xaa + match decodeBody master42 nonce11 badRoot enc.body enc.info.paddingLen + (FormatBits.ofUInt8 4) false with + | .error .baoAuthenticationFailed => pure () + | .error e => fail s!"bao wrong root: expected baoAuthenticationFailed, got {repr e}" + | .ok _ => fail "bao wrong root: ok" + IO.println "wrong bao root → baoAuthenticationFailed ok" + + -- FEC uneven via pipeline fec decode + match fecDecodeStep (ofList [1, 2, 3]) 0 true with + | .error .unevenShards => pure () + | .error e => fail s!"uneven pipe: expected unevenShards, got {repr e}" + | .ok _ => fail "uneven pipe: ok" + IO.println "fecDecodeStep uneven → unevenShards ok" + + -- ofFecError / ofBaoError / ofCryptoError exact maps (remaining taxonomy) + expectTrue "map tooFew" (ofFecError .tooFewShards == .tooFewShards) + expectTrue "map emptyShard" (ofFecError .emptyShard == .emptyShard) + expectTrue "map incorrectSize" (ofFecError .incorrectShardSize == .incorrectShardSize) + expectTrue "map badGeometry" (ofFecError .badGeometry == .badGeometry) + expectTrue "map paddingTooLarge" (ofFecError .paddingTooLarge == .paddingTooLarge) + expectTrue "map singular" (ofFecError .singularMatrix == .singularMatrix) + expectTrue "map trunc" (ofBaoError .truncatedResponse == .truncatedResponse) + expectTrue "map trail" (ofBaoError .trailingData == .trailingData) + expectTrue "map prefix" (ofBaoError .invalidPrefix == .invalidPrefix) + expectTrue "map rootLen" (ofBaoError .invalidRootLength == .invalidRootLength) + expectTrue "map sliceIdx" (ofBaoError .invalidSliceIndex == .invalidSliceIndex) + expectTrue "map sliceCnt" (ofBaoError .invalidSliceCount == .invalidSliceCount) + expectTrue "map ctLen" (ofCryptoError .invalidCiphertextLength == .invalidCiphertextLength) + -- Map-only residual (not reachable as distinct product pipeline paths without + -- fabricating lower-layer inputs): tooFewShards, emptyShard, incorrectShardSize, + -- singularMatrix, invalidSliceIndex, invalidSliceCount — see SPEC-MATRIX / PROOFS. + IO.println "PipelineError taxonomy maps ok" + + -- Stream bounds + expectTrue "stripe retain" (maxFecStripeRetain stripeUnit == 32768) + expectTrue "empty retain" (maxFecStripeRetain 0 == 0) + expectTrue "one-byte retain" (maxFecStripeRetain 1 == 32768) + expectTrue "chunk eq slice" (chunkBytes == sliceLen) + IO.println "stream stripe bounds ok" + + -- Scrub: requires verification + match scrubInboardArchive (utf8 "x") (replicate 32 0) (FormatBits.ofUInt8 0) with + | .error .scrubRequiresVerification => pure () + | .error e => fail s!"scrub noV: expected scrubRequiresVerification, got {repr e}" + | .ok _ => fail "scrub noV: ok" + IO.println "scrubRequiresVerification ok" + + -- Scrub: pristine → unnecessaryScrub + match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error e => fail s!"enc for scrub: {repr e}" + | .ok enc => + match scrubInboardArchive enc.body enc.baoHash (FormatBits.ofUInt8 4) with + | .error .unnecessaryScrub => pure () + | .error e => fail s!"pristine scrub: expected unnecessaryScrub, got {repr e}" + | .ok _ => fail "pristine scrub: ok" + IO.println "unnecessaryScrub ok" + + -- Scrub: knockout data shards, recover via RS + Bao root + match Carbonado.Fec.Inboard.encodeInboard (utf8 "hello") with + | .error e => fail s!"fec for scrub: {repr e}" + | .ok (fecBody, pad, _) => + let (root, art) := encodeInboardForFormat 12 fecBody + match scrubWithMissing fecBody root pad 12 [0, 1, 2, 3] with + | .ok rec => + expectTrue "scrub recover len" (rec.size == art.size) + expectTrue "scrub recover bytes" (toHex rec == toHex art) + | .error e => fail s!"scrub knockout: {repr e}" + -- Too many missing (5) → invalidScrubbedHash + match scrubWithMissing fecBody root pad 12 [0, 1, 2, 3, 4] with + | .error .invalidScrubbedHash => pure () + | .error e => fail s!"scrub too many: expected invalidScrubbedHash, got {repr e}" + | .ok _ => fail "scrub too many: ok" + -- Empty FEC body → badGeometry (no panic) + match scrubWithMissing ByteArray.empty (replicate 32 0) 0 12 [0] with + | .error .badGeometry => pure () + | .error e => fail s!"scrub empty: expected badGeometry, got {repr e}" + | .ok _ => fail "scrub empty: ok" + match scrubAfterKnockout ByteArray.empty (replicate 32 0) 0 12 [0] with + | .error .badGeometry => pure () + | .error e => fail s!"scrubAfter empty: expected badGeometry, got {repr e}" + | .ok _ => fail "scrubAfter empty: ok" + IO.println "scrub knockout recovery + invalidScrubbedHash ok" + + -- Sharding + let n0 := replicate 16 0x11 + let n1 := replicate 16 0x22 + let n2 := replicate 16 0x33 + match roundtripShards master42 (utf8 "abcdefghij") (FormatBits.ofUInt8 0) 4 #[n0, n1, n2] with + | .ok true => pure () + | .ok false => fail "shards mismatch" + | .error e => fail s!"shards: {repr e}" + match encodeShards master42 (utf8 "ab") (FormatBits.ofUInt8 0) 0 #[n0] zeroSlhPk zeroMeta with + | .error .emptySegment => pure () + | .error e => fail s!"budget0: expected emptySegment, got {repr e}" + | .ok _ => fail "budget0: ok" + -- insufficientNonces (distinct from invalidNonceLength) + match encodeShards master42 (utf8 "abcdefghij") (FormatBits.ofUInt8 0) 4 + #[n0] zeroSlhPk zeroMeta with + | .error .insufficientNonces => pure () + | .error e => fail s!"few nonces: expected insufficientNonces, got {repr e}" + | .ok _ => fail "few nonces: ok" + match validateChunkSequence #[0, 2] with + | .error .invalidChunkSequence => pure () + | .error e => fail s!"gap seq: expected invalidChunkSequence, got {repr e}" + | .ok _ => fail "gap seq: ok" + match validateChunkSequence #[0, 0] with + | .error .invalidChunkSequence => pure () + | .error e => fail s!"dup seq: expected invalidChunkSequence, got {repr e}" + | .ok _ => fail "dup seq: ok" + -- Structure label disagrees with verified header chunk_index + match encodeShards master42 (utf8 "abcdefgh") (FormatBits.ofUInt8 0) 4 + #[n0, n1] zeroSlhPk zeroMeta with + | .error e => fail s!"enc shards for label: {repr e}" + | .ok shards => + if shards.size ≥ 1 then + let s0 := shards[0]! + let lied := { s0 with chunkIndex := 99 } + match decodeShards master42 #[lied] with + | .error .invalidChunkSequence => pure () + | .error e => fail s!"label lie: expected invalidChunkSequence, got {repr e}" + | .ok _ => fail "label lie: ok" + IO.println "shard roundtrip + sequence errors ok" + + -- Encrypted formats odd + expectTrue "c15 odd" (formatC15.toUInt8 % 2 == 1) + expectTrue "c14 even" (formatC14.toUInt8 % 2 == 0) + IO.println "encrypted formats odd ok" + + IO.println "pipeline stack ok" + + ------------------------------------------------------------------ + -- Program F: zstd compress + SLH1 wire / bind-to-root + ------------------------------------------------------------------ + expectTrue "zstd level 20" (zstdLevel == 20) + expectTrue "zstd magic len" (zstdMagic.length == 4) + -- Status mapping (pure) + expectTrue "status empty" (match decodeStatusPayload ByteArray.empty with + | .error .invalidInput => true | _ => false) + expectTrue "status 1" (match decodeStatusPayload (ofList [1]) with + | .error .compressionFailed => true | _ => false) + expectTrue "status 2" (match decodeStatusPayload (ofList [2]) with + | .error .decompressionFailed => true | _ => false) + expectTrue "status 3" (match decodeStatusPayload (ofList [3]) with + | .error .outputTooLarge => true | _ => false) + expectTrue "status 4" (match decodeStatusPayload (ofList [4]) with + | .error .invalidInput => true | _ => false) + IO.println "zstd status mapping ok" + + -- Pipeline ofZstdError maps (distinct) + expectTrue "map compressionFailed" + (ofZstdError ZstdError.compressionFailed == PipelineError.compressionFailed) + expectTrue "map decompressionFailed" + (ofZstdError ZstdError.decompressionFailed == PipelineError.decompressionFailed) + expectTrue "map outputTooLarge" + (ofZstdError ZstdError.outputTooLarge == PipelineError.decompressOutputTooLarge) + expectTrue "map invalidInput" + (ofZstdError ZstdError.invalidInput == PipelineError.zstdInvalidInput) + IO.println "PipelineError zstd maps ok" + + -- AOT real zstd: hello golden from ZSTD_compress API (level 20) + let hello := utf8 "hello" + match compressLevel20 hello with + | .error e => fail s!"zstd compress hello: {repr e}" + | .ok ct => + expectTrue "zstd hello magic" (hasZstdMagic ct) + expectHex "zstd hello level20" ct "28b52ffd200529000068656c6c6f" + match decompress ct with + | .error e => fail s!"zstd decompress hello: {repr e}" + | .ok pt => expectTrue "zstd hello roundtrip" (ctEq pt hello) + -- Empty compress golden + match compressLevel20 ByteArray.empty with + | .error e => fail s!"zstd empty compress: {repr e}" + | .ok ct => + expectHex "zstd empty level20" ct "28b52ffd2000010000" + match decompress ct with + | .error e => fail s!"zstd empty decompress: {repr e}" + | .ok pt => expectTrue "zstd empty roundtrip" (ctEq pt ByteArray.empty) + -- Corrupt frame → decompressionFailed (not lumped) + match decompress (ofList [0x00, 0x01, 0x02, 0x03]) with + | .error .decompressionFailed => pure () + | .error e => fail s!"corrupt zstd: expected decompressionFailed, got {repr e}" + | .ok _ => fail "corrupt zstd: ok" + -- Tiny maxOut on non-empty frame → outputTooLarge when content known + match compressLevel20 hello with + | .error e => fail s!"zstd for max: {repr e}" + | .ok ct => + match decompressWithMax ct 1 with + | .error .outputTooLarge => pure () + | .error e => fail s!"maxOut: expected outputTooLarge, got {repr e}" + | .ok _ => fail "maxOut: ok" + -- Highly compressible: many zeros → smaller than input under real zstd + let zeros := replicate 4096 0 + match compressLevel20 zeros with + | .error e => fail s!"zstd zeros: {repr e}" + | .ok ct => + expectTrue "zstd zeros shrinks" (ct.size < zeros.size) + match decompress ct with + | .error e => fail s!"zstd zeros dec: {repr e}" + | .ok pt => expectTrue "zstd zeros roundtrip" (ctEq pt zeros) + IO.println "zstd goldens + roundtrip + error paths ok" + + -- Pipeline c2 (compression only) roundtrip under AOT zstd + match roundtripBody master42 nonce11 hello (FormatBits.ofUInt8 2) with + | .ok true => pure () + | .ok false => fail "c2 zstd pipeline mismatch" + | .error e => fail s!"c2 zstd pipeline: {repr e}" + -- c6 = compression + verification + match roundtripBody master42 nonce11 hello (FormatBits.ofUInt8 6) with + | .ok true => pure () + | .ok false => fail "c6 zstd+bao mismatch" + | .error e => fail s!"c6 zstd+bao: {repr e}" + -- Headered + compression: c3 (encrypted|compression) and c7 (E|C|V) + match roundtripHeadered master42 nonce11 hello (FormatBits.ofUInt8 3) with + | .ok true => pure () + | .ok false => fail "headered c3 zstd mismatch" + | .error e => fail s!"headered c3 zstd: {repr e}" + match roundtripHeadered master42 nonce11 hello (FormatBits.ofUInt8 7) with + | .ok true => pure () + | .ok false => fail "headered c7 zstd mismatch" + | .error e => fail s!"headered c7 zstd: {repr e}" + IO.println "pipeline compression formats c2/c6 + headered c3/c7 ok" + + -- SLH1 wire + expectTrue "slh magic" (ctEq slh1MagicBA (ofList [0x53, 0x4c, 0x48, 0x31])) + expectTrue "slh sidecar len" (slh1SidecarLen == 7860) + match buildSidecar (replicate slh1SignatureLen 0) with + | .error e => fail s!"build sidecar: {repr e}" + | .ok sc => + expectTrue "sidecar wire len" (sc.size == 7860) + match parseSidecar sc with + | .ok sig => expectTrue "parse sig zeros" (ctEq sig (replicate slh1SignatureLen 0)) + | .error e => fail s!"parse sidecar: {repr e}" + match parseSidecar (ofList [1, 2, 3]) with + | .error .invalidSidecarLength => pure () + | .error e => fail s!"short sidecar: expected invalidSidecarLength, got {repr e}" + | .ok _ => fail "short sidecar: ok" + match parseSidecar (replicate slh1SidecarLen 0) with + | .error .badSlhMagic => pure () + | .error e => fail s!"bad magic: expected badSlhMagic, got {repr e}" + | .ok _ => fail "bad magic: ok" + match buildSidecar (ofList [1]) with + | .error .invalidSignatureLength => pure () + | .error e => fail s!"short sig: expected invalidSignatureLength, got {repr e}" + | .ok _ => fail "short sig: ok" + IO.println "SLH1 wire framing ok" + + -- Bind-to-root (mock oracle) + full-length wire roundtrip (AOT only) + let rootA := replicate hashLen 0xaa + let rootB := replicate hashLen 0xbb + let pk := replicate slhPublicKeyLen 0x11 + let goodSig := replicate slh1SignatureLen 0xcd + let badSig := replicate slh1SignatureLen 0x00 + match buildSidecar goodSig with + | .error e => fail s!"build full sidecar: {repr e}" + | .ok sc => + expectTrue "full sidecar len" (sc.size == 7860) + match parseSidecar sc with + | .ok sig => expectTrue "full parse sig" (ctEq sig goodSig) + | .error e => fail s!"parse full sidecar: {repr e}" + -- wrong magic at exact length + let badMag := appendBA (ofList [0, 0, 0, 0]) goodSig + match parseSidecar badMag with + | .error .badSlhMagic => pure () + | .error e => fail s!"bad magic full: expected badSlhMagic, got {repr e}" + | .ok _ => fail "bad magic full: ok" + match verifyBoundToExpected (mockOracleFor rootA goodSig) pk rootA rootB goodSig with + | .error .verificationFailed => pure () + | .error e => fail s!"wrong root: expected verificationFailed, got {repr e}" + | .ok _ => fail "wrong root: ok" + match verifyBoundToExpected (mockOracleFor rootA goodSig) pk rootA rootA goodSig with + | .ok () => pure () + | .error e => fail s!"correct root: {repr e}" + match verifyBoundToExpected (mockOracleFor rootA goodSig) pk rootA rootA badSig with + | .error .verificationFailed => pure () + | .error e => fail s!"bad sig: expected verificationFailed, got {repr e}" + | .ok _ => fail "bad sig: ok" + match signRoot (replicate 128 0x42) rootA with + | .error .signatureUnavailable => pure () + | .error e => fail s!"sign: expected signatureUnavailable, got {repr e}" + | .ok _ => fail "sign: ok" + match signRoot (replicate 128 0x42) (ofList [1]) with + | .error .invalidRootLength => pure () + | .error e => fail s!"sign root len: expected invalidRootLength, got {repr e}" + | .ok _ => fail "sign root len: ok" + match mkBinding (ofList [1]) rootA goodSig with + | .error .invalidPublicKeyLength => pure () + | .error e => fail s!"pk len: expected invalidPublicKeyLength, got {repr e}" + | .ok _ => fail "pk len: ok" + IO.println "SLH bind-to-root + unavailable sign ok" + + IO.println "program F stack ok" + + -- ── Program G: Adamantine + Filepack + Directory ── + expectTrue "adamantine magic len" (adamantineMagic.length == 13) + expectTrue "adamantine header len" (adamantineHeaderLen == 19) + let adamPayload := utf8 "catalog-body" + let adamWire := encodeAdamantine adamPayload adamantineFmtPublic 0 + match decodeAdamantine adamWire with + | .error e => fail s!"adamantine decode: {repr e}" + | .ok (p, h) => + expectTrue "adamantine payload" (ctEq p adamPayload) + expectTrue "adamantine fmt" (h.carbonadoFmt == adamantineFmtPublic) + expectTrue "adamantine flags" (h.flags == 0) + match decodeAdamantine (encodeAdamantine ByteArray.empty adamantineFmtPublic 2) with + | .error (.invalidFlags 2) => pure () + | .error e => fail s!"adam flags: expected invalidFlags 2, got {repr e}" + | .ok _ => fail "adam flags: ok" + match decodeAdamantine (encodeAdamantine ByteArray.empty 0 0) with + | .error (.invalidCarbonadoFormat 0) => pure () + | .error e => fail s!"adam fmt: expected invalidCarbonadoFormat, got {repr e}" + | .ok _ => fail "adam fmt: ok" + match decodeAdamantine (appendBA adamantineMagicDevV2 (replicate 7 0)) with + | .error (.unsupportedVersion 2 0) => pure () + | .error e => fail s!"adam dev2: expected unsupportedVersion 2 0, got {repr e}" + | .ok _ => fail "adam dev2: ok" + match decodeAdamantine (ofList [1, 2, 3]) with + | .error .invalidHeader => pure () + | .error e => fail s!"adam short: expected invalidHeader, got {repr e}" + | .ok _ => fail "adam short: ok" + match buildPayload (utf8 "man") (utf8 "bun") with + | .error e => fail s!"buildPayload: {repr e}" + | .ok pl => + match splitPayload pl with + | .error e => fail s!"splitPayload: {repr e}" + | .ok (m, b) => + expectTrue "payload man" (ctEq m (utf8 "man")) + expectTrue "payload bun" (ctEq b (utf8 "bun")) + IO.println "adamantine wire ok" + + -- Path validation + match validateRelPath "" with + | .error .emptyRelPath => pure () + | _ => fail "rel empty" + match validateRelPath "a/../b" with + | .error .relPathTraversal => pure () + | _ => fail "rel traversal" + match validateRelPath "/abs" with + | .error .relPathAbsolute => pure () + | _ => fail "rel absolute" + match validateRelPath "a\\b" with + | .error .relPathBackslash => pure () + | _ => fail "rel backslash" + match validateRelPath "a//b" with + | .error .relPathEmptyComponent => pure () + | _ => fail "rel empty component" + match validateRelPath "ok/file.txt" with + | .ok () => pure () + | _ => fail "rel ok" + IO.println "filepack path rules ok" + + -- Outboard c12/c14 roundtrip + let pubMaster := replicate 32 0 + match roundtripOutboard pubMaster nonce11 (utf8 "hello-outboard") (FormatBits.ofUInt8 12) with + | .error e => fail s!"outboard c12: {repr e}" + | .ok false => fail "outboard c12 mismatch" + | .ok true => pure () + match roundtripOutboard pubMaster nonce11 (utf8 "hello-c14") (FormatBits.ofUInt8 14) with + | .error e => fail s!"outboard c14: {repr e}" + | .ok false => fail "outboard c14 mismatch" + | .ok true => pure () + match roundtripOutboard master42 nonce11 (utf8 "secret-seg") (FormatBits.ofUInt8 13) with + | .error e => fail s!"outboard c13: {repr e}" + | .ok false => fail "outboard c13 mismatch" + | .ok true => pure () + IO.println "outboard segment roundtrip ok" + + -- Directory pure roundtrip (public) + let dirFiles : Array DirFile := #[ + { relPath := "a.txt", content := utf8 "alpha" }, + { relPath := "sub/b.txt", content := utf8 "beta" } + ] + let dirOpts : DirectoryEncodeOptions := { + catalogEncrypted := false + segmentPolicy := .forceCompressed + } + match roundtripDirectory pubMaster dirFiles dirOpts #[] with + | .error e => fail s!"directory public roundtrip: {repr e}" + | .ok false => fail "directory public content mismatch" + | .ok true => pure () + -- Encrypted directory + let encOpts : DirectoryEncodeOptions := { + catalogEncrypted := true + segmentPolicy := .forceCompressed + } + -- 2 files + 1 catalog nonce + let n0 := replicate 16 0x10 + let n1 := replicate 16 0x20 + let n2 := replicate 16 0x30 + match roundtripDirectory master42 dirFiles encOpts #[n0, n1, n2] with + | .error e => fail s!"directory encrypted roundtrip: {repr e}" + | .ok false => fail "directory encrypted content mismatch" + | .ok true => pure () + -- Path traversal rejected at encode + let badFiles : Array DirFile := #[{ relPath := "../evil", content := utf8 "x" }] + match encodeDirectory pubMaster badFiles dirOpts #[] with + | .error .pathTraversal => pure () + | .error e => fail s!"traversal: expected pathTraversal, got {repr e}" + | .ok _ => fail "traversal: ok" + match encodeDirectory pubMaster #[{ relPath := "/abs", content := utf8 "x" }] dirOpts #[] with + | .error .pathAbsolute => pure () + | .error e => fail s!"absolute: expected pathAbsolute, got {repr e}" + | .ok _ => fail "absolute: ok" + -- Zero master on encrypted rejected + match encodeDirectory pubMaster dirFiles encOpts #[n0, n1, n2] with + | .error .zeroMasterKeyNotAllowed => pure () + | .error e => fail s!"zero master enc: expected zeroMasterKeyNotAllowed, got {repr e}" + | .ok _ => fail "zero master enc: ok" + -- Non-zero master on public rejected + match encodeDirectory master42 dirFiles dirOpts #[] with + | .error .encryptedDirectoryNotRequested => pure () + | .error e => fail s!"public non-zero: expected encryptedDirectoryNotRequested, got {repr e}" + | .ok _ => fail "public non-zero: ok" + -- Catalog name parse + match parseCatalogName "aa.adam.c14" with + | .error .invalidCatalogPath => pure () + | _ => fail "short catalog name" + -- requireOts rejected at encode (not minting undecodeable archives) + match encodeDirectory pubMaster dirFiles { catalogEncrypted := false, segmentPolicy := .forceRaw, requireOts := true } #[] with + | .error .otsFeatureRequired => pure () + | .error e => fail s!"requireOts encode: expected otsFeatureRequired, got {repr e}" + | .ok _ => fail "requireOts encode: ok" + IO.println "directory pure encode/decode ok" + + -- Exact-variant decode failures (AOT; after a good encode) + match encodeDirectory pubMaster + #[{ relPath := "only.txt", content := utf8 "payload" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } #[] with + | .error e => fail s!"encode for tamper tests: {repr e}" + | .ok arch => + -- catalogBaoRootMismatch: wrong root in structure vs filename + let badRootArch := { arch with catalogBaoRoot := replicate 32 0xab } + match decodeDirectory pubMaster badRootArch with + | .error .catalogBaoRootMismatch => pure () + | .error e => fail s!"catalog root: expected catalogBaoRootMismatch, got {repr e}" + | .ok _ => fail "catalog root: ok" + -- missingSegment: drop all segment artifacts + let noSegs := { arch with segments := #[] } + match decodeDirectory pubMaster noSegs with + | .error .missingSegment => pure () + | .error e => fail s!"missing segment: expected missingSegment, got {repr e}" + | .ok _ => fail "missing segment: ok" + -- segmentMainLenMismatch: truncate main bytes + if arch.segments.size > 0 then + let s0 := arch.segments[0]! + let shortMain := + if s0.main.size > 0 then s0.main.extract 0 (s0.main.size - 1) else s0.main + let badLenSegs := #[ { s0 with main := shortMain } ] + match decodeDirectory pubMaster { arch with segments := badLenSegs } with + | .error .segmentMainLenMismatch => pure () + | .error e => fail s!"main len: expected segmentMainLenMismatch, got {repr e}" + | .ok _ => fail "main len: ok" + -- contentBlake3Mismatch exact helper (integrated into decodeDirectory) + match checkContentBlake3 (utf8 "hello") (replicate 32 0) with + | .error .contentBlake3Mismatch => pure () + | .error e => fail s!"blake3 helper: expected contentBlake3Mismatch, got {repr e}" + | .ok _ => fail "blake3 helper: ok" + match checkContentBlake3 ByteArray.empty (Carbonado.Bao.Blake3.hash ByteArray.empty) with + | .ok () => pure () + | .error e => fail s!"blake3 helper ok path: {repr e}" + IO.println "directory exact failure modes ok" + + -- DirectoryError maps (1:1) + expectTrue "map traversal" (ofFilepackError .relPathTraversal == DirectoryError.pathTraversal) + expectTrue "map absolute" (ofFilepackError .relPathAbsolute == DirectoryError.pathAbsolute) + expectTrue "map backslash" (ofFilepackError .relPathBackslash == DirectoryError.pathBackslash) + expectTrue "map null" (ofFilepackError .relPathNullByte == DirectoryError.pathNullByte) + expectTrue "map empty component" (ofFilepackError .relPathEmptyComponent == DirectoryError.pathEmptyComponent) + expectTrue "map tooManySegments" (ofFilepackError .tooManySegments == DirectoryError.tooManySegments) + expectTrue "map otsProofTooLarge" (ofFilepackError .otsProofTooLarge == DirectoryError.otsProofTooLarge) + expectTrue "map adam flags" (ofAdamantineError (.invalidFlags 3) == .invalidAdamantineFlags 3) + expectTrue "map adam magic" (ofAdamantineError .invalidMagic == .invalidAdamantineMagic) + -- Bundle semantics exact + match validateSegmentBundleSemantics 0x0C { + segmentBaoRoot := replicate 32 0, chunkIndex := 0, mainLen := 1, + verificationOutboardOffset := 0, verificationOutboardLen := 0, + fecParityOffset := 0, fecParityLen := 0 } with + | .error .missingFecParity => pure () + | .error e => fail s!"bundle missing fec: {repr e}" + | .ok _ => fail "bundle missing fec: ok" + match validateSegmentBundleSemantics 0x0C { + segmentBaoRoot := replicate 32 0, chunkIndex := 0, mainLen := 1, + verificationOutboardOffset := 0, verificationOutboardLen := 0, + fecParityOffset := 0, fecParityLen := 1 } with + | .error .fecParityLenMismatch => pure () + | .error e => fail s!"bundle fec len: {repr e}" + | .ok _ => fail "bundle fec len: ok" + IO.println "directory error taxonomy ok" + + IO.println "program G stack ok" + IO.println s!"version = {Carbonado.Cli.versionString}" + +/-- Entry: CLI subcommands or default demo (flake `checks.demo`). -/ +def main (args : List String) : IO UInt32 := do + match args with + | [] | ["demo"] => + runDemo + pure 0 + | cmd :: rest => + Carbonado.Cli.runCommand cmd rest diff --git a/Carbonado/Outboard.lean b/Carbonado/Outboard.lean new file mode 100644 index 0000000..5510280 --- /dev/null +++ b/Carbonado/Outboard.lean @@ -0,0 +1,204 @@ +/- + Outboard body encode/decode (Program G). + + Matches Rust `encoding::encode_outboard` / `decoding::decode_outboard` pure surface: + * compress → encrypt (embedded nonce layout when encrypted) → bare main + * FEC parity is a **sidecar** (parity shards only); main is unpadded logical body + * keyed Bao **post-order outboard** over bare main when Verification bit set + + Directory segments always use Verification|Fec (c12–c15). Catalogs use inboard + headered path (`Pipeline.encodeHeadered`), not this module. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Crypto.EtM +import Carbonado.Fec.Inboard +import Carbonado.Fec.RS +import Carbonado.Bao.Product +import Carbonado.Pipeline +import Carbonado.Compress + +namespace Carbonado.Outboard + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Crypto.EtM +open Carbonado.Fec.Inboard +open Carbonado.Fec.RS +open Carbonado.Bao.Product +open Carbonado.Pipeline +open Carbonado.Compress + +/-- Outboard encode result (bare main + optional sidecars). -/ +structure OutboardEncoded where + main : ByteArray + verificationOutboard : ByteArray + fecParity : ByteArray + baoHash : ByteArray + paddingLen : Nat + chunkLen : Nat + deriving DecidableEq + +/-- + Encode FEC parity sidecar only; main remains unpadded input. + + Returns `(parity_concat of shards 4..7, padding_len, chunk_len)`. +-/ +def encodeOutboardParity (input : ByteArray) : Except PipelineError (ByteArray × Nat × Nat) := + if input.size == 0 then + .ok (ByteArray.empty, 0, 0) + else + match encodeInboard input with + | .error e => .error (ofFecError e) + | .ok (body, pad, chunk) => + -- body = 8 × chunk; parity = last 4 shards + let parityStart := fecK * chunk + if body.size != fecM * chunk then + .error .unevenShards + else + .ok (body.extract parityStart body.size, pad, chunk) + +/-- + Decode with main + parity sidecars (undamaged path: all data present in main). + + Pads main to stripe geometry, rebuilds k data + m-k parity shards, reconstructs, + strips padding. +-/ +def decodeOutboardFec (main parity : ByteArray) (padding : Nat) : + Except PipelineError ByteArray := + if main.size == 0 && padding == 0 then + if parity.size == 0 then .ok ByteArray.empty + else .error .unevenShards + else if parity.size == 0 then + .error .emptyShard + else if parity.size % (fecM - fecK) != 0 then + .error .unevenShards + else + let shardLen := parity.size / (fecM - fecK) + if shardLen == 0 then + .error .emptyShard + else + let paddedTotal := shardLen * fecK + if padding > paddedTotal then + .error .paddingTooLarge + else + let logicalLen := paddedTotal - padding + -- Copy main into padded buffer (zeros after main). + let padded := + if main.size ≥ paddedTotal then + main.extract 0 paddedTotal + else + padWithZeros main paddedTotal + Id.run do + let mut opts : Array (Option ByteArray) := Array.mkEmpty fecM + for i in [:fecK] do + let start := i * shardLen + opts := opts.push (some (padded.extract start (start + shardLen))) + for j in [:fecM - fecK] do + let start := j * shardLen + opts := opts.push (some (parity.extract start (start + shardLen))) + match reconstructLogical opts padding with + | .error e => pure (.error (ofFecError e)) + | .ok data => pure (.ok data) + +/-- + Outboard encode body (embedded-nonce encrypt when Encrypted). + + `nonce` is required when `format.encrypted` (pure model has no CSPRNG). +-/ +def encodeOutboardBody (master nonce plaintext : ByteArray) (format : FormatBits) : + Except PipelineError OutboardEncoded := + let formatByte := format.toUInt8 + match compressStep plaintext format.compression with + | .error e => .error e + | .ok (afterComp, _) => + let encRes : Except PipelineError ByteArray := + if format.encrypted then + -- Embedded layout for bare mains (matches encoding::encode_outboard). + encryptStep master nonce afterComp false + else + .ok afterComp + match encRes with + | .error e => .error e + | .ok bareMain => + match + (if format.fec then encodeOutboardParity bareMain + else .ok (ByteArray.empty, 0, 0)) + with + | .error e => .error e + | .ok (fecParity, paddingLen, chunkLen) => + if format.verification then + let (root, ob) := encodeOutboardForFormat formatByte bareMain + .ok { + main := bareMain + verificationOutboard := ob + fecParity := fecParity + baoHash := root + paddingLen := paddingLen + chunkLen := chunkLen + } + else + .ok { + main := bareMain + verificationOutboard := ByteArray.empty + fecParity := fecParity + baoHash := zeroHash + paddingLen := paddingLen + chunkLen := chunkLen + } + +/-- + Outboard decode: Bao verify → FEC reconstruct → decrypt embedded → decompress. + + `padding` must match encode-time padding (directory uses `calcPaddingLen main_len`). +-/ +def decodeOutboardBody (master root main verOutboard fecParity : ByteArray) + (padding : Nat) (format : FormatBits) : Except PipelineError ByteArray := + let formatByte := format.toUInt8 + -- Bao verify first when verification bit set (empty post-order outboard is valid for single-leaf). + let afterBao : Except PipelineError ByteArray := + if format.verification then + match verifyOutboardForFormat formatByte root main verOutboard with + | .error e => .error (ofBaoError e) + | .ok () => .ok main + else + .ok main + match afterBao with + | .error e => .error e + | .ok main' => + let afterFec : Except PipelineError ByteArray := + if format.fec then + if main'.size == 0 then + .ok ByteArray.empty + else if fecParity.size == 0 then + .error .emptyShard + else + decodeOutboardFec main' fecParity padding + else + .ok main' + match afterFec with + | .error e => .error e + | .ok afterF => + -- Embedded-nonce decrypt when encrypted. + match decryptStep master ByteArray.empty afterF format.encrypted false with + | .error e => .error e + | .ok afterDec => + decompressStep afterDec format.compression + +/-- Round-trip outboard for a format. -/ +def roundtripOutboard (master nonce plaintext : ByteArray) (format : FormatBits) : + Except PipelineError Bool := + match encodeOutboardBody master nonce plaintext format with + | .error e => .error e + | .ok enc => + match decodeOutboardBody master enc.baoHash enc.main enc.verificationOutboard + enc.fecParity enc.paddingLen format with + | .error e => .error e + | .ok pt => .ok (ctEq pt plaintext) + +/-- Padding for directory segment decode: `calcPaddingLen(main_len)` when FEC. -/ +def paddingForMainLen (mainLen : Nat) (fec : Bool) : Nat := + if !fec || mainLen == 0 then 0 + else (calcPaddingLen mainLen).paddingLen + +end Carbonado.Outboard diff --git a/Carbonado/Pipeline.lean b/Carbonado/Pipeline.lean new file mode 100644 index 0000000..9dc77db --- /dev/null +++ b/Carbonado/Pipeline.lean @@ -0,0 +1,450 @@ +/- + Full Carbonado encode/decode pipeline (Programs E–F). + + Normative stage order (AGENTS.md): + encode: compress → encrypt → FEC → keyed Bao + decode: keyed Bao → FEC → decrypt → decompress + + Compression (Program F): zstd level 20 when the Compression bit is set + (`Carbonado.Compress`; AOT links real zstd; interpreter uses identity fallback — + see LIMITS). Format bit still affects keyed Bao roots (verification key domain). + + Nonce layouts: + * `headerPathEncrypt = true` → `[tag|ct]` (Header carries `payload_nonce`) + * `headerPathEncrypt = false` → `[nonce|tag|ct]` (low-level / encoding::encode) + + **MAC-before-decrypt:** payload EtM is verified only after Bao/FEC reverse, and + `decryptWithNonce` / `decryptEmbeddedNonce` still refuse keystream until MAC ok. + **Header-MAC-before-body:** `decodeHeadered` verifies header before body decode. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Crypto.EtM +import Carbonado.Fec.RS +import Carbonado.Fec.Inboard +import Carbonado.Bao.Tree +import Carbonado.Bao.Product +import Carbonado.Header +import Carbonado.Compress + +namespace Carbonado.Pipeline + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Crypto.EtM +open Carbonado.Fec.RS +open Carbonado.Fec.Inboard +open Carbonado.Bao.Tree +open Carbonado.Bao.Product +open Carbonado.Header +open Carbonado.Compress + +/-- Strict pipeline error taxonomy (exact-match in tests; no lumped diagnostics). -/ +inductive PipelineError where + -- Header + | invalidHeaderLength + | badMagic + | headerAuthenticationFailed + | invalidFieldLength + /-- Body shorter than authenticated `encoded_len` (Rust maps this to InvalidHeaderLength). -/ + | truncatedBody + -- Crypto / EtM + | invalidKeyLength + | invalidCiphertextLength + | invalidNonceLength + | payloadAuthenticationFailed + -- FEC + | unevenShards + | tooFewShards + | emptyShard + | incorrectShardSize + | badGeometry + | paddingTooLarge + | singularMatrix + -- Bao + | baoAuthenticationFailed + | truncatedResponse + | trailingData + | invalidPrefix + | invalidRootLength + | invalidSliceIndex + | invalidSliceCount + -- Compress (Program F) + | compressionFailed + | decompressionFailed + | decompressOutputTooLarge + /-- Invalid zstd input/params (encode or decode; not compress-only). -/ + | zstdInvalidInput + -- Scrub / shard + | scrubRequiresVerification + | unnecessaryScrub + | invalidScrubbedHash + | invalidChunkSequence + | emptySegment + /-- Caller supplied fewer nonces than segments (distinct from per-nonce size ≠ 16). -/ + | insufficientNonces + deriving DecidableEq, Repr + +def ofHeaderError : HeaderError → PipelineError + | .invalidHeaderLength => .invalidHeaderLength + | .badMagic => .badMagic + | .headerAuthenticationFailed => .headerAuthenticationFailed + | .invalidKeyLength => .invalidKeyLength + | .invalidFieldLength => .invalidFieldLength + +def ofCryptoError : CryptoError → PipelineError + | .invalidKeyLength => .invalidKeyLength + | .invalidCiphertextLength => .invalidCiphertextLength + | .invalidNonceLength => .invalidNonceLength + | .authenticationFailed => .payloadAuthenticationFailed + +def ofFecError : FecError → PipelineError + | .unevenShards => .unevenShards + | .tooFewShards => .tooFewShards + | .emptyShard => .emptyShard + | .incorrectShardSize => .incorrectShardSize + | .badGeometry => .badGeometry + | .paddingTooLarge => .paddingTooLarge + | .singularMatrix => .singularMatrix + +def ofBaoError : BaoError → PipelineError + | .authenticationFailed => .baoAuthenticationFailed + | .truncatedResponse => .truncatedResponse + | .trailingData => .trailingData + | .invalidPrefix => .invalidPrefix + | .invalidRootLength => .invalidRootLength + | .invalidSliceIndex => .invalidSliceIndex + | .invalidSliceCount => .invalidSliceCount + +/-- Map zstd errors without collapsing distinct modes. -/ +def ofZstdError : ZstdError → PipelineError + | .compressionFailed => .compressionFailed + | .decompressionFailed => .decompressionFailed + | .outputTooLarge => .decompressOutputTooLarge + | .invalidInput => .zstdInvalidInput + +/-- Encode bookkeeping (Rust `EncodeInfo`; factors omitted as pure-Nat model). -/ +structure EncodeInfo where + inputLen : Nat + outputLen : Nat + bytesCompressed : Nat + bytesEncrypted : Nat + bytesEcc : Nat + bytesVerifiable : Nat + paddingLen : Nat + chunkLen : Nat + verifiableSliceCount : Nat + chunkSliceCount : Nat + deriving DecidableEq, Repr + +/-- Body encode result (inboard; no Header prepended). -/ +structure Encoded where + body : ByteArray + /-- Keyed Bao root when verification bit set; else 32 zero bytes. -/ + baoHash : ByteArray + info : EncodeInfo + deriving DecidableEq + +/-- Zero hash used when Verification bit is clear. -/ +def zeroHash : ByteArray := replicate hashLen 0 + +/-- Zero SLH public key slot. -/ +def zeroSlhPk : ByteArray := replicate slhPublicKeyLen 0 + +/-- Zero metadata. -/ +def zeroMeta : ByteArray := replicate 8 0 + +/-- + Compress step: zstd-20 when bit set. + + **AOT product:** calls `compressLevel20` (real zstd via `@[extern]`). + **native_decide / elaborator:** do not evaluate this on compression formats — + CarbonadoTest format-matrix theorems use non-compression formats; AOT Main + exercises c2/c6/c14/c15 with real zstd (LIMITS). +-/ +def compressStep (plaintext : ByteArray) (compression : Bool) : + Except PipelineError (ByteArray × Nat) := + if !compression then + .ok (plaintext, 0) + else + match compressLevel20 plaintext with + | .error e => .error (ofZstdError e) + | .ok ct => .ok (ct, ct.size) + +/-- Decompress step: zstd when bit set (same AOT / native_decide caveats). -/ +def decompressStep (data : ByteArray) (compression : Bool) : + Except PipelineError ByteArray := + if !compression then + .ok data + else + match decompress data with + | .error e => .error (ofZstdError e) + | .ok pt => .ok pt + +/-- Encrypt step (header-path or embedded). -/ +def encryptStep (master nonce plaintext : ByteArray) (headerPath : Bool) : + Except PipelineError ByteArray := + if headerPath then + match encryptWithNonce master nonce plaintext with + | .error e => .error (ofCryptoError e) + | .ok blob => .ok blob + else + match encryptEmbeddedNonce master nonce plaintext with + | .error e => .error (ofCryptoError e) + | .ok blob => .ok blob + +/-- Decrypt step after Bao/FEC reverse (MAC-before-decrypt inside EtM). -/ +def decryptStep (master nonceOrEmpty ciphertext : ByteArray) (encrypted headerPath : Bool) : + Except PipelineError ByteArray := + if !encrypted then + .ok ciphertext + else if headerPath then + match decryptWithNonce master nonceOrEmpty ciphertext with + | .ok pt => .ok pt + | .error e => .error (ofCryptoError e) + else + match decryptEmbeddedNonce master ciphertext with + | .ok pt => .ok pt + | .error e => .error (ofCryptoError e) + +/-- FEC encode step. -/ +def fecEncodeStep (input : ByteArray) (fec : Bool) : + Except PipelineError (ByteArray × Nat × Nat × Nat) := + if !fec then + .ok (input, 0, 0, 0) + else + match Carbonado.Fec.Inboard.encodeInboard input with + | .error e => .error (ofFecError e) + | .ok (body, pad, chunk) => + .ok (body, pad, chunk, body.size) + +/-- FEC decode step. -/ +def fecDecodeStep (body : ByteArray) (padding : Nat) (fec : Bool) : + Except PipelineError ByteArray := + if !fec then + .ok body + else + match Carbonado.Fec.Inboard.decodeInboard body padding with + | .error e => .error (ofFecError e) + | .ok pt => .ok pt + +/-- Bao inboard encode step (format byte keys the tree). -/ +def baoEncodeStep (data : ByteArray) (formatByte : UInt8) (verification : Bool) : + ByteArray × ByteArray × Nat := + if !verification then + (data, zeroHash, data.size) + else + let (root, art) := encodeInboardForFormat formatByte data + (art, root, art.size) + +/-- Bao inboard decode/verify step. -/ +def baoDecodeStep (body root : ByteArray) (formatByte : UInt8) (verification : Bool) : + Except PipelineError ByteArray := + if !verification then + .ok body + else + match decodeInboardForFormat formatByte root body with + | .error e => .error (ofBaoError e) + | .ok data => .ok data + +/-- + Encode logical plaintext through the format pipeline (body only). + + Matches `encoding::encode` / `stream_encode_buffer` when `headerPathEncrypt = false`. + Nonce is required when `format.encrypted` (caller-supplied; pure model has no CSPRNG). +-/ +def encodeBody (master nonce plaintext : ByteArray) (format : FormatBits) + (headerPathEncrypt : Bool) : Except PipelineError Encoded := + let formatByte := format.toUInt8 + let inputLen := plaintext.size + match compressStep plaintext format.compression with + | .error e => .error e + | .ok (afterComp, bytesCompressed) => + let encRes : Except PipelineError ByteArray := + if format.encrypted then + encryptStep master nonce afterComp headerPathEncrypt + else + .ok afterComp + match encRes with + | .error e => .error e + | .ok encryptedBody => + let bytesEncrypted := if format.encrypted then encryptedBody.size else 0 + match fecEncodeStep encryptedBody format.fec with + | .error e => .error e + | .ok (afterFec, paddingLen, chunkLen, bytesEcc) => + let (verifiable, baoHash, bytesVerifiable) := + baoEncodeStep afterFec formatByte format.verification + let verifiableSliceCount := + if format.fec then bytesEcc / sliceLen else 0 + let chunkSliceCount := + if format.fec then verifiableSliceCount / fecM else 0 + .ok { + body := verifiable + baoHash := baoHash + info := { + inputLen := inputLen + outputLen := bytesVerifiable + bytesCompressed := bytesCompressed + bytesEncrypted := bytesEncrypted + bytesEcc := bytesEcc + bytesVerifiable := bytesVerifiable + paddingLen := paddingLen + chunkLen := chunkLen + verifiableSliceCount := verifiableSliceCount + chunkSliceCount := chunkSliceCount + } + } + +/-- + Decode body (reverse pipeline). + + `padding` is the encode-time padding (from Header / EncodeInfo). + `nonce` is used only for header-path encrypted decrypt; ignored for embedded layout. +-/ +def decodeBody (master nonce hash body : ByteArray) (padding : Nat) (format : FormatBits) + (headerPathEncrypt : Bool) : Except PipelineError ByteArray := + let formatByte := format.toUInt8 + match baoDecodeStep body hash formatByte format.verification with + | .error e => .error e + | .ok afterBao => + match fecDecodeStep afterBao padding format.fec with + | .error e => .error e + | .ok afterFec => + match decryptStep master nonce afterFec format.encrypted headerPathEncrypt with + | .error e => .error e + | .ok afterDec => + decompressStep afterDec format.compression + +/-- Max value for u32 length fields (`UInt32.toNat (UInt32.ofNat n)` identity). -/ +def u32Max : Nat := 4294967295 + +/-- Encode Nat length into UInt32 if in range; else `invalidFieldLength`. -/ +def natToU32Field (n : Nat) : Except PipelineError UInt32 := + if n > u32Max then .error .invalidFieldLength + else .ok (UInt32.ofNat n) + +/-- Headered encode: body + authenticated 177-byte Header (header-path encrypt). -/ +def encodeHeadered (master nonce plaintext : ByteArray) (format : FormatBits) + (chunkIndex : UInt32) (slhPublicKey metadata : ByteArray) : + Except PipelineError (Header × ByteArray) := + match encodeBody master nonce plaintext format true with + | .error e => .error e + | .ok enc => + match natToU32Field enc.info.outputLen, natToU32Field enc.info.paddingLen with + | .error e, _ => .error e + | _, .error e => .error e + | .ok encLenU32, .ok padU32 => + match Header.new master nonce enc.baoHash slhPublicKey format.toUInt8 + chunkIndex encLenU32 padU32 metadata with + | .error e => .error (ofHeaderError e) + | .ok hdr => + match hdr.toBytes with + | .error e => .error (ofHeaderError e) + | .ok hdrBytes => + .ok (hdr, appendBA hdrBytes enc.body) + +/-- + Headered decode: **header MAC verified first**, then body with `payload_nonce`. + + Enforces authenticated `encoded_len` (Rust `file::decode`): body must be ≥ + `encoded_len`; only the first `encoded_len` bytes enter the pipeline (trailers + ignored). Short body → `truncatedBody`. +-/ +def decodeHeadered (master archive : ByteArray) : Except PipelineError ByteArray := + if archive.size < headerLen then + .error .invalidHeaderLength + else + let hdrBytes := archive.extract 0 headerLen + let bodyAll := archive.extract headerLen archive.size + match parseAndVerify master hdrBytes with + | .error e => .error (ofHeaderError e) + | .ok hdr => + let need := UInt32.toNat hdr.encodedLen + if bodyAll.size < need then + .error .truncatedBody + else + let body := bodyAll.extract 0 need + let format := FormatBits.ofUInt8 hdr.format + let padding := UInt32.toNat hdr.paddingLen + decodeBody master hdr.payloadNonce hdr.hash body padding format true + +/-- + Headered decode returning the verified Header (for sharding: authenticated + `chunk_index` rebinding). Same `encoded_len` bound as `decodeHeadered`. +-/ +def decodeHeaderedWithHeader (master archive : ByteArray) : + Except PipelineError (Header × ByteArray) := + if archive.size < headerLen then + .error .invalidHeaderLength + else + let hdrBytes := archive.extract 0 headerLen + let bodyAll := archive.extract headerLen archive.size + match parseAndVerify master hdrBytes with + | .error e => .error (ofHeaderError e) + | .ok hdr => + let need := UInt32.toNat hdr.encodedLen + if bodyAll.size < need then + .error .truncatedBody + else + let body := bodyAll.extract 0 need + let format := FormatBits.ofUInt8 hdr.format + let padding := UInt32.toNat hdr.paddingLen + match decodeBody master hdr.payloadNonce hdr.hash body padding format true with + | .error e => .error e + | .ok pt => .ok (hdr, pt) + +/-- Round-trip helper for format matrix (body, embedded nonce when encrypted). -/ +def roundtripBody (master nonce plaintext : ByteArray) (format : FormatBits) : + Except PipelineError Bool := + match encodeBody master nonce plaintext format false with + | .error e => .error e + | .ok enc => + match decodeBody master nonce enc.baoHash enc.body enc.info.paddingLen format false with + | .error e => .error e + | .ok pt => .ok (ctEq pt plaintext) + +/-- Round-trip headered archive. -/ +def roundtripHeadered (master nonce plaintext : ByteArray) (format : FormatBits) : + Except PipelineError Bool := + match encodeHeadered master nonce plaintext format 0 zeroSlhPk zeroMeta with + | .error e => .error e + | .ok (_hdr, archive) => + match decodeHeadered master archive with + | .error e => .error e + | .ok pt => .ok (ctEq pt plaintext) + +/-- All 16 format codes as FormatBits. -/ +def allFormats : List FormatBits := + (List.range 16).map (fun n => FormatBits.ofUInt8 (UInt8.ofNat n)) + +/-- + Roundtrip every format (zstd when Compression bit set; encrypted uses given nonce). + + Propagates the first `PipelineError` (does not swallow errors into `.ok false`). + Returns `.ok false` only when a roundtrip succeeds but plaintext mismatches. +-/ +def formatMatrixRoundtrip (master nonce plaintext : ByteArray) : Except PipelineError Bool := + Id.run do + let mut err : Option PipelineError := none + let mut allMatch := true + for fmt in allFormats do + if err.isNone then + match roundtripBody master nonce plaintext fmt with + | .error e => err := some e + | .ok b => if !b then allMatch := false + match err with + | some e => pure (.error e) + | none => pure (.ok allMatch) + +/-- Unencrypted formats are even (restate Constants invariant for pipeline). -/ +theorem encrypted_bit_is_odd (f : FormatBits) : + f.encrypted = true → f.toUInt8 % 2 = 1 := by + intro h + cases f with + | mk e c v z => + simp [FormatBits.toUInt8, formatBitEncrypted, formatBitCompression, + formatBitVerification, formatBitFec] at h ⊢ + subst h + cases c <;> cases v <;> cases z <;> native_decide + +end Carbonado.Pipeline diff --git a/Carbonado/Scrub.lean b/Carbonado/Scrub.lean new file mode 100644 index 0000000..1938c13 --- /dev/null +++ b/Carbonado/Scrub.lean @@ -0,0 +1,194 @@ +/- + Scrub: pure combinatorial FEC recovery + re-encode + keyed Bao root compare. + + Normative spirit (Rust `decoding::scrub`): + * Requires Verification bit + * Pristine inboard (Bao verifies) → `unnecessaryScrub` + * Damaged: try subsets of FEC shards (≥ k present), reconstruct logical, + re-encode FEC + Bao, accept first body whose root matches the oracle hash + + Pure model uses shard split of the **inboard FEC body** under the Bao artifact. + When Verification is set, the on-disk body is Bao-inboard wrapping FEC bytes; + scrub first peels Bao when possible via slice/extract. For the pure Lean model we + support two entry points: + 1. `scrubFecBody` — FEC body already exposed (after successful partial extract) + 2. `scrubInboard` — full Bao+FEC inboard: try decode; on fail, brute-force over + FEC-layer candidates obtained by re-encoding search from corrupted body + interpreted as raw FEC concat when geometry matches + + For FEC+Verification formats the encode layout is Bao(FEC(data)). On damage, + we search over RS subsets of the **inner** FEC body when the caller supplies it + (`scrubFecThenBao`), which matches the Rust path after slice extract of chunks. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Fec.RS +import Carbonado.Fec.Inboard +import Carbonado.Bao.Product +import Carbonado.Pipeline + +namespace Carbonado.Scrub + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Fec.RS +open Carbonado.Fec.Inboard +open Carbonado.Bao.Product +open Carbonado.Pipeline + +/-- Popcount of a Nat mask (low 8 bits used). -/ +def popcount8 (mask : Nat) : Nat := + Id.run do + let mut c : Nat := 0 + for i in [:8] do + if (mask >>> i) % 2 == 1 then + c := c + 1 + pure c + +/-- + Try one mask of present shard indices; reconstruct, re-encode FEC, re-Bao, compare root. + Returns `some` recovered Bao-inboard body on success. + + Requires `shards.size = fecM` (caller must guard). +-/ +def tryMask (shards : Array ByteArray) (padding : Nat) (formatByte : UInt8) + (wantRoot : ByteArray) (mask : Nat) : Option ByteArray := + if shards.size != fecM then + none + else if popcount8 mask < fecK then + none + else + Id.run do + let mut opts : Array (Option ByteArray) := Array.mkEmpty fecM + for i in [:fecM] do + if (mask >>> i) % 2 == 1 then + opts := opts.push (some (shards[i]!)) + else + opts := opts.push none + match reconstructLogical opts padding with + | .error _ => pure none + | .ok logical => + match Carbonado.Fec.Inboard.encodeInboard logical with + | .error _ => pure none + | .ok (fecBody, pad', _) => + if pad' != padding then + pure none + else + let (root, art) := encodeInboardForFormat formatByte fecBody + if ctEq root wantRoot then + some art + else + none + +/-- Search all 8-bit masks with ≥ k present shards. -/ +def searchMasks (shards : Array ByteArray) (padding : Nat) (formatByte : UInt8) + (wantRoot : ByteArray) : Option ByteArray := + if shards.size != fecM then + none + else + Id.run do + let mut found : Option ByteArray := none + for mask in [:256] do + if found.isNone then + match tryMask shards padding formatByte wantRoot mask with + | some art => found := some art + | none => pure () + pure found + +/-- Require exactly `fecM` shards (empty body from `inboardToShards` → badGeometry). -/ +def requireFecShards (shards : Array ByteArray) : Except PipelineError Unit := + if shards.size != fecM then + .error .badGeometry + else + .ok () + +/-- + Scrub from FEC inboard body (8-shard concat) + expected Bao root of re-encoded form. + + Used when FEC shards are already available (Rust: after slice extract). + `wantRoot` is the keyed Bao root over the **FEC body** (same as archive hash when + Verification wraps FEC only — i.e. format with V+FEC). +-/ +def scrubFecThenBao (fecBody wantRoot : ByteArray) (padding : Nat) (formatByte : UInt8) : + Except PipelineError ByteArray := + match inboardToShards fecBody with + | .error e => .error (ofFecError e) + | .ok shards => + match requireFecShards shards with + | .error e => .error e + | .ok () => + match searchMasks shards padding formatByte wantRoot with + | some art => .ok art + | none => .error .invalidScrubbedHash + +/-- + Full scrub entry for Verification formats. + + * No verification bit → `scrubRequiresVerification` + * Bao verifies → `unnecessaryScrub` + * Else → `invalidScrubbedHash` (opaque Bao without FEC extract) +-/ +def scrubInboardArchive (body wantRoot : ByteArray) (format : FormatBits) : + Except PipelineError ByteArray := + if !format.verification then + .error .scrubRequiresVerification + else + match decodeInboardForFormat format.toUInt8 wantRoot body with + | .ok _ => .error .unnecessaryScrub + | .error .authenticationFailed => + .error .invalidScrubbedHash + | .error e => .error (ofBaoError e) + +/-- + Scrub path for tests: zero-fill missing shard slots on the FEC body, then full mask search. + + Empty / short FEC body → `badGeometry` (no panic). +-/ +def scrubAfterKnockout (fecBody wantRoot : ByteArray) (padding : Nat) + (formatByte : UInt8) (missing : List Nat) : Except PipelineError ByteArray := + match inboardToShards fecBody with + | .error e => .error (ofFecError e) + | .ok shards => + match requireFecShards shards with + | .error e => .error e + | .ok () => + if missing.any (fun i => decide (i ≥ fecM)) then + .error .badGeometry + else + Id.run do + let mut damagedShards : Array ByteArray := Array.mkEmpty fecM + for i in [:fecM] do + if missing.contains i then + damagedShards := damagedShards.push (replicate (shards[i]!).size 0) + else + damagedShards := damagedShards.push (shards[i]!) + let damagedBody := concatShards damagedShards + pure (scrubFecThenBao damagedBody wantRoot padding formatByte) + +/-- Knockout recovery using only the complement of `missing` (no full-mask fallback). + + Returns `invalidScrubbedHash` when the chosen present set cannot reconstruct a body + whose re-encoded Bao root matches `wantRoot` (e.g. more than 4 shards missing). + Empty / short FEC body → `badGeometry` (no panic). +-/ +def scrubWithMissing (fecBody wantRoot : ByteArray) (padding : Nat) + (formatByte : UInt8) (missing : List Nat) : Except PipelineError ByteArray := + match inboardToShards fecBody with + | .error e => .error (ofFecError e) + | .ok shards => + match requireFecShards shards with + | .error e => .error e + | .ok () => + if missing.any (fun i => decide (i ≥ fecM)) then + .error .badGeometry + else + Id.run do + let mut mask : Nat := 0 + for i in [:fecM] do + if !(missing.contains i) then + mask := mask + (1 <<< i) + match tryMask shards padding formatByte wantRoot mask with + | some art => pure (.ok art) + | none => pure (.error .invalidScrubbedHash) + +end Carbonado.Scrub diff --git a/Carbonado/Shard.lean b/Carbonado/Shard.lean new file mode 100644 index 0000000..e77ba93 --- /dev/null +++ b/Carbonado/Shard.lean @@ -0,0 +1,196 @@ +/- + Multi-segment sharding model (Program E). + + Matches Rust `stream/shard.rs` pure surface: + * Split logical plaintext by `segmentPlaintextBudget` + * Each segment encoded independently with `chunk_index` 0..n-1 bound under header_mac + * Decode rebinds order from **verified** `header.chunkIndex` (not unauthenticated labels) +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Header +import Carbonado.Pipeline + +namespace Carbonado.Shard + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Header +open Carbonado.Pipeline + +/-- Default budget: u32::MAX = 2^32 - 1 (same bookkeeping ceiling as Rust). -/ +def defaultSegmentPlaintextBudget : Nat := 4294967295 + +/-- One encoded shard (authenticated header + body). + + `chunkIndex` is a convenience label filled at encode time; **decode must not** + trust it for ordering — use the verified Header inside `archive` instead. +-/ +structure Shard where + chunkIndex : UInt32 + header : Header + /-- Full archive bytes: header wire || body. -/ + archive : ByteArray + deriving DecidableEq, Inhabited + +/-- Split plaintext into consecutive budget-sized segments (last may be short). + + `budget = 0` → empty list (caller should use a positive budget). + Empty plaintext → single empty segment (one shard) so encode still produces chunk 0. +-/ +def splitByBudget (plaintext : ByteArray) (budget : Nat) : Array ByteArray := + if budget == 0 then + #[] + else if plaintext.size == 0 then + #[ByteArray.empty] + else + Id.run do + let mut out : Array ByteArray := Array.mkEmpty ((plaintext.size + budget - 1) / budget) + let mut off : Nat := 0 + while off < plaintext.size do + let end_ := min (off + budget) plaintext.size + out := out.push (plaintext.extract off end_) + off := end_ + pure out + +/-- + Encode multi-segment archive set. + + `nonces[i]` is the payload_nonce for segment `i` (header-path encrypt). + Requires `nonces.size ≥` segment count after split. + Too few nonces → `insufficientNonces` (not `invalidNonceLength`). +-/ +def encodeShards (master plaintext : ByteArray) (format : FormatBits) + (budget : Nat) (nonces : Array ByteArray) + (slhPublicKey metadata : ByteArray) : + Except PipelineError (Array Shard) := + let segments := splitByBudget plaintext budget + if segments.size == 0 then + .error .emptySegment + else if nonces.size < segments.size then + .error .insufficientNonces + else + Id.run do + let mut out : Array Shard := Array.mkEmpty segments.size + let mut err : Option PipelineError := none + for i in [:segments.size] do + if err.isNone then + if i > u32Max then + err := some .invalidFieldLength + else + let nonce := nonces[i]! + let seg := segments[i]! + match encodeHeadered master nonce seg format (UInt32.ofNat i) slhPublicKey metadata with + | .error e => err := some e + | .ok (hdr, archive) => + out := out.push { + chunkIndex := UInt32.ofNat i + header := hdr + archive := archive + } + match err with + | some e => pure (.error e) + | none => pure (.ok out) + +/-- Check indices form contiguous `0 .. n-1` (any order accepted after placement). -/ +def validateChunkSequence (indices : Array UInt32) : Except PipelineError Unit := + if indices.size == 0 then + .error .emptySegment + else + Id.run do + let n := indices.size + let mut seen : Array Bool := Array.mkEmpty n + for _ in [:n] do + seen := seen.push false + let mut ok := true + for i in [:n] do + let idx := UInt32.toNat (indices[i]!) + if idx ≥ n then + ok := false + else if seen[idx]! then + ok := false + else + seen := seen.set! idx true + for i in [:n] do + if !seen[i]! then + ok := false + pure (if ok then .ok () else .error .invalidChunkSequence) + +/-- + Decode shard set by **verified** header `chunk_index` under `header_mac`. + + Structure `Shard.chunkIndex` is ignored for ordering. If a non-empty external + label is present and disagrees with the verified index, returns + `invalidChunkSequence`. Concatenates plaintext in authenticated index order. +-/ +def decodeShards (master : ByteArray) (shards : Array Shard) : + Except PipelineError ByteArray := + if shards.size == 0 then + .error .emptySegment + else + Id.run do + let n := shards.size + -- Decode each archive; collect (verifiedIndex, plaintext) + let mut verifiedIndices : Array UInt32 := Array.mkEmpty n + let mut plaintexts : Array ByteArray := Array.mkEmpty n + let mut err : Option PipelineError := none + for i in [:n] do + if err.isNone then + let s := shards[i]! + match decodeHeaderedWithHeader master s.archive with + | .error e => err := some e + | .ok (hdr, pt) => + -- If structure label disagrees with authenticated index, fail. + if s.chunkIndex != hdr.chunkIndex then + err := some .invalidChunkSequence + else + verifiedIndices := verifiedIndices.push hdr.chunkIndex + plaintexts := plaintexts.push pt + match err with + | some e => pure (.error e) + | none => + match validateChunkSequence verifiedIndices with + | .error e => pure (.error e) + | .ok () => + -- Place plaintext by verified index + let mut slots : Array (Option ByteArray) := Array.mkEmpty n + for _ in [:n] do + slots := slots.push none + for i in [:n] do + let idx := UInt32.toNat (verifiedIndices[i]!) + slots := slots.set! idx (some (plaintexts[i]!)) + let mut out := ByteArray.empty + let mut placeErr : Option PipelineError := none + for i in [:n] do + if placeErr.isNone then + match slots[i]! with + | none => placeErr := some .invalidChunkSequence + | some pt => out := out.append pt + match placeErr with + | some e => pure (.error e) + | none => pure (.ok out) + +/-- Round-trip multi-segment encode/decode. -/ +def roundtripShards (master plaintext : ByteArray) (format : FormatBits) + (budget : Nat) (nonces : Array ByteArray) : Except PipelineError Bool := + match encodeShards master plaintext format budget nonces zeroSlhPk zeroMeta with + | .error e => .error e + | .ok shards => + match decodeShards master shards with + | .error e => .error e + | .ok pt => .ok (ctEq pt plaintext) + +/-- Budget split theorems. -/ +theorem split_empty_budget : + (splitByBudget (ofList [1, 2, 3]) 0).size = 0 := by + native_decide + +theorem split_hello_budget_2 : + (splitByBudget (utf8 "hello") 2).size = 3 := by + native_decide + +theorem split_empty_plaintext : + (splitByBudget ByteArray.empty 10).size = 1 := by + native_decide + +end Carbonado.Shard diff --git a/Carbonado/Slh.lean b/Carbonado/Slh.lean new file mode 100644 index 0000000..9903c2b --- /dev/null +++ b/Carbonado/Slh.lean @@ -0,0 +1,276 @@ +/- + SLH-DSA-SHA2-128s sidecar wire format + Bao-root binding (Program F). + + Normative (AGENTS §2.3): + * Sidecar: `SLH1` (4) + raw signature (7856) = 7860 bytes + * Public key (32 B) lives in Header.slh_public_key, not the sidecar + * Signature is over the 32-byte Bao root of the target container + + Real SPHINCS+/libbitcoinpqc sign-verify is **not** linked in this program + (libbitcoinpqc submodule empty / heavy cmake — see LIMITS). This module + provides fail-closed wire codec, binding model, and theorems. Optional FFI + can replace the oracle later without changing the wire API. + + Large-array roundtrips (7856 B sig) are gated in AOT Main, not `native_decide` + (elaboration cost). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util + +namespace Carbonado.Slh + +open Carbonado.Constants +open Carbonado.Crypto.Util + +/-- Strict SLH/sidecar error taxonomy (exact-match in tests). -/ +inductive SlhError where + /-- Sidecar byte length ≠ 7860. -/ + | invalidSidecarLength + /-- First 4 bytes ≠ `SLH1`. -/ + | badSlhMagic + /-- Signature material length ≠ 7856. -/ + | invalidSignatureLength + /-- Public key length ≠ 32. -/ + | invalidPublicKeyLength + /-- Claimed Bao root length ≠ 32. -/ + | invalidRootLength + /-- Signature verification failed (oracle returned false). -/ + | verificationFailed + /-- Oracle/sign path refused (no real crypto linked, bad params, etc.). -/ + | signatureUnavailable + deriving DecidableEq, Repr + +/-- Detached SLH1 sidecar contents (signature only; pk is out-of-band). -/ +structure SlhSidecar where + signature : ByteArray + deriving DecidableEq + +/-- Binding of public key + Bao root + signature (product verification view). -/ +structure SlhBinding where + publicKey : ByteArray + baoRoot : ByteArray + signature : ByteArray + deriving DecidableEq + +/-- Expected SLH1 magic as ByteArray. -/ +def slh1MagicBA : ByteArray := + ofList slh1Magic + +/-- True iff `wire` begins with the 4-byte `SLH1` magic (no length check). -/ +def slh1MagicPrefix (wire : ByteArray) : Bool := + wire.size ≥ 4 && ctEq (wire.extract 0 4) slh1MagicBA + +/-- + Magic check for an **exact-length** sidecar body (after size gate). + Distinct from `invalidSidecarLength` — used so `badSlhMagic` is theorem-tested + without allocating 7860 bytes under `native_decide`. +-/ +def parseMagicAtExactLen (bytes : ByteArray) : Except SlhError ByteArray := + if !slh1MagicPrefix bytes then + .error .badSlhMagic + else + .ok (bytes.extract 4 bytes.size) + +/-- Build on-disk sidecar bytes from a raw 7856-byte signature. -/ +def buildSidecar (signature : ByteArray) : Except SlhError ByteArray := + if signature.size != slh1SignatureLen then + .error .invalidSignatureLength + else + .ok (appendBA slh1MagicBA signature) + +/-- + Parse and validate SLH1 sidecar wire. + Returns the raw 7856-byte signature (matches Rust `read_slh_sidecar`). + Order: length → magic → extract (short good-magic prefix → length error first). +-/ +def parseSidecar (bytes : ByteArray) : Except SlhError ByteArray := + if bytes.size != slh1SidecarLen then + .error .invalidSidecarLength + else + parseMagicAtExactLen bytes + +/-- Validate field sizes and construct a binding (no crypto). -/ +def mkBinding (publicKey baoRoot signature : ByteArray) : Except SlhError SlhBinding := + if publicKey.size != slhPublicKeyLen then + .error .invalidPublicKeyLength + else if baoRoot.size != hashLen then + .error .invalidRootLength + else if signature.size != slh1SignatureLen then + .error .invalidSignatureLength + else + .ok { publicKey := publicKey, baoRoot := baoRoot, signature := signature } + +/-- + Build binding from Header pk + root + sidecar file bytes. + Fail-closed on wire errors before any verify oracle is consulted. +-/ +def bindingFromSidecar (publicKey baoRoot sidecarBytes : ByteArray) : + Except SlhError SlhBinding := + match parseSidecar sidecarBytes with + | .error e => .error e + | .ok sig => mkBinding publicKey baoRoot sig + +/-- + Verify that a signature is bound to a **specific** Bao root. + + `verifyOracle pk message sig` is the SLH-DSA verify predicate (true = accept). +-/ +def verifyBound (verifyOracle : ByteArray → ByteArray → ByteArray → Bool) + (publicKey claimedRoot signature : ByteArray) : Except SlhError Unit := + match mkBinding publicKey claimedRoot signature with + | .error e => .error e + | .ok b => + if verifyOracle b.publicKey b.baoRoot b.signature then + .ok () + else + .error .verificationFailed + +/-- + Bind-to-root check: signature must verify over `expectedRoot`, and the + claimed message root must equal `expectedRoot` (ctEq). Wrong root → + `verificationFailed` even if a confused oracle would accept another message. +-/ +def verifyBoundToExpected (verifyOracle : ByteArray → ByteArray → ByteArray → Bool) + (publicKey expectedRoot claimedRoot signature : ByteArray) : + Except SlhError Unit := + if expectedRoot.size != hashLen then + .error .invalidRootLength + else if claimedRoot.size != hashLen then + .error .invalidRootLength + else if !ctEq expectedRoot claimedRoot then + .error .verificationFailed + else + verifyBound verifyOracle publicKey claimedRoot signature + +/-- + Mock oracle for pure tests: accepts iff `sig` equals `acceptedSig` and + `message` equals `acceptedRoot`. Used only for binding model tests. +-/ +def mockOracleFor (acceptedRoot acceptedSig : ByteArray) + (_pk message sig : ByteArray) : Bool := + ctEq message acceptedRoot && ctEq sig acceptedSig + +/-- Placeholder sign: always `signatureUnavailable` until libbitcoinpqc is linked. -/ +def signRoot (_secretKeyEntropy root : ByteArray) : Except SlhError ByteArray := + if root.size != hashLen then + .error .invalidRootLength + else + .error .signatureUnavailable + +/-! ## Theorems: wire framing + bind-to-root (small native_decide cases) -/ + +theorem slh1_sidecar_len : slh1SidecarLen = 7860 := slh1SidecarLen_eq + +theorem slh1_sig_len : slh1SignatureLen = 7856 := by native_decide + +/-- Short sidecar → invalidSidecarLength (not badSlhMagic). -/ +theorem parse_short_length : + (match parseSidecar (ofList [0x53, 0x4c, 0x48, 0x31]) with + | .error .invalidSidecarLength => true + | _ => false) = true := by + native_decide + +/-- Empty sidecar → invalidSidecarLength. -/ +theorem parse_empty : + (match parseSidecar ByteArray.empty with + | .error .invalidSidecarLength => true + | _ => false) = true := by + native_decide + +/-- Wrong 4-byte prefix is not SLH1 magic. -/ +theorem zeros_not_slh1_magic : + slh1MagicPrefix (ofList [0, 0, 0, 0]) = false := by + native_decide + +/-- Correct magic list is accepted by prefix check. -/ +theorem slh1_magic_prefix_ok : + slh1MagicPrefix slh1MagicBA = true := by + native_decide + +/-- Exact-length magic gate: zeros → badSlhMagic (not invalidSidecarLength). -/ +theorem parse_magic_bad : + (match parseMagicAtExactLen (ofList [0, 0, 0, 0]) with + | .error .badSlhMagic => true + | _ => false) = true := by + native_decide + +/-- Exact-length magic gate: good magic → extract of empty remaining payload. -/ +theorem parse_magic_good_empty_payload : + (match parseMagicAtExactLen slh1MagicBA with + | .ok b => b.size == 0 + | .error _ => false) = true := by + native_decide + +/-- + When size is exact and magic fails, `parseSidecar` is `badSlhMagic`. + (AOT Main also exercises full 7860-byte all-zero wire.) +-/ +theorem parse_bad_magic_when_exact + (bytes : ByteArray) + (hlen : bytes.size = slh1SidecarLen) + (hmag : slh1MagicPrefix bytes = false) : + parseSidecar bytes = .error .badSlhMagic := by + simp [parseSidecar, parseMagicAtExactLen, hlen, hmag] + +/-- buildSidecar rejects wrong signature length. -/ +theorem build_bad_sig_len : + (match buildSidecar (ofList [1, 2, 3]) with + | .error .invalidSignatureLength => true + | _ => false) = true := by + native_decide + +/-- mkBinding rejects short public key (short sig also; pk checked first). -/ +theorem bind_bad_pk : + (match mkBinding (ofList [1]) (replicate hashLen 0) (ofList [1]) with + | .error .invalidPublicKeyLength => true + | _ => false) = true := by + native_decide + +/-- mkBinding rejects short root after pk ok. -/ +theorem bind_bad_root : + (match mkBinding (replicate slhPublicKeyLen 0) (ofList [1]) (ofList [1]) with + | .error .invalidRootLength => true + | _ => false) = true := by + native_decide + +/-- mkBinding rejects short signature after pk+root ok. -/ +theorem bind_bad_sig : + (match mkBinding (replicate slhPublicKeyLen 0) (replicate hashLen 0) (ofList [1]) with + | .error .invalidSignatureLength => true + | _ => false) = true := by + native_decide + +/-- + Wrong claimed root vs expected → verificationFailed **before** oracle/sig size. + (Size gates are separate; this proves the ctEq root check is fail-closed.) +-/ +theorem wrong_root_fails : + (let rootA := replicate hashLen 0xaa + let rootB := replicate hashLen 0xbb + let pk := replicate slhPublicKeyLen 0x11 + -- Short sig: would be invalidSignatureLength if roots matched; roots differ first. + let sig := ofList [0xcd] + match verifyBoundToExpected (mockOracleFor rootA sig) pk rootA rootB sig with + | .error .verificationFailed => true + | _ => false) = true := by + native_decide + +/-- signRoot is unavailable without linked PQC (fail-closed). -/ +theorem sign_unavailable : + (match signRoot (replicate 128 0x42) (replicate hashLen 0) with + | .error .signatureUnavailable => true + | _ => false) = true := by + native_decide + +/-- signRoot rejects bad root length before unavailability. -/ +theorem sign_bad_root : + (match signRoot (replicate 128 0x42) (ofList [1]) with + | .error .invalidRootLength => true + | _ => false) = true := by + native_decide + +/-- Magic list matches ASCII SLH1. -/ +theorem slh1_magic_bytes : + slh1Magic = [0x53, 0x4c, 0x48, 0x31] := slh1Magic_eq_literal + +end Carbonado.Slh diff --git a/Carbonado/Stream.lean b/Carbonado/Stream.lean new file mode 100644 index 0000000..5daaf5e --- /dev/null +++ b/Carbonado/Stream.lean @@ -0,0 +1,107 @@ +/- + Pure stream / buffer-bounds model for Carbonado pipeline stages (Program E). + + This is **not** an async IO runtime. It records the chunk/stripe geometry the + product uses so buffer ceilings are explicit and (where practical) proved. + + Memory axes (do not conflate — AGENTS.md): + * Streaming / memory: O(chunk) spool residual; **O(stripe)** FEC residual + * Bao slice stream decode: O(response) (Program D) + * Parallelism / async: out of scope here +-/ +import Carbonado.Constants +import Carbonado.Fec.Inboard +import Carbonado.Pipeline + +namespace Carbonado.Stream + +open Carbonado.Constants +open Carbonado.Fec.Inboard +open Carbonado.Pipeline + +/-- One logical stripe unit (16 KiB plaintext before FEC expansion). -/ +def stripeBytes : Nat := stripeUnit + +/-- Bytes retained for one full inboard FEC stripe after encode (8 × chunk). + + For a full stripe of `stripeUnit` logical bytes: chunk = 4096, body = 32768 = 2×stripe. +-/ +def inboardStripeBytes (logicalLen : Nat) : Nat := + let geo := calcPaddingLen logicalLen + if logicalLen == 0 then 0 else geo.chunkLen * fecM + +/-- Maximum retained buffer for a single-stripe FEC encode/decode residual. -/ +def maxFecStripeRetain (logicalLen : Nat) : Nat := + inboardStripeBytes logicalLen + +/-- Chunk / leaf size (4 KiB) for non-FEC streaming geometry. -/ +def chunkBytes : Nat := sliceLen + +/-- + Pure stripe transducer: map each `stripeUnit`-sized logical window through `f`. + + Concatenates results. Used as the abstract model of streaming FEC encode + (product may implement multi-stripe later; single-segment residual is O(stripe)). +-/ +def mapStripes (input : ByteArray) (f : ByteArray → Except PipelineError ByteArray) : + Except PipelineError ByteArray := + if input.size == 0 then + .ok ByteArray.empty + else + Id.run do + let mut out := ByteArray.empty + let mut off : Nat := 0 + let mut err : Option PipelineError := none + while off < input.size && err.isNone do + let end_ := min (off + stripeUnit) input.size + let piece := input.extract off end_ + match f piece with + | .error e => err := some e + | .ok part => out := out.append part + off := end_ + match err with + | some e => pure (.error e) + | none => pure (.ok out) + +/-- Encode one logical window with FEC inboard (stripe residual). -/ +def encodeFecStripe (window : ByteArray) : Except PipelineError ByteArray := + match Carbonado.Fec.Inboard.encodeInboard window with + | .error e => .error (ofFecError e) + | .ok (body, _, _) => .ok body + +/-- Full multi-stripe FEC encode model (concat of per-stripe inboards). + + Note: Rust product currently uses segment-wide RS geometry (one pad for whole + body). This multi-stripe model documents the streaming bound alternative; + pipeline `encodeBody` still uses segment-wide `encodeInboard` for parity. +-/ +def encodeFecStriped (input : ByteArray) : Except PipelineError ByteArray := + mapStripes input encodeFecStripe + +/-- Bound: one full stripe expands to exactly `2 * stripeUnit` inboard bytes. -/ +theorem full_stripe_inboard_len : + inboardStripeBytes stripeUnit = 2 * stripeUnit := by + native_decide + +/-- Bound: one full stripe retain ceiling equals inboard length. -/ +theorem full_stripe_retain : + maxFecStripeRetain stripeUnit = 2 * stripeUnit := by + native_decide + +/-- Empty input retains nothing. -/ +theorem empty_stripe_retain : maxFecStripeRetain 0 = 0 := by + native_decide + +/-- Single-byte logical pad geometry → one stripe of retain 32768. -/ +theorem one_byte_stripe_retain : + maxFecStripeRetain 1 = 32768 := by + native_decide + +/-- Chunk bound for non-FEC leaf processing is `sliceLen`. -/ +theorem chunk_eq_slice : chunkBytes = sliceLen := rfl + +/-- Stripe unit is `fecK` leaves. -/ +theorem stripe_eq_k_slices : stripeBytes = fecK * sliceLen := by + native_decide + +end Carbonado.Stream diff --git a/CarbonadoTest.lean b/CarbonadoTest.lean new file mode 100644 index 0000000..451c6ec --- /dev/null +++ b/CarbonadoTest.lean @@ -0,0 +1,27 @@ +import Carbonado.Constants +import Carbonado.Crypto +import Carbonado.Fec +import Carbonado.Bao +import Carbonado.Header +import Carbonado.Compress +import Carbonado.Slh +import Carbonado.Pipeline +import Carbonado.Stream +import Carbonado.Scrub +import Carbonado.Shard +import Carbonado.Adamantine +import Carbonado.Filepack +import Carbonado.Outboard +import Carbonado.Directory +import Carbonado.Cli +import CarbonadoTest.Scaffold +import CarbonadoTest.EtM +import CarbonadoTest.Fec +import CarbonadoTest.Bao +import CarbonadoTest.Pipeline +import CarbonadoTest.Compress +import CarbonadoTest.Slh +import CarbonadoTest.Directory + +/- Test root. Name is deliberately *not* `Tests/` — collides with Rust `tests/` + on case-insensitive filesystems (macOS APFS). -/ diff --git a/CarbonadoTest/Bao.lean b/CarbonadoTest/Bao.lean new file mode 100644 index 0000000..4982997 --- /dev/null +++ b/CarbonadoTest/Bao.lean @@ -0,0 +1,235 @@ +/- + Program D — keyed Bao tests: BLAKE3, verification keys, roots, inboard/outboard, + stream slice decode, strict error paths (every BaoError variant exact-matched). + + Dependency direction: CarbonadoTest → Carbonado only. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Bao +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.Bao + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Bao.Blake3 +open Carbonado.Bao.Tree +open Carbonado.Bao.Product + +/-! ## Geometry -/ + +theorem leaf_eq_slice : leafBytes = sliceLen := leafBytes_eq_sliceLen + +theorem slice_len_4096 : sliceLen = 4096 := by native_decide + +theorem bao_chunk_log_2 : baoChunkLog = 2 := by native_decide + +/-! ## BLAKE3 goldens -/ + +theorem blake3_empty : + toHex (hash ByteArray.empty) = + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" := + hash_empty + +theorem blake3_abc : + toHex (hash (utf8 "abc")) = + "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85" := + hash_abc + +/-! ## Verification keys (format domain) -/ + +theorem vkey_c4 : + toHex (carbonadoVerificationKey 4) = + "6f1b6d31098f44f98e31231fe4244d532d1263556a45fe370d74cae1a447ffbf" := by + native_decide + +theorem vkey_c6 : + toHex (carbonadoVerificationKey 6) = + "3923848b90f499febc394b34cbac1f3c2d9a98a89b93beb4bb6361d1db5d4615" := by + native_decide + +theorem vkey_domain : + toHex (carbonadoVerificationKey 4) ≠ toHex (carbonadoVerificationKey 6) := + verification_key_format_domain + +theorem root_format_domain : + let data := ofList [0, 1, 2, 3, 4] + toHex (rootForFormat 4 data) ≠ toHex (rootForFormat 6 data) := + root_commits_to_format + +/-! ## Roots -/ + +theorem hello_root : + toHex (rootForFormat 4 (utf8 "hello")) = + "f8a7892045a78f933cca82f9ef17046c453ad166e5463e5e93c88cf614443d86" := by + native_decide + +theorem hello_root_is_keyed_hash : + let data := utf8 "hello" + let key := carbonadoVerificationKey 4 + toHex (rootForFormat 4 data) = toHex (keyedHash key data) := + hello_root_eq_keyed_hash + +private def pat100 : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:100] do + out := out.push (UInt8.ofNat (i % 251)) + pure out + +theorem pat100_root_c4 : + toHex (rootForFormat 4 pat100) = + "27e8845ee8cfeed082734d4409991f9db4e9e4d0476de3ed196d89f4a2f37077" := by + native_decide + +theorem pat100_root_c6 : + toHex (rootForFormat 6 pat100) = + "1e478e01260caf8df6ad29f09c754a5e8423ee5ceceee4209db543828416147f" := by + native_decide + +/-! ## Inboard goldens + roundtrip -/ + +theorem empty_inboard_roundtrip : + (match encodeInboardForFormat 4 ByteArray.empty with + | (root, art) => + match decodeInboardForFormat 4 root art with + | .ok d => d.size == 0 + | .error _ => false) = true := + encode_decode_empty_c4 + +theorem hello_inboard_artifact : + (let (_root, art) := encodeInboardForFormat 4 (utf8 "hello") + toHex art = "050000000000000068656c6c6f") = true := by + native_decide + +theorem pat100_inboard_roundtrip : + (let (root, art) := encodeInboardForFormat 4 pat100 + match decodeInboardForFormat 4 root art with + | .ok d => toHex d == toHex pat100 + | .error _ => false) = true := by + native_decide + +/-- Encode is deterministic. -/ +theorem encode_deterministic_hello : + (let a := encodeInboardForFormat 4 (utf8 "hello") + let b := encodeInboardForFormat 4 (utf8 "hello") + toHex a.1 == toHex b.1 && toHex a.2 == toHex b.2) = true := by + native_decide + +/-- Stream slice decode recovers authenticated bytes (pat100 = single leaf, full range). -/ +theorem stream_slice_decode_pat100 : + (let (root, _art) := encodeInboardForFormat 4 pat100 + let (_r, resp) := encodeSliceForFormat 4 pat100 0 1 + match decodeSliceForFormat 4 root 100 0 1 resp with + | .ok d => toHex d == toHex pat100 + | .error _ => false) = true := by + native_decide + +/-! ## Strict error observers (every BaoError constructor) -/ + +private def isAuthFail : Except BaoError α → Bool + | .error .authenticationFailed => true + | _ => false + +private def isTrunc : Except BaoError α → Bool + | .error .truncatedResponse => true + | _ => false + +private def isTrailing : Except BaoError α → Bool + | .error .trailingData => true + | _ => false + +private def isInvalidPrefix : Except BaoError α → Bool + | .error .invalidPrefix => true + | _ => false + +private def isInvalidRoot : Except BaoError α → Bool + | .error .invalidRootLength => true + | _ => false + +private def isInvalidSlice : Except BaoError α → Bool + | .error .invalidSliceIndex => true + | _ => false + +private def isInvalidCount : Except BaoError α → Bool + | .error .invalidSliceCount => true + | _ => false + +theorem wrong_format_auth_fail : + (let (root, art) := encodeInboardForFormat 4 pat100 + isAuthFail (decodeInboardForFormat 6 root art)) = true := by + native_decide + +theorem short_prefix_error : + isInvalidPrefix (contentLenPrefix (ofList [1, 2, 3])) = true := by + native_decide + +theorem invalid_root_length_error : + (let (_root, art) := encodeInboardForFormat 4 pat100 + isInvalidRoot (decodeInboardForFormat 4 (ofList [0]) art)) = true := by + native_decide + +theorem invalid_slice_index_error : + (let (root, art) := encodeInboardForFormat 4 pat100 + isInvalidSlice (verifySliceInboardForFormat 4 root art 5 1)) = true := by + native_decide + +/-- content_len claims 100 bytes but response only has 99 → truncatedResponse. -/ +theorem truncated_body_is_trunc : + (let (root, art) := encodeInboardForFormat 4 pat100 + let short := art.extract 0 (art.size - 1) + isTrunc (decodeInboardForFormat 4 root short)) = true := by + native_decide + +/-- Extra byte after valid inboard → trailingData (not truncatedResponse). -/ +theorem trailing_body_is_trailing : + (let (root, art) := encodeInboardForFormat 4 pat100 + let long := art.push 0xaa + isTrailing (decodeInboardForFormat 4 root long)) = true := by + native_decide + +theorem tampered_body_auth_fail : + (let (root, art) := encodeInboardForFormat 4 pat100 + let bad := art.set! 10 (art.get! 10 ^^^ 0x01) + isAuthFail (decodeInboardForFormat 4 root bad)) = true := by + native_decide + +/-- Stream decode count=0 → invalidSliceCount. -/ +theorem slice_count_zero_error : + (let (root, _art) := encodeInboardForFormat 4 pat100 + let (_r, resp) := encodeSliceForFormat 4 pat100 0 1 + isInvalidCount (decodeSliceForFormat 4 root 100 0 0 resp)) = true := by + native_decide + +/-- Corrupt inboard + count=0 extract must not succeed (auth-first). -/ +theorem count_zero_corrupt_inboard_auth_fail : + (let (root, art) := encodeInboardForFormat 4 pat100 + let bad := art.set! 10 (art.get! 10 ^^^ 0x01) + isAuthFail (verifySliceInboardForFormat 4 root bad 0 0)) = true := by + native_decide + +/-- Stream slice wrong format key → authenticationFailed. -/ +theorem stream_slice_wrong_key_auth_fail : + (let (root, _art) := encodeInboardForFormat 4 pat100 + let (_r, resp) := encodeSliceForFormat 4 pat100 0 1 + isAuthFail (decodeSliceForFormat 6 root 100 0 1 resp)) = true := by + native_decide + +/-- Stream slice truncated → truncatedResponse. -/ +theorem stream_slice_truncated : + (let (root, _art) := encodeInboardForFormat 4 pat100 + let (_r, resp) := encodeSliceForFormat 4 pat100 0 1 + let short := resp.extract 0 (min 3 resp.size) + isTrunc (decodeSliceForFormat 4 root 100 0 1 short)) = true := by + native_decide + +/-- Stream slice trailing garbage → trailingData. -/ +theorem stream_slice_trailing : + (let (root, _art) := encodeInboardForFormat 4 pat100 + let (_r, resp) := encodeSliceForFormat 4 pat100 0 1 + let long := resp.push 0xcc + isTrailing (decodeSliceForFormat 4 root 100 0 1 long)) = true := by + native_decide + +end CarbonadoTest.Bao diff --git a/CarbonadoTest/Compress.lean b/CarbonadoTest/Compress.lean new file mode 100644 index 0000000..75b39b1 --- /dev/null +++ b/CarbonadoTest/Compress.lean @@ -0,0 +1,98 @@ +/- + Program F — zstd compress API + status mapping tests. + + Dependency direction: CarbonadoTest → Carbonado only. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Compress +import Carbonado.Pipeline +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.Compress + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Compress +open Carbonado.Pipeline + +theorem magic_len : zstdMagic.length = 4 := zstdMagic_length + +theorem ofStatus_1 : ofStatus 1 = .compressionFailed := ofStatus_compress +theorem ofStatus_2 : ofStatus 2 = .decompressionFailed := ofStatus_decompress +theorem ofStatus_3 : ofStatus 3 = .outputTooLarge := ofStatus_too_large +theorem ofStatus_4 : ofStatus 4 = .invalidInput := ofStatus_invalid_4 +theorem ofStatus_unknown : ofStatus 99 = .invalidInput := ofStatus_invalid_99 + +theorem empty_raw : + (match decodeStatusPayload ByteArray.empty with + | .error .invalidInput => true + | _ => false) = true := decode_empty_raw + +theorem status_1 : + (match decodeStatusPayload (ofList [1]) with + | .error .compressionFailed => true + | _ => false) = true := decode_status_1 + +theorem status_2 : + (match decodeStatusPayload (ofList [2]) with + | .error .decompressionFailed => true + | _ => false) = true := decode_status_2 + +theorem status_3 : + (match decodeStatusPayload (ofList [3]) with + | .error .outputTooLarge => true + | _ => false) = true := decode_status_3 + +theorem status_4 : + (match decodeStatusPayload (ofList [4]) with + | .error .invalidInput => true + | _ => false) = true := decode_status_4 + +theorem status_ok : + (match decodeStatusPayload (ofList [0, 0x68, 0x69]) with + | .ok b => ctEq b (ofList [0x68, 0x69]) + | .error _ => false) = true := decode_status_ok_hello + +theorem framing_identity : + (match decodeStatusPayload (statusOkPayload (ofList [1, 2, 3])) with + | .ok b => ctEq b (ofList [1, 2, 3]) + | .error _ => false) = true := statusOk_payload_identity + +/-- Pipeline ofZstdError maps are injective per mode. -/ +theorem map_zstd_compress : + ofZstdError ZstdError.compressionFailed = PipelineError.compressionFailed := rfl + +theorem map_zstd_decompress : + ofZstdError ZstdError.decompressionFailed = PipelineError.decompressionFailed := rfl + +theorem map_zstd_too_large : + ofZstdError ZstdError.outputTooLarge = PipelineError.decompressOutputTooLarge := rfl + +theorem map_zstd_invalid : + ofZstdError ZstdError.invalidInput = PipelineError.zstdInvalidInput := rfl + +theorem pipeline_map_decompress : + ofZstdError (ofStatus 2) = PipelineError.decompressionFailed := rfl + +theorem pipeline_map_too_large : + ofZstdError (ofStatus 3) = PipelineError.decompressOutputTooLarge := rfl + +/-- compressStep with bit clear is identity (no zstd). -/ +theorem compress_bit_clear : + (match compressStep (ofList [9, 8, 7]) false with + | .ok (b, n) => ctEq b (ofList [9, 8, 7]) && n == 0 + | .error _ => false) = true := by + native_decide + +/-- decompressStep bit clear is identity. -/ +theorem decompress_bit_clear : + (match decompressStep (ofList [9, 8, 7]) false with + | .ok b => ctEq b (ofList [9, 8, 7]) + | .error _ => false) = true := by + native_decide + +/-- Level constant is 20. -/ +theorem level_20 : zstdLevel = 20 := by native_decide + +end CarbonadoTest.Compress diff --git a/CarbonadoTest/Directory.lean b/CarbonadoTest/Directory.lean new file mode 100644 index 0000000..12b98b0 --- /dev/null +++ b/CarbonadoTest/Directory.lean @@ -0,0 +1,327 @@ +/- + Program G — Adamantine, Filepack, Directory tests. + + Dependency: CarbonadoTest → Carbonado only. + + Exact-match every product-reachable DirectoryError / FilepackError path where + cheap; FEC-heavy tamper cases are AOT-gated in Main. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Adamantine +import Carbonado.Filepack +import Carbonado.Outboard +import Carbonado.Directory +import Carbonado.Pipeline +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.Directory + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Adamantine +open Carbonado.Filepack +open Carbonado.Outboard +open Carbonado.Directory +open Carbonado.Pipeline + +private def pubMaster : ByteArray := replicate 32 0 +private def master42 : ByteArray := replicate 32 0x42 + +/-! ## Adamantine wire -/ + +theorem adam_magic_len : adamantineMagic.length = 13 := adamantineMagic_length + +theorem adam_header_19 : adamantineHeaderLen = 19 := adamantineHeaderLen_eq + +theorem adam_roundtrip_empty : + (match decodeAdamantine (encodeAdamantine ByteArray.empty adamantineFmtPublic 0) with + | .ok (p, h) => p.size == 0 && h.carbonadoFmt == adamantineFmtPublic && h.flags == 0 + | .error _ => false) = true := + encode_decode_empty_public + +theorem adam_invalid_flags : + (match decodeAdamantine (encodeAdamantine ByteArray.empty adamantineFmtPublic 2) with + | .error (.invalidFlags 2) => true + | _ => false) = true := + invalid_flags_bit1 + +theorem adam_invalid_fmt : + (match decodeAdamantine (encodeAdamantine ByteArray.empty 0 0) with + | .error (.invalidCarbonadoFormat 0) => true + | _ => false) = true := + invalid_fmt_c0 + +theorem adam_short : + (match decodeAdamantine (ofList [1, 2, 3]) with + | .error .invalidHeader => true + | _ => false) = true := + short_header + +theorem adam_dev_v2 : + (match decodeAdamantine (appendBA adamantineMagicDevV2 (replicate 7 0)) with + | .error (.unsupportedVersion 2 0) => true + | _ => false) = true := + dev_v2_rejected + +/-! ## Path rules (fail-closed) -/ + +theorem path_empty : + (match validateRelPath "" with | .error .emptyRelPath => true | _ => false) = true := + rel_empty + +theorem path_traversal : + (match validateRelPath "a/../b" with | .error .relPathTraversal => true | _ => false) = true := + rel_traversal + +theorem path_absolute : + (match validateRelPath "/etc/passwd" with | .error .relPathAbsolute => true | _ => false) = + true := + rel_absolute + +theorem path_backslash : + (match validateRelPath "a\\b" with | .error .relPathBackslash => true | _ => false) = true := + rel_backslash + +theorem path_ok : + (match validateRelPath "src/main.lean" with | .ok () => true | _ => false) = true := + rel_ok + +theorem path_empty_comp : + (match validateRelPath "a//b" with | .error .relPathEmptyComponent => true | _ => false) = + true := + rel_empty_component + +theorem path_null : + (match validateRelPath ("a" ++ String.singleton (Char.ofNat 0) ++ "b") with + | .error .relPathNullByte => true + | _ => false) = true := + rel_null + +/-! ## Error maps (1:1, no collapse) -/ + +theorem map_traversal : ofFilepackError .relPathTraversal = .pathTraversal := ofFilepack_traversal + +theorem map_absolute : ofFilepackError .relPathAbsolute = .pathAbsolute := ofFilepack_absolute + +theorem map_backslash : ofFilepackError .relPathBackslash = .pathBackslash := ofFilepack_backslash + +theorem map_null : ofFilepackError .relPathNullByte = .pathNullByte := ofFilepack_null + +theorem map_empty_component : + ofFilepackError .relPathEmptyComponent = .pathEmptyComponent := ofFilepack_empty_component + +theorem map_too_many_segments : + ofFilepackError .tooManySegments = .tooManySegments := ofFilepack_too_many_segments + +theorem map_ots_too_large : + ofFilepackError .otsProofTooLarge = .otsProofTooLarge := ofFilepack_ots_too_large + +theorem map_adam_flags : + ofAdamantineError (.invalidFlags 2) = .invalidAdamantineFlags 2 := ofAdamantine_flags + +theorem map_adam_magic : ofAdamantineError .invalidMagic = .invalidAdamantineMagic := + ofAdamantine_magic + +/-! ## Filepack / policy -/ + +theorem cfp2_magic : + cfp2Magic = [0x43, 0x46, 0x50, 0x32] := by + native_decide + +theorem filepack_legacy_c4 : + (match validateSegmentFormat 0x04 false with + | .error (.legacySegmentFormat 0x04) => true + | _ => false) = true := by + native_decide + +theorem filepack_seg_enc_mismatch : + (match validateSegmentFormat segmentFormatPublicRaw true with + | .error (.segmentFormatMismatch 0x0C) => true + | _ => false) = true := by + native_decide + +theorem filepack_seg_enc_mismatch_named : + (match validateSegmentFormat (0x0C : UInt8) true with + | .error (.segmentFormatMismatch 0x0C) => true + | _ => false) = true := by + native_decide + +/-! ## Master policy + path at encode (cheap) -/ + +theorem directory_zero_master_encrypted : + (match encodeDirectory pubMaster + #[{ relPath := "a", content := utf8 "z" }] + { catalogEncrypted := true, segmentPolicy := .forceRaw } + #[replicate 16 1, replicate 16 2] with + | .error .zeroMasterKeyNotAllowed => true + | _ => false) = true := by + native_decide + +theorem directory_nonzero_master_public : + (match encodeDirectory master42 + #[{ relPath := "a", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } + #[] with + | .error .encryptedDirectoryNotRequested => true + | _ => false) = true := by + native_decide + +theorem directory_rejects_traversal : + (match encodeDirectory pubMaster + #[{ relPath := "../x", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } + #[] with + | .error .pathTraversal => true + | _ => false) = true := by + native_decide + +theorem directory_rejects_absolute : + (match encodeDirectory pubMaster + #[{ relPath := "/etc/passwd", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } + #[] with + | .error .pathAbsolute => true + | _ => false) = true := by + native_decide + +theorem directory_rejects_backslash : + (match encodeDirectory pubMaster + #[{ relPath := "a\\b", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } + #[] with + | .error .pathBackslash => true + | _ => false) = true := by + native_decide + +theorem directory_empty_path : + (match encodeDirectory pubMaster + #[{ relPath := "", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } + #[] with + | .error .pathEmpty => true + | _ => false) = true := by + native_decide + +theorem directory_null_path : + (match encodeDirectory pubMaster + #[{ relPath := "a" ++ String.singleton (Char.ofNat 0) ++ "b", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw } + #[] with + | .error .pathNullByte => true + | _ => false) = true := by + native_decide + +/-- requireOts rejected at encode (no undecodeable archives). -/ +theorem directory_require_ots_encode_rejected : + (match encodeDirectory pubMaster + #[{ relPath := "a", content := utf8 "z" }] + { catalogEncrypted := false, segmentPolicy := .forceRaw, requireOts := true } + #[] with + | .error .otsFeatureRequired => true + | _ => false) = true := by + native_decide + +/-! ## FEC geometry -/ + +theorem padding_main_zero : + paddingForMainLen 0 true = 0 := by + native_decide + +theorem padding_main_one : + paddingForMainLen 1 true = 16383 := by + native_decide + +theorem expected_fec_parity_empty : expectedFecParityLen 0 = 0 := by native_decide + +theorem expected_fec_parity_one : expectedFecParityLen 1 = 16384 := by native_decide + +/-- Empty main + nonzero FEC parity → unexpectedFecParity. -/ +theorem bundle_sem_zero_main_with_fec : + (match validateSegmentBundleSemantics segmentFormatPublicRaw { + segmentBaoRoot := replicate 32 0 + chunkIndex := 0 + mainLen := 0 + verificationOutboardOffset := 0 + verificationOutboardLen := 0 + fecParityOffset := 0 + fecParityLen := 1 + } with + | .error .unexpectedFecParity => true + | _ => false) = true := by + native_decide + +/-- Nonzero main + zero FEC parity → missingFecParity. -/ +theorem bundle_sem_missing_fec : + (match validateSegmentBundleSemantics segmentFormatPublicRaw { + segmentBaoRoot := replicate 32 0 + chunkIndex := 0 + mainLen := 1 + verificationOutboardOffset := 0 + verificationOutboardLen := 0 + fecParityOffset := 0 + fecParityLen := 0 + } with + | .error .missingFecParity => true + | _ => false) = true := by + native_decide + +/-- Wrong FEC parity length → fecParityLenMismatch. -/ +theorem bundle_sem_fec_len_mismatch : + (match validateSegmentBundleSemantics segmentFormatPublicRaw { + segmentBaoRoot := replicate 32 0 + chunkIndex := 0 + mainLen := 1 + verificationOutboardOffset := 0 + verificationOutboardLen := 0 + fecParityOffset := 0 + fecParityLen := 1 + } with + | .error .fecParityLenMismatch => true + | _ => false) = true := by + native_decide + +/-- FEC present on non-FEC format → unexpectedFecParity. -/ +theorem bundle_sem_fec_on_non_fec_format : + (match validateSegmentBundleSemantics 0x04 { + segmentBaoRoot := replicate 32 0 + chunkIndex := 0 + mainLen := 1 + verificationOutboardOffset := 0 + verificationOutboardLen := 0 + fecParityOffset := 0 + fecParityLen := 16 + } with + | .error .unexpectedFecParity => true + | _ => false) = true := by + native_decide + +/-- Catalog name too short → invalidCatalogPath. -/ +theorem catalog_name_short : + (match parseCatalogName "aa.adam.c14" with + | .error .invalidCatalogPath => true + | _ => false) = true := by + native_decide + +/-- contentBlake3Mismatch exact (wrong hash). -/ +theorem content_blake3_mismatch : + (match checkContentBlake3 (utf8 "x") (replicate 32 0) with + | .error .contentBlake3Mismatch => true + | _ => false) = true := by + native_decide + +/-- contentBlake3 ok path for empty. -/ +theorem content_blake3_empty_ok : + (match checkContentBlake3 ByteArray.empty (Carbonado.Bao.Blake3.hash ByteArray.empty) with + | .ok () => true + | _ => false) = true := by + native_decide + +/-- invalidHashLength when expected hash size wrong. -/ +theorem content_blake3_bad_hash_len : + (match checkContentBlake3 (utf8 "x") (ofList [1, 2, 3]) with + | .error .invalidHashLength => true + | _ => false) = true := by + native_decide + +end CarbonadoTest.Directory diff --git a/CarbonadoTest/EtM.lean b/CarbonadoTest/EtM.lean new file mode 100644 index 0000000..12e156f --- /dev/null +++ b/CarbonadoTest/EtM.lean @@ -0,0 +1,305 @@ +/- + Program B — EtM tests: structural theorems + golden / failure-mode checks. + + Dependency direction: CarbonadoTest → Carbonado only. +-/ +import Carbonado.Constants +import Carbonado.Crypto +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.EtM + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Crypto.SHA512 +open Carbonado.Crypto.HMAC +open Carbonado.Crypto.AESCTR +open Carbonado.Crypto.EtM + +/-! ## Re-export MAC-before-decrypt theorems into the test tree -/ + +theorem mac_tag_fail_is_auth_error + (aesKey macKey nonce tag ct : ByteArray) + (h : ctEq tag (computePayloadTag macKey nonce ct) = false) : + decryptAfterMacCheck aesKey macKey nonce tag ct = .error .authenticationFailed := + decryptAfterMacCheck_tag_fail aesKey macKey nonce tag ct h + +theorem mac_ok_implies_ctEq + (aesKey macKey nonce tag ct pt : ByteArray) + (h : decryptAfterMacCheck aesKey macKey nonce tag ct = .ok pt) : + ctEq tag (computePayloadTag macKey nonce ct) = true := + decryptAfterMacCheck_ok_implies_mac aesKey macKey nonce tag ct pt h + +theorem auth_fail_plaintext_none + (aesKey macKey nonce tag ct : ByteArray) + (h : ctEq tag (computePayloadTag macKey nonce ct) = false) : + (decryptAfterMacCheck aesKey macKey nonce tag ct).plaintext? = none := + decryptAfterMacCheck_auth_fail_no_plaintext aesKey macKey nonce tag ct h + +theorem short_input_is_invalidCiphertextLength + (master nonce input : ByteArray) + (hs : (input.size < hmacTagLen) = true) : + decryptWithNonce master nonce input = .error .invalidCiphertextLength := + decryptWithNonce_short_input master nonce input hs + +theorem short_master_is_invalidKeyLength + (master nonce input : ByteArray) + (hs : (input.size < hmacTagLen) = false) + (hm : (master.size < minMasterLen) = true) : + decryptWithNonce master nonce input = .error .invalidKeyLength := + decryptWithNonce_short_master master nonce input hs hm + +theorem bad_nonce_is_invalidNonceLength + (master nonce input : ByteArray) + (hs : (input.size < hmacTagLen) = false) + (hm : (master.size < minMasterLen) = false) + (hn : (nonce.size != nonceLen) = true) : + decryptWithNonce master nonce input = .error .invalidNonceLength := + decryptWithNonce_bad_nonce master nonce input hs hm hn + +/-! ## Bool observers for `native_decide` (DecryptResult has no DecidableEq). -/ + +private def isAuthFailed : DecryptResult → Bool + | .error .authenticationFailed => true + | _ => false + +private def isInvalidCiphertextLength : DecryptResult → Bool + | .error .invalidCiphertextLength => true + | _ => false + +private def isInvalidKeyLength : DecryptResult → Bool + | .error .invalidKeyLength => true + | _ => false + +private def isInvalidNonceLength : DecryptResult → Bool + | .error .invalidNonceLength => true + | _ => false + +private def encryptOkHex (master nonce pt : ByteArray) : String := + match encryptWithNonce master nonce pt with + | .ok blob => toHex blob + | .error _ => "" + +private def encryptErrIsInvalidNonce (master nonce pt : ByteArray) : Bool := + match encryptWithNonce master nonce pt with + | .error .invalidNonceLength => true + | _ => false + +private def roundtripOk (master nonce pt : ByteArray) : Bool := + match encryptWithNonce master nonce pt with + | .error _ => false + | .ok blob => + match decryptWithNonce master nonce blob with + | .ok pt' => toHex pt' == toHex pt + | .error _ => false + +private def embeddedRoundtripOk (master nonce pt : ByteArray) : Bool := + match encryptEmbeddedNonce master nonce pt with + | .error _ => false + | .ok blob => + match decryptEmbeddedNonce master blob with + | .ok pt' => toHex pt' == toHex pt + | .error _ => false + +private def tamperTagFails (master nonce pt : ByteArray) : Bool := + match encryptWithNonce master nonce pt with + | .error _ => false + | .ok blob => + let bad := blob.set! 0 (blob.get! 0 ^^^ 1) + isAuthFailed (decryptWithNonce master nonce bad) + +private def tamperBodyFails (master nonce pt : ByteArray) : Bool := + match encryptWithNonce master nonce pt with + | .error _ => false + | .ok blob => + if blob.size ≤ hmacTagLen then false + else + let bad := blob.set! hmacTagLen (blob.get! hmacTagLen ^^^ 1) + isAuthFailed (decryptWithNonce master nonce bad) + +private def wrongKeyFails (master wrong nonce pt : ByteArray) : Bool := + match encryptWithNonce master nonce pt with + | .error _ => false + | .ok blob => isAuthFailed (decryptWithNonce wrong nonce blob) + +private def headerMacHex (master auth : ByteArray) : String := + match computeHeaderMac master auth with + | .ok tag => toHex tag + | .error _ => "" + +private def verifyHeaderMacBool (master auth tag : ByteArray) : Bool := + match verifyHeaderMac master auth tag with + | .ok b => b + | .error _ => false + +private def master42 : ByteArray := replicate 32 0x42 +private def nonce11 : ByteArray := replicate 16 0x11 + +private def nistKey : ByteArray := ofList [ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4] + +private def nistCtr : ByteArray := ofList [ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff] + +private def nistPt : ByteArray := ofList [ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10] + +/-- Empty SHA-512 digest (FIPS). -/ +theorem sha512_empty_hex : + toHex (Carbonado.Crypto.SHA512.hash ByteArray.empty) = + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" := by + native_decide + +/-- SHA-512("abc"). -/ +theorem sha512_abc_hex : + toHex (hashString "abc") = + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" := by + native_decide + +/-- HMAC-SHA512 RFC 4231 test case 1. -/ +theorem hmac_rfc4231_1 : + toHex (hmacSHA512 (replicate 20 0x0b) (utf8 "Hi There")) = + "87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854" := by + native_decide + +/-- AES-256-CTR NIST SP 800-38A F.5.5. -/ +theorem aes_ctr_nist_f55 : + toHex (ctrXor nistKey nistCtr nistPt) = + "601ec313775789a5b7a7f504bbf3d228f443e3ca4d62b59aca84e990cacaf5c52b0930daa23de94ce87017ba2d84988ddfc9c58db67aada613c2dd08457941a6" := by + native_decide + +/-- Subkey `aes-ctr` under master 0x42×32. -/ +theorem subkey_aes_ctr_golden : + (match deriveSubkey master42 "aes-ctr" with + | .ok sk => toHex sk + | .error _ => "") = + "6f15fb9936ca3e4d2ecc5bd80bcc06c12d67361b72dcf5e8edc8312092f42a28494d106e3340595717f67ab0ec91b0b8d0ea653853ca129a4515ea6df74a5ca7" := by + native_decide + +/-- Subkey `etm-hmac` under master 0x42×32. -/ +theorem subkey_etm_hmac_golden : + (match deriveSubkey master42 "etm-hmac" with + | .ok sk => toHex sk + | .error _ => "") = + "d9795accc69d8966b12f051575d3efc53725697a14d8117ffcef149eea20bb859e025d2e76f5b47bb1bc73af3c34aceb317ba0d5d00e5b3f36e7f21accae3609" := by + native_decide + +/-- Subkey `header-auth` under master 0x42×32. -/ +theorem subkey_header_auth_golden : + (match deriveSubkey master42 "header-auth" with + | .ok sk => toHex sk + | .error _ => "") = + "af1f7d5c23422538fab14c8343eaef42918230ba04fef171176b3c01a89e9bab0ae60b90586d9863f4a4231d91eb984516f0be04c20d4c7e784f0fe459de2b19" := by + native_decide + +/-- EtM empty plaintext header-path golden. -/ +theorem etm_empty_golden : + encryptOkHex master42 nonce11 ByteArray.empty = + "74e137726bc9f0a9e55add833d1ac0c187bb366f22f0a2be1189536828d77dfc2d021e8aad99fab802c664db3b8ec1e0c46198f44dd3f3e9321bded8263a6aa2" := by + native_decide + +/-- EtM `hello` golden. -/ +theorem etm_hello_golden : + encryptOkHex master42 nonce11 (utf8 "hello") = + "1d05aa600755696228225aeada6672b65266554ef6a2e2b5f4a083870ad00f534747fbd18d98f4c6e449a4b64e954b77f65bd63eba1a9a0e080cca3c296760b08f0a0e143c" := by + native_decide + +/-- EtM multi-block golden (fox sentence). -/ +theorem etm_multi_golden : + encryptOkHex master42 nonce11 (utf8 "The quick brown fox jumps over the lazy dog") = + "38cddad202dc0f7daacd53b35573b43a3c79bbcfa44aee039d661c92d5be8f16701b996b7474d66610627fb714376524cfcbb4aa30d8e7062ca9b61cf32eccd9b307075822eda7e16b52f04354c83f70cd6b5a9ab6817560693ae25a0cdd1883752eeccaff8bb0228d84c6" := by + native_decide + +/-- EtM empty roundtrip. -/ +theorem etm_empty_roundtrip : + roundtripOk master42 nonce11 ByteArray.empty = true := by + native_decide + +/-- EtM hello roundtrip. -/ +theorem etm_hello_roundtrip : + roundtripOk master42 nonce11 (utf8 "hello") = true := by + native_decide + +/-- Embedded-nonce hello roundtrip. -/ +theorem etm_embedded_hello_roundtrip : + embeddedRoundtripOk master42 nonce11 (utf8 "hello") = true := by + native_decide + +/-- Embedded short input → `invalidCiphertextLength`. -/ +theorem etm_embedded_short_ct : + isInvalidCiphertextLength (decryptEmbeddedNonce master42 (replicate 20 0)) = true := by + native_decide + +/-- Tampered tag fails with `authenticationFailed`. -/ +theorem etm_tampered_tag_auth_failed : + tamperTagFails master42 nonce11 (utf8 "hi") = true := by + native_decide + +/-- Ciphertext-body tamper fails with `authenticationFailed`. -/ +theorem etm_tampered_body_auth_failed : + tamperBodyFails master42 nonce11 (utf8 "hello") = true := by + native_decide + +/-- Wrong master fails with `authenticationFailed`. -/ +theorem etm_wrong_key_auth_failed : + wrongKeyFails master42 (replicate 32 0x43) nonce11 (utf8 "hi") = true := by + native_decide + +/-- Short ciphertext → `invalidCiphertextLength`. -/ +theorem etm_short_ct_length : + isInvalidCiphertextLength (decryptWithNonce master42 nonce11 (replicate 8 0)) = true := by + native_decide + +/-- Dual-invalid short master + short CT → CT length first (Rust parity). -/ +theorem etm_dual_short_prefers_ct_length : + isInvalidCiphertextLength + (decryptWithNonce (replicate 16 0x42) nonce11 (replicate 8 0)) = true := by + native_decide + +/-- Short master (CT long enough) → `invalidKeyLength`. -/ +theorem etm_short_master_key : + isInvalidKeyLength (decryptWithNonce (replicate 16 0x42) nonce11 (replicate 64 0)) = true := by + native_decide + +/-- Bad nonce length on encrypt → `invalidNonceLength`. -/ +theorem etm_encrypt_bad_nonce : + encryptErrIsInvalidNonce master42 (replicate 8 0x11) (utf8 "hi") = true := by + native_decide + +/-- Bad nonce length on decrypt → `invalidNonceLength`. -/ +theorem etm_decrypt_bad_nonce : + isInvalidNonceLength (decryptWithNonce master42 (replicate 8 0x11) (replicate 64 0)) = true := by + native_decide + +/-- Empty master rejected by deriveSubkey. -/ +theorem derive_empty_master : + (match deriveSubkey ByteArray.empty "aes-ctr" with + | .error .invalidKeyLength => true + | _ => false) = true := by + native_decide + +/-- Header MAC over MAGIC golden. -/ +theorem header_mac_magic_golden : + headerMacHex master42 (utf8 "CARBONADO20\n") = + "c02b40016162e5abf37a007183f2117a46fb74175529188dc98786c9cab370691c7903dcf552765f7764ec2c392af0863b618ea295ed026e8a47b304f0127937" := by + native_decide + +/-- `verifyHeaderMac` false path on flipped tag. -/ +theorem header_mac_verify_false : + (match computeHeaderMac master42 (utf8 "CARBONADO20\n") with + | .ok tag => + verifyHeaderMacBool master42 (utf8 "CARBONADO20\n") + (tag.set! 0 (tag.get! 0 ^^^ 1)) = false + | .error _ => false) = true := by + native_decide + +/-- Digest length is full 64 bytes; tag constant matches. -/ +theorem digest_and_tag_len : + (Carbonado.Crypto.SHA512.hash ByteArray.empty).size = 64 ∧ hmacTagLen = 64 := by + native_decide + +end CarbonadoTest.EtM diff --git a/CarbonadoTest/Fec.lean b/CarbonadoTest/Fec.lean new file mode 100644 index 0000000..14dbbfc --- /dev/null +++ b/CarbonadoTest/Fec.lean @@ -0,0 +1,307 @@ +/- + Program C — RS 4/8 FEC tests: geometry theorems, goldens, reconstruct, errors. + + Dependency direction: CarbonadoTest → Carbonado only. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Fec +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.Fec + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Fec.Galois +open Carbonado.Fec.Matrix +open Carbonado.Fec.RS +open Carbonado.Fec.Inboard + +/-! ## Re-export geometry theorems -/ + +theorem padding_zero : calcPaddingLen 0 = { paddingLen := 0, chunkLen := 0 } := + calcPaddingLen_zero + +theorem padding_one : calcPaddingLen 1 = { paddingLen := 16383, chunkLen := 4096 } := + calcPaddingLen_one + +theorem padding_stripe : calcPaddingLen 16384 = { paddingLen := 0, chunkLen := 4096 } := + calcPaddingLen_stripe + +theorem padding_stripe_plus : + calcPaddingLen 16385 = { paddingLen := 16383, chunkLen := 8192 } := + calcPaddingLen_stripe_plus_one + +theorem padding_100 : calcPaddingLen 100 = { paddingLen := 16284, chunkLen := 4096 } := + calcPaddingLen_100 + +theorem padding_4096 : calcPaddingLen 4096 = { paddingLen := 12288, chunkLen := 4096 } := + calcPaddingLen_4096 + +theorem padded_aligns : + (List.map paddedLen [1, 100, 4096, 16384, 16385]).all + (fun p => p % stripeUnit == 0) = true := + paddedLen_aligns_samples + +theorem rs_geometry : + carbonadoRS.dataShards = fecK ∧ carbonadoRS.parityShards = fecM - fecK := + carbonadoRS_geometry + +theorem rs_constructs : + (match carbonadoRSExcept with | .ok _ => true | .error _ => false) = true := + carbonadoRS_constructs + +/-! ## GF goldens -/ + +theorem gf_mul_53_ca : mul 0x53 0xca = 0x8f := mul_0x53_0xca +theorem gf_div_2_3 : div 2 3 = 0xf5 := div_2_3 +theorem gf_exp_2_3 : exp 2 3 = 8 := exp_2_3 + +/-! ## Bool observers for strict error matches (FecError has DecidableEq). -/ + +private def isUneven : Except FecError α → Bool + | .error .unevenShards => true + | _ => false + +private def isTooFew : Except FecError α → Bool + | .error .tooFewShards => true + | _ => false + +private def isEmptyShard : Except FecError α → Bool + | .error .emptyShard => true + | _ => false + +private def isIncorrectSize : Except FecError α → Bool + | .error .incorrectShardSize => true + | _ => false + +private def isBadGeometry : Except FecError α → Bool + | .error .badGeometry => true + | _ => false + +private def isPaddingTooLarge : Except FecError α → Bool + | .error .paddingTooLarge => true + | _ => false + +private def isSingular : Except FecError α → Bool + | .error .singularMatrix => true + | _ => false + +/-- Build 1-byte data shards [1],[2],[3],[4] + zero parity placeholders. -/ +private def shardsLen1 : Array ByteArray := + #[ofList [1], ofList [2], ofList [3], ofList [4], + ofList [0], ofList [0], ofList [0], ofList [0]] + +/-- Encode 1-byte shards; parity must be 0x45 0x5e 0x67 0x78 (pin crate golden). -/ +theorem encode_len1_parity : + (match carbonadoRS.encode shardsLen1 with + | .error _ => false + | .ok enc => + (enc[4]!).get! 0 == 0x45 && + (enc[5]!).get! 0 == 0x5e && + (enc[6]!).get! 0 == 0x67 && + (enc[7]!).get! 0 == 0x78) = true := by + native_decide + +/-- Encode is deterministic: two encodes of the same data match. -/ +theorem encode_deterministic_len1 : + (match carbonadoRS.encode shardsLen1, carbonadoRS.encode shardsLen1 with + | .ok a, .ok b => + (a[4]!).get! 0 == (b[4]!).get! 0 && + (a[5]!).get! 0 == (b[5]!).get! 0 && + (a[6]!).get! 0 == (b[6]!).get! 0 && + (a[7]!).get! 0 == (b[7]!).get! 0 + | _, _ => false) = true := by + native_decide + +/-- Reconstruct after dropping data shards 0 and 1. -/ +private def optsDrop01 : Array (Option ByteArray) := + match carbonadoRS.encode shardsLen1 with + | .error _ => #[] + | .ok enc => + #[none, none, some (enc[2]!), some (enc[3]!), + some (enc[4]!), some (enc[5]!), some (enc[6]!), some (enc[7]!)] + +theorem reconstruct_drop_data_01 : + (match carbonadoRS.reconstruct optsDrop01 with + | .error _ => false + | .ok full => + (full[0]!).get! 0 == 1 && + (full[1]!).get! 0 == 2 && + (full[2]!).get! 0 == 3 && + (full[3]!).get! 0 == 4) = true := by + native_decide + +/-- Parity-only reconstruct (drop all four data shards). -/ +private def optsParityOnly : Array (Option ByteArray) := + match carbonadoRS.encode shardsLen1 with + | .error _ => #[] + | .ok enc => + #[none, none, none, none, + some (enc[4]!), some (enc[5]!), some (enc[6]!), some (enc[7]!)] + +theorem reconstruct_parity_only : + (match carbonadoRS.reconstruct optsParityOnly with + | .error _ => false + | .ok full => + (full[0]!).get! 0 == 1 && + (full[1]!).get! 0 == 2 && + (full[2]!).get! 0 == 3 && + (full[3]!).get! 0 == 4) = true := by + native_decide + +/-- Mixed knockout {0,2,5,7} via reconstructAfterKnockout (cheap len-1 shards). -/ +private def encodedLen1 : Array ByteArray := + match carbonadoRS.encode shardsLen1 with + | .error _ => #[] + | .ok enc => enc + +theorem reconstruct_mixed_knockout : + (match reconstructAfterKnockout encodedLen1 [0, 2, 5, 7] 0 with + | .error _ => false + | .ok pt => + pt.size == 4 && + pt.get! 0 == 1 && pt.get! 1 == 2 && pt.get! 2 == 3 && pt.get! 3 == 4) = true := by + native_decide + +/-- Too few shards → `tooFewShards`. -/ +theorem too_few_shards_error : + isTooFew (carbonadoRS.reconstruct + #[some (ofList [1]), some (ofList [2]), some (ofList [3]), + none, none, none, none, none]) = true := by + native_decide + +/-- Empty present shard → `emptyShard`. -/ +theorem empty_shard_error : + isEmptyShard (carbonadoRS.reconstruct + #[some ByteArray.empty, some (ofList [1]), some (ofList [2]), + some (ofList [3]), none, none, none, none]) = true := by + native_decide + +/-- Uneven shard sizes → `incorrectShardSize`. -/ +theorem incorrect_shard_size_error : + isIncorrectSize (carbonadoRS.reconstruct + #[some (ofList [1]), some (ofList [1, 2]), some (ofList [3]), + some (ofList [4]), none, none, none, none]) = true := by + native_decide + +/-- Wrong shard count → `badGeometry`. -/ +theorem bad_geometry_error : + isBadGeometry (carbonadoRS.reconstruct + #[some (ofList [1]), some (ofList [2])]) = true := by + native_decide + +/-- Uneven inboard body length → `unevenShards`. -/ +theorem uneven_inboard_error : + isUneven (decodeInboard (ofList [1, 2, 3]) 0) = true := by + native_decide + +/-- Padding larger than reconstructed data → `paddingTooLarge`. -/ +theorem padding_too_large_error : + (match carbonadoRS.encode shardsLen1 with + | .error _ => false + | .ok enc => + isPaddingTooLarge (stripPadding + #[enc[0]!, enc[1]!, enc[2]!, enc[3]!] 5)) = true := by + native_decide + +/-- Zero matrix invert → `singularMatrix` (taxonomy surface; product RS rows stay invertible). -/ +theorem singular_matrix_error : + isSingular (invertOrSingular (Matrix.zeros 2 2)) = true := by + native_decide + +theorem singular_matrix_exact : + (match invertOrSingular (Matrix.zeros 2 2) with + | .error .singularMatrix => true + | _ => false) = true := + invertOrSingular_zeros + +/-- `ReedSolomon.new` zero data/parity → `badGeometry`. -/ +theorem new_zero_data_badGeometry : + isBadGeometry (ReedSolomon.new 0 4) = true := by + native_decide + +theorem new_zero_parity_badGeometry : + isBadGeometry (ReedSolomon.new 4 0) = true := by + native_decide + +/-- Encode wrong shard count → `badGeometry`. -/ +theorem encode_bad_geometry : + isBadGeometry (carbonadoRS.encode #[ofList [1], ofList [2]]) = true := by + native_decide + +/-- Encode empty first shard → `emptyShard`. -/ +theorem encode_empty_shard : + isEmptyShard (carbonadoRS.encode + #[ByteArray.empty, ofList [1], ofList [2], ofList [3], + ofList [0], ofList [0], ofList [0], ofList [0]]) = true := by + native_decide + +/-- Encode mismatched lengths → `incorrectShardSize`. -/ +theorem encode_incorrect_size : + isIncorrectSize (carbonadoRS.encode + #[ofList [1], ofList [1, 2], ofList [3], ofList [4], + ofList [0], ofList [0], ofList [0], ofList [0]]) = true := by + native_decide + +/-- Out-of-range knockout index → `badGeometry`. -/ +theorem knockout_oob_badGeometry : + isBadGeometry (reconstructAfterKnockout encodedLen1 [0, 8] 0) = true := by + native_decide + +/-- Empty encode/decode roundtrip. -/ +theorem empty_inboard_roundtrip : + (match encodeInboard ByteArray.empty with + | .ok (body, pad, chunk) => + body.size == 0 && pad == 0 && chunk == 0 && + (match decodeInboard body 0 with + | .ok pt => pt.size == 0 + | _ => false) + | _ => false) = true := by + native_decide + +/-- Small 8-byte sequential shard encode golden (pin crate). -/ +private def shardsSeq8 : Array ByteArray := + #[ofList [0,1,2,3,4,5,6,7], + ofList [8,9,10,11,12,13,14,15], + ofList [16,17,18,19,20,21,22,23], + ofList [24,25,26,27,28,29,30,31], + ofList [0,0,0,0,0,0,0,0], + ofList [0,0,0,0,0,0,0,0], + ofList [0,0,0,0,0,0,0,0], + ofList [0,0,0,0,0,0,0,0]] + +theorem encode_seq8_parity0 : + (match carbonadoRS.encode shardsSeq8 with + | .error _ => false + | .ok enc => + (enc[4]!).get! 0 == 0x20 && + (enc[4]!).get! 1 == 0x21 && + (enc[4]!).get! 7 == 0x27 && + (enc[7]!).get! 0 == 0x38 && + (enc[7]!).get! 7 == 0x3f) = true := by + native_decide + +/-- verify returns true on well-formed encode output. -/ +theorem verify_good_len1 : + (match carbonadoRS.encode shardsLen1 with + | .error _ => false + | .ok enc => + match carbonadoRS.verify enc with + | .ok true => true + | _ => false) = true := by + native_decide + +/-- verify returns false when a parity byte is flipped. -/ +theorem verify_bad_parity_len1 : + (match carbonadoRS.encode shardsLen1 with + | .error _ => false + | .ok enc => + let flipped := ofList [(enc[4]!).get! 0 ^^^ 0x01] + let bad := enc.set! 4 flipped + match carbonadoRS.verify bad with + | .ok false => true + | _ => false) = true := by + native_decide + +end CarbonadoTest.Fec diff --git a/CarbonadoTest/Pipeline.lean b/CarbonadoTest/Pipeline.lean new file mode 100644 index 0000000..679217f --- /dev/null +++ b/CarbonadoTest/Pipeline.lean @@ -0,0 +1,410 @@ +/- + Program E — pipeline, header, stream bounds, scrub, shard tests. + + Dependency direction: CarbonadoTest → Carbonado only. + + Coverage: + * Path tests for product-reachable PipelineError variants (encode/decode/scrub/shard). + * Map `rfl` theorems for lower-layer injections (of*Error). + * Residual map-only variants not reachable without fabricated lower-layer inputs: + tooFewShards, emptyShard, incorrectShardSize, singularMatrix, + invalidSliceIndex, invalidSliceCount (documented in SPEC-MATRIX). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Header +import Carbonado.Compress +import Carbonado.Pipeline +import Carbonado.Stream +import Carbonado.Scrub +import Carbonado.Shard +import Carbonado.Fec.Inboard +import Carbonado.Bao.Product +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.Pipeline + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Header +open Carbonado.Compress +open Carbonado.Pipeline +open Carbonado.Stream +open Carbonado.Scrub +open Carbonado.Shard +open Carbonado.Fec.Inboard +open Carbonado.Bao.Product + +private def master42 : ByteArray := replicate 32 0x42 +private def nonce11 : ByteArray := replicate 16 0x11 + +private def pat (n : Nat) : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:n] do + out := out.push (UInt8.ofNat (i % 251)) + pure out + +/-! ## Stream bounds -/ + +theorem full_stripe_inboard : inboardStripeBytes stripeUnit = 2 * stripeUnit := + full_stripe_inboard_len + +theorem full_stripe_ret : maxFecStripeRetain stripeUnit = 2 * stripeUnit := + full_stripe_retain + +theorem empty_ret : maxFecStripeRetain 0 = 0 := empty_stripe_retain + +theorem one_byte_ret : maxFecStripeRetain 1 = 32768 := one_byte_stripe_retain + +theorem stripe_k_slices : stripeBytes = fecK * sliceLen := stripe_eq_k_slices + +/-! ## Shard split -/ + +theorem split_budget_0 : (splitByBudget (ofList [1, 2, 3]) 0).size = 0 := + split_empty_budget + +theorem split_hello_2 : (splitByBudget (utf8 "hello") 2).size = 3 := + split_hello_budget_2 + +theorem split_empty_pt : (splitByBudget ByteArray.empty 10).size = 1 := + split_empty_plaintext + +/-! ## Header auth_data length -/ + +theorem auth_data_113 : + magicBytes.length + nonceLen + hashLen + slhPublicKeyLen + 1 + 4 + 4 + 4 + 8 = 113 := + authData_len_formula + +/-! ## Format matrix roundtrips (native_decide on small payloads) -/ + +theorem roundtrip_c0_hello : + (match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) with + | .ok b => b + | .error _ => false) = true := by + native_decide + +theorem roundtrip_c4_hello : + (match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) with + | .ok b => b + | .error _ => false) = true := by + native_decide + +theorem roundtrip_c5_hello : + (match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 5) with + | .ok b => b + | .error _ => false) = true := by + native_decide + +theorem roundtrip_c12_hello : + (match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 12) with + | .ok b => b + | .error _ => false) = true := by + native_decide + +/-- c13 = encrypted|verification|fec (no compression bit) — pure-decidable. -/ +theorem roundtrip_c13_hello : + (match roundtripBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 13) with + | .ok b => b + | .error _ => false) = true := by + native_decide + +/-- + Non-compression format matrix (c0,c1,c4,c5,c8,c9,c12,c13). + Compression formats (bit 2) require AOT zstd `@[extern]` — gated in Main `demo` + (c2/c6/c14/c15), not `native_decide` (LIMITS). +-/ +private def nonCompressionFormats : List FormatBits := + [FormatBits.ofUInt8 0, FormatBits.ofUInt8 1, FormatBits.ofUInt8 4, FormatBits.ofUInt8 5, + FormatBits.ofUInt8 8, FormatBits.ofUInt8 9, FormatBits.ofUInt8 12, FormatBits.ofUInt8 13] + +private def nonCompressionMatrix (master nonce plaintext : ByteArray) : Bool := + Id.run do + let mut allMatch := true + for fmt in nonCompressionFormats do + match roundtripBody master nonce plaintext fmt with + | .error _ => allMatch := false + | .ok b => if !b then allMatch := false + pure allMatch + +theorem format_matrix_no_compress_hi : + nonCompressionMatrix master42 nonce11 (utf8 "hi") = true := by + native_decide + +theorem headered_c5_hello : + (match roundtripHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 5) with + | .ok b => b + | .error _ => false) = true := by + native_decide + +/-! ## Header wire length -/ + +theorem header_new_wire_177 : + (match Header.new master42 nonce11 (replicate 32 0xcd) (replicate 32 0) 5 0 100 0 + (replicate 8 0) with + | .error _ => 0 + | .ok h => + match h.toBytes with + | .ok b => b.size + | .error _ => 0) = 177 := by + native_decide + +theorem header_verify_good : + (match Header.new master42 nonce11 (replicate 32 0xcd) (replicate 32 0) 5 0 100 0 + (replicate 8 0) with + | .error _ => false + | .ok h => + match h.toBytes with + | .error _ => false + | .ok b => + match parseAndVerify master42 b with + | .ok _ => true + | .error _ => false) = true := by + native_decide + +/-! ## Strict PipelineError mapping (every variant reachable / distinct) -/ + +theorem map_header_invalid_len : + ofHeaderError .invalidHeaderLength = .invalidHeaderLength := rfl + +theorem map_header_bad_magic : + ofHeaderError .badMagic = .badMagic := rfl + +theorem map_header_auth : + ofHeaderError .headerAuthenticationFailed = .headerAuthenticationFailed := rfl + +theorem map_header_key : + ofHeaderError .invalidKeyLength = .invalidKeyLength := rfl + +theorem map_header_field : + ofHeaderError .invalidFieldLength = .invalidFieldLength := rfl + +theorem map_crypto_key : + ofCryptoError .invalidKeyLength = .invalidKeyLength := rfl + +theorem map_crypto_ct : + ofCryptoError .invalidCiphertextLength = .invalidCiphertextLength := rfl + +theorem map_crypto_nonce : + ofCryptoError .invalidNonceLength = .invalidNonceLength := rfl + +theorem map_crypto_auth : + ofCryptoError .authenticationFailed = .payloadAuthenticationFailed := rfl + +theorem map_fec_uneven : + ofFecError .unevenShards = .unevenShards := rfl + +theorem map_fec_too_few : + ofFecError .tooFewShards = .tooFewShards := rfl + +theorem map_fec_empty : + ofFecError .emptyShard = .emptyShard := rfl + +theorem map_fec_size : + ofFecError .incorrectShardSize = .incorrectShardSize := rfl + +theorem map_fec_geo : + ofFecError .badGeometry = .badGeometry := rfl + +theorem map_fec_pad : + ofFecError .paddingTooLarge = .paddingTooLarge := rfl + +theorem map_fec_sing : + ofFecError .singularMatrix = .singularMatrix := rfl + +theorem map_bao_auth : + ofBaoError .authenticationFailed = .baoAuthenticationFailed := rfl + +theorem map_bao_trunc : + ofBaoError .truncatedResponse = .truncatedResponse := rfl + +theorem map_bao_trail : + ofBaoError .trailingData = .trailingData := rfl + +theorem map_bao_prefix : + ofBaoError .invalidPrefix = .invalidPrefix := rfl + +theorem map_bao_root : + ofBaoError .invalidRootLength = .invalidRootLength := rfl + +theorem map_bao_slice_idx : + ofBaoError .invalidSliceIndex = .invalidSliceIndex := rfl + +theorem map_bao_slice_cnt : + ofBaoError .invalidSliceCount = .invalidSliceCount := rfl + +theorem map_zstd_compress : + ofZstdError ZstdError.compressionFailed = PipelineError.compressionFailed := rfl + +theorem map_zstd_decompress : + ofZstdError ZstdError.decompressionFailed = PipelineError.decompressionFailed := rfl + +theorem map_zstd_too_large : + ofZstdError ZstdError.outputTooLarge = PipelineError.decompressOutputTooLarge := rfl + +theorem map_zstd_invalid : + ofZstdError ZstdError.invalidInput = PipelineError.zstdInvalidInput := rfl + +/-! ## Pipeline error paths (exact match) -/ + +theorem short_header_decode : + (match decodeHeadered master42 (ofList [1, 2, 3]) with + | .error .invalidHeaderLength => true + | _ => false) = true := by + native_decide + +theorem bad_magic_decode : + (match parseAndVerify master42 (replicate headerLen 0) with + | .error .badMagic => true + | _ => false) = true := by + native_decide + +theorem bad_nonce_encrypt : + (match encodeBody master42 (replicate 8 0) (utf8 "hi") (FormatBits.ofUInt8 1) false with + | .error .invalidNonceLength => true + | _ => false) = true := by + native_decide + +theorem short_master_encrypt : + (match encodeBody (replicate 16 0) nonce11 (utf8 "hi") (FormatBits.ofUInt8 1) false with + | .error .invalidKeyLength => true + | _ => false) = true := by + native_decide + +theorem scrub_requires_v : + (match scrubInboardArchive (utf8 "x") (replicate 32 0) (FormatBits.ofUInt8 0) with + | .error .scrubRequiresVerification => true + | _ => false) = true := by + native_decide + +theorem empty_segment_err : + (match encodeShards master42 (utf8 "ab") (FormatBits.ofUInt8 0) 0 #[nonce11] + zeroSlhPk zeroMeta with + | .error .emptySegment => true + | _ => false) = true := by + native_decide + +theorem invalid_chunk_seq : + (match validateChunkSequence #[0, 0] with + | .error .invalidChunkSequence => true + | _ => false) = true := by + native_decide + +theorem invalid_field_header : + (match Header.new master42 nonce11 (ofList [1]) (replicate 32 0) 0 0 0 0 (replicate 8 0) with + | .error .invalidFieldLength => true + | _ => false) = true := by + native_decide + +/-- Short body vs authenticated encoded_len → truncatedBody. -/ +theorem truncated_body_path : + (match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) 0 + zeroSlhPk zeroMeta with + | .error _ => false + | .ok (_h, arch) => + if arch.size ≤ headerLen + 1 then false + else + match decodeHeadered master42 (arch.extract 0 (headerLen + 1)) with + | .error .truncatedBody => true + | _ => false) = true := by + native_decide + +/-- Trailer after encoded_len ignored (c0). -/ +theorem trailer_ignored_c0 : + (match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) 0 + zeroSlhPk zeroMeta with + | .error _ => false + | .ok (_h, arch) => + match decodeHeadered master42 (appendBA arch (ofList [0xaa, 0xbb])) with + | .ok pt => ctEq pt (utf8 "hello") + | .error _ => false) = true := by + native_decide + +/-- Composition: short encrypted blob → invalidCiphertextLength. -/ +theorem short_ct_composition : + (match decodeBody master42 nonce11 zeroHash (replicate 10 0) 0 + (FormatBits.ofUInt8 1) false with + | .error .invalidCiphertextLength => true + | _ => false) = true := by + native_decide + +/-- Composition: FEC pad > empty body → paddingTooLarge. -/ +theorem padding_too_large_composition : + (match decodeBody master42 nonce11 zeroHash ByteArray.empty 5 + (FormatBits.ofUInt8 8) false with + | .error .paddingTooLarge => true + | _ => false) = true := by + native_decide + +/-- Composition: short Bao prefix → invalidPrefix. -/ +theorem invalid_prefix_composition : + (match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error _ => false + | .ok enc => + match decodeBody master42 nonce11 enc.baoHash (enc.body.extract 0 4) + enc.info.paddingLen (FormatBits.ofUInt8 4) false with + | .error .invalidPrefix => true + | _ => false) = true := by + native_decide + +/-- Scrub knockout recovery for hello under c12 (V|F). -/ +theorem scrub_knockout_hello : + (match Carbonado.Fec.Inboard.encodeInboard (utf8 "hello") with + | .error _ => false + | .ok (fecBody, pad, _) => + let (root, art) := encodeInboardForFormat 12 fecBody + match scrubWithMissing fecBody root pad 12 [0, 1, 2, 3] with + | .ok rec => ctEq rec art + | .error _ => false) = true := by + native_decide + +/-- Too many missing shards → invalidScrubbedHash (path test). -/ +theorem scrub_invalid_hash_too_many : + (match Carbonado.Fec.Inboard.encodeInboard (utf8 "hello") with + | .error _ => false + | .ok (fecBody, pad, _) => + let (root, _) := encodeInboardForFormat 12 fecBody + match scrubWithMissing fecBody root pad 12 [0, 1, 2, 3, 4] with + | .error .invalidScrubbedHash => true + | _ => false) = true := by + native_decide + +/-- Empty FEC body scrub → badGeometry (no panic). -/ +theorem scrub_empty_bad_geometry : + (match scrubWithMissing ByteArray.empty (replicate 32 0) 0 12 [0] with + | .error .badGeometry => true + | _ => false) = true := by + native_decide + +/-- Unnecessary scrub when archive is pristine. -/ +theorem scrub_pristine_unnecessary : + (match encodeBody master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 4) false with + | .error _ => false + | .ok enc => + match scrubInboardArchive enc.body enc.baoHash (FormatBits.ofUInt8 4) with + | .error .unnecessaryScrub => true + | _ => false) = true := by + native_decide + +/-- Shards roundtrip small budget. -/ +theorem shards_roundtrip : + (match roundtripShards master42 (utf8 "abcdefghij") (FormatBits.ofUInt8 0) 4 + #[nonce11, replicate 16 0x22, replicate 16 0x33] with + | .ok b => b + | .error _ => false) = true := by + native_decide + +/-- Too few nonces → insufficientNonces (not invalidNonceLength). -/ +theorem insufficient_nonces_path : + (match encodeShards master42 (utf8 "abcdefghij") (FormatBits.ofUInt8 0) 4 + #[nonce11] zeroSlhPk zeroMeta with + | .error .insufficientNonces => true + | _ => false) = true := by + native_decide + +/-- Encrypted formats use odd codes. -/ +theorem c15_odd : formatC15.toUInt8 % 2 = 1 := by native_decide + +theorem c14_even : formatC14.toUInt8 % 2 = 0 := by native_decide + +end CarbonadoTest.Pipeline diff --git a/CarbonadoTest/Scaffold.lean b/CarbonadoTest/Scaffold.lean new file mode 100644 index 0000000..ff6de60 --- /dev/null +++ b/CarbonadoTest/Scaffold.lean @@ -0,0 +1,28 @@ +/- + Program A scaffold tests. + + Named `CarbonadoTest` (not `Tests/`) so it does not collide with legacy Rust + `tests/` on case-insensitive filesystems (Darwin APFS default). + + Product theorems live next to definitions under `Carbonado/`. + This module re-states critical wire invariants so the test tree is non-empty + and ready for Programs B–G vector / parity modules. + + Dependency direction: CarbonadoTest → Carbonado only (never reverse). +-/ +import Carbonado.Constants + +namespace CarbonadoTest.Scaffold + +open Carbonado.Constants + +theorem magic_len : magicBytes.length = 12 := magicBytes_length +theorem header_177 : headerLen = 177 := rfl +theorem slice_4k : sliceLen = 4096 := rfl +theorem fec_4_of_8 : fecK = 4 ∧ fecM = 8 := ⟨rfl, rfl⟩ +theorem stripe_16k : stripeUnit = 16384 := stripeUnit_eq +theorem slh_sidecar_7860 : slh1SidecarLen = 7860 := slh1SidecarLen_eq +theorem c14_public : formatC14.toUInt8 = 14 := formatC14_byte +theorem c15_encrypted : formatC15.toUInt8 = 15 := formatC15_byte + +end CarbonadoTest.Scaffold diff --git a/CarbonadoTest/Slh.lean b/CarbonadoTest/Slh.lean new file mode 100644 index 0000000..00c641e --- /dev/null +++ b/CarbonadoTest/Slh.lean @@ -0,0 +1,99 @@ +/- + Program F — SLH1 wire + Bao-root binding theorems. + + Large 7856-byte signature roundtrips are AOT Main only (not native_decide). +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Slh +import CarbonadoTest.Scaffold + +namespace CarbonadoTest.Slh + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Slh + +theorem sidecar_len : slh1SidecarLen = 7860 := slh1_sidecar_len +theorem sig_len : slh1SignatureLen = 7856 := slh1_sig_len +theorem magic_ascii : slh1Magic = [0x53, 0x4c, 0x48, 0x31] := slh1_magic_bytes + +theorem short_sidecar : + (match parseSidecar (ofList [0x53, 0x4c, 0x48, 0x31]) with + | .error .invalidSidecarLength => true + | _ => false) = true := parse_short_length + +theorem empty_sidecar : + (match parseSidecar ByteArray.empty with + | .error .invalidSidecarLength => true + | _ => false) = true := parse_empty + +/-- badSlhMagic path (exact-length magic gate; full 7860 B also in AOT Main). -/ +theorem bad_magic_prefix : + (match parseMagicAtExactLen (ofList [0, 0, 0, 0]) with + | .error .badSlhMagic => true + | _ => false) = true := parse_magic_bad + +theorem good_magic_prefix : + (match parseMagicAtExactLen slh1MagicBA with + | .ok b => b.size == 0 + | .error _ => false) = true := parse_magic_good_empty_payload + +theorem zeros_not_magic : slh1MagicPrefix (ofList [0, 0, 0, 0]) = false := + zeros_not_slh1_magic + +theorem build_bad_len : + (match buildSidecar (ofList [1, 2, 3]) with + | .error .invalidSignatureLength => true + | _ => false) = true := build_bad_sig_len + +theorem bad_pk : + (match mkBinding (ofList [1]) (replicate hashLen 0) (ofList [1]) with + | .error .invalidPublicKeyLength => true + | _ => false) = true := bind_bad_pk + +theorem bad_root : + (match mkBinding (replicate slhPublicKeyLen 0) (ofList [1]) (ofList [1]) with + | .error .invalidRootLength => true + | _ => false) = true := bind_bad_root + +theorem bad_sig : + (match mkBinding (replicate slhPublicKeyLen 0) (replicate hashLen 0) (ofList [1]) with + | .error .invalidSignatureLength => true + | _ => false) = true := bind_bad_sig + +theorem wrong_root_path : + (let rootA := replicate hashLen 0xaa + let rootB := replicate hashLen 0xbb + let pk := replicate slhPublicKeyLen 0x11 + let sig := ofList [0xcd] + match verifyBoundToExpected (mockOracleFor rootA sig) pk rootA rootB sig with + | .error .verificationFailed => true + | _ => false) = true := wrong_root_fails + +theorem sign_unavail : + (match signRoot (replicate 128 0x42) (replicate hashLen 0) with + | .error .signatureUnavailable => true + | _ => false) = true := sign_unavailable + +theorem sign_bad_root_len : + (match signRoot (replicate 128 0x42) (ofList [1]) with + | .error .invalidRootLength => true + | _ => false) = true := sign_bad_root + +/-- bindingFromSidecar: short file → invalidSidecarLength. -/ +theorem binding_short_sidecar : + (match bindingFromSidecar (replicate slhPublicKeyLen 0) (replicate hashLen 0) + (ofList [1, 2, 3]) with + | .error .invalidSidecarLength => true + | _ => false) = true := by + native_decide + +/-- Long sidecar (5 bytes) → invalidSidecarLength. -/ +theorem parse_too_long_short : + (match parseSidecar (ofList [0, 1, 2, 3, 4]) with + | .error .invalidSidecarLength => true + | _ => false) = true := by + native_decide + +end CarbonadoTest.Slh diff --git a/Cargo.toml b/Cargo.toml index a07b5d1..cbe164d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,12 +79,19 @@ getrandom = { version = "0.2", features = ["js"] } # bitcoinpqc 0.4 (WASM-capable build.rs); SLH-DSA-SHA2-128s sidecars. # Local mirror lag: `.cargo/config.toml` patches 0.4 until 2026-07-18 (see lift checklist there). bitcoinpqc = { version = "0.4", optional = true } +# Dual-backend: Lean AOT C library (docs/ABI.md, docs/TEST_CONTRACT.md). +carbonado-sys = { path = "carbonado-sys", optional = true } # Note: ecies + libsecp256k1-core + nostr + secp256k1 removed (clean break from old ECIES design and Nostr bech32/npub helpers). # Future quantum-resistant key formats (qpub etc.) will be handled separately. [features] -default = ["pqc", "ots", "cli", "parallel"] +# Default: pure Rust engine + existing features. Dual-backend: see backend-rust / backend-lean. +default = ["backend-rust", "pqc", "ots", "cli", "parallel"] +# Pure Rust implementation (default engine). +backend-rust = [] +# Lean AOT engine via carbonado-sys / libcarbonado (G8). Requires CARBONADO_LEAN_LIB. +backend-lean = ["dep:carbonado-sys"] pqc = ["dep:bitcoinpqc"] # OpenTimestamps stub stamping (offline/testable; no network calendar in default build). ots = [] diff --git a/carbonado-sys/Cargo.toml b/carbonado-sys/Cargo.toml new file mode 100644 index 0000000..5bfcb00 --- /dev/null +++ b/carbonado-sys/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "carbonado-sys" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "FFI bindings to Lean AOT libcarbonado (dual-backend parity)" +publish = false + +[dependencies] + +[build-dependencies] diff --git a/carbonado-sys/build.rs b/carbonado-sys/build.rs new file mode 100644 index 0000000..3d75e46 --- /dev/null +++ b/carbonado-sys/build.rs @@ -0,0 +1,36 @@ +//! Link against Nix-built `libcarbonado` when `CARBONADO_LEAN_LIB` / `CARBONADO_LEAN_INCLUDE` +//! are set (or `OUT_DIR` sibling after `nix build .#libcarbonado` + env). +//! +//! ```bash +//! nix build .#libcarbonado +//! export CARBONADO_LEAN_LIB=$PWD/result/lib +//! export CARBONADO_LEAN_INCLUDE=$PWD/result/include +//! cargo test -p carbonado --features backend-lean +//! ``` + +use std::env; +use std::path::PathBuf; + +fn main() { + println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_LIB"); + println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_INCLUDE"); + + let lib = env::var_os("CARBONADO_LEAN_LIB").map(PathBuf::from); + let include = env::var_os("CARBONADO_LEAN_INCLUDE").map(PathBuf::from); + + if let Some(inc) = include { + println!("cargo:include={}", inc.display()); + } + + if let Some(lib_dir) = lib { + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rustc-link-lib=static=carbonado"); + // Lean/zstd static archive may need system libs when full Lean objects are linked later. + println!("cargo:rustc-link-lib=pthread"); + println!("cargo:rustc-link-lib=m"); + println!("cargo:rustc-link-lib=dl"); + } else { + // Allow crate to compile docs/check without the AOT lib; link fails at use if missing. + println!("cargo:warning=CARBONADO_LEAN_LIB unset; carbonado-sys will not link libcarbonado"); + } +} diff --git a/carbonado-sys/src/lib.rs b/carbonado-sys/src/lib.rs new file mode 100644 index 0000000..316d126 --- /dev/null +++ b/carbonado-sys/src/lib.rs @@ -0,0 +1,94 @@ +//! Low-level FFI to Lean AOT `libcarbonado` (see `include/carbonado.h`, `docs/ABI.md`). + +#![allow(non_camel_case_types)] + +use std::os::raw::{c_int, c_void}; + +pub const CARBONADO_ABI_VERSION: u32 = 1; + +pub const CARBONADO_OK: c_int = 0; +pub const CARBONADO_ERR_INVALID_ARGUMENT: c_int = 1; +pub const CARBONADO_ERR_INVALID_KEY_LENGTH: c_int = 2; +pub const CARBONADO_ERR_AUTHENTICATION: c_int = 3; +pub const CARBONADO_ERR_INVALID_MAGIC: c_int = 4; +pub const CARBONADO_ERR_INVALID_HEADER: c_int = 5; +pub const CARBONADO_ERR_FEC: c_int = 6; +pub const CARBONADO_ERR_BAO: c_int = 7; +pub const CARBONADO_ERR_ZSTD: c_int = 8; +pub const CARBONADO_ERR_SCRUB_UNNECESSARY: c_int = 9; +pub const CARBONADO_ERR_SCRUB_FAILED: c_int = 10; +pub const CARBONADO_ERR_NOT_IMPLEMENTED: c_int = 11; +pub const CARBONADO_ERR_INTERNAL: c_int = 12; + +extern "C" { + pub fn carbonado_abi_version() -> u32; + pub fn carbonado_free(p: *mut c_void); + pub fn carbonado_encode( + master: *const u8, + master_len: usize, + plaintext: *const u8, + plaintext_len: usize, + format: u8, + nonce: *const u8, + nonce_len: usize, + out: *mut *mut u8, + out_len: *mut usize, + hash_out: *mut u8, + ) -> c_int; + pub fn carbonado_decode( + master: *const u8, + master_len: usize, + hash: *const u8, + hash_len: usize, + body: *const u8, + body_len: usize, + padding: u32, + format: u8, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_encode_headered( + master: *const u8, + master_len: usize, + plaintext: *const u8, + plaintext_len: usize, + format: u8, + nonce: *const u8, + nonce_len: usize, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_decode_headered( + master: *const u8, + master_len: usize, + archive: *const u8, + archive_len: usize, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_verification_key(format: u8, key_out: *mut u8) -> c_int; +} + +/// Safe wrapper: free a buffer returned by libcarbonado. +/// +/// # Safety +/// `p` must be null or a pointer returned by libcarbonado. +pub unsafe fn free(p: *mut u8) { + carbonado_free(p as *mut c_void); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn abi_version_matches_header() { + // Only runs when linked against libcarbonado (CARBONADO_LEAN_LIB set). + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + eprintln!("skip: CARBONADO_LEAN_LIB unset"); + return; + } + let v = unsafe { carbonado_abi_version() }; + assert_eq!(v, CARBONADO_ABI_VERSION); + } +} diff --git a/docs/ABI.md b/docs/ABI.md new file mode 100644 index 0000000..705acd2 --- /dev/null +++ b/docs/ABI.md @@ -0,0 +1,196 @@ +# carbonado C ABI (dual-backend) + +Stable C interface for the **Lean AOT engine** (`libcarbonado`). Rust `backend-lean` links this library (`carbonado-sys`) and is required to expose the **same high-level Rust API** as `backend-rust` so that `tests/` is one contract. + +**Normative sources (must stay in sync):** + +| Artifact | Role | +|----------|------| +| [`include/carbonado.h`](../include/carbonado.h) | C declarations (v0 surface) | +| [`carbonado-sys/src/lib.rs`](../carbonado-sys/src/lib.rs) | Rust FFI bindings + error constants | +| [`nix/native/carbonado_abi.c`](../nix/native/carbonado_abi.c) | C stubs / weak symbols in the native archive | +| [`Carbonado/Ffi.lean`](../Carbonado/Ffi.lean) | Lean pure helpers + planned `@[export]` surface | +| This document | Ownership, versioning, error codes, link instructions | + +**ABI version:** `1` (`CARBONADO_ABI_VERSION`). Bump major on breaking changes (symbol rename, error-code reuse, semantic change of successful outputs). + +--- + +## Memory ownership + +| Pattern | Rule | +|---------|------| +| Input buffers | Caller owns; not freed by libcarbonado | +| Output buffers | Returned via `uint8_t **out` + `size_t *out_len`; allocated with the same allocator family as `carbonado_free` (malloc); **caller frees with `carbonado_free`** (or takes ownership via `Vec::from_raw_parts` on the Rust side — do not double-free) | +| Errors | Integer codes only on the hot path; no heap error strings in v0 | +| Null | Null input pointers with non-zero lengths → `CARBONADO_ERR_INVALID_ARGUMENT` (when implemented) | + +```c +void carbonado_free(void *p); /* free(NULL) is a no-op */ +``` + +--- + +## Versioning + +```c +#define CARBONADO_ABI_VERSION 1u +uint32_t carbonado_abi_version(void); /* returns CARBONADO_ABI_VERSION */ +``` + +Lean: `Carbonado.Ffi.abiVersion` / `@[export carbonado_abi_version]`. + +--- + +## Error codes (v0) + +Stable integers shared by `include/carbonado.h`, `carbonado-sys`, and `Carbonado.Ffi`. Map to `CarbonadoError` in `src/backend/mod.rs` (`lean::map_err`). Unknown codes → generic failure. + +| Code | Name | Meaning | Approximate Rust mapping | +|-----:|------|---------|--------------------------| +| 0 | `CARBONADO_OK` | Success | `Ok` | +| 1 | `CARBONADO_ERR_INVALID_ARGUMENT` | Null/lengths/nonce size/sequence | bad args; nonce length; empty segment | +| 2 | `CARBONADO_ERR_INVALID_KEY_LENGTH` | Master not 32 or 64 bytes | `InvalidKeyLength` (or current stand-in until dedicated variant) | +| 3 | `CARBONADO_ERR_AUTHENTICATION` | Header MAC / payload EtM / Bao auth | `AuthenticationFailed` (+ header MAC fails) | +| 4 | `CARBONADO_ERR_INVALID_MAGIC` | Bad `CARBONADO20\n` (or related magic) | `InvalidMagicNumber` | +| 5 | `CARBONADO_ERR_INVALID_HEADER` | Truncated/malformed header or body bounds | `InvalidHeaderLength` / truncated body | +| 6 | `CARBONADO_ERR_FEC` | RS geometry / shard errors | `UnevenFecChunks` / FEC failures | +| 7 | `CARBONADO_ERR_BAO` | Keyed Bao verify / slice stream errors | Bao / verification failures | +| 8 | `CARBONADO_ERR_ZSTD` | Compress/decompress failures | `ZstdError` | +| 9 | `CARBONADO_ERR_SCRUB_UNNECESSARY` | Scrub not needed | `UnnecessaryScrub` | +| 10 | `CARBONADO_ERR_SCRUB_FAILED` | Scrub cannot recover / requires verification | `InvalidScrubbedHash` / `ScrubRequiresVerification` | +| 11 | `CARBONADO_ERR_NOT_IMPLEMENTED` | Surface not exported or still stubbed | fail closed (Phase 0–1 stubs) | +| 12 | `CARBONADO_ERR_INTERNAL` | Unexpected / allocator / invariant | internal | + +**Collapse rule:** Fine-grained Lean `PipelineError` variants map through `Carbonado.Ffi.ofPipelineError` into these codes at the C boundary. Distinct failure modes that tests assert via `matches!` must either keep distinct codes or get refined Rust-side mapping before those tests are on the lean allowlist. Do **not** map unrelated failures to a single diagnostic variant permanently. + +**Phase 0–1 honesty:** Until real exports are linked, all encode/decode/verification_key C entry points return `CARBONADO_ERR_NOT_IMPLEMENTED` (weak stubs in `carbonado_abi.c`). + +--- + +## Core v0 functions (in `include/carbonado.h`) + +These are the **only** product symbols declared in the header today. Signatures must match the header byte-for-byte in meaning. + +### Lifecycle + +```c +uint32_t carbonado_abi_version(void); +void carbonado_free(void *p); +``` + +### Encode (low-level buffer ≈ Rust `encoding::encode` body) + +Low-level layout: when encrypted, the body uses the embedded-nonce blob shape Rust low-level paths use (`[nonce|tag|ct]` inside the encrypt stage as applicable). For **public** formats, `nonce` may be null / `nonce_len == 0`. For **encrypted** formats, `nonce` must be 16 bytes (tests use fixed nonces for determinism). + +```c +/* out: verifiable body only (no Carbonado Header). hash_out: 32-byte Bao root. */ +int carbonado_encode( + const uint8_t *master, size_t master_len, /* 32 or 64 */ + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, /* 16 if encrypted; else 0/null */ + uint8_t **out, size_t *out_len, + uint8_t hash_out[32] +); +``` + +### Decode (low-level ≈ Rust `decoding::decode`) + +```c +int carbonado_decode( + const uint8_t *master, size_t master_len, + const uint8_t *hash, size_t hash_len, /* 32 */ + const uint8_t *body, size_t body_len, + uint32_t padding, + uint8_t format, + uint8_t **out, size_t *out_len +); +``` + +### Headered encode/decode (≈ Rust `file::encode` / `file::decode`) + +```c +/* Full file: Header (177 B) || body. Bao root lives in the header. */ +int carbonado_encode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, /* 16 when Encrypted bit set */ + uint8_t **out, size_t *out_len +); + +int carbonado_decode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *archive, size_t archive_len, + uint8_t **out, size_t *out_len +); +``` + +Lean pure analogues (not yet live C malloc wrappers): `Carbonado.Ffi.encodeHeaderedBytes` / `decodeHeaderedBytes`. + +### Verification key + +```c +/* Format-keyed Bao key: blake3::derive_key("carbonado-v2/verification", &[format]). */ +int carbonado_verification_key(uint8_t format, uint8_t key_out[32]); +``` + +Lean pure: `Carbonado.Ffi.verificationKeyBytes`. + +--- + +## Future C surface (not in `include/carbonado.h` v0) + +The following are **planned** for later ABI revisions when Phase 2+ test classes need them. They are **not** declared in the current header; do not document them as exported. + +| Future symbol (illustrative) | Rust analogue | Target phase | +|------------------------------|---------------|--------------| +| `carbonado_encode_outboard` / `carbonado_decode_outboard` | `encode_outboard` / `decode_outboard` | Phase 2 | +| `carbonado_scrub` / `carbonado_scrub_outboard` | `scrub` / `scrub_outboard` | Phase 2 | +| `carbonado_verify_slice` / `carbonado_extract_slice` | `verify_slice` / `extract_slice` | Phase 2 | +| Directory / Adamantine catalog helpers | `encode_directory` / `decode_directory` | Phase 3 (rkyv) | +| SLH sign/verify | `crypto::slh_dsa_*` | Phase 4 (G10) | + +Until exported, Lean or Rust may implement these **above** the v0 body/headered ABI without new C symbols, but the dual-backend bar for those tests is green only when both backends produce matching results. + +--- + +## Implementation status (Phase 0 close) + +| Symbol | Lean pure | C in `libcarbonado` | `carbonado-sys` | Rust `backend-lean` dispatch | +|--------|-----------|---------------------|-----------------|------------------------------| +| `carbonado_abi_version` | `@[export]` in `Ffi.lean` | **implemented** (`carbonado_abi.c`) | bound | `lean::abi_version` | +| `carbonado_free` | — | **implemented** | bound | used on free paths | +| `carbonado_encode` | planned / helpers partial | **weak stub → NOT_IMPLEMENTED** | bound | not yet on crate-root `encode` | +| `carbonado_decode` | planned | **weak stub → NOT_IMPLEMENTED** | bound | not yet on crate-root `decode` | +| `carbonado_encode_headered` | pure `encodeHeaderedBytes` | **weak stub → NOT_IMPLEMENTED** | bound | `lean::encode_headered` wrapper exists; needs live lib | +| `carbonado_decode_headered` | pure `decodeHeaderedBytes` | **weak stub → NOT_IMPLEMENTED** | bound | `lean::decode_headered` wrapper exists; needs live lib | +| `carbonado_verification_key` | pure `verificationKeyBytes` | **weak stub → NOT_IMPLEMENTED** | bound | not yet wired to crate root | +| outboard / scrub / slice C | partial Lean modules | **not in header** | — | later | + +**Phase 1 definition of done (engineering, not this doc pass):** real non-stub implementations for the v0 encode/decode/verification_key symbols in the linked archive; Phase 1 test allowlist green on `backend-lean`; `backend-rust` still full green. + +Update this table as Phase 1 lands. + +--- + +## Linking + +```text +# After: nix build .#libcarbonado +export CARBONADO_LEAN_LIB=$PWD/result/lib +export CARBONADO_LEAN_INCLUDE=$PWD/result/include +cargo test --no-default-features --features "backend-lean,pqc,ots" +# typical link line: -L $CARBONADO_LEAN_LIB -lcarbonado -lpthread -ldl -lm +``` + +Exact `cargo` `rustc-link-*` flags live in [`carbonado-sys/build.rs`](../carbonado-sys/build.rs). If `CARBONADO_LEAN_LIB` is unset, `carbonado-sys` warns and does not link — encode/decode cannot succeed. + +**Known residual (not Phase 0):** flake/`libcarbonado` packaging and full Lean export linkage may still need Phase 1 work; Phase 0 does not require `nix build .#libcarbonado` green for doc closure. + +--- + +## Mutual exclusion of Cargo features + +Enable **exactly one** of `backend-rust` or `backend-lean` per build (`src/backend/mod.rs` `compile_error!`). Dual-backend CI runs two invocations, not one binary with both engines. diff --git a/docs/GAPS.md b/docs/GAPS.md new file mode 100644 index 0000000..89ff047 --- /dev/null +++ b/docs/GAPS.md @@ -0,0 +1,54 @@ +# carbonado — gaps + +Living inventory. IDs are durable; close only when theorems and/or parity gates are green. + +## Dual-backend model (north star) + +| Role | Location | +|------|----------| +| First-class engine | **Rust** (`src/`, default `backend-rust`) — production library + CLI | +| Normative contract | **Rust `tests/`** — both backends must pass the same suite | +| Second engine | **Lean 4 AOT** (`Carbonado/`, `libcarbonado` via C ABI) — proofs + wire-compatible implementation | +| Build / proofs | Nix flakes (`nix flake check`, `libcarbonado` package) | +| Oracles | `ref/` pins + parity drivers | + +**Parity bar (G8):** `cargo test` with `backend-rust` and with `backend-lean` (links Lean AOT C). Same tests; not separate Lean-only demos. + +See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), AGENTS.md dual-backend block. + +| ID | Gap | Status | +|----|-----|--------| +| G0 | Lean+Nix scaffold | **closed** (Program A) | +| G1 | `ref/` pins + dual-backend SSOT clarity | **partial** (pins present; dual-backend docs closed at P0; optional `ref/carbonado-rust` freeze still open) | +| G2 | EtM Lean | **closed** (Program B) | +| G3 | RS 4/8 Lean | **closed** (Program C) | +| G4 | Keyed Bao Lean | **closed** (Program D) | +| G5 | Pipeline / stream / scrub / shard | **closed** (Program E) | +| G6 | zstd link + SLH product | **partial** (zstd closed; SLH FFI open — see G10) | +| G7 | Adamantine + CLI | **partial** (Lean CFP2 path closed; **rkyv wire for dual-suite open** — Phase 3) | +| **G8** | **Dual-backend: C ABI + `cargo test --features backend-lean` full suite** | **open** (P0 inventory/docs **closed**; P1+ engineering open) | +| G9 | Cross-backend encode/decode matrix (Rust↔Lean) | **open** (depends on G8 Phase 2 body/headered stability) | +| G10 | libbitcoinpqc real SLH sign/verify in libcarbonado | **open** (wire+binding in Lean; FFI not linked) | +| G11 | Live CI matrix both backends | **open** (depends on G8 Phase 1+ allowlist → full; freeze at P5) | + +## Dual-backend phases (G8 breakdown) + +| Phase | Work | Status | +|-------|------|--------| +| **P0** | Test contract inventory, ABI.md, GAPS/AGENTS dual-backend, cross-doc consistency | **closed** (2026-07; docs-only) | +| P1 | C ABI v0 live exports + libcarbonado link + `backend-lean` core allowlist green | **open** (stubs still `NOT_IMPLEMENTED`) | +| P2 | Format matrix + scrub/outboard/stream + cross-backend buffer (G9 start) | **open** | +| P3 | rkyv-compatible catalog + directory suite (G7 residual) | **open** | +| P4 | PQC (G10) + CLI dual path | **open** | +| P5 | CI freeze both backends; G8 + G11 closed | **open** | + +### P0 deliverables (evidence of close) + +| Deliverable | Location | +|-------------|----------| +| Full `tests/*.rs` classification + API map + Phase 1 allowlist | [TEST_CONTRACT.md](./TEST_CONTRACT.md) | +| C ABI ownership, error codes, v0 symbols, stub honesty, link notes | [ABI.md](./ABI.md) + `include/carbonado.h` | +| Dual-backend SSOT rules | AGENTS.md (top block); this file | +| Cross-doc model (no “Lean replaces Rust” product rule) | [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), [PROOFS.md](./PROOFS.md) | + +P0 does **not** require live encode/decode through `backend-lean` or full `nix build .#libcarbonado` product export maturity. diff --git a/docs/LIMITS.md b/docs/LIMITS.md new file mode 100644 index 0000000..27cddb6 --- /dev/null +++ b/docs/LIMITS.md @@ -0,0 +1,137 @@ +# carbonado — limits (honest) + +## Current product surface + +### Dual-backend (normative product model) + +| Backend | Status | +|---------|--------| +| **Rust** (`src/`, default `backend-rust`) | First-class production library + CLI; full `cargo test` | +| **Lean AOT** (`Carbonado/`, `libcarbonado`, optional `backend-lean`) | Second engine: proofs + wire/C ABI; dual-suite phased (G8) | +| **Rust `tests/`** | Normative behavioral contract for **both** engines | + +- AOT CLI (`packages.carbonado` / `nix run`) runs **Programs A–G**: constants, EtM, FEC, keyed Bao, full pipeline (c0–c15), Header wire, scrub, stream bounds, multi-segment shards, **zstd-20 compression (linked)**, **SLH1 sidecar wire + bind-to-root model**, **Adamantine 1.0 directories**, **encode/decode/slh CLI**. +- Rust tree (`src/`, `tests/`, …) **stays** first-class (AGENTS dual-backend). Optional historical pin under `ref/carbonado-rust` is G1 residual — not a license to delete `src/` or `tests/`. +- Lean theorem/test tree is **`CarbonadoTest/`** (not `Tests/`) so it does not collide with Rust `tests/` on case-insensitive filesystems (Darwin APFS). +- Dependency direction is **CarbonadoTest → Carbonado** only. + +## Program B crypto (shipped) + +- Full Lean: SHA-512, HMAC-SHA512, AES-256-CTR (Ctr128BE), subkeys, payload EtM (both layouts), header MAC. +- MAC-before-decrypt is a **control-flow theorem** on `decryptAfterMacCheck` (tag verify before keystream). Not a constant-time proof. +- Parity is bit-match goldens vs RustCrypto/`src/crypto.rs` semantics (embedded + etm-vectors driver), not a live Nix `diff` harness against a Rust binary yet. +- **Low-level AESCTR** (`expandKey256` / `ctrXor`) is unchecked: short key/nonce panic via `get!`. EtM validates first. +- Lean exposes `invalidNonceLength` because nonces are `ByteArray`; Rust’s typed `[u8; 16]` cannot be wrong-sized at the same API. + +## Program C FEC (shipped) + +- Full Lean GF(2^8) (poly 0x1d log/exp tables), systematic RS matrix (Vandermonde × inv(top)), encode + reconstruct matching `reed-solomon-erasure` 5.0.3. +- Carbonado geometry: `calcPaddingLen` / `stripeUnit=16384` / inboard 8×`chunk_len` concat; `encodeInboard` / `decodeInboard` / `reconstructAfterKnockout`. +- **Stripe memory:** encode and decode **materialize O(stripe)** — for one segment-wide stripe that is `O(padded_len × 2)` shard buffers (8 × chunk_len). Same residual class as Rust `FecInboardEncoder` / `FecInboardWriteAt`. Documented and theorem-bounded in `Carbonado.Stream` (`maxFecStripeRetain`). +- Outboard parity-sidecar encode API not yet a separate product surface (inboard concat covers encode; split parity is trivial slice of shards 4..7). +- Parity is bit-match goldens vs pin crate / `rs-vectors` driver, not a live Nix `diff` harness yet. + +## Program D keyed Bao (shipped) + +- Full Lean BLAKE3 reference (hash / keyed_hash / derive_key + hazmat subtree/parent CVs) ported from the BLAKE3 reference algorithm; parity vs `ref/blake3` 1.8.5 portable semantics and bao-vectors. +- Keyed Bao product paths: format verification key, root, inboard `[u64le|response]`, post-order outboard, **stream slice decode** (`decodeSliceResponse` / `decodeSliceForFormat`) against `(key, root, contentLen)` via `decodeRec` — returns authenticated bytes from the response, **not** a re-encode oracle over trusted plaintext. +- Inboard slice extract (`verifySliceInboard*`) always runs full `decodeInboard` **before** any `count = 0` empty return (auth-first). Stream decode rejects `count = 0` with `invalidSliceCount`. +- Error taxonomy: short stream → `truncatedResponse`; overlong stream → `trailingData` (distinct). +- Tree model uses **leaf-group** recursion (4096 B) matching bao-tree `BlockSize::from_chunk_log(2)` IO. +- **Not claimed:** SIMD BLAKE3 throughput; O(slice) memory for **inboard** slice extract (full inboard materialize then extract — same class as some Rust inboard paths); standalone slice responses are O(response) for stream decode; async/tokio bao-tree APIs; pre-order outboard layout. +- **Constant-time:** logical `ctEq` only on hash compares; not a CT proof. +- Parity is bit-match goldens vs `ref/bao-tree` @ lock + `bao-vectors` driver, not a live Nix `diff` harness yet. + +## Program E pipeline / scrub / shard (shipped) + +- **Pipeline order:** compress → encrypt → FEC → keyed Bao (reverse on decode). Modules: `Carbonado.Pipeline`, `Header`, `Stream`, `Scrub`, `Shard`. +- **Header:** 177 B wire codec; `header_mac` verified before body (`decodeHeadered`). Authenticated `encoded_len` bounds the body (`truncatedBody` if short; trailers after `encoded_len` ignored). Public metadata only. +- **Nonce layouts:** header-path `[tag|ct]` vs low-level `[nonce|tag|ct]` as in EtM; pure model takes caller-supplied nonce (no CSPRNG). +- **MAC-before-decrypt:** EtM still refuses keystream until MAC ok; pipeline only decrypts after Bao/FEC reverse. +- **Scrub:** pure RS combinatorial search on FEC body + re-encode + Bao root compare. Does **not** implement Rust seekable slice extract entry; tests use `scrubWithMissing` / `scrubFecThenBao` after FEC body is known. Opaque Bao-only damage without FEC extract → `invalidScrubbedHash`. +- **Stream model:** pure stripe transducer + proved O(stripe) retain bounds; product `encodeBody` still uses segment-wide RS geometry (same as Rust residual). Multi-stripe encode model is documented alternative, not default parity path. +- **Sharding:** pure multi-segment headered encode/decode with contiguous `chunk_index` validation. +- **Outboard product pipeline** (`.out`/`.par` high-level file APIs) not fully composed as a separate encode/decode surface in Lean yet (Bao outboard primitives exist from D). +- Parity: format-matrix roundtrips in Lean AOT; live Nix vs `ref/carbonado-rust` product-matrix still open (G8). + +## Program F zstd + SLH (shipped with declared residuals) + +### Zstd (linked) + +- **AOT product:** `nix/native` builds a **static** `libcarbonado_native.a` = FFI glue + single-threaded libzstd objects from pinned commit **`f8745da6…` / v1.5.7** (same as `ref/zstd`; flake `zstdPinned` fetchFromGitHub; no shared `-lzstd`). Level **20** (`Carbonado.Compress.zstdLevel`). +- **Pipeline:** Compression bit → `compressLevel20` / `decompress` in `compressStep` / `decompressStep`; errors map 1:1 via `ofZstdError` → `compressionFailed` | `decompressionFailed` | `decompressOutputTooLarge` | `zstdInvalidInput` (no lumped catch-all). +- **DoS cap:** decompressed output ≤ 256 MiB (`maxDecompressedLen`, matches Rust `MAX_SEGMENT_MAIN_LEN`). +- **Interpreter / `native_decide`:** `@[extern]` bodies are identity fallbacks; **do not** `native_decide` compression formats (extern needs native symbols). Pure tests: status decode + bit-clear paths + non-compression format matrix. **Real** zstd + c2/c6/c14/c15 gated by AOT `demo` (`ZSTD_compress` API goldens empty/hello). +- **Not claimed:** streaming zstd (buffer API only); multi-threaded zstd; dictionary compression. + +### SLH-DSA sidecars (wire + binding; no real PQC FFI yet) + +- **Wire:** `Carbonado.Slh` — `SLH1` + 7856 B sig = 7860 B; parse/build fail-closed (`invalidSidecarLength` vs `badSlhMagic` vs `invalidSignatureLength` distinct). +- **Binding:** `verifyBound` / `verifyBoundToExpected` — signature is over the 32-byte Bao root; wrong root → `verificationFailed`; pk size / root size / sig size have distinct errors. +- **Sign:** `signRoot` returns `signatureUnavailable` (fail-closed) until libbitcoinpqc is linked. +- **Why no real SLH yet:** `ref/bitcoinpqc` pin is present but nested `libbitcoinpqc` submodule is empty; full cmake+secp+SLH link is deferred. Product integrates **wire + theorems + Header.slh_public_key slot**; real sign/verify oracle is the next deepen step. +- **Mock oracles** in tests only; never used as production crypto. + +## External C (declared) + +| Component | Status | +|-----------|--------| +| zstd | **Linked** static via `nix/native` + flake `zstdPinned` (commit `f8745da6…` / same as `ref/zstd` v1.5.7); no shared libzstd | +| SLH-DSA-SHA2-128s | Wire + binding in Lean; **FFI not linked** (libbitcoinpqc residual) | + +## Program G Adamantine + CLI (shipped with declared residuals) + +### Adamantine envelope (wire-compatible) + +- Magic `ADAMANTINE10\n` (13 B), header 19 B, `carbonado_fmt` c14/c15, flags bit0 `REQUIRE_OTS` only. +- Payload framing matches Rust: `[u32 LE man_len][man][u32 LE bun_len][bun]`. +- Dev `ADAMANTINE1\n` / `ADAMANTINE2\n` rejected with `unsupportedVersion`. + +### Filepack manifest — **CFP2 Lean-native, not rkyv** + +- Logical fields match FilepackManifest v2 (version, format_level, entries, SegmentRef, content_blake3, optional OTS). +- Wire body magic `CFP2` + deterministic LE layout (`Carbonado.Filepack`). +- **Not** byte-identical to Rust rkyv `FilepackManifestWire`. Adamantine *envelope* framing is shared; manifest body interop with Rust-produced catalogs needs a converter (future). Product CLI is CFP2 end-to-end. + +### Directory model + +- Catalog: inboard headered `{root}.adam.c14`/`.adam.c15`; segments: bare mains `{root}.c12`–`.c15`; centralized Bao+FEC bundle in Adamantine payload. +- Path rules fail-closed: empty, `..`, absolute `/`, `\`, empty components, NUL, length cap. +- Content BLAKE3 checked after segment recovery. +- Segment policy Auto/ForceRaw/ForceCompressed/ForceC12–C15; legacy c4–c7 rejected. +- OTS: `REQUIRE_OTS` flag → fail-closed `otsFeatureRequired` (no OTS stamps in Lean path). +- Master policy: zero master only for public; non-zero only for encrypted. + +### CLI + +- `demo` / no-args: full A–G self-test (flake `checks.demo`). +- `encode`/`decode` single-file (headered) and directory; single-file default name `{bao_root_hex}.c{fmt:02x}` (AGENTS hex). +- `slh parse`: wire only (exit 0 on valid frame). `slh verify`: **exit 1** until real SLH-DSA FFI (never soft-success). +- Nonces: `/dev/urandom` for encrypted encode. +- Directory default outdir `{input}-archive/`. +- Encode rejects `requireOts` (`otsFeatureRequired`); does not mint undecodeable archives. +- CLI encode rejects symlink source entries (`symlinkNotAllowed`); decode refuses write-through symlinks when detectible. + +## Dual-backend (G8 — P0 closed, engineering open) + +| Backend | Status | +|---------|--------| +| `backend-rust` (default) | Full Rust engine; full `cargo test` (must never regress) | +| `backend-lean` | **Scaffolding** — header, `carbonado-sys`, feature flags, weak C stubs (`NOT_IMPLEMENTED`); pure Lean FFI helpers exist; **not** full suite | +| Cross encode/decode Rust↔Lean | Not yet (G9; after Phase 2) | +| Docs / inventory (Phase 0) | **Closed** — [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md) | + +Rust-only for now: `async`, `async-tokio`, and (by default) multi-thread `parallel` paths. Lean uses serial RS. + +## Not claimed yet + +- **Constant-time** crypto proofs (logical `ctEq` only) +- Secret zeroization proofs / automatic zeroize of master keys +- WASM product target +- Throughput parity with AES-NI / SIMD RS / SIMD BLAKE3 Rust paths (optimize after correctness) +- Real SLH-DSA sign/verify via libbitcoinpqc (G10) +- Byte-identical rkyv FilepackManifest interop with Rust directories (required for directory dual-suite — Phase 3 / G7) +- Live C ABI encode/decode (stubs return `NOT_IMPLEMENTED` until Phase 1) +- Full `cargo test --features backend-lean` (G8 open; Phase 1 allowlist first) +- Live CI matrix both backends (G11) +- Live Nix product-matrix vs optional frozen `ref/carbonado-rust` diff --git a/docs/PARITY.md b/docs/PARITY.md new file mode 100644 index 0000000..5470f37 --- /dev/null +++ b/docs/PARITY.md @@ -0,0 +1,136 @@ +# carbonado — parity and `ref/` pins + +## Method (dual-backend) + +1. **Primary parity bar (G8):** the same Rust tests under `tests/` pass on **`backend-rust`** and **`backend-lean`** (Lean AOT `libcarbonado` via C ABI). See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md). +2. **Component oracles:** pin reference trees under `ref/`; keep offline drivers (`etm-vectors`, `rs-vectors`, `bao-vectors`) for fast regression against Lean goldens / AOT demos. +3. **Cross-backend tests (G9):** Rust encode → Lean decode and reverse once ABI encode/decode are stable (after G8 Phase 2). + +**SSOT roles (do not invert):** + +| Layer | Role | +|-------|------| +| Rust `src/` + default `backend-rust` | First-class production engine | +| Rust `tests/` | Normative behavioral contract for **both** backends | +| Lean `Carbonado/` + AOT `libcarbonado` | Second engine: proofs + wire/C-ABI compatible implementation | +| `ref/` | Pinned third-party oracles and vector drivers | + +Pin the exact trees the Rust product used; Lean AOT must remain wire-compatible with that contract. Rust is **not** demoted to “oracle only” while dual-backend work is in progress (G1 optional freeze is a pin, not a product deletion). + +## Pins (from Carbonado `Cargo.lock` / Surmount) + +| ref path | Source | Pin (commit / tag) | +|----------|--------|--------------------| +| `ref/bao-tree` | `https://github.com/SurmountSystems/bao-tree.git` | **`02916e784bb0afe0fd5a73c291c8c5335865e166`** (Cargo.lock; branch `76-keyed-bao`) | +| `ref/reed-solomon-erasure` | `https://github.com/darrenldl/reed-solomon-erasure.git` | tag **`v5.0.3`** → **`9f974918f8c598eee351406c36fa0295f4bb4d69`** | +| `ref/rustcrypto-block-ciphers` | `https://github.com/RustCrypto/block-ciphers.git` | tag **`aes-v0.8.4`** → **`f2dbee516b4d0cf4cb4f3045d09e35b5fd80087b`** | +| `ref/rustcrypto-macs` | `https://github.com/RustCrypto/MACs.git` | tag **`hmac-v0.12.1`** → **`46797e3b44973a30edb9d7f3a3ebb41810061d90`** | +| `ref/rustcrypto-hashes` | `https://github.com/RustCrypto/hashes.git` | tag **`sha2-v0.10.9`** → **`82c36a428f8d6f05f3bfccdedb243e9d1f85359d`** | +| `ref/blake3` | `https://github.com/BLAKE3-team/BLAKE3.git` | tag **`1.8.5`** → **`93a431c78a52d7ccf0f366f106467f5070e6075e`** | +| `ref/zstd` | `https://github.com/facebook/zstd.git` | tag **`v1.5.7`** → **`f8745da6ff1ad1e7bab384bd1f9d742439278e99`** — **product SSOT** for static libzstd in `nix/native` (not nixpkgs.src); Rust crate was zstd 0.13.3 | +| `ref/bitcoinpqc` | `https://github.com/cryptoquick/libbitcoinpqc-bindings.git` | **`7936b56f15e86b6764947c9298215ecfe38b712b`** | +| `ref/crates/ctr-0.9.2` | crates.io `ctr` 0.9.2 | checksum `0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835` — **vendored (Program B)** | +| `ref/carbonado-rust` | optional pin of live tree (`src/`, `tests/`, …) | freeze commit **pending** (G1); live tree remains first-class | +| `ref/parity-harness` | in-repo | `drivers/etm-vectors` (B); `drivers/rs-vectors` (C); `drivers/bao-vectors` (D); directory/CFP2 vectors deferred (G residual) | + +## Submodules + +Declared in [`.gitmodules`](../.gitmodules). Checked-out commits must match the table above +(`git submodule status` / `git -C ref/ rev-parse HEAD`). + +## Program B EtM parity + +1. **Oracle driver:** `ref/parity-harness/drivers/etm-vectors` (RustCrypto aes+ctr+hmac+sha2, same labels/domains as `src/crypto.rs`). +2. **Goldens embedded** in `Carbonado/Main.lean` and `CarbonadoTest/EtM.lean` (`native_decide` + AOT `demo` greps). +3. Vectors cover: SHA-512 empty/abc/fox, HMAC RFC 4231-1, NIST SP 800-38A AES-256-CTR F.5.5, Carbonado subkeys (`aes-ctr`/`etm-hmac`/`header-auth` under master `0x42×32`), header-path EtM blobs, low-level `[nonce\|tag\|ct]`, header MAC samples. +4. **Decrypt error order** matches Rust `symmetric_decrypt_with_nonce`: ciphertext length → master length → (Lean-only) nonce length → MAC. Rust takes `[u8; 16]` nonces so it has no nonce-length branch; Lean’s `invalidNonceLength` is the ByteArray API analogue. + +Regenerate goldens: + +```bash +cd ref/parity-harness/drivers/etm-vectors && cargo run --quiet +``` + +## Program C RS 4/8 parity + +1. **Oracle driver:** `ref/parity-harness/drivers/rs-vectors` (path dep on `ref/reed-solomon-erasure` @ v5.0.3). +2. **Goldens embedded** in `Carbonado/Main.lean` and `CarbonadoTest/Fec.lean`. +3. Vectors cover: + - GF(2^8) mul/div/exp samples (poly 0x1d) + - `calc_padding_len` for 0/1/100/4096/16384/16385 + - RS encode 1-byte shards → parity `45 5e 67 78` + - sequential 8-byte shards parity heads + - Carbonado inboard `hello` (pad 16379, chunk 4096, body 32768) + - pattern `i%251` len 100 parity0 head `001b362d6c775a41` +4. Reconstruct: parity-only (drop data 0–3) and mixed knockouts exercised in AOT Main. + +Regenerate goldens: + +```bash +cd ref/parity-harness/drivers/rs-vectors && cargo run --quiet +``` + +## Program D keyed Bao parity + +1. **Oracle driver:** `ref/parity-harness/drivers/bao-vectors` (path dep on `ref/bao-tree` @ `02916e78…`, `blake3` 1.x; `Cargo.lock` committed). +2. **Goldens embedded** in `Carbonado/Main.lean` and `CarbonadoTest/Bao.lean`. +3. Vectors cover: + - BLAKE3 `hash(empty)`, `hash(abc)` + - `carbonado_verification_key` for formats 0/4/6/12/14/15 via `blake3::derive_key("carbonado-v2/verification", &[format])` + - Keyed roots for patterned lengths (0…8192) under c4; format domain separation c4/c6/c14 on pat100 + - Inboard `[u64le content_len | response]` for empty/1/hello/pat100/4096/5000/**12288** (three-leaf) + - Post-order outboard for 5000 B (64 B) and 12288 B (128 B nested parents) + - Slice encode/stream-decode: first leaf of 5000 → 4160 B; middle leaf of 12288 → 4224 B (`keyed_decode_ranges` without full trusted body) + - Wrong format key fails decode (`LeafHashMismatch` / `ParentHashMismatch` → Lean `authenticationFailed`) +4. Invariants: full-file root ≡ `blake3::keyed_hash(key, data)`; 4 KiB leaf geometry (`BlockSize::from_chunk_log(2)`). +5. Lean stream API: `decodeSliceResponse` / `decodeSliceForFormat` (not re-encode oracle). + +Regenerate goldens: + +```bash +cd ref/parity-harness/drivers/bao-vectors && cargo run --quiet +``` + +## Program E pipeline parity + +1. **Composition:** Lean `Carbonado.Pipeline.encodeBody` / `decodeBody` mirrors Rust `stream_encode_buffer` / `stream_decode_buffer` stage order (compress → encrypt → FEC → Bao). +2. **Compression:** Program F links zstd-20 for the Compression bit (see below). Structure + format-byte keying match. +3. **Header:** 177 B layout matches `src/file.rs` `Header::LEN` / `try_to_vec` / `TryFrom<&[u8]>`; header MAC formula matches EtM goldens already in Program B. +4. **Scrub:** pure RS subset + re-encode + Bao root (same oracle idea as `decoding::scrub`); no seekable slice extract entry in Lean yet. +5. **Shards:** pure multi-segment model vs `stream/shard.rs` (`chunk_index` sequence, headered segments). +6. **Optional driver:** `ref/parity-harness/drivers/pipeline-vectors` may be added later for live Rust↔Lean body goldens; until then AOT format-matrix + CarbonadoTest `native_decide` roundtrips are the gate. +7. **Stream bounds:** Lean theorems on O(stripe) retain (`Carbonado.Stream`); documents residual shared with Rust FEC path. + +## Program F zstd + SLH parity + +1. **zstd pin / product SSOT:** commit **`f8745da6…`** (tag v1.5.7). Checked out as `ref/zstd` submodule for oracle/review; flake **fetches the same rev+hash** into `nix/native` (`zstdPinned` in `flake.nix`) and **statically** compiles `lib/common|compress|decompress` + FFI into `libcarbonado_native.a` (level 20, single-threaded, no shared `-lzstd`). nixpkgs is only for the host toolchain / Lean headers — **not** the zstd source pin. Updating zstd requires: submodule checkout, `flake.nix` rev/hash, and PARITY table together. +2. **Goldens (AOT `demo`):** API frames for empty (`28b52ffd2000010000`) and `hello` (`28b52ffd200529000068656c6c6f`); corrupt frame → `decompressionFailed`; tight `maxOut` → `outputTooLarge`; zeros shrink; pipeline c2/c6 + **headered c3/c7**; full format matrix incl. compression at runtime. +3. **Interpreter residual:** Lean `@[extern]` bodies are identity for elaborator; real frames only in AOT (LIMITS). CarbonadoTest `native_decide` covers non-compression formats + status maps. +4. **SLH1 wire:** magic `SLH1`, signature 7856 B, sidecar 7860 B — matches `src/crypto.rs` `SLH1_*` and AGENTS §2.3. Pure suite: length errors + `parseMagicAtExactLen` / `badSlhMagic` gate theorems; full-length build/parse + all-zero magic in `demo`. +5. **Bind-to-root:** Lean model requires signature message = Bao root (`verifyBoundToExpected`); wrong root → `verificationFailed`. Real SLH-DSA verify via libbitcoinpqc **not** linked (nested submodule empty). +6. **Optional future drivers:** `ref/parity-harness/drivers/zstd-vectors`, `slh-vectors` once PQC FFI lands. + +## Adding a gate + +1. Vectors under `ref/parity-harness/` or `CarbonadoTest/` goldens +2. Nix derivation comparing Lean AOT vs ref binary/library +3. Register in `flake.nix` `checks` and [SPEC-MATRIX.md](SPEC-MATRIX.md) + +## Submodule init + +```bash +git submodule update --init --recursive +# bao-tree must be at the lock commit: +git -C ref/bao-tree checkout 02916e784bb0afe0fd5a73c291c8c5335865e166 +``` + +CI must checkout recursively once submodules are recorded on the default branch. + +## carbonado-rust freeze strategy (optional pin — G1 residual) + +Dual-backend model keeps **live** Rust under `src/` and `tests/` as first-class. An optional historical pin is separate: + +1. Keep production Rust under `src/`, `tests/`, etc. (do not delete for “Lean purity”). +2. Optionally add submodule or subtree `ref/carbonado-rust` at a named freeze commit for long-lived oracle/goldens isolation. +3. **Do not** treat Lean as a replacement that removes the Rust engine: G8 requires both backends against the same `tests/`. +4. Record any freeze SHA here and in [GAPS.md](GAPS.md) G1 when created. diff --git a/docs/PROOFS.md b/docs/PROOFS.md new file mode 100644 index 0000000..e58b6ac --- /dev/null +++ b/docs/PROOFS.md @@ -0,0 +1,44 @@ +# carbonado — proof inventory + +**Policy:** product Lean under `Carbonado/` and `CarbonadoTest/` must contain **no** proof holes. + +**Dual-backend:** Lean theorems prove properties of the Lean engine and wire model. **Bit-match / behavioral parity** with production is additionally enforced by the Rust suite on `backend-lean` (G8) and `ref/` oracles ([PARITY.md](./PARITY.md)). Proofs do **not** replace `tests/`; they complement them. + +**Hole vocabulary (forbidden):** `sorry`, `admit` (Lean alias for `sorry`). +Enforced by `nix build .#checks.x86_64-linux.no-sorry` / `nix flake check` (and explicit check builds). The gate fails closed if those directories or their `*.lean` roots are missing. + +## Counts + +| Module | Theorems / notes | +|--------|------------------| +| `Carbonado/Constants.lean` | `magicBytes_length`, `magicBytes_eq_literal`, `sliceLen_eq_bao_group`, `stripeUnit_eq`, `fecM_eq_twice_fecK`, `slh1Magic_*`, `slh1SidecarLen_eq`, full `SubkeyLabel` registry, `formatBits_roundtrip`, `unencrypted_format_even`, `formatC14_byte`, `formatC15_byte`, `format_codes_roundtrip`, `headerLen_sum` | +| `Carbonado/Crypto/EtM.lean` | **MAC-before-decrypt:** `decryptAfterMacCheck_tag_fail`, `decryptAfterMacCheck_ok_implies_mac`, `decryptAfterMacCheck_auth_fail_no_plaintext`, `decryptResult_ok_implies_plaintext`. **Guard taxonomy:** `decryptWithNonce_short_input` (CT length first), `decryptWithNonce_short_master` (master after CT gate), `decryptWithNonce_bad_nonce` (`invalidNonceLength`) | +| `Carbonado/Fec/Galois.lean` | GF mul/div/exp goldens vs pin: `mul_1_1`, `mul_2_3`, `mul_0x53_0xca`, `mul_0xff_1`, `mul_7_11`, `div_2_3`, `exp_2_3`, `exp_0x53_3`, `mul_comm_7_11` | +| `Carbonado/Fec/RS.lean` | `carbonadoRS_constructs`, `carbonadoRS_geometry` (aligned with `fecK`/`fecM`); `invertOrSingular_zeros` / `invertOrSingular_identity` | +| `Carbonado/Fec/Inboard.lean` | `calcPaddingLen_{zero,one,stripe,stripe_plus_one,100,4096}`, `paddedLen_aligns_samples` | +| `Carbonado/Bao/Blake3.lean` | `hash_empty`, `hash_abc` (official BLAKE3 goldens) | +| `Carbonado/Bao/Tree.lean` | `leafBytes_eq_sliceLen`, `root_eq_keyed_hash`; stream `decodeSliceResponse` / auth-first inboard extract | +| `Carbonado/Bao/Product.lean` | `verification_key_format_domain`, `root_commits_to_format`, `encode_decode_empty_c4`, `hello_root_eq_keyed_hash` | +| `Carbonado/Header.lean` | `authData_len_formula` (113 B); `headerLen_eq_177` | +| `Carbonado/Pipeline.lean` | `encrypted_bit_is_odd`; full encode/decode composition; strict `PipelineError` taxonomy incl. zstd modes | +| `Carbonado/Stream.lean` | `full_stripe_inboard_len`, `full_stripe_retain`, `empty_stripe_retain`, `one_byte_stripe_retain`, `chunk_eq_slice`, `stripe_eq_k_slices` | +| `Carbonado/Scrub.lean` | Pure RS mask search + Bao root oracle (AOT + CarbonadoTest) | +| `Carbonado/Shard.lean` | `split_empty_budget`, `split_hello_budget_2`, `split_empty_plaintext` | +| `Carbonado/Compress.lean` | `zstdMagic_length`; `ofStatus_*`; `decode_status_*` for every status code; `statusOk_payload_identity` (pure framing helper; not an `@[extern]` decide) | +| `Carbonado/Slh.lean` | Wire: `parse_short_length`, `parse_empty`, `build_bad_sig_len`, `slh1_magic_bytes`, **`parse_magic_bad` / `zeros_not_slh1_magic` / `parse_bad_magic_when_exact`** (`badSlhMagic` path). Binding: `bind_bad_{pk,root,sig}`, **`wrong_root_fails`**, `sign_unavailable`, `sign_bad_root`. Full 7856 B wire: AOT Main | +| `CarbonadoTest/Scaffold.lean` | Re-exports / restates wire invariants | +| `CarbonadoTest/EtM.lean` | Re-exports MAC + guard theorems; **`native_decide` matrix** for crypto goldens | +| `CarbonadoTest/Fec.lean` | Geometry + RS; all 7 `FecError` paths | +| `CarbonadoTest/Bao.lean` | Geometry; BLAKE3; stream slice; **exact** every `BaoError` | +| `CarbonadoTest/Pipeline.lean` | Non-compression format matrix + path tests; **exact maps** incl. `ofZstdError` → `zstdInvalidInput` | +| `CarbonadoTest/Compress.lean` | Every `ZstdError` status map; bit-clear compress/decompress; pipeline maps | +| `CarbonadoTest/Slh.lean` | Short-path every `SlhError` **except** full-length AOT-only wire (length/magic/sig/pk/root/unavailable/verification via prefix+gate theorems; full 7860 B `badSlhMagic` + roundtrip in Main) | +| `Carbonado/Adamantine.lean` | `adamantineMagic_*`, `adamantineHeaderLen_eq`; encode/decode empty public; `invalid_flags_bit1`; `invalid_fmt_c0`; `short_header`; `dev_v2_rejected` | +| `Carbonado/Filepack.lean` | `cfp2Magic_length`; path: `rel_empty`, `rel_traversal`, `rel_absolute`, `rel_backslash`, `rel_ok`, `rel_empty_component` | +| `Carbonado/Outboard.lean` | Outboard encode/decode + FEC parity sidecar (AOT roundtrips in Main) | +| `Carbonado/Directory.lean` | `ofFilepack_*` / `ofAdamantine_*` maps; pure encode/decode (AOT); path fail-closed | +| `Carbonado/Cli.lean` | Product CLI dispatch (IO; no theorems) | +| `CarbonadoTest/Directory.lean` | Restates Adamantine/path maps; master-policy + path reject exact variants; padding helpers | +| `Carbonado/Main.lean` | AOT Programs A–G; full zstd goldens; SLH full wire; directory pure roundtrip; gated by `checks.demo` greps | + +Refresh with theorem line counts (beastdb PROOFS pattern) as parity gates deepen. diff --git a/docs/SPEC-MATRIX.md b/docs/SPEC-MATRIX.md new file mode 100644 index 0000000..5bca1d1 --- /dev/null +++ b/docs/SPEC-MATRIX.md @@ -0,0 +1,22 @@ +# carbonado — specification matrix + +Every product capability maps to Lean module(s), parity gate(s), and proof status. + +| Capability | Lean | Parity gate | Proof status | +|------------|------|-------------|--------------| +| Magic / header sizes / format bits | `Carbonado.Constants`, `CarbonadoTest.Scaffold` | `demo` / AOT Main | **scaffold theorems** (no sorry/admit) | +| Header wire + header_mac | `Carbonado.Header` (177 B parse/build/verify), `Carbonado.Crypto.EtM.computeHeaderMac` | golden in `demo` + `ref/parity-harness/drivers/etm-vectors` | **Program E:** wire codec + header-MAC-before-body; auth_data 113 B formula theorem | +| AES-CTR + HMAC-SHA512 EtM | `Carbonado.Crypto.{SHA512,HMAC,AESCTR,EtM}` | `demo` goldens; driver `ref/parity-harness/drivers/etm-vectors` | **Program B closed**: MAC-before-decrypt theorems; SHA/HMAC/AES-CTR/EtM goldens; roundtrip + tamper + wrong-key | +| RS 4/8 + geometry | `Carbonado.Fec.{Galois,Matrix,RS,Inboard}`, `CarbonadoTest.Fec` | `demo` goldens; driver `ref/parity-harness/drivers/rs-vectors` | **Program C closed**: GF; geometry; encode/reconstruct; all 7 `FecError` variants | +| Keyed Bao 4 KiB | `Carbonado.Bao.{Blake3,Tree,Product}`, `CarbonadoTest.Bao` | `demo` goldens; driver `ref/parity-harness/drivers/bao-vectors` | **Program D closed**: stream slice decode; all `BaoError` variants | +| Pipeline c0–c15 | `Carbonado.Pipeline`, `CarbonadoTest.Pipeline` | `demo` format matrix; optional future `product-matrix` vs rust | **Program E+F**: compress(zstd-20 when bit set)→encrypt→FEC→Bao + reverse; headered + body paths; `encoded_len` bound; MAC-before-decrypt + header-MAC-before-body; strict `PipelineError` incl. zstd modes | +| Scrub | `Carbonado.Scrub` | demo knockout recovery | **Program E:** pure RS subset search + re-encode + Bao root compare; `unnecessaryScrub` / `scrubRequiresVerification` / `invalidScrubbedHash` | +| Outboard | `Carbonado.Bao` create/verify; `Carbonado.Outboard` product body (bare main + FEC parity + verification sidecar) | bao-vectors + `demo` outboard segment roundtrip | **Program D+G**: post-order Bao outboard; directory segments via `encodeOutboardBody` / `decodeOutboardBody` | +| Streaming bounds | `Carbonado.Stream` | demo greps + theorems | **Program E:** O(stripe) FEC retain theorems (`maxFecStripeRetain`); pure stripe transducer model | +| Sharding | `Carbonado.Shard` | demo multi-segment roundtrip | **Program E:** budget split + `chunk_index` sequence + headered segments | +| Zstd-20 compress | `Carbonado.Compress`, `CarbonadoTest.Compress` | `demo` API goldens (empty/hello); pipeline c2/c6 | **Program F closed**: linked zstd; status taxonomy; interpreter identity fallback (LIMITS) | +| SLH1 sidecars | `Carbonado.Slh`, `CarbonadoTest.Slh` | `demo` wire + bind-to-root | **Program F closed** for wire/binding theorems; real SLH-DSA FFI residual (LIMITS) | +| Adamantine directory | `Carbonado.Adamantine`, `Filepack`, `Outboard`, `Directory`, `CarbonadoTest.Directory` | `demo` Program G greps; pure roundtrip AOT | **Program G closed** for product path: Adamantine10 + CFP2 manifest + outboard segments + fail-closed paths + content BLAKE3. rkyv body residual (LIMITS) | +| CLI | `Carbonado.Cli`, `Carbonado.Main` | `demo` + CLI subcommands | **Program G:** encode/decode file+dir; single-file default `{bao_root_hex}.c{fmt:02x}`; dir default `{input}-archive/`; `slh parse` wire; `slh verify` fail-closed exit 1 until FFI | + +Expand rows until full product parity with dual-backend G8 (`backend-rust` + `backend-lean` on `tests/`) and optional `ref/carbonado-rust` freeze is closed. Component rows above track Lean+Nix proof/oracle gates; dual-suite status is [GAPS.md](./GAPS.md) G8 / [TEST_CONTRACT.md](./TEST_CONTRACT.md). diff --git a/docs/TEST_CONTRACT.md b/docs/TEST_CONTRACT.md new file mode 100644 index 0000000..7a8b76e --- /dev/null +++ b/docs/TEST_CONTRACT.md @@ -0,0 +1,140 @@ +# carbonado — Rust test contract (dual-backend) + +The Rust integration suite under `tests/` is the **normative behavioral contract**. +Both `backend-rust` (default) and `backend-lean` (Lean AOT `libcarbonado` via C ABI) must pass the **same** tests, growing from a Phase 1 allowlist to the full suite. + +See [ABI.md](./ABI.md), [PARITY.md](./PARITY.md), [GAPS.md](./GAPS.md) G8, [LIMITS.md](./LIMITS.md). + +**Invariant:** never regress `backend-rust` `cargo test` while landing Lean paths. + +## How backends relate to this suite + +| Feature flag | Engine | Expected of this suite | +|--------------|--------|------------------------| +| `backend-rust` (default) | Pure Rust (`src/encoding`, `src/decoding`, `src/file`, …) | Full green (always) | +| `backend-lean` | Lean AOT via `carbonado-sys` / `libcarbonado` | Phased allowlist → full green (G8) | + +```bash +# Normative default +cargo test + +# Dual-backend (after Phase 1 wiring + linked lib) +# nix build .#libcarbonado +# export CARBONADO_LEAN_LIB=$PWD/result/lib CARBONADO_LEAN_INCLUDE=$PWD/result/include +cargo test --no-default-features --features "backend-lean,pqc,ots" +``` + +Helpers under `tests/common/` are not separate contract files; they support the files below. + +--- + +## Classification (every `tests/*.rs` file) + +| Class | File | lean-backend target phase | Notes | +|-------|------|---------------------------|-------| +| **core** | `codec.rs` | Phase 1–2 | Low-level `encode`/`decode`, slice, scrub, header layout, samples | +| **core** | `format.rs` | Phase 1–2 | Full format matrix (inboard + outboard + scrub) | +| **core** | `format_amplification.rs` | Phase 2 | Size/geometry amplification via `file::encode` | +| **core** | `header_tamper.rs` | Phase 1–2 | Header field flips → auth / layout failures | +| **core** | `bao_keyed_contract.rs` | Phase 1–2 | Verification key, keyed roots, slice verify paths | +| **core** | `adversarial_proptest.rs` | Phase 2 | Proptest outboard/header adversarial | +| **core** | `deprecation_aliases.rs` | Phase 3+ | Type/const aliases only (no encode path) | +| **fec_scrub** | `fec_chaos.rs` | Phase 2 | Distributed knockouts inboard/outboard | +| **fec_scrub** | `fec_scrub_matrix.rs` | Phase 2 | Scrub matrix public/encrypted FEC | +| **fec_scrub** | `shard_fec_scrub.rs` | Phase 2 | Per-segment scrub after sharding | +| **fec_scrub** | `udp_fec_sim.rs` | Phase 2 | Datagram FEC sim + directory scrub path | +| **fec_scrub** | `apocalypse.rs` | Phase 2 | Large-sample encode/scrub chaos | +| **stream** | `streaming.rs` | Phase 2 | Stream encode/decode buffer + outboard | +| **stream** | `streaming_limits.rs` | Phase 2 | Bounds, FEC encoder, crypto stream, scrub | +| **stream** | `seekable_slices.rs` | Phase 2 | O(slice) verify inboard/outboard | +| **shard** | `sharding.rs` | Phase 2 | `encode_shard_stream` / `decode_shards_stream` | +| **directory** | `directory_archive.rs` | Phase 3 | Adamantine 1.0 + rkyv catalog + scrub_outboard | +| **directory** | `filepack_interop.rs` | Phase 3 | Filepack / CBOR interop + directory decode | +| **directory** | `format_policy.rs` | Phase 3 | Segment format policy (no I/O encode) | +| **cli** | `bin_cli.rs` | Phase 4 | Prebuilt `carbonado` binary CLI | +| **cli** | `bin_smoke.rs` | Phase 4 | CLI smoke encode/decode | +| **cli** | `bin_heuristics.rs` | Phase 4 | Filename heuristics + CLI | +| **pqc** | `slh_outboard.rs` | Phase 4 | SLH-DSA sidecars + header `slh_public_key` | +| **async** | `streaming_async.rs` | **rust-only** (unless declared) | `stream_decode_async` | +| **parallel** | `parallel_determinism.rs` | rust-only or serial lean path | RS parallel vs serial determinism | +| **parallel** | `serial_fec_path.rs` | Phase 2 (serial) | Serial FEC encoder vs buffer path | + +**Inventory count:** 26 integration test files under `tests/*.rs` (complete as of Phase 0 close). + +--- + +## Primary public APIs used by tests + +Mapped from actual `use carbonado::…` imports in `tests/*.rs`. C ABI column is the dual-backend export target ([ABI.md](./ABI.md)). + +| API / type | Typical tests | C ABI priority | +|------------|---------------|----------------| +| `encode` / `decode` (crate root = `encoding`/`decoding`) | codec, format, header_tamper, fec_*, apocalypse, udp_fec_sim, parallel_determinism | **v0** (`carbonado_encode` / `carbonado_decode`) | +| `encode_outboard` / `decode_outboard` | format, fec_*, bao_keyed, streaming*, directory, adversarial | **v1+** (not in `include/carbonado.h` v0) | +| `scrub` / `scrub_outboard` | codec, format, fec_*, apocalypse, streaming_limits, shard_fec_scrub, directory | **v1+** | +| `verify_slice` / `extract_slice` | codec, seekable_slices, bao_keyed | **v1+** | +| `verify_slice_inboard_seekable` / `verify_slice_outboard` | bao_keyed, seekable_slices | **v1+** | +| `carbonado_verification_key` | bao_keyed_contract | **v0** | +| `file::encode` / `file::decode` / `Header` | format, format_amplification, header_tamper, streaming_limits, slh_outboard, adversarial | **v0** (`carbonado_encode_headered` / `carbonado_decode_headered`) | +| `file::encode_stream` / `decode_stream` | streaming, streaming_limits | Phase 2 (may stay Rust-side over buffer ABI) | +| `file::encode_directory` / `encode_directory_with_options` / `decode_directory` | directory_archive, filepack_interop, udp_fec_sim | Phase 3 (+ rkyv wire) | +| `stream_encode_buffer` / `stream_decode_buffer` (+ outboard buffer variants) | streaming*, bao_keyed, parallel_determinism | Phase 2 | +| `stream::fec::*` / `stream::parallel::*` | streaming_limits, serial_fec_path, parallel_determinism | rust-internal / serial lean | +| `encode_shard_stream` / `decode_shards_stream` | sharding, shard_fec_scrub | Phase 2 | +| Adamantine / filepack_manifest / format_policy | directory_*, filepack_interop, format_policy | Phase 3 | +| `crypto::slh_*` / sidecar helpers | slh_outboard | Phase 4 (G10) | +| `ots::*` | directory_archive (feature `ots`) | Phase 4 | +| Deprecation aliases (`PackIndex`, …) | deprecation_aliases | n/a (API surface only) | +| `stream_decode_async` | streaming_async | **rust-only** initially | +| CLI binary (`src/bin/carbonado.rs`) | bin_* | Phase 4 | + +### Error-contract note (both backends) + +Tests that `matches!` ultra-specific `CarbonadoError` variants require a stable C-code → Rust mapping ([ABI.md](./ABI.md) error table). Phase 1 may collapse some Lean `PipelineError` variants into broader ABI codes; refine mapping before claiming full-suite green on failure-mode tests (`header_tamper`, scrub unnecessary vs failed, etc.). + +--- + +## Phase 1 allowlist (first green `backend-lean` gate) + +**Honest status at Phase 0 close:** C symbols exist in `include/carbonado.h` and `carbonado-sys`, but `nix/native/carbonado_abi.c` weak stubs return `CARBONADO_ERR_NOT_IMPLEMENTED` for all encode/decode/verification_key entry points. Lean has pure helpers in `Carbonado/Ffi.lean` (`encodeHeaderedBytes`, `decodeHeaderedBytes`, `ofPipelineError`) — **not** yet linked as live C exports in the archive used by `backend-lean`. Phase 1 work is: real `@[export]` / link, Rust dispatch into `src/backend/lean`, then the allowlist below. + +### Phase 1 scope (concrete) + +1. **Public (even) formats only** for first green: e.g. c0, c2, c4, c6, c12, c14 — no encryption / no random nonce dependency until headered encrypted path is deterministic under test nonces. +2. **Buffer / headered APIs only** (match C ABI v0): + - `carbonado_abi_version` / `carbonado_free` + - `carbonado_verification_key` + - `carbonado_encode` / `carbonado_decode` (low-level body; Rust `encoding::encode` / `decoding::decode` shape) + - `carbonado_encode_headered` / `carbonado_decode_headered` (Rust `file::encode` / `file::decode` shape) +3. **Suggested first test targets** (grow in CI / justfile as green): + - Subset of `tests/codec.rs` (roundtrip + basic failure) **or** a dedicated `tests/lean_backend_smoke.rs` reusing `tests/common` helpers + - `tests/bao_keyed_contract.rs` cases that only need verification key + comparable encode roots + - Selected `tests/header_tamper.rs` auth-fail cases once headered encode is real (not stub) +4. **Explicitly out of Phase 1:** outboard, scrub, seekable slice C exports, directory/rkyv, CLI, SLH FFI, async, parallel RS. + +### Phase 1 non-goals + +- Full `tests/` green on `backend-lean` +- Changing normative wire format +- Replacing or deleting the Rust engine + +Document the live allowlist in CI / justfile as it grows. Full suite remains the G8 end state (Phase 5). + +--- + +## Later phases (test-suite coverage) + +| Phase | Test classes unlocked | Depends on | +|-------|----------------------|------------| +| 2 | fec_scrub, stream, shard, remaining core | scrub/outboard/slice ABI or Rust-side composition over body ABI; format matrix | +| 3 | directory, format_policy, filepack_interop, deprecation_aliases | rkyv-compatible catalog wire (not Lean-only CFP2) | +| 4 | cli, pqc (slh_outboard), ots paths | libbitcoinpqc in libcarbonado (G10); CLI dual path | +| 5 | CI freeze both backends; G8 closed | full suite + docs freeze | + +--- + +## Maintenance + +- New `tests/*.rs` files **must** be added to the classification table above in the same PR. +- New public encode/decode surfaces used by tests must be listed in the API table and, if dual-backend-relevant, in [ABI.md](./ABI.md). +- Prefer strict `matches!` on specific `CarbonadoError` variants for failure-mode tests; when ABI collapse prevents 1:1 mapping, document backend-aware expectations rather than loosening asserts permanently. diff --git a/docs/VISION.md b/docs/VISION.md new file mode 100644 index 0000000..4b14c26 --- /dev/null +++ b/docs/VISION.md @@ -0,0 +1,31 @@ +# carbonado — vision (dual-backend: Rust + Lean 4 AOT + Nix) + +**Mission:** Apocalypse-resistant archival format for consensus-critical data. + +**Product model:** Rust remains a **first-class** production engine (`src/`, default `backend-rust`). Lean 4 AOT (`Carbonado/`, `libcarbonado`) is a **second engine**: machine-checked proofs plus a wire- and C-ABI-compatible implementation. Both must pass the same Rust `tests/` (G8 dual-backend parity). + +## Prove everything + +Each product claim is either: + +- machine-checked in Lean (no `sorry` in product), and/or +- bit-matched via the Rust suite on `backend-lean` and/or pinned `ref/` oracles (CI parity gates). + +Lean covers encode/decode, EtM, FEC, keyed Bao, scrub, streaming geometry, sharding, Adamantine directories, outboard, SLH sidecars, CLI — **in addition to**, not as a deletion of, the Rust engine. + +## Method + +1. Pin references under `ref/` (submodules, exact commits from Cargo.lock / Surmount forks). +2. Lean algorithms + theorems; keep Rust `src/`/`tests/` first-class. +3. AOT to C via Lean’s backend (never hand-edit generated C); expose C ABI for `backend-lean`. +4. Nix links AOT objects and allowed external C (zstd, libbitcoinpqc) until replaced. +5. Parity: `ref/` drivers + dual-backend `cargo test` ([TEST_CONTRACT.md](./TEST_CONTRACT.md), [PARITY.md](./PARITY.md)). + +## Precedents + +- **beastdb** — Lean product + Nix AOT packaging +- **verik1** — prove and bit-match production crypto against `ref/` + +## Priority + +Truth, correctness, depth, quality — over schedule. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..39782de --- /dev/null +++ b/flake.lock @@ -0,0 +1,117 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_2" + }, + "locked": { + "lastModified": 1765835352, + "narHash": "sha256-XswHlK/Qtjasvhd1nOa1e8MgZ8GS//jBoTqWtrS1Giw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "a34fae9c08a15ad73f295041fec82323541400a9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "lean4-nix": { + "inputs": { + "flake-parts": "flake-parts_2", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1782490315, + "narHash": "sha256-dUZ98JBxn+zvYoRag0IfTDHLxVXACytxvdxluWjeI58=", + "owner": "lenianiva", + "repo": "lean4-nix", + "rev": "7711df0784e335267b6aa57a5c8ada62a6bef0bf", + "type": "github" + }, + "original": { + "owner": "lenianiva", + "repo": "lean4-nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1765779637, + "narHash": "sha256-KJ2wa/BLSrTqDjbfyNx70ov/HdgNBCBBSQP3BIzKnv4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "1306659b587dc277866c7b69eb97e5f07864d8c4", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1782614948, + "narHash": "sha256-ePjCwr1sNm9NYUqywL7QfK3JnlS015msC+eBu2zKlp8=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "db3f255737b94216eb71cce308e2912cf6bc2d7c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixpkgs-lib_2": { + "locked": { + "lastModified": 1765674936, + "narHash": "sha256-k00uTP4JNfmejrCLJOwdObYC9jHRrr/5M/a/8L2EIdo=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "2075416fcb47225d9b68ac469a5c4801a9c4dd85", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "lean4-nix": "lean4-nix", + "nixpkgs": [ + "lean4-nix", + "nixpkgs" + ] + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..04f834f --- /dev/null +++ b/flake.nix @@ -0,0 +1,332 @@ +{ + description = "carbonado — apocalypse-resistant archival format (Lean 4 AOT + Nix)"; + + inputs = { + nixpkgs.follows = "lean4-nix/nixpkgs"; + flake-parts.url = "github:hercules-ci/flake-parts"; + lean4-nix.url = "github:lenianiva/lean4-nix"; + }; + + outputs = inputs @ { + nixpkgs, + flake-parts, + lean4-nix, + ... + }: + flake-parts.lib.mkFlake {inherit inputs;} { + systems = [ + "aarch64-darwin" + "aarch64-linux" + "x86_64-darwin" + "x86_64-linux" + ]; + + perSystem = { + system, + pkgs, + ... + }: let + # Product source: Lean modules + flake metadata. ref/ and Rust legacy stay out + # of the Lean package src filter where possible. + # Note: Lean test tree is `CarbonadoTest/` (not `Tests/`) to avoid colliding + # with Rust `tests/` on case-insensitive filesystems (Darwin). + productSrc = pkgs.lib.cleanSourceWith { + src = ./.; + filter = path: type: let + base = baseNameOf path; + in + !(base == "ref" && type == "directory") + && !(base == "target" && type == "directory") + && !(base == "src" && type == "directory") + && !(base == "tests" && type == "directory") + && !(base == "benches" && type == "directory") + && !(base == "examples" && type == "directory") + && !(base == ".git" && type == "directory") + && !(base == "result" || pkgs.lib.hasPrefix "result-" base) + && pkgs.lib.cleanSourceFilter path type; + }; + + # Match proof holes: `sorry` and its Lean alias `admit`. + holePattern = ''(^|[^a-zA-Z_])(sorry|admit)([^a-zA-Z_]|$)''; + + # Program F: static zstd from the **same pin as ref/zstd** (v1.5.7 / + # f8745da6…) + FFI glue → libcarbonado_native.a. Fetched by fixed rev/hash + # so flake purity does not require the submodule worktree to be git-tracked + # in the parent tree; SHA must stay in lockstep with docs/PARITY.md. + zstdPinned = pkgs.fetchFromGitHub { + owner = "facebook"; + repo = "zstd"; + rev = "f8745da6ff1ad1e7bab384bd1f9d742439278e99"; + hash = "sha256-tNFWIT9ydfozB8dWcmTMuZLCQmQudTFJIkSr0aG7S44="; + }; + carbonadoNative = import ./nix/native { + inherit pkgs; + leanAll = pkgs.lean.lean-all; + zstdSrc = zstdPinned; + carbonadoInclude = ./include; + }; + + leanPkg = pkgs.lean.buildLeanPackage { + name = "carbonado"; + # Separate roots so CarbonadoTest compiles without product → test imports. + # lean4-nix only discovers modules under the root name of each entry. + roots = [ + "Carbonado.Main" + "CarbonadoTest.Scaffold" + "CarbonadoTest.EtM" + "CarbonadoTest.Fec" + "CarbonadoTest.Bao" + "CarbonadoTest.Pipeline" + "CarbonadoTest.Compress" + "CarbonadoTest.Slh" + "CarbonadoTest.Directory" + ]; + src = productSrc; + debug = false; + leancFlags = ["-O3" "-DNDEBUG"]; + # Static zstd + FFI (no shared libzstd — avoids lld shlib-undefined/pthread). + staticLibDeps = [carbonadoNative]; + linkFlags = []; + }; + + noSorry = + pkgs.runCommand "carbonado-no-sorry" { + src = productSrc; + } '' + set -euo pipefail + # Fail-closed: product Lean trees must exist in productSrc. + for dir in Carbonado CarbonadoTest; do + if [ ! -d "$src/$dir" ]; then + echo "carbonado: missing required directory $dir/ in product source" >&2 + exit 1 + fi + lean_count=$(find "$src/$dir" -type f -name '*.lean' | wc -l) + if [ "$lean_count" -lt 1 ]; then + echo "carbonado: $dir/ has no .lean files" >&2 + exit 1 + fi + if grep -R --include='*.lean' -nE '${holePattern}' "$src/$dir"; then + echo "carbonado: proof hole (sorry/admit) found in $dir Lean sources" >&2 + exit 1 + fi + done + for f in Carbonado.lean CarbonadoTest.lean; do + if [ ! -f "$src/$f" ]; then + echo "carbonado: missing root module $f" >&2 + exit 1 + fi + if grep -nE '${holePattern}' "$src/$f"; then + echo "carbonado: proof hole (sorry/admit) found in $f" >&2 + exit 1 + fi + done + mkdir -p $out + echo ok > $out/result + ''; + + toolingPurity = import ./nix/tooling-purity.nix { + inherit pkgs; + src = productSrc; + }; + + # Run AOT binary as a check (constants + EtM + FEC + Bao + pipeline + Program F). + demo = + pkgs.runCommand "carbonado-demo" { + nativeBuildInputs = [leanPkg.executable]; + } '' + set -euo pipefail + ${leanPkg.executable}/bin/carbonado | tee $out + grep -q "scaffold constants ok" $out + grep -q "headerLen = 177" $out + grep -q "sliceLen = 4096" $out + grep -q "leafBytes = 4096" $out + grep -q "verificationContext = carbonado-v2/verification" $out + grep -q "sample public c14 format byte = 14" $out + grep -q "sample encrypted c15 format byte = 15" $out + grep -q "fecK = 4 fecM = 8 stripeUnit = 16384" $out + grep -q "sha512 goldens ok" $out + grep -q "hmac goldens ok" $out + grep -q "aes-ctr nist golden ok" $out + grep -q "subkey goldens ok" $out + grep -q "etm header-path goldens + roundtrip ok" $out + grep -q "etm low-level layout ok" $out + grep -q "tampered tag → authenticationFailed ok" $out + grep -q "wrong key → authenticationFailed ok" $out + grep -q "short ciphertext → invalidCiphertextLength ok" $out + grep -q "short master → invalidKeyLength ok" $out + grep -q "bad nonce → invalidNonceLength ok" $out + grep -q "ct body tamper → authenticationFailed ok" $out + grep -q "header mac goldens ok" $out + grep -q "header mac verify false path ok" $out + grep -q "etm stack ok" $out + grep -q "gf goldens ok" $out + grep -q "padding geometry ok" $out + grep -q "rs encode/reconstruct goldens ok" $out + grep -q "inboard hello roundtrip + knockout ok" $out + grep -q "inboard pattern roundtrip ok" $out + grep -q "unevenShards ok" $out + grep -q "tooFewShards ok" $out + grep -q "emptyShard ok" $out + grep -q "incorrectShardSize ok" $out + grep -q "badGeometry ok" $out + grep -q "paddingTooLarge ok" $out + grep -q "singularMatrix ok" $out + grep -q "encode/new guards ok" $out + grep -q "verify good/bad ok" $out + grep -q "knockout oob badGeometry ok" $out + grep -q "fec stack ok" $out + grep -q "blake3 goldens ok" $out + grep -q "verification key goldens ok" $out + grep -q "keyed root goldens ok" $out + grep -q "inboard encode/decode ok" $out + grep -q "outboard encode/verify ok" $out + grep -q "slice encode/stream-decode ok" $out + grep -q "three-leaf tree ok" $out + grep -q "wrong format key → authenticationFailed ok" $out + grep -q "slice wrong key → authenticationFailed ok" $out + grep -q "truncated response → truncatedResponse ok" $out + grep -q "truncated slice → truncatedResponse ok" $out + grep -q "trailing data → trailingData ok" $out + grep -q "slice trailing data → trailingData ok" $out + grep -q "short prefix → invalidPrefix ok" $out + grep -q "bad root length → invalidRootLength ok" $out + grep -q "bad slice index → invalidSliceIndex ok" $out + grep -q "slice count 0 → invalidSliceCount ok" $out + grep -q "tampered body → authenticationFailed ok" $out + grep -q "tampered slice → authenticationFailed ok" $out + grep -q "bao stack ok" $out + grep -q "header wire + verify ok" $out + grep -q "badMagic → HeaderError/PipelineError.badMagic ok" $out + grep -q "short header → invalidHeaderLength ok" $out + grep -q "invalidFieldLength ok" $out + grep -q "format matrix c0–c15 roundtrip ok" $out + grep -q "headered + c12/c15 roundtrip ok" $out + grep -q "encoded_len truncatedBody + trailer ignore ok" $out + grep -q "payload tamper → payloadAuthenticationFailed ok" $out + grep -q "composition invalidCiphertextLength + paddingTooLarge + bao trunc ok" $out + grep -q "pipeline invalidNonceLength + invalidKeyLength ok" $out + grep -q "wrong bao root → baoAuthenticationFailed ok" $out + grep -q "fecDecodeStep uneven → unevenShards ok" $out + grep -q "PipelineError taxonomy maps ok" $out + grep -q "stream stripe bounds ok" $out + grep -q "scrubRequiresVerification ok" $out + grep -q "unnecessaryScrub ok" $out + grep -q "scrub knockout recovery + invalidScrubbedHash ok" $out + grep -q "shard roundtrip + sequence errors ok" $out + grep -q "encrypted formats odd ok" $out + grep -q "pipeline stack ok" $out + # Program F + grep -q "zstd status mapping ok" $out + grep -q "PipelineError zstd maps ok" $out + grep -q "zstd goldens + roundtrip + error paths ok" $out + grep -q "pipeline compression formats c2/c6 + headered c3/c7 ok" $out + grep -q "SLH1 wire framing ok" $out + grep -q "SLH bind-to-root + unavailable sign ok" $out + grep -q "program F stack ok" $out + # Program G + grep -q "adamantine wire ok" $out + grep -q "filepack path rules ok" $out + grep -q "outboard segment roundtrip ok" $out + grep -q "directory pure encode/decode ok" $out + grep -q "directory exact failure modes ok" $out + grep -q "directory error taxonomy ok" $out + grep -q "program G stack ok" $out + grep -q "version = lean-program-g-0" $out + ''; + + # Strip only with GNU strip (Linux). Darwin strip rejects --strip-unneeded. + carbonadoRelease = + if pkgs.stdenv.isLinux + then + pkgs.runCommand "carbonado-release" { + nativeBuildInputs = [pkgs.binutils]; + } '' + set -euo pipefail + mkdir -p $out/bin + cp ${leanPkg.executable}/bin/carbonado $out/bin/carbonado + chmod u+w $out/bin/carbonado + strip --strip-unneeded $out/bin/carbonado + '' + else + pkgs.runCommand "carbonado-release" {} '' + set -euo pipefail + mkdir -p $out/bin + cp ${leanPkg.executable}/bin/carbonado $out/bin/carbonado + # Darwin/BSD strip does not use GNU long options; ship unstripped release. + echo "carbonado-release: non-Linux host; leaving binary unstripped" >&2 + ''; + in { + _module.args.pkgs = import nixpkgs { + inherit system; + overlays = [(lean4-nix.readToolchainFile ./lean-toolchain)]; + }; + + # Dual-backend: static lib + header for Rust `backend-lean` / carbonado-sys. + libcarbonado = + pkgs.runCommand "libcarbonado" {} '' + set -euo pipefail + mkdir -p $out/lib $out/include + cp ${carbonadoNative}/libcarbonado_native.a $out/lib/libcarbonado.a + cp ${carbonadoNative}/include/carbonado.h $out/include/ + cp ${carbonadoNative}/libcarbonado_native.a $out/lib/libcarbonado_native.a + ''; + + leanAbiCheck = + pkgs.runCommand "carbonado-lean-abi" { + nativeBuildInputs = [pkgs.binutils]; + } '' + set -euo pipefail + test -f ${libcarbonado}/include/carbonado.h + test -f ${libcarbonado}/lib/libcarbonado.a + # Symbols from C ABI stubs (encode may be weak NOT_IMPLEMENTED). + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_abi_version + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_free + echo ok > $out + ''; + + packages = { + default = leanPkg.executable; + carbonado = leanPkg.executable; + carbonado-release = carbonadoRelease; + libcarbonado = libcarbonado; + }; + + apps.default = { + type = "app"; + program = "${leanPkg.executable}/bin/carbonado"; + meta.description = "Carbonado Lean 4 AOT product binary (Programs A–G: Adamantine dirs + CLI)"; + }; + + checks = { + no-sorry = noSorry; + tooling-purity = toolingPurity; + demo = demo; + # Building the package is itself a check of Lean compile (includes CarbonadoTest roots). + carbonado = leanPkg.executable; + lean-abi = leanAbiCheck; + }; + + devShells.default = pkgs.mkShell { + packages = with pkgs; [ + cacert + git + gnupg + ripgrep + scc + # Host elan/lake may be used; lean4-nix provides leanc via package builds. + ]; + shellHook = '' + export SSL_CERT_FILE="${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + export GIT_SSL_CAINFO="$SSL_CERT_FILE" + if [ -f lean-toolchain ]; then + echo "carbonado Lean 4 + Nix dev shell" + echo "Lean toolchain pin: $(cat lean-toolchain)" + echo "Build: nix build .#carbonado" + echo "Check: nix flake check" + echo "Run: nix run" + fi + ''; + }; + }; + }; +} diff --git a/include/carbonado.h b/include/carbonado.h new file mode 100644 index 0000000..633d9e3 --- /dev/null +++ b/include/carbonado.h @@ -0,0 +1,88 @@ +/** + * carbonado C ABI — Lean AOT engine (libcarbonado) + * + * See docs/ABI.md for ownership, error codes, and versioning. + * ABI version 1 (v0 surface). + */ +#ifndef CARBONADO_H +#define CARBONADO_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define CARBONADO_ABI_VERSION 1u + +#define CARBONADO_OK 0 +#define CARBONADO_ERR_INVALID_ARGUMENT 1 +#define CARBONADO_ERR_INVALID_KEY_LENGTH 2 +#define CARBONADO_ERR_AUTHENTICATION 3 +#define CARBONADO_ERR_INVALID_MAGIC 4 +#define CARBONADO_ERR_INVALID_HEADER 5 +#define CARBONADO_ERR_FEC 6 +#define CARBONADO_ERR_BAO 7 +#define CARBONADO_ERR_ZSTD 8 +#define CARBONADO_ERR_SCRUB_UNNECESSARY 9 +#define CARBONADO_ERR_SCRUB_FAILED 10 +#define CARBONADO_ERR_NOT_IMPLEMENTED 11 +#define CARBONADO_ERR_INTERNAL 12 + +/** Returns CARBONADO_ABI_VERSION. */ +uint32_t carbonado_abi_version(void); + +/** Free a buffer returned by libcarbonado (malloc family). */ +void carbonado_free(void *p); + +/** + * Low-level encode (Rust encoding::encode body shape). + * On success: *out is malloc'd body, hash_out is 32-byte Bao root. + * Encrypted formats require nonce_len == 16. + */ +int carbonado_encode( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, + uint8_t **out, size_t *out_len, + uint8_t hash_out[32]); + +/** + * Low-level decode of a verifiable body (hash + padding + format). + */ +int carbonado_decode( + const uint8_t *master, size_t master_len, + const uint8_t *hash, size_t hash_len, + const uint8_t *body, size_t body_len, + uint32_t padding, + uint8_t format, + uint8_t **out, size_t *out_len); + +/** + * Headered encode: full file Header || body (Rust file::encode shape). + */ +int carbonado_encode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, + uint8_t **out, size_t *out_len); + +/** + * Headered decode: full file archive → plaintext. + */ +int carbonado_decode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *archive, size_t archive_len, + uint8_t **out, size_t *out_len); + +/** Format-keyed verification key (32 bytes). */ +int carbonado_verification_key(uint8_t format, uint8_t key_out[32]); + +#ifdef __cplusplus +} +#endif + +#endif /* CARBONADO_H */ diff --git a/lake-manifest.json b/lake-manifest.json new file mode 100644 index 0000000..20b28fb --- /dev/null +++ b/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.2.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "carbonado", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/lakefile.toml b/lakefile.toml new file mode 100644 index 0000000..9afe999 --- /dev/null +++ b/lakefile.toml @@ -0,0 +1,13 @@ +name = "carbonado" +version = "0.1.0" +defaultTargets = ["carbonado"] + +[[lean_lib]] +name = "Carbonado" + +[[lean_lib]] +name = "CarbonadoTest" + +[[lean_exe]] +name = "carbonado" +root = "Carbonado.Main" diff --git a/lean-toolchain b/lean-toolchain new file mode 100644 index 0000000..af9e5d3 --- /dev/null +++ b/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.30.0 diff --git a/nix/native/carbonado_abi.c b/nix/native/carbonado_abi.c new file mode 100644 index 0000000..3b7d713 --- /dev/null +++ b/nix/native/carbonado_abi.c @@ -0,0 +1,73 @@ +/** + * C ABI surface for libcarbonado (docs/ABI.md, include/carbonado.h). + * + * Phase 1: version + free + thin wrappers. Full encode/decode path is driven from + * Lean `@[export]` symbols when linked into the AOT image; until those symbols are + * part of the shared static archive used by carbonado-sys, encode/decode return + * CARBONADO_ERR_NOT_IMPLEMENTED so Rust can fail closed instead of linking garbage. + */ +#include +#include +#include + +#include "carbonado.h" + +uint32_t carbonado_abi_version(void) { + return CARBONADO_ABI_VERSION; +} + +void carbonado_free(void *p) { + free(p); +} + +/* Weak stubs: real implementations may be provided by Lean @[export] objects + * when the full static archive is linked. These provide a defined symbol so + * partial links still resolve. */ +__attribute__((weak)) int carbonado_encode( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, + uint8_t **out, size_t *out_len, + uint8_t hash_out[32]) { + (void)master; (void)master_len; (void)plaintext; (void)plaintext_len; + (void)format; (void)nonce; (void)nonce_len; (void)out; (void)out_len; (void)hash_out; + return CARBONADO_ERR_NOT_IMPLEMENTED; +} + +__attribute__((weak)) int carbonado_decode( + const uint8_t *master, size_t master_len, + const uint8_t *hash, size_t hash_len, + const uint8_t *body, size_t body_len, + uint32_t padding, + uint8_t format, + uint8_t **out, size_t *out_len) { + (void)master; (void)master_len; (void)hash; (void)hash_len; + (void)body; (void)body_len; (void)padding; (void)format; (void)out; (void)out_len; + return CARBONADO_ERR_NOT_IMPLEMENTED; +} + +__attribute__((weak)) int carbonado_encode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, + uint8_t **out, size_t *out_len) { + (void)master; (void)master_len; (void)plaintext; (void)plaintext_len; + (void)format; (void)nonce; (void)nonce_len; (void)out; (void)out_len; + return CARBONADO_ERR_NOT_IMPLEMENTED; +} + +__attribute__((weak)) int carbonado_decode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *archive, size_t archive_len, + uint8_t **out, size_t *out_len) { + (void)master; (void)master_len; (void)archive; (void)archive_len; + (void)out; (void)out_len; + return CARBONADO_ERR_NOT_IMPLEMENTED; +} + +__attribute__((weak)) int carbonado_verification_key(uint8_t format, uint8_t key_out[32]) { + (void)format; (void)key_out; + return CARBONADO_ERR_NOT_IMPLEMENTED; +} diff --git a/nix/native/carbonado_zstd.c b/nix/native/carbonado_zstd.c new file mode 100644 index 0000000..da2a66c --- /dev/null +++ b/nix/native/carbonado_zstd.c @@ -0,0 +1,158 @@ +/* + * Carbonado zstd FFI (Program F). + * + * Wire format to Lean: ByteArray = [status_u8 | payload...] + * status 0 = success; payload is compressed/decompressed bytes + * status 1 = compression failed + * status 2 = decompression failed + * status 3 = decompressed output exceeds max + * status 4 = invalid input / size error + * + * Linked into the AOT product via flake staticLibDeps: + * libcarbonado_native.a = this FFI + static libzstd objects from ref/zstd. + * No shared -lzstd. Lean elaborator uses the `@[extern]` body (identity fallback). + */ +#include +#include +#include +#include +#include + +/* Match Rust MAX_SEGMENT_MAIN_LEN (filepack_manifest). */ +#ifndef CARBONADO_ZSTD_MAX_DECOMPRESSED +#define CARBONADO_ZSTD_MAX_DECOMPRESSED ((size_t)256 * 1024 * 1024) +#endif + +enum { + ST_OK = 0, + ST_COMPRESS_FAILED = 1, + ST_DECOMPRESS_FAILED = 2, + ST_OUTPUT_TOO_LARGE = 3, + ST_INVALID_INPUT = 4 +}; + +static lean_obj_res mk_status(uint8_t status, const uint8_t *payload, size_t payload_len) { + size_t total = 1 + payload_len; + lean_obj_res out = lean_alloc_sarray(1, total, total); + uint8_t *p = lean_sarray_cptr(out); + p[0] = status; + if (payload_len > 0 && payload != NULL) { + memcpy(p + 1, payload, payload_len); + } + return out; +} + +static lean_obj_res mk_status_only(uint8_t status) { + return mk_status(status, NULL, 0); +} + +/* + * carbonado_zstd_compress : @& ByteArray → UInt32 → ByteArray + * Lean calling convention: borrowed ByteArray, UInt32 by value. + */ +LEAN_EXPORT lean_obj_res carbonado_zstd_compress(b_lean_obj_arg input, uint32_t level) { + size_t in_size = lean_sarray_size(input); + const void *in_data = lean_sarray_cptr(input); + + if (level > 22u) { + return mk_status_only(ST_INVALID_INPUT); + } + + size_t bound = ZSTD_compressBound(in_size); + if (bound == 0 && in_size > 0) { + return mk_status_only(ST_COMPRESS_FAILED); + } + + /* Allocate payload buffer (status byte added later). */ + void *buf = malloc(bound == 0 ? 1 : bound); + if (buf == NULL) { + return mk_status_only(ST_COMPRESS_FAILED); + } + + size_t n = ZSTD_compress(buf, bound == 0 ? 1 : bound, in_data, in_size, (int)level); + if (ZSTD_isError(n)) { + free(buf); + return mk_status_only(ST_COMPRESS_FAILED); + } + + lean_obj_res out = mk_status(ST_OK, (const uint8_t *)buf, n); + free(buf); + return out; +} + +/* + * carbonado_zstd_decompress : @& ByteArray → UInt64 → ByteArray + * max_out caps decompressed size (DoS guard); 0 means use CARBONADO_ZSTD_MAX_DECOMPRESSED. + */ +LEAN_EXPORT lean_obj_res carbonado_zstd_decompress(b_lean_obj_arg input, uint64_t max_out) { + size_t in_size = lean_sarray_size(input); + const void *in_data = lean_sarray_cptr(input); + + size_t cap = max_out == 0 ? CARBONADO_ZSTD_MAX_DECOMPRESSED : (size_t)max_out; + if (cap == 0) { + return mk_status_only(ST_INVALID_INPUT); + } + + unsigned long long frame_size = ZSTD_getFrameContentSize(in_data, in_size); + if (frame_size == ZSTD_CONTENTSIZE_ERROR) { + return mk_status_only(ST_DECOMPRESS_FAILED); + } + + size_t out_cap; + if (frame_size != ZSTD_CONTENTSIZE_UNKNOWN) { + if (frame_size > cap) { + return mk_status_only(ST_OUTPUT_TOO_LARGE); + } + out_cap = (size_t)frame_size; + if (out_cap == 0) { + /* Empty content still needs a successful decompress of the frame. */ + out_cap = 1; + } + } else { + /* Unknown size: grow heuristically up to cap. */ + out_cap = in_size * 3 + 64; + if (out_cap > cap) { + out_cap = cap; + } + if (out_cap == 0) { + out_cap = 1; + } + } + + void *buf = malloc(out_cap); + if (buf == NULL) { + return mk_status_only(ST_DECOMPRESS_FAILED); + } + + size_t n = ZSTD_decompress(buf, out_cap, in_data, in_size); + if (ZSTD_isError(n)) { + /* Retry with larger buffer if content size was unknown and we under-allocated. */ + if (frame_size == ZSTD_CONTENTSIZE_UNKNOWN && out_cap < cap) { + free(buf); + out_cap = cap; + buf = malloc(out_cap); + if (buf == NULL) { + return mk_status_only(ST_DECOMPRESS_FAILED); + } + n = ZSTD_decompress(buf, out_cap, in_data, in_size); + if (ZSTD_isError(n)) { + free(buf); + /* Distinguish capacity vs corrupt when possible. */ + if (ZSTD_getErrorCode(n) == ZSTD_error_dstSize_tooSmall) { + return mk_status_only(ST_OUTPUT_TOO_LARGE); + } + return mk_status_only(ST_DECOMPRESS_FAILED); + } + } else { + free(buf); + if (ZSTD_getErrorCode(n) == ZSTD_error_dstSize_tooSmall) { + return mk_status_only(ST_OUTPUT_TOO_LARGE); + } + return mk_status_only(ST_DECOMPRESS_FAILED); + } + } + + lean_obj_res out = mk_status(ST_OK, (const uint8_t *)buf, n); + free(buf); + return out; +} diff --git a/nix/native/default.nix b/nix/native/default.nix new file mode 100644 index 0000000..e3c8a7a --- /dev/null +++ b/nix/native/default.nix @@ -0,0 +1,100 @@ +# Static FFI glue for Carbonado AOT (zstd wrappers + libzstd objects). +# Output: $out/libcarbonado_native.a (linked via buildLeanPackage.staticLibDeps). +# +# Embeds single-threaded libzstd from the **pinned** `ref/zstd` tree (v1.5.7) +# so product frames track the git submodule SSOT — not a floating nixpkgs.src. +# Static archive only (no shared -lzstd / pthread shlib issues under lld). +{ + pkgs, + leanAll, # pkgs.lean.lean-all — provides lean/lean.h + zstdSrc, # flake: ./ref/zstd (must be checked-out submodule pin) + carbonadoInclude ? ../.. + "/include", # repo include/carbonado.h (ABI) +}: +pkgs.stdenv.mkDerivation { + pname = "carbonado-native"; + version = "0.1.0"; + src = ./.; + + nativeBuildInputs = [pkgs.binutils]; + + # Fail-closed: every .c compile must succeed (no `|| true` / silent stderr). + buildPhase = '' + runHook preBuild + set -euo pipefail + + ZSTD_LIB="${zstdSrc}/lib" + ABI_INC="${carbonadoInclude}" + if [ ! -f "$ABI_INC/carbonado.h" ]; then + echo "carbonado-native: missing $ABI_INC/carbonado.h" >&2 + exit 1 + fi + if [ ! -d "$ZSTD_LIB" ]; then + echo "carbonado-native: missing zstd lib dir at $ZSTD_LIB (init ref/zstd submodule)" >&2 + exit 1 + fi + if [ ! -f "$ZSTD_LIB/zstd.h" ]; then + echo "carbonado-native: missing $ZSTD_LIB/zstd.h" >&2 + exit 1 + fi + + # Portable single-thread objects (no assembly). Explicit loops — fail on first error. + compile_dir() { + local dir="$1" + local f base + for f in "$dir"/*.c; do + [ -f "$f" ] || continue + base=$(basename "$f" .c) + echo " CC $base.c" + $CC -c -O2 -fPIC -DZSTD_DISABLE_ASM \ + -I"$ZSTD_LIB" -I"$ZSTD_LIB/common" \ + "$f" -o "$base.o" + done + } + + echo "carbonado-native: compiling libzstd (common/compress/decompress) from ref pin" + compile_dir "$ZSTD_LIB/common" + compile_dir "$ZSTD_LIB/compress" + compile_dir "$ZSTD_LIB/decompress" + # dictBuilder not required for ZSTD_compress / ZSTD_decompress buffer API. + + echo "carbonado-native: compiling carbonado_zstd.c" + $CC -c -O2 -fPIC \ + -I${leanAll}/include \ + -I"$ZSTD_LIB" \ + carbonado_zstd.c \ + -o carbonado_zstd.o + + echo "carbonado-native: compiling carbonado_abi.c (C ABI v0 stubs)" + $CC -c -O2 -fPIC \ + -I"$ABI_INC" \ + carbonado_abi.c \ + -o carbonado_abi.o + + # Fail-closed: must have more than just the FFI object. + ocount=$(ls -1 ./*.o 2>/dev/null | wc -l) + if [ "$ocount" -lt 10 ]; then + echo "carbonado-native: expected many zstd objects, found $ocount" >&2 + ls -la ./*.o >&2 || true + exit 1 + fi + + ar rcs libcarbonado_native.a ./*.o + echo "carbonado-native: archived $ocount objects → libcarbonado_native.a" + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + # lean4-nix staticLibDeps expects $out/libcarbonado_native.a (archive root). + mkdir -p $out/lib $out/include + cp libcarbonado_native.a $out/ + cp libcarbonado_native.a $out/lib/ + ln -sf libcarbonado_native.a $out/lib/libcarbonado.a + cp "${carbonadoInclude}/carbonado.h" $out/include/ + runHook postInstall + ''; + + meta = { + description = "Carbonado Lean AOT native glue (static zstd + C ABI stubs)"; + }; +} diff --git a/nix/tooling-purity.nix b/nix/tooling-purity.nix new file mode 100644 index 0000000..802f174 --- /dev/null +++ b/nix/tooling-purity.nix @@ -0,0 +1,96 @@ +# checks.tooling-purity — product Lean/Nix tree must not grow non-ref impurity. +# Transition: existing Rust under src/, tests/, benches/, examples/ is legacy product +# until moved to ref/carbonado-rust. This check: +# * requires product Lean roots to exist and be Lean-only +# * bans product shell/python glue outside nix/ and ref/ +# * allowlists known top-level roots (transitional Rust included) +{ pkgs, src }: +pkgs.runCommand "carbonado-tooling-purity" { + inherit src; + nativeBuildInputs = [pkgs.findutils pkgs.gnugrep pkgs.coreutils]; +} '' + set -euo pipefail + cd "$src" + + # Fail-closed: product Lean trees must be present in productSrc. + for dir in Carbonado CarbonadoTest; do + if [ ! -d "$dir" ]; then + echo "carbonado tooling-purity: missing required directory $dir/" >&2 + exit 1 + fi + lean_count=$(find "$dir" -type f -name '*.lean' | wc -l) + if [ "$lean_count" -lt 1 ]; then + echo "carbonado tooling-purity: $dir/ has no .lean files" >&2 + exit 1 + fi + bad=$(find "$dir" -type f ! -name '*.lean' 2>/dev/null || true) + if [ -n "''${bad}" ]; then + echo "carbonado tooling-purity: non-Lean files under $dir/:" >&2 + echo "$bad" >&2 + exit 1 + fi + done + + for f in Carbonado.lean CarbonadoTest.lean; do + if [ ! -f "$f" ]; then + echo "carbonado tooling-purity: missing root module $f" >&2 + exit 1 + fi + done + + # Forbidden product glue paths (use Nix or ref/). + if [ -d scripts ] || [ -d tools ]; then + echo "carbonado tooling-purity: scripts/ and tools/ are forbidden product roots" >&2 + exit 1 + fi + for f in Makefile; do + if [ -e "$f" ]; then + echo "carbonado tooling-purity: forbidden product path $f (use Nix or ref/)" >&2 + exit 1 + fi + done + # Root-level shell/python product glue (ref/ and nix/ may have their own). + sh_py=$(find . -maxdepth 1 -type f \( -name '*.sh' -o -name '*.py' \) 2>/dev/null || true) + if [ -n "''${sh_py}" ]; then + echo "carbonado tooling-purity: forbidden root shell/python product glue:" >&2 + echo "$sh_py" >&2 + exit 1 + fi + + # Positive allowlist for top-level names. + # Transitional Rust (src, tests, benches, examples, Cargo.*) until freeze. + # productSrc often excludes those; allowlist still names them for full-tree runs. + is_allowed() { + local base="$1" + case "$base" in + .|..) return 0 ;; + .git|.cargo|.github|.vscode|.gitignore|.gitmodules) return 0 ;; + Carbonado|CarbonadoTest|nix|docs|doc|ref|src|tests|benches|examples|target) return 0 ;; + Carbonado.lean|CarbonadoTest.lean) return 0 ;; + flake.nix|flake.lock|lean-toolchain|justfile) return 0 ;; + # Optional Lake manifest for local Lean IDE/`lake build` (Nix remains SSOT package). + lakefile.toml|lakefile.lean|lake-manifest.json) return 0 ;; + AGENTS.md|README.md|LICENSE|CHANGELOG.md|Cargo.toml|Cargo.lock) return 0 ;; + result|result-*) return 0 ;; + *.md) return 0 ;; # project notes / reviews at root + esac + # Dotfiles are tooling metadata, not product glue. + case "$base" in + .*) return 0 ;; + esac + return 1 + } + + for entry in * .[!.]* ..?*; do + [ -e "$entry" ] || continue + base=$(basename "$entry") + if ! is_allowed "$base"; then + echo "carbonado tooling-purity: unexpected top-level product path: $base" >&2 + echo " (allowlist is intentional; move glue to nix/ or ref/, or update allowlist)" >&2 + exit 1 + fi + done + + mkdir -p $out + echo ok > $out/result +'' diff --git a/ref/README.md b/ref/README.md new file mode 100644 index 0000000..66afad5 --- /dev/null +++ b/ref/README.md @@ -0,0 +1,29 @@ +# `ref/` — reference implementations (not product source) + +Any language. Used to **prove and bit-match** Lean AOT analogues (verik1 / beastdb pattern). + +Product code lives only under `Carbonado/` (Lean) and is built with Nix flakes. + +See [docs/PARITY.md](../docs/PARITY.md) for pin table and [docs/SPEC-MATRIX.md](../docs/SPEC-MATRIX.md) for coverage. + +## Submodules (see [docs/PARITY.md](../docs/PARITY.md) for SHAs) + +| Path | Purpose | Status | +|------|---------|--------| +| `bao-tree` | Surmount keyed Bao fork | **pinned** | +| `reed-solomon-erasure` | RS 4/8 | **pinned** | +| `rustcrypto-block-ciphers` | AES 0.8.4 | **pinned** | +| `rustcrypto-macs` | HMAC 0.12.1 | **pinned** | +| `rustcrypto-hashes` | SHA-2 0.10.9 | **pinned** | +| `blake3` | Hash / Bao leaves | **pinned** | +| `zstd` | Compression C (Nix-linked) | **pinned** | +| `bitcoinpqc` | SLH-DSA-SHA2-128s bindings | **pinned** | +| `carbonado-rust` | Frozen historical Rust product | pending freeze | +| `parity-harness/` | Compare drivers | skeleton (README) | +| `crates/` | crates.io vendors (e.g. `ctr` 0.9.2) | skeleton (README) | + +Initialize: + +```bash +git submodule update --init --recursive +``` diff --git a/ref/bao-tree b/ref/bao-tree new file mode 160000 index 0000000..02916e7 --- /dev/null +++ b/ref/bao-tree @@ -0,0 +1 @@ +Subproject commit 02916e784bb0afe0fd5a73c291c8c5335865e166 diff --git a/ref/bitcoinpqc b/ref/bitcoinpqc new file mode 160000 index 0000000..7936b56 --- /dev/null +++ b/ref/bitcoinpqc @@ -0,0 +1 @@ +Subproject commit 7936b56f15e86b6764947c9298215ecfe38b712b diff --git a/ref/blake3 b/ref/blake3 new file mode 160000 index 0000000..93a431c --- /dev/null +++ b/ref/blake3 @@ -0,0 +1 @@ +Subproject commit 93a431c78a52d7ccf0f366f106467f5070e6075e diff --git a/ref/crates/README.md b/ref/crates/README.md new file mode 100644 index 0000000..a98580e --- /dev/null +++ b/ref/crates/README.md @@ -0,0 +1,11 @@ +# `ref/crates` — crates.io-only vendors + +Some Cargo.lock pins are not git monorepo tags (or left the monorepo). Vendor exact +crates.io sources here when a Lean parity program needs them. + +| Crate | Version | Checksum (Cargo.lock) | Status | +|-------|---------|----------------------|--------| +| `ctr` | 0.9.2 | `0369ee1ad6718345…` | **vendored** (`ref/crates/ctr-0.9.2`, Program B) | + +Do not use crates.io at build time for product gates; pin trees under this directory +(or a submodule) so builds are hermetic. diff --git a/ref/crates/ctr-0.9.2/.cargo_vcs_info.json b/ref/crates/ctr-0.9.2/.cargo_vcs_info.json new file mode 100644 index 0000000..f1e1b03 --- /dev/null +++ b/ref/crates/ctr-0.9.2/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "65320945adb1b6f7290e61b033f9e04ccdb2581b" + }, + "path_in_vcs": "ctr" +} \ No newline at end of file diff --git a/ref/crates/ctr-0.9.2/CHANGELOG.md b/ref/crates/ctr-0.9.2/CHANGELOG.md new file mode 100644 index 0000000..b60f31d --- /dev/null +++ b/ref/crates/ctr-0.9.2/CHANGELOG.md @@ -0,0 +1,79 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## 0.9.2 (2022-09-30) +### Changed +- Implement `Clone` directly for `CtrCore`, so it would work with non-`Clone` flavors ([#24]) + +[#24]: https://github.com/RustCrypto/block-modes/pull/24 + +## 0.9.1 (2022-02-17) +### Fixed +- Minimal versions build ([#9]) + +[#9]: https://github.com/RustCrypto/block-modes/pull/9 + +## 0.9.0 (2022-02-10) +### Changed +- Update `cipher` dependency to v0.4 and move crate +to the [RustCrypto/block-modes] repository ([#2]) + +[#2]: https://github.com/RustCrypto/block-modes/pull/2 +[RustCrypto/block-modes]: https://github.com/RustCrypto/block-modes + +## 0.8.0 (2021-07-08) +### Changed +- Make implementation generic over block size (previously it +was generic only over 128-bit block ciphers). Breaking changes +in the `CtrFlavor` API. ([#252]). + +[#252]: https://github.com/RustCrypto/stream-ciphers/pull/252 + +## 0.7.0 (2020-04-29) +### Changed +- Generic implementation of CTR ([#195]) +- Removed `Ctr32LE` mask bit ([#197]) +- Bump `cipher` dependency to v0.3 ([#226]) + +[#195]: https://github.com/RustCrypto/stream-ciphers/pull/195 +[#197]: https://github.com/RustCrypto/stream-ciphers/pull/197 +[#226]: https://github.com/RustCrypto/stream-ciphers/pull/226 + +## 0.6.0 (2020-10-16) +### Added +- `Ctr32BE` and `Ctr32LE` ([#170]) + +### Changed +- Replace `block-cipher`/`stream-cipher` with `cipher` crate ([#177]) + +[#177]: https://github.com/RustCrypto/stream-ciphers/pull/177 +[#170]: https://github.com/RustCrypto/stream-ciphers/pull/170 + +## 0.5.0 (2020-08-26) +### Changed +- Bump `stream-cipher` dependency to v0.7, implement the `FromBlockCipher` trait ([#161], [#164]) + +[#161]: https://github.com/RustCrypto/stream-ciphers/pull/161 +[#164]: https://github.com/RustCrypto/stream-ciphers/pull/164 + +## 0.4.0 (2020-06-06) +### Changed +- Upgrade to the `stream-cipher` v0.4 crate ([#116], [#138]) +- Upgrade to Rust 2018 edition ([#116]) + +[#138]: https://github.com/RustCrypto/stream-ciphers/pull/138 +[#116]: https://github.com/RustCrypto/stream-ciphers/pull/121 + +## 0.3.2 (2019-03-11) + +## 0.3.0 (2018-11-01) + +## 0.2.0 (2018-10-13) + +## 0.1.1 (2018-10-13) + +## 0.1.0 (2018-07-30) diff --git a/ref/crates/ctr-0.9.2/Cargo.toml b/ref/crates/ctr-0.9.2/Cargo.toml new file mode 100644 index 0000000..2feab94 --- /dev/null +++ b/ref/crates/ctr-0.9.2/Cargo.toml @@ -0,0 +1,68 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.56" +name = "ctr" +version = "0.9.2" +authors = ["RustCrypto Developers"] +description = "CTR block modes of operation" +documentation = "https://docs.rs/ctr" +readme = "README.md" +keywords = [ + "crypto", + "block-mode", + "stream-cipher", + "ciphers", +] +categories = [ + "cryptography", + "no-std", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/RustCrypto/block-modes" +resolver = "1" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[dependencies.cipher] +version = "0.4.2" + +[dev-dependencies.aes] +version = "0.8" + +[dev-dependencies.cipher] +version = "0.4.2" +features = ["dev"] + +[dev-dependencies.hex-literal] +version = "0.3.3" + +[dev-dependencies.kuznyechik] +version = "0.8" + +[dev-dependencies.magma] +version = "0.8" + +[features] +alloc = ["cipher/alloc"] +block-padding = ["cipher/block-padding"] +std = [ + "cipher/std", + "alloc", +] +zeroize = ["cipher/zeroize"] diff --git a/ref/crates/ctr-0.9.2/Cargo.toml.orig b/ref/crates/ctr-0.9.2/Cargo.toml.orig new file mode 100644 index 0000000..5783fff --- /dev/null +++ b/ref/crates/ctr-0.9.2/Cargo.toml.orig @@ -0,0 +1,33 @@ +[package] +name = "ctr" +version = "0.9.2" +description = "CTR block modes of operation" +authors = ["RustCrypto Developers"] +license = "MIT OR Apache-2.0" +edition = "2021" +rust-version = "1.56" +readme = "README.md" +documentation = "https://docs.rs/ctr" +repository = "https://github.com/RustCrypto/block-modes" +keywords = ["crypto", "block-mode", "stream-cipher", "ciphers"] +categories = ["cryptography", "no-std"] + +[dependencies] +cipher = "0.4.2" + +[dev-dependencies] +aes = "0.8" +magma = "0.8" +kuznyechik = "0.8" +cipher = { version = "0.4.2", features = ["dev"] } +hex-literal = "0.3.3" + +[features] +alloc = ["cipher/alloc"] +std = ["cipher/std", "alloc"] +block-padding = ["cipher/block-padding"] +zeroize = ["cipher/zeroize"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/ref/crates/ctr-0.9.2/LICENSE-APACHE b/ref/crates/ctr-0.9.2/LICENSE-APACHE new file mode 100644 index 0000000..78173fa --- /dev/null +++ b/ref/crates/ctr-0.9.2/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/ref/crates/ctr-0.9.2/LICENSE-MIT b/ref/crates/ctr-0.9.2/LICENSE-MIT new file mode 100644 index 0000000..d19d409 --- /dev/null +++ b/ref/crates/ctr-0.9.2/LICENSE-MIT @@ -0,0 +1,26 @@ +Copyright (c) 2018-2022 RustCrypto Developers +Copyright (c) 2018 Artyom Pavlov + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/ref/crates/ctr-0.9.2/README.md b/ref/crates/ctr-0.9.2/README.md new file mode 100644 index 0000000..a2a3043 --- /dev/null +++ b/ref/crates/ctr-0.9.2/README.md @@ -0,0 +1,59 @@ +# RustCrypto: CTR + +[![crate][crate-image]][crate-link] +[![Docs][docs-image]][docs-link] +![Apache2/MIT licensed][license-image] +![Rust Version][rustc-image] +[![Project Chat][chat-image]][chat-link] +[![Build Status][build-image]][build-link] + +Generic implementation of the [Counter][CTR] (CTR) block cipher mode of operation. + + + +See [documentation][cipher-doc] of the `cipher` crate for additional information. + +## Minimum Supported Rust Version + +Rust **1.56** or higher. + +Minimum supported Rust version can be changed in the future, but it will be +done with a minor version bump. + +## SemVer Policy + +- All on-by-default features of this library are covered by SemVer +- MSRV is considered exempt from SemVer as noted above + +## License + +Licensed under either of: + + * [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + * [MIT license](http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. + +[//]: # (badges) + +[crate-image]: https://img.shields.io/crates/v/ctr.svg +[crate-link]: https://crates.io/crates/ctr +[docs-image]: https://docs.rs/ctr/badge.svg +[docs-link]: https://docs.rs/ctr/ +[license-image]: https://img.shields.io/badge/license-Apache2.0/MIT-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.56+-blue.svg +[chat-image]: https://img.shields.io/badge/zulip-join_chat-blue.svg +[chat-link]: https://rustcrypto.zulipchat.com/#narrow/stream/308460-block-modes +[build-image]: https://github.com/RustCrypto/block-modes/workflows/ctr/badge.svg?branch=master&event=push +[build-link]: https://github.com/RustCrypto/block-modes/actions?query=workflow%3Actr+branch%3Amaster + +[//]: # (general links) + +[CTR]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR) +[cipher-doc]: https://docs.rs/cipher/ diff --git a/ref/crates/ctr-0.9.2/benches/aes128.rs b/ref/crates/ctr-0.9.2/benches/aes128.rs new file mode 100644 index 0000000..ad30e34 --- /dev/null +++ b/ref/crates/ctr-0.9.2/benches/aes128.rs @@ -0,0 +1,26 @@ +#![feature(test)] +extern crate test; + +cipher::stream_cipher_bench!( + ctr::Ctr32LE; + ctr_32le_aes128_stream_bench1_16b 16; + ctr_32le_aes128_stream_bench2_256b 256; + ctr_32le_aes128_stream_bench3_1kib 1024; + ctr_32le_aes128_stream_bench4_16kib 16384; +); + +cipher::stream_cipher_bench!( + ctr::Ctr64LE; + ctr_64le_aes128_stream_bench1_16b 16; + ctr_64le_aes128_stream_bench2_256b 256; + ctr_64le_aes128_stream_bench3_1kib 1024; + ctr_64le_aes128_stream_bench4_16kib 16384; +); + +cipher::stream_cipher_bench!( + ctr::Ctr128BE; + ctr_128be_aes128_stream_bench1_16b 16; + ctr_128be_aes128_stream_bench2_256b 256; + ctr_128be_aes128_stream_bench3_1kib 1024; + ctr_128be_aes128_stream_bench4_16kib 16384; +); diff --git a/ref/crates/ctr-0.9.2/src/backend.rs b/ref/crates/ctr-0.9.2/src/backend.rs new file mode 100644 index 0000000..a94fd43 --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/backend.rs @@ -0,0 +1,83 @@ +use crate::CtrFlavor; +use cipher::{ + generic_array::ArrayLength, Block, BlockBackend, BlockClosure, BlockSizeUser, ParBlocks, + ParBlocksSizeUser, StreamBackend, StreamClosure, +}; + +struct Backend<'a, F, B> +where + F: CtrFlavor, + B: BlockBackend, +{ + ctr_nonce: &'a mut F::CtrNonce, + backend: &'a mut B, +} + +impl<'a, F, B> BlockSizeUser for Backend<'a, F, B> +where + F: CtrFlavor, + B: BlockBackend, +{ + type BlockSize = B::BlockSize; +} + +impl<'a, F, B> ParBlocksSizeUser for Backend<'a, F, B> +where + F: CtrFlavor, + B: BlockBackend, +{ + type ParBlocksSize = B::ParBlocksSize; +} + +impl<'a, F, B> StreamBackend for Backend<'a, F, B> +where + F: CtrFlavor, + B: BlockBackend, +{ + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + let tmp = F::next_block(self.ctr_nonce); + self.backend.proc_block((&tmp, block).into()); + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut ParBlocks) { + let mut tmp = ParBlocks::::default(); + for block in tmp.iter_mut() { + *block = F::next_block(self.ctr_nonce); + } + self.backend.proc_par_blocks((&tmp, blocks).into()); + } +} + +pub(crate) struct Closure<'a, F, BS, SC> +where + F: CtrFlavor, + BS: ArrayLength, + SC: StreamClosure, +{ + pub(crate) ctr_nonce: &'a mut F::CtrNonce, + pub(crate) f: SC, +} + +impl<'a, F, BS, SC> BlockSizeUser for Closure<'a, F, BS, SC> +where + F: CtrFlavor, + BS: ArrayLength, + SC: StreamClosure, +{ + type BlockSize = BS; +} + +impl<'a, F, BS, SC> BlockClosure for Closure<'a, F, BS, SC> +where + F: CtrFlavor, + BS: ArrayLength, + SC: StreamClosure, +{ + #[inline(always)] + fn call>(self, backend: &mut B) { + let Self { ctr_nonce, f } = self; + f.call(&mut Backend:: { ctr_nonce, backend }) + } +} diff --git a/ref/crates/ctr-0.9.2/src/ctr_core.rs b/ref/crates/ctr-0.9.2/src/ctr_core.rs new file mode 100644 index 0000000..c193596 --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/ctr_core.rs @@ -0,0 +1,156 @@ +use crate::{backend::Closure, CtrFlavor}; +use cipher::{ + crypto_common::{InnerUser, IvSizeUser}, + AlgorithmName, BlockCipher, BlockEncryptMut, BlockSizeUser, InnerIvInit, Iv, IvState, + StreamCipherCore, StreamCipherSeekCore, StreamClosure, +}; +use core::fmt; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::ZeroizeOnDrop; + +/// Generic CTR block mode instance. +pub struct CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + cipher: C, + ctr_nonce: F::CtrNonce, +} + +impl BlockSizeUser for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + type BlockSize = C::BlockSize; +} + +impl StreamCipherCore for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + #[inline] + fn remaining_blocks(&self) -> Option { + F::remaining(&self.ctr_nonce) + } + + #[inline] + fn process_with_backend(&mut self, f: impl StreamClosure) { + let Self { cipher, ctr_nonce } = self; + cipher.encrypt_with_backend_mut(Closure:: { ctr_nonce, f }); + } +} + +impl StreamCipherSeekCore for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + type Counter = F::Backend; + + #[inline] + fn get_block_pos(&self) -> Self::Counter { + F::as_backend(&self.ctr_nonce) + } + + #[inline] + fn set_block_pos(&mut self, pos: Self::Counter) { + F::set_from_backend(&mut self.ctr_nonce, pos); + } +} + +impl InnerUser for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + type Inner = C; +} + +impl IvSizeUser for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + type IvSize = C::BlockSize; +} + +impl InnerIvInit for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + #[inline] + fn inner_iv_init(cipher: C, iv: &Iv) -> Self { + Self { + cipher, + ctr_nonce: F::from_nonce(iv), + } + } +} + +impl IvState for CtrCore +where + C: BlockEncryptMut + BlockCipher, + F: CtrFlavor, +{ + #[inline] + fn iv_state(&self) -> Iv { + F::current_block(&self.ctr_nonce) + } +} + +impl AlgorithmName for CtrCore +where + C: BlockEncryptMut + BlockCipher + AlgorithmName, + F: CtrFlavor, +{ + fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Ctr")?; + f.write_str(F::NAME)?; + f.write_str("<")?; + ::write_alg_name(f)?; + f.write_str(">") + } +} + +impl Clone for CtrCore +where + C: BlockEncryptMut + BlockCipher + Clone, + F: CtrFlavor, +{ + #[inline] + fn clone(&self) -> Self { + Self { + cipher: self.cipher.clone(), + ctr_nonce: self.ctr_nonce.clone(), + } + } +} + +impl fmt::Debug for CtrCore +where + C: BlockEncryptMut + BlockCipher + AlgorithmName, + F: CtrFlavor, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Ctr")?; + f.write_str(F::NAME)?; + f.write_str("<")?; + ::write_alg_name(f)?; + f.write_str("> { ... }") + } +} + +#[cfg(feature = "zeroize")] +#[cfg_attr(docsrs, doc(cfg(feature = "zeroize")))] +impl ZeroizeOnDrop for CtrCore +where + C: BlockEncryptMut + BlockCipher + ZeroizeOnDrop, + F: CtrFlavor, + F::CtrNonce: ZeroizeOnDrop, +{ +} diff --git a/ref/crates/ctr-0.9.2/src/flavors.rs b/ref/crates/ctr-0.9.2/src/flavors.rs new file mode 100644 index 0000000..30361c8 --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/flavors.rs @@ -0,0 +1,44 @@ +//! CTR mode flavors + +use cipher::{ + generic_array::{ArrayLength, GenericArray}, + Counter, +}; + +mod ctr128; +mod ctr32; +mod ctr64; + +pub use ctr128::{Ctr128BE, Ctr128LE}; +pub use ctr32::{Ctr32BE, Ctr32LE}; +pub use ctr64::{Ctr64BE, Ctr64LE}; + +/// Trait implemented by different CTR flavors. +pub trait CtrFlavor> { + /// Inner representation of nonce. + type CtrNonce: Clone; + /// Backend numeric type + type Backend: Counter; + /// Flavor name + const NAME: &'static str; + + /// Return number of remaining blocks. + /// + /// If result does not fit into `usize`, returns `None`. + fn remaining(cn: &Self::CtrNonce) -> Option; + + /// Generate block for given `nonce` and current counter value. + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray; + + /// Generate block for given `nonce` and current counter value. + fn current_block(cn: &Self::CtrNonce) -> GenericArray; + + /// Initialize from bytes. + fn from_nonce(block: &GenericArray) -> Self::CtrNonce; + + /// Convert from a backend value + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend); + + /// Convert to a backend value + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend; +} diff --git a/ref/crates/ctr-0.9.2/src/flavors/ctr128.rs b/ref/crates/ctr-0.9.2/src/flavors/ctr128.rs new file mode 100644 index 0000000..9d10385 --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/flavors/ctr128.rs @@ -0,0 +1,158 @@ +//! 128-bit counter falvors. +use super::CtrFlavor; +use cipher::{ + generic_array::{ArrayLength, GenericArray}, + typenum::{PartialDiv, PartialQuot, Unsigned, U16}, +}; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; + +type ChunkSize = U16; +type Chunks = PartialQuot; +const CS: usize = ChunkSize::USIZE; + +#[derive(Clone)] +pub struct CtrNonce128> { + ctr: u128, + nonce: GenericArray, +} + +#[cfg(feature = "zeroize")] +impl> Drop for CtrNonce128 { + fn drop(&mut self) { + self.ctr.zeroize(); + self.nonce.zeroize(); + } +} + +#[cfg(feature = "zeroize")] +impl> ZeroizeOnDrop for CtrNonce128 {} + +/// 128-bit big endian counter flavor. +pub enum Ctr128BE {} + +impl CtrFlavor for Ctr128BE +where + B: ArrayLength + PartialDiv, + Chunks: ArrayLength, +{ + type CtrNonce = CtrNonce128>; + type Backend = u128; + const NAME: &'static str = "128BE"; + + #[inline] + fn remaining(cn: &Self::CtrNonce) -> Option { + (core::u128::MAX - cn.ctr).try_into().ok() + } + + #[inline(always)] + fn current_block(cn: &Self::CtrNonce) -> GenericArray { + let mut block = GenericArray::::default(); + for i in 0..Chunks::::USIZE { + let t = if i == Chunks::::USIZE - 1 { + cn.ctr.wrapping_add(cn.nonce[i]).to_be_bytes() + } else { + cn.nonce[i].to_ne_bytes() + }; + block[CS * i..][..CS].copy_from_slice(&t); + } + block + } + + #[inline] + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray { + let block = Self::current_block(cn); + cn.ctr = cn.ctr.wrapping_add(1); + block + } + + #[inline] + fn from_nonce(block: &GenericArray) -> Self::CtrNonce { + let mut nonce = GenericArray::>::default(); + for i in 0..Chunks::::USIZE { + let chunk = block[CS * i..][..CS].try_into().unwrap(); + nonce[i] = if i == Chunks::::USIZE - 1 { + u128::from_be_bytes(chunk) + } else { + u128::from_ne_bytes(chunk) + } + } + let ctr = 0; + Self::CtrNonce { ctr, nonce } + } + + #[inline] + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend { + cn.ctr + } + + #[inline] + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend) { + cn.ctr = v; + } +} + +/// 128-bit big endian counter flavor. +pub enum Ctr128LE {} + +impl CtrFlavor for Ctr128LE +where + B: ArrayLength + PartialDiv, + Chunks: ArrayLength, +{ + type CtrNonce = CtrNonce128>; + type Backend = u128; + const NAME: &'static str = "128LE"; + + #[inline] + fn remaining(cn: &Self::CtrNonce) -> Option { + (core::u128::MAX - cn.ctr).try_into().ok() + } + + #[inline(always)] + fn current_block(cn: &Self::CtrNonce) -> GenericArray { + let mut block = GenericArray::::default(); + for i in 0..Chunks::::USIZE { + let t = if i == 0 { + cn.ctr.wrapping_add(cn.nonce[i]).to_le_bytes() + } else { + cn.nonce[i].to_ne_bytes() + }; + block[CS * i..][..CS].copy_from_slice(&t); + } + block + } + + #[inline] + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray { + let block = Self::current_block(cn); + cn.ctr = cn.ctr.wrapping_add(1); + block + } + + #[inline] + fn from_nonce(block: &GenericArray) -> Self::CtrNonce { + let mut nonce = GenericArray::>::default(); + for i in 0..Chunks::::USIZE { + let chunk = block[CS * i..][..CS].try_into().unwrap(); + nonce[i] = if i == 0 { + u128::from_le_bytes(chunk) + } else { + u128::from_ne_bytes(chunk) + } + } + let ctr = 0; + Self::CtrNonce { ctr, nonce } + } + + #[inline] + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend { + cn.ctr + } + + #[inline] + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend) { + cn.ctr = v; + } +} diff --git a/ref/crates/ctr-0.9.2/src/flavors/ctr32.rs b/ref/crates/ctr-0.9.2/src/flavors/ctr32.rs new file mode 100644 index 0000000..17cb011 --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/flavors/ctr32.rs @@ -0,0 +1,158 @@ +//! 32-bit counter falvors. +use super::CtrFlavor; +use cipher::{ + generic_array::{ArrayLength, GenericArray}, + typenum::{PartialDiv, PartialQuot, Unsigned, U4}, +}; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; + +type ChunkSize = U4; +type Chunks = PartialQuot; +const CS: usize = ChunkSize::USIZE; + +#[derive(Clone)] +pub struct CtrNonce32> { + ctr: u32, + nonce: GenericArray, +} + +#[cfg(feature = "zeroize")] +impl> Drop for CtrNonce32 { + fn drop(&mut self) { + self.ctr.zeroize(); + self.nonce.zeroize(); + } +} + +#[cfg(feature = "zeroize")] +impl> ZeroizeOnDrop for CtrNonce32 {} + +/// 32-bit big endian counter flavor. +pub enum Ctr32BE {} + +impl CtrFlavor for Ctr32BE +where + B: ArrayLength + PartialDiv, + Chunks: ArrayLength, +{ + type CtrNonce = CtrNonce32>; + type Backend = u32; + const NAME: &'static str = "32BE"; + + #[inline] + fn remaining(cn: &Self::CtrNonce) -> Option { + (core::u32::MAX - cn.ctr).try_into().ok() + } + + #[inline(always)] + fn current_block(cn: &Self::CtrNonce) -> GenericArray { + let mut block = GenericArray::::default(); + for i in 0..Chunks::::USIZE { + let t = if i == Chunks::::USIZE - 1 { + cn.ctr.wrapping_add(cn.nonce[i]).to_be_bytes() + } else { + cn.nonce[i].to_ne_bytes() + }; + block[CS * i..][..CS].copy_from_slice(&t); + } + block + } + + #[inline] + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray { + let block = Self::current_block(cn); + cn.ctr = cn.ctr.wrapping_add(1); + block + } + + #[inline] + fn from_nonce(block: &GenericArray) -> Self::CtrNonce { + let mut nonce = GenericArray::>::default(); + for i in 0..Chunks::::USIZE { + let chunk = block[CS * i..][..CS].try_into().unwrap(); + nonce[i] = if i == Chunks::::USIZE - 1 { + u32::from_be_bytes(chunk) + } else { + u32::from_ne_bytes(chunk) + } + } + let ctr = 0; + Self::CtrNonce { ctr, nonce } + } + + #[inline] + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend { + cn.ctr + } + + #[inline] + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend) { + cn.ctr = v; + } +} + +/// 32-bit big endian counter flavor. +pub enum Ctr32LE {} + +impl CtrFlavor for Ctr32LE +where + B: ArrayLength + PartialDiv, + Chunks: ArrayLength, +{ + type CtrNonce = CtrNonce32>; + type Backend = u32; + const NAME: &'static str = "32LE"; + + #[inline] + fn remaining(cn: &Self::CtrNonce) -> Option { + (core::u32::MAX - cn.ctr).try_into().ok() + } + + #[inline(always)] + fn current_block(cn: &Self::CtrNonce) -> GenericArray { + let mut block = GenericArray::::default(); + for i in 0..Chunks::::USIZE { + let t = if i == 0 { + cn.ctr.wrapping_add(cn.nonce[i]).to_le_bytes() + } else { + cn.nonce[i].to_ne_bytes() + }; + block[CS * i..][..CS].copy_from_slice(&t); + } + block + } + + #[inline] + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray { + let block = Self::current_block(cn); + cn.ctr = cn.ctr.wrapping_add(1); + block + } + + #[inline] + fn from_nonce(block: &GenericArray) -> Self::CtrNonce { + let mut nonce = GenericArray::>::default(); + for i in 0..Chunks::::USIZE { + let chunk = block[CS * i..][..CS].try_into().unwrap(); + nonce[i] = if i == 0 { + u32::from_le_bytes(chunk) + } else { + u32::from_ne_bytes(chunk) + } + } + let ctr = 0; + Self::CtrNonce { ctr, nonce } + } + + #[inline] + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend { + cn.ctr + } + + #[inline] + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend) { + cn.ctr = v; + } +} diff --git a/ref/crates/ctr-0.9.2/src/flavors/ctr64.rs b/ref/crates/ctr-0.9.2/src/flavors/ctr64.rs new file mode 100644 index 0000000..3e63abe --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/flavors/ctr64.rs @@ -0,0 +1,158 @@ +//! 64-bit counter falvors. +use super::CtrFlavor; +use cipher::{ + generic_array::{ArrayLength, GenericArray}, + typenum::{PartialDiv, PartialQuot, Unsigned, U8}, +}; + +#[cfg(feature = "zeroize")] +use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; + +type ChunkSize = U8; +type Chunks = PartialQuot; +const CS: usize = ChunkSize::USIZE; + +#[derive(Clone)] +pub struct CtrNonce64> { + ctr: u64, + nonce: GenericArray, +} + +#[cfg(feature = "zeroize")] +impl> Drop for CtrNonce64 { + fn drop(&mut self) { + self.ctr.zeroize(); + self.nonce.zeroize(); + } +} + +#[cfg(feature = "zeroize")] +impl> ZeroizeOnDrop for CtrNonce64 {} + +/// 64-bit big endian counter flavor. +pub enum Ctr64BE {} + +impl CtrFlavor for Ctr64BE +where + B: ArrayLength + PartialDiv, + Chunks: ArrayLength, +{ + type CtrNonce = CtrNonce64>; + type Backend = u64; + const NAME: &'static str = "64BE"; + + #[inline] + fn remaining(cn: &Self::CtrNonce) -> Option { + (core::u64::MAX - cn.ctr).try_into().ok() + } + + #[inline(always)] + fn current_block(cn: &Self::CtrNonce) -> GenericArray { + let mut block = GenericArray::::default(); + for i in 0..Chunks::::USIZE { + let t = if i == Chunks::::USIZE - 1 { + cn.ctr.wrapping_add(cn.nonce[i]).to_be_bytes() + } else { + cn.nonce[i].to_ne_bytes() + }; + block[CS * i..][..CS].copy_from_slice(&t); + } + block + } + + #[inline] + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray { + let block = Self::current_block(cn); + cn.ctr = cn.ctr.wrapping_add(1); + block + } + + #[inline] + fn from_nonce(block: &GenericArray) -> Self::CtrNonce { + let mut nonce = GenericArray::>::default(); + for i in 0..Chunks::::USIZE { + let chunk = block[CS * i..][..CS].try_into().unwrap(); + nonce[i] = if i == Chunks::::USIZE - 1 { + u64::from_be_bytes(chunk) + } else { + u64::from_ne_bytes(chunk) + } + } + let ctr = 0; + Self::CtrNonce { ctr, nonce } + } + + #[inline] + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend { + cn.ctr + } + + #[inline] + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend) { + cn.ctr = v; + } +} + +/// 64-bit big endian counter flavor. +pub enum Ctr64LE {} + +impl CtrFlavor for Ctr64LE +where + B: ArrayLength + PartialDiv, + Chunks: ArrayLength, +{ + type CtrNonce = CtrNonce64>; + type Backend = u64; + const NAME: &'static str = "64LE"; + + #[inline] + fn remaining(cn: &Self::CtrNonce) -> Option { + (core::u64::MAX - cn.ctr).try_into().ok() + } + + #[inline(always)] + fn current_block(cn: &Self::CtrNonce) -> GenericArray { + let mut block = GenericArray::::default(); + for i in 0..Chunks::::USIZE { + let t = if i == 0 { + cn.ctr.wrapping_add(cn.nonce[i]).to_le_bytes() + } else { + cn.nonce[i].to_ne_bytes() + }; + block[CS * i..][..CS].copy_from_slice(&t); + } + block + } + + #[inline] + fn next_block(cn: &mut Self::CtrNonce) -> GenericArray { + let block = Self::current_block(cn); + cn.ctr = cn.ctr.wrapping_add(1); + block + } + + #[inline] + fn from_nonce(block: &GenericArray) -> Self::CtrNonce { + let mut nonce = GenericArray::>::default(); + for i in 0..Chunks::::USIZE { + let chunk = block[CS * i..][..CS].try_into().unwrap(); + nonce[i] = if i == 0 { + u64::from_le_bytes(chunk) + } else { + u64::from_ne_bytes(chunk) + } + } + let ctr = 0; + Self::CtrNonce { ctr, nonce } + } + + #[inline] + fn as_backend(cn: &Self::CtrNonce) -> Self::Backend { + cn.ctr + } + + #[inline] + fn set_from_backend(cn: &mut Self::CtrNonce, v: Self::Backend) { + cn.ctr = v; + } +} diff --git a/ref/crates/ctr-0.9.2/src/lib.rs b/ref/crates/ctr-0.9.2/src/lib.rs new file mode 100644 index 0000000..1199dbc --- /dev/null +++ b/ref/crates/ctr-0.9.2/src/lib.rs @@ -0,0 +1,90 @@ +//! Generic implementations of [CTR mode][1] for block ciphers. +//! +//! +//! +//! +//! Mode functionality is accessed using traits from re-exported [`cipher`] crate. +//! +//! # ⚠️ Security Warning: Hazmat! +//! +//! This crate does not ensure ciphertexts are authentic! Thus ciphertext integrity +//! is not verified, which can lead to serious vulnerabilities! +//! +//! # Example +//! ``` +//! use aes::cipher::{KeyIvInit, StreamCipher, StreamCipherSeek}; +//! use hex_literal::hex; +//! +//! type Aes128Ctr64LE = ctr::Ctr64LE; +//! +//! let key = [0x42; 16]; +//! let iv = [0x24; 16]; +//! let plaintext = *b"hello world! this is my plaintext."; +//! let ciphertext = hex!( +//! "3357121ebb5a29468bd861467596ce3da59bdee42dcc0614dea955368d8a5dc0cad4" +//! ); +//! +//! // encrypt in-place +//! let mut buf = plaintext.to_vec(); +//! let mut cipher = Aes128Ctr64LE::new(&key.into(), &iv.into()); +//! cipher.apply_keystream(&mut buf); +//! assert_eq!(buf[..], ciphertext[..]); +//! +//! // CTR mode can be used with streaming messages +//! let mut cipher = Aes128Ctr64LE::new(&key.into(), &iv.into()); +//! for chunk in buf.chunks_mut(3) { +//! cipher.apply_keystream(chunk); +//! } +//! assert_eq!(buf[..], plaintext[..]); +//! +//! // CTR mode supports seeking. The parameter is zero-based _bytes_ counter (not _blocks_). +//! cipher.seek(0u32); +//! +//! // encrypt/decrypt from buffer to buffer +//! // buffer length must be equal to input length +//! let mut buf1 = [0u8; 34]; +//! cipher +//! .apply_keystream_b2b(&plaintext, &mut buf1) +//! .unwrap(); +//! assert_eq!(buf1[..], ciphertext[..]); +//! +//! let mut buf2 = [0u8; 34]; +//! cipher.seek(0u32); +//! cipher.apply_keystream_b2b(&buf1, &mut buf2).unwrap(); +//! assert_eq!(buf2[..], plaintext[..]); +//! ``` +//! +//! [1]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#CTR + +#![no_std] +#![doc( + html_logo_url = "https://raw.githubusercontent.com/RustCrypto/media/6ee8e381/logo.svg", + html_favicon_url = "https://raw.githubusercontent.com/RustCrypto/media/6ee8e381/logo.svg" +)] +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![warn(missing_docs, rust_2018_idioms)] + +pub mod flavors; + +mod backend; +mod ctr_core; + +pub use cipher; +pub use flavors::CtrFlavor; + +use cipher::StreamCipherCoreWrapper; +pub use ctr_core::CtrCore; + +/// CTR mode with 128-bit big endian counter. +pub type Ctr128BE = StreamCipherCoreWrapper>; +/// CTR mode with 128-bit little endian counter. +pub type Ctr128LE = StreamCipherCoreWrapper>; +/// CTR mode with 64-bit big endian counter. +pub type Ctr64BE = StreamCipherCoreWrapper>; +/// CTR mode with 64-bit little endian counter. +pub type Ctr64LE = StreamCipherCoreWrapper>; +/// CTR mode with 32-bit big endian counter. +pub type Ctr32BE = StreamCipherCoreWrapper>; +/// CTR mode with 32-bit little endian counter. +pub type Ctr32LE = StreamCipherCoreWrapper>; diff --git a/ref/crates/ctr-0.9.2/tests/ctr128/data/aes128-ctr.blb b/ref/crates/ctr-0.9.2/tests/ctr128/data/aes128-ctr.blb new file mode 100644 index 0000000000000000000000000000000000000000..d721e4ec446b1b483a24b8d09cf2f5fbab81ce22 GIT binary patch literal 2039 zcmVH?%aNi8yBGE*I`MX`=^z^J|NF@kJUiIX=f(q8%1gUF{hWzy^!-5oQY^tuW<U?m#sZovnrI(**apt@qOP*f4v(mR;ceUOHo>@f7=FA3RgxlzRub}+@p2SE z0uenRz+gp=VKPP zvdnETAeRe(uJ=O*0=z=k)U}c9nP|@p*zhK%NuM2u z7EpLz6#s!+94J1}_>mn8-({Ms`vs9!UkWn7E$2mzP#m`txEX-4+AiN%8UnZ^0Qc!< zMNvmTX^74|oMbb1 zN;2LiQ3u`lso$7dK*|0T6`eqNM0T<-zmL}>&^Q$pcGCNl$63Om01TYPFq&}{VY2~U zoc2D^l7OlxF9+BfcB{bo71!uf_|wzZOK}=8yF}^I8@WfjXcN4NOD-<|n8c=aIw0#L zs4Mh7>BE@PU?)-PM9{7B6oTM79ENRR50qTl4*T$&S;VyVTxin>9w~DHY^I~Be0t>* za0$Jm1DJQalt6y#0E4i?`n*yt{ECVs0y*^Ea%Dk=t4)5N-+kHa2vJ9F{pGTN9CL^Q zr{yjrus&>UH!_ME6_=-Tv>)MD>pa=X)(&72g}Nfn9zQ<-%>jZvqiH4L?6(ElSj%uOc|?^gTWK#4cV}!9N#uO*(+z zDYjqEyPO;lV`*#~x^;^`VWuTtn0R>wJ~92RJwk=9fQ`&%)3JiU&f6yXbY|!!6kjd= zBVa&XIR_ln_$Hl@QDpk2fp~=Lg<F8Ov=5OD_Y$SJTicl=>+Uu?%$;$UIR709UZ0X)N+!th^a+R@30cThRdK=jTxE6j zx|IeDuZA$*(s&miYRP}^mOvfs_Davc(=tV4@!gt%@b(DRSwgW`rn8ED=Ci6_f9F|) z678#Iyh(wGU{2VoXq7(Q|6yr&fVZ__EP9J=bLfk_$$-GYIvvZgi-*{#QaBa!=lsc+ zDPC&T;o!q0{G$c0Z>KRH1;>ilI2o?LSLr_?`Rl{iozGqq0``I7ZVTIyCjTJ}7%$n$ zYbcO*riRuX8e}>nzly%X@iigB5z=1k)&9-lV^EXZwoR!AGWPptgPrmVmSw@3?dGPJ z-9ySuW_-O0Qq4%HkbVz6ZblP$rKHZGP8}orD`fc$n%TUj4zLeD0gI)owLXxfSZH6} zwoi3`BInXV_BL3ifM;;GFm>rNZ@+|Yt=YTEQ$=)tSjqgbhs@&}XsQ~QD8;S*~OoV3rygG{dU{aG^Wa4`2%$GKz$Iu6cz z9q?DNOb7L*f%%Z;s7jv6e9NTvrB6<&1od^Au9w~cyQpWnM_WhM=z}^ z16Pg0_r&YPS%5;q*hzA{P;vEFK!bH(0T)~>k{LM$x3c+Lj?iHp;>kaX_q zfT~L-@$g;{@C!gqvX8m^IaIylJJjSZhCGRA{5oW!ap{C+>6(8%e(KkqxnyjevE=7< VYVN$AuMsq5Y*Li)97ec1(!p2h`XB%R literal 0 HcmV?d00001 diff --git a/ref/crates/ctr-0.9.2/tests/ctr128/data/aes256-ctr.blb b/ref/crates/ctr-0.9.2/tests/ctr128/data/aes256-ctr.blb new file mode 100644 index 0000000000000000000000000000000000000000..47daaf84de9a0c16a03f2ad5c386108d2f241b62 GIT binary patch literal 2055 zcmV+i2>AB^K!E|&noS)YQo+U;QkV3>&EDlaF@Zr9!lb7W61EdBf*_5O)*U?$3!~rD z@+OvSk--Ys_KU}Z@{#X$a2V%NZ8IrjqwW&=HOc`LFHiEx@4+e?}MBA0+YFA zKqA)c5fRVn!(EDcZ(i|4#LuChX82Fubg`$8i`UjI9iA?X4;RKu^d3$!0PLXdq8d!22;*tgN2RyRC z{)dnXr_W%Q`;G6~6LRn)MzdhUq4J$MIxoaqO&t4TS_DO{7`~v$$*nk_fe^^B#C_LH z#-wobS>Jjk$FOsLi?N6#QoP4R=k9#8Hxdx9v7C`V08;8W$(OBNwfSJDbf&ng;wR=@ z2Kf7yBdlY2EovSk+AT8cG0kjCK>r4;vu zbwoRJCxI;8~y40+ZAq|Z~lr*)qLLjUy4GybsI2touX({J*NcnZzDd# z9}E90$o5;gKU&fnBP~Mf7Tczul4EUU7s2c3w8vHMA_3$t#C9YzTE?YWY(&wKJ1F;% z>0c}EH&fc>)7H_IOk#^ndHjkqTgqgZq^lj7fY#J26rQ`m*Iwrn$K>o1r;#WBW zg$?oiDqRBt5kkR(f3pM05Xbr6skkiN8snYLo+D!+Ez4gESWd~Y&9P`>KU;4U>uShd zG9#ZuijqN;tcE}@-ZHOk-gy}P_5ny#sBJ0Hp{EsHhmy=1M^ozRno1J8bT`NEy{xp~ zB4NLv%RfI5sE#-LM+MZn;<5Lpj!?2T5@kjYzY-?W=yWhKE|3Ik(bL{J=xsCG0TdqU zKBQIs1jp}GtoboN-3QnfkpI*x`euPFvC_)AaS@Cq+!gEUOY|e&Rq5U97#Tb4`9@p~ zN6C>#cEx}Yt_E~kT~`^9+P*tS&;npOIKd?|4NofQneHhXrrH#qqFT9I#Aa!Xf!{ic zX}hB0nthLBf`qbCzn{_UCC~Uy!M$MDpOb zRnl>jS#I1H2VrFGMpM$>yU|XcCwkC7-Eq-dgu+rd*UMk+wath-vxh24!3)x7!@6{p z2vis=i{^|w;GMwBR9&+Gt4@J;GP{8 z;p6a`eALC*tu{g>xT4SOXJ(mLp*Y*Jbdg9)y~JMj`_CmNMq{!j#n8tk}Kd@55}f=RipoI53OdD52-(OS}Eq!%pnU zfwAs|^eMJmFa2KLgOY*FTiv;NcFV{rA4Y`+Iksp~9Wurivh_toMTk_4LZ{v04Z2GF z%nO>;nAeu`%lmiN*eAt%p;)4*$|X!`s|oxqt7!SLRffE#luZZ+H)UjL`dB&6BMf@u zQ#Rv6Cay!h=h}_W^BiNVV?B36q}`}=sU4l|Z61-J4M6;0+a-~&IEa^uF)WW6&2;;i zWj0Qa*cN-k6g^8zmC$z1E!_3gZ3=~~M|Ls$v3bYpLhQ0HHHH&`wd)m#h7`4;*O_FS z69S{A7sz!?D-&y@{)rv>I9CkdVGul6N1!&x#y8o)7G@LBf`kPZtE8Mf0fKS~H3p<# zKlFU>=zUQ*eRkGwMJi@SFB{mrILlJd^DyhW@`cDjNQqqjZx;~15q#z=A~3jY8A literal 0 HcmV?d00001 diff --git a/ref/crates/ctr-0.9.2/tests/ctr128/mod.rs b/ref/crates/ctr-0.9.2/tests/ctr128/mod.rs new file mode 100644 index 0000000..c5e0206 --- /dev/null +++ b/ref/crates/ctr-0.9.2/tests/ctr128/mod.rs @@ -0,0 +1,12 @@ +use aes::{Aes128, Aes256}; +use ctr::{flavors, Ctr128BE, CtrCore}; + +cipher::stream_cipher_test!(aes128_ctr_core, "aes128-ctr", Ctr128BE); +cipher::stream_cipher_test!(aes256_ctr_core, "aes256-ctr", Ctr128BE); +cipher::stream_cipher_seek_test!(aes128_ctr_seek, Ctr128BE); +cipher::stream_cipher_seek_test!(aes256_ctr_seek, Ctr128BE); +cipher::iv_state_test!( + aes128_ctr_iv_state, + CtrCore, + apply_ks, +); diff --git a/ref/crates/ctr-0.9.2/tests/ctr32/be.rs b/ref/crates/ctr-0.9.2/tests/ctr32/be.rs new file mode 100644 index 0000000..98b1302 --- /dev/null +++ b/ref/crates/ctr-0.9.2/tests/ctr32/be.rs @@ -0,0 +1,86 @@ +//! Counter Mode with a 32-bit big endian counter + +use cipher::{KeyIvInit, StreamCipher, StreamCipherSeek, StreamCipherSeekCore}; +use hex_literal::hex; + +type Aes128Ctr = ctr::Ctr32BE; + +const KEY: &[u8; 16] = &hex!("000102030405060708090A0B0C0D0E0F"); +const NONCE1: &[u8; 16] = &hex!("11111111111111111111111111111111"); +const NONCE2: &[u8; 16] = &hex!("222222222222222222222222FFFFFFFE"); + +#[test] +fn counter_incr() { + let mut ctr = Aes128Ctr::new(KEY.into(), NONCE1.into()); + assert_eq!(ctr.get_core().get_block_pos(), 0); + + let mut buffer = [0u8; 64]; + ctr.apply_keystream(&mut buffer); + + assert_eq!(ctr.get_core().get_block_pos(), 4); + assert_eq!( + &buffer[..], + &hex!( + "35D14E6D3E3A279CF01E343E34E7DED36EEADB04F42E2251AB4377F257856DBA" + "0AB37657B9C2AA09762E518FC9395D5304E96C34CCD2F0A95CDE7321852D90C0" + )[..] + ); +} + +#[test] +fn counter_seek() { + let mut ctr = Aes128Ctr::new(KEY.into(), NONCE1.into()); + ctr.seek(16); + assert_eq!(ctr.get_core().get_block_pos(), 1); + + let mut buffer = [0u8; 64]; + ctr.apply_keystream(&mut buffer); + + assert_eq!(ctr.get_core().get_block_pos(), 5); + assert_eq!( + &buffer[..], + &hex!( + "6EEADB04F42E2251AB4377F257856DBA0AB37657B9C2AA09762E518FC9395D53" + "04E96C34CCD2F0A95CDE7321852D90C0F7441EAB3811A03FDBD162AEC402F5AA" + )[..] + ); +} + +#[test] +fn keystream_xor() { + let mut ctr = Aes128Ctr::new(KEY.into(), NONCE1.into()); + let mut buffer = [1u8; 64]; + + ctr.apply_keystream(&mut buffer); + assert_eq!( + &buffer[..], + &hex!( + "34D04F6C3F3B269DF11F353F35E6DFD26FEBDA05F52F2350AA4276F356846CBB" + "0BB27756B8C3AB08772F508EC8385C5205E86D35CDD3F1A85DDF7220842C91C1" + )[..] + ); +} + +#[test] +fn counter_wrap() { + let mut ctr = Aes128Ctr::new(KEY.into(), NONCE2.into()); + assert_eq!(ctr.get_core().get_block_pos(), 0); + + let mut buffer = [0u8; 64]; + ctr.apply_keystream(&mut buffer); + + assert_eq!(ctr.get_core().get_block_pos(), 4); + assert_eq!( + &buffer[..], + &hex!( + "58FC849D1CF53C54C63E1B1D15CB3C8AAA335F72135585E9FF943F4DAC77CB63" + "BD1AE8716BE69C3B4D886B222B9B4E1E67548EF896A96E2746D8CA6476D8B183" + )[..] + ); +} + +cipher::iv_state_test!( + iv_state, + ctr::CtrCore, + apply_ks, +); diff --git a/ref/crates/ctr-0.9.2/tests/ctr32/le.rs b/ref/crates/ctr-0.9.2/tests/ctr32/le.rs new file mode 100644 index 0000000..21bf358 --- /dev/null +++ b/ref/crates/ctr-0.9.2/tests/ctr32/le.rs @@ -0,0 +1,96 @@ +//! Counter Mode with a 32-bit little endian counter + +use cipher::{ + consts::U16, generic_array::GenericArray, KeyIvInit, StreamCipher, StreamCipherSeek, + StreamCipherSeekCore, +}; +use hex_literal::hex; + +type Aes128Ctr = ctr::Ctr32LE; + +const KEY: &[u8; 16] = &hex!("000102030405060708090A0B0C0D0E0F"); +const NONCE1: &[u8; 16] = &hex!("11111111111111111111111111111111"); +const NONCE2: &[u8; 16] = &hex!("FEFFFFFF222222222222222222222222"); + +/// Compute nonce as used by AES-GCM-SIV +fn nonce(bytes: &[u8; 16]) -> GenericArray { + let mut n = *bytes; + n[15] |= 0x80; + n.into() +} + +#[test] +fn counter_incr() { + let mut ctr = Aes128Ctr::new(KEY.into(), &nonce(NONCE1)); + assert_eq!(ctr.get_core().get_block_pos(), 0); + + let mut buffer = [0u8; 64]; + ctr.apply_keystream(&mut buffer); + + // assert_eq!(ctr.current_ctr(), 4); + assert_eq!( + &buffer[..], + &hex!( + "2A0680B210CAD45E886D7EF6DAB357C9F18B39AFF6930FDB2D9FCE34261FF699" + "EB96774669D24B560C9AD028C5C39C4580775A82065256B4787DC91C6942B700" + )[..] + ); +} + +#[test] +fn counter_seek() { + let mut ctr = Aes128Ctr::new(KEY.into(), &nonce(NONCE1)); + ctr.seek(16); + assert_eq!(ctr.get_core().get_block_pos(), 1); + + let mut buffer = [0u8; 64]; + ctr.apply_keystream(&mut buffer); + + assert_eq!(ctr.get_core().get_block_pos(), 5); + assert_eq!( + &buffer[..], + &hex!( + "F18B39AFF6930FDB2D9FCE34261FF699EB96774669D24B560C9AD028C5C39C45" + "80775A82065256B4787DC91C6942B7001564DDA1B07DCED9201AB71BAF06905B" + )[..] + ); +} + +#[test] +fn keystream_xor() { + let mut ctr = Aes128Ctr::new(KEY.into(), &nonce(NONCE1)); + let mut buffer = [1u8; 64]; + + ctr.apply_keystream(&mut buffer); + assert_eq!( + &buffer[..], + &hex!( + "2B0781B311CBD55F896C7FF7DBB256C8F08A38AEF7920EDA2C9ECF35271EF798" + "EA97764768D34A570D9BD129C4C29D4481765B83075357B5797CC81D6843B601" + )[..] + ); +} + +#[test] +fn counter_wrap() { + let mut ctr = Aes128Ctr::new(KEY.into(), &nonce(NONCE2)); + assert_eq!(ctr.get_core().get_block_pos(), 0); + + let mut buffer = [0u8; 64]; + ctr.apply_keystream(&mut buffer); + + assert_eq!(ctr.get_core().get_block_pos(), 4); + assert_eq!( + &buffer[..], + &hex!( + "A1E649D8B382293DC28375C42443BB6A226BAADC9E9CCA8214F56E07A4024E06" + "6355A0DA2E08FB00112FFA38C26189EE55DD5B0B130ED87096FE01B59A665A60" + )[..] + ); +} + +cipher::iv_state_test!( + iv_state, + ctr::CtrCore, + apply_ks, +); diff --git a/ref/crates/ctr-0.9.2/tests/ctr32/mod.rs b/ref/crates/ctr-0.9.2/tests/ctr32/mod.rs new file mode 100644 index 0000000..f9c17d4 --- /dev/null +++ b/ref/crates/ctr-0.9.2/tests/ctr32/mod.rs @@ -0,0 +1,9 @@ +//! Counter Mode with a 32-bit counter. +//! +//! NOTE: AES-128-CTR test vectors used by these tests were generated by first +//! integration testing the implementation in the contexts of AES-GCM and +//! AES-GCM-SIV, with the former tested against the NIST CAVS vectors, and the +//! latter against the RFC8452 test vectors. + +mod be; +mod le; diff --git a/ref/crates/ctr-0.9.2/tests/gost/mod.rs b/ref/crates/ctr-0.9.2/tests/gost/mod.rs new file mode 100644 index 0000000..31e5cec --- /dev/null +++ b/ref/crates/ctr-0.9.2/tests/gost/mod.rs @@ -0,0 +1,55 @@ +use cipher::{KeyIvInit, StreamCipher}; +use hex_literal::hex; + +type MagmaCtr = ctr::Ctr32BE; +type KuznyechikCtr = ctr::Ctr64BE; + +/// Test vectors from GOST R 34.13-2015 (Section A.1.2) +#[test] +fn kuznyechik() { + let key = hex!( + "8899aabbccddeeff0011223344556677" + "fedcba98765432100123456789abcdef" + ); + let nonce = hex!("1234567890abcef00000000000000000"); + let mut pt = hex!( + "1122334455667700ffeeddccbbaa9988" + "00112233445566778899aabbcceeff0a" + "112233445566778899aabbcceeff0a00" + "2233445566778899aabbcceeff0a0011" + ); + let ct = hex!( + "f195d8bec10ed1dbd57b5fa240bda1b8" + "85eee733f6a13e5df33ce4b33c45dee4" + "a5eae88be6356ed3d5e877f13564a3a5" + "cb91fab1f20cbab6d1c6d15820bdba73" + ); + let mut cipher = KuznyechikCtr::new(&key.into(), &nonce.into()); + cipher.apply_keystream(&mut pt); + assert_eq!(pt[..], ct[..]); +} + +/// Test vectors from GOST R 34.13-2015 (Section A.2.2) +#[test] +fn magma() { + let key = hex!( + "ffeeddccbbaa99887766554433221100" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff" + ); + let nonce = hex!("1234567800000000"); + let mut pt = hex!( + "92def06b3c130a59" + "db54c704f8189d20" + "4a98fb2e67a8024c" + "8912409b17b57e41" + ); + let ct = hex!( + "4e98110c97b7b93c" + "3e250d93d6e85d69" + "136d868807b2dbef" + "568eb680ab52a12d" + ); + let mut cipher = MagmaCtr::new(&key.into(), &nonce.into()); + cipher.apply_keystream(&mut pt); + assert_eq!(pt[..], ct[..]); +} diff --git a/ref/crates/ctr-0.9.2/tests/mod.rs b/ref/crates/ctr-0.9.2/tests/mod.rs new file mode 100644 index 0000000..0f91835 --- /dev/null +++ b/ref/crates/ctr-0.9.2/tests/mod.rs @@ -0,0 +1,5 @@ +//! Counter Mode Tests + +mod ctr128; +mod ctr32; +mod gost; diff --git a/ref/parity-harness/README.md b/ref/parity-harness/README.md new file mode 100644 index 0000000..ae61882 --- /dev/null +++ b/ref/parity-harness/README.md @@ -0,0 +1,33 @@ +# `ref/parity-harness` + +Compare drivers for Lean AOT vs pinned `ref/` implementations (verik1 style). + +## Layout + +```text +ref/parity-harness/ + drivers/ + etm-vectors/ # Program B: RustCrypto EtM goldens (aes/ctr/hmac/sha2) + rs-vectors/ # Program C: reed-solomon-erasure 4/8 + Carbonado padding + README.md +``` + +## EtM vectors (Program B) + +```bash +cd ref/parity-harness/drivers/etm-vectors +cargo run --quiet +``` + +Emits SHA-512, HMAC-SHA512, AES-256-CTR NIST, Carbonado subkeys, EtM blobs, and header MAC samples matching Lean goldens in `Carbonado/Main.lean` and `CarbonadoTest/EtM.lean`. + +## RS vectors (Program C) + +```bash +cd ref/parity-harness/drivers/rs-vectors +cargo run --quiet +``` + +Emits GF(2^8) samples, `calc_padding_len` geometry, RS 4/8 encode goldens, and Carbonado inboard encode heads matching `Carbonado/Main.lean` and `CarbonadoTest/Fec.lean`. Depends on `ref/reed-solomon-erasure` (v5.0.3 pin). + +Register each new gate under `flake.nix` `checks` and [docs/SPEC-MATRIX.md](../../docs/SPEC-MATRIX.md). diff --git a/ref/parity-harness/drivers/bao-vectors/Cargo.lock b/ref/parity-harness/drivers/bao-vectors/Cargo.lock new file mode 100644 index 0000000..b2c0bc6 --- /dev/null +++ b/ref/parity-harness/drivers/bao-vectors/Cargo.lock @@ -0,0 +1,340 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + +[[package]] +name = "bao-tree" +version = "0.16.0" +dependencies = [ + "blake3", + "bytes", + "genawaiter", + "positioned-io", + "range-collections", + "self_cell", + "smallvec", +] + +[[package]] +name = "bao-vectors" +version = "0.1.0" +dependencies = [ + "bao-tree", + "blake3", + "hex", +] + +[[package]] +name = "binary-merge" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "futures-core", + "genawaiter-macro", + "genawaiter-proc-macro", + "proc-macro-hack", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + +[[package]] +name = "genawaiter-proc-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" +dependencies = [ + "proc-macro-error", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "inplace-vec-builder" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf64c2edc8226891a71f127587a2861b132d2b942310843814d5001d99a1d307" +dependencies = [ + "smallvec", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "positioned-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ec4b80060f033312b99b6874025d9503d2af87aef2dd4c516e253fbfcdada7" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "proc-macro-error" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "syn-mid", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "range-collections" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "861706ea9c4aded7584c5cd1d241cec2ea7f5f50999f236c22b65409a1f1a0d0" +dependencies = [ + "binary-merge", + "inplace-vec-builder", + "ref-cast", + "smallvec", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea305d57546cc8cd04feb14b62ec84bf17f50e3f7b12560d7bfa9265f39d9ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[patch.unused]] +name = "bitcoinpqc" +version = "0.4.0" +source = "git+https://github.com/cryptoquick/libbitcoinpqc-bindings.git?rev=7936b56#7936b56f15e86b6764947c9298215ecfe38b712b" diff --git a/ref/parity-harness/drivers/bao-vectors/Cargo.toml b/ref/parity-harness/drivers/bao-vectors/Cargo.toml new file mode 100644 index 0000000..65df182 --- /dev/null +++ b/ref/parity-harness/drivers/bao-vectors/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "bao-vectors" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +blake3 = { version = "1", default-features = false, features = ["std"] } +hex = "0.4" +bao-tree = { path = "../../../bao-tree", default-features = false, features = ["validate"] } diff --git a/ref/parity-harness/drivers/bao-vectors/src/main.rs b/ref/parity-harness/drivers/bao-vectors/src/main.rs new file mode 100644 index 0000000..d073d43 --- /dev/null +++ b/ref/parity-harness/drivers/bao-vectors/src/main.rs @@ -0,0 +1,199 @@ +//! Golden vectors for Carbonado keyed Bao (bao-tree 76-keyed-bao, 4 KiB groups). +use std::io::Cursor; + +use bao_tree::{ + blake3, + io::{ + outboard::{EmptyOutboard, PostOrderMemOutboard}, + sync::{keyed_decode_ranges, keyed_encode_ranges_validated, keyed_outboard_post_order}, + }, + BaoTree, BlockSize, ChunkNum, ChunkRanges, +}; + +const BAO_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(2); +const CTX: &str = "carbonado-v2/verification"; + +fn hex(b: &[u8]) -> String { + hex::encode(b) +} + +fn verification_key(format: u8) -> [u8; 32] { + blake3::derive_key(CTX, &[format]) +} + +fn patterned(len: usize) -> Vec { + (0..len).map(|i| (i % 251) as u8).collect() +} + +fn inboard_encode(data: &[u8], format: u8) -> (Vec, [u8; 32]) { + let key = verification_key(format); + let content_len = data.len() as u64; + let tree = BaoTree::new(content_len, BAO_BLOCK_SIZE); + let mut sidecar = Vec::new(); + let root = keyed_outboard_post_order(Cursor::new(data), tree, &mut sidecar, &key).unwrap(); + let ob = PostOrderMemOutboard { + root, + tree, + data: sidecar, + }; + let mut out = Vec::new(); + out.extend_from_slice(&content_len.to_le_bytes()); + keyed_encode_ranges_validated(data, &ob, &ChunkRanges::all(), &mut out, &key).unwrap(); + (out, *root.as_bytes()) +} + +fn outboard_encode(data: &[u8], format: u8) -> (Vec, [u8; 32]) { + let key = verification_key(format); + let ob = PostOrderMemOutboard::create_keyed(data, BAO_BLOCK_SIZE, &key); + (ob.data, *ob.root.as_bytes()) +} + +fn main() { + println!("=== verification keys ==="); + for f in [0u8, 4, 6, 12, 14, 15] { + println!("key_c{:02x} = {}", f, hex(&verification_key(f))); + } + + println!("\n=== blake3 goldens ==="); + println!("hash(empty) = {}", hex(blake3::hash(b"").as_bytes())); + println!("hash(abc) = {}", hex(blake3::hash(b"abc").as_bytes())); + let k0 = verification_key(4); + println!("keyed_hash(c4, empty) = {}", hex(blake3::keyed_hash(&k0, b"").as_bytes())); + println!("keyed_hash(c4, hello) = {}", hex(blake3::keyed_hash(&k0, b"hello").as_bytes())); + let pat100 = patterned(100); + println!("keyed_hash(c4, pat100) = {}", hex(blake3::keyed_hash(&k0, &pat100).as_bytes())); + let pat4096 = patterned(4096); + println!("keyed_hash(c4, pat4096) = {}", hex(blake3::keyed_hash(&k0, &pat4096).as_bytes())); + let pat5000 = patterned(5000); + println!("keyed_hash(c4, pat5000) = {}", hex(blake3::keyed_hash(&k0, &pat5000).as_bytes())); + + println!("\n=== keyed roots (create_keyed == keyed_hash) ==="); + for &len in &[0usize, 1, 100, 1024, 4095, 4096, 4097, 8192] { + let data = patterned(len); + let key = verification_key(4); + let root = PostOrderMemOutboard::create_keyed(&data, BAO_BLOCK_SIZE, &key).root; + let direct = blake3::keyed_hash(&key, &data); + assert_eq!(root, direct); + println!("root_c4_len{} = {}", len, hex(root.as_bytes())); + } + // format domain separation same data + let data = patterned(100); + for f in [4u8, 6, 14] { + let key = verification_key(f); + let root = PostOrderMemOutboard::create_keyed(&data, BAO_BLOCK_SIZE, &key).root; + println!("root_c{:02x}_len100 = {}", f, hex(root.as_bytes())); + } + + println!("\n=== outboard sidecars ==="); + for &len in &[0usize, 1, 100, 4096, 5000] { + let data = patterned(len); + let (ob, root) = outboard_encode(&data, 4); + println!("out_c4_len{} root={} len={} hex={}", len, hex(&root), ob.len(), hex(&ob)); + } + + println!("\n=== inboard (prefix+response) ==="); + for &len in &[0usize, 1, 5, 100, 4096, 5000] { + let data = if len == 5 { + b"hello".to_vec() + } else { + patterned(len) + }; + let (ib, root) = inboard_encode(&data, 4); + println!( + "in_c4_len{} root={} total={} head32={}", + if len == 5 { 5 } else { len }, + hex(&root), + ib.len(), + hex(&ib[..ib.len().min(32)]) + ); + if ib.len() <= 200 { + println!(" full = {}", hex(&ib)); + } else { + println!(" tail16 = {}", hex(&ib[ib.len() - 16..])); + } + } + + // wrong format fails decode + println!("\n=== wrong key fails ==="); + let data = patterned(100); + let (ib, root) = inboard_encode(&data, 4); + let key_wrong = verification_key(6); + let content_len = u64::from_le_bytes(ib[0..8].try_into().unwrap()); + let tree = BaoTree::new(content_len, BAO_BLOCK_SIZE); + let mut ob = EmptyOutboard { + tree, + root: blake3::Hash::from(root), + }; + let mut out = vec![0u8; content_len as usize]; + let res = keyed_decode_ranges( + Cursor::new(&ib[8..]), + &ChunkRanges::all(), + &mut out[..], + &mut ob, + &key_wrong, + ); + println!("decode_wrong_key_err = {:?}", res.err().map(|e| format!("{e}"))); + + // slice range: first 4 KiB of 5000-byte file + println!("\n=== slice first group (5000 bytes, chunks 0..4) ==="); + let data = patterned(5000); + let key = verification_key(4); + let tree = BaoTree::new(5000, BAO_BLOCK_SIZE); + let mut sidecar = Vec::new(); + let root = keyed_outboard_post_order(Cursor::new(&data), tree, &mut sidecar, &key).unwrap(); + let ob = PostOrderMemOutboard { + root, + tree, + data: sidecar, + }; + // chunk group = 4 chunks of 1024 = 4096 bytes → ChunkNum 0..4 + let ranges = ChunkRanges::from(ChunkNum(0)..ChunkNum(4)); + let mut slice_enc = Vec::new(); + keyed_encode_ranges_validated(&data[..], &ob, &ranges, &mut slice_enc, &key).unwrap(); + println!("slice_c4_5000_0_4 root={} enc_len={} head48={}", hex(root.as_bytes()), slice_enc.len(), hex(&slice_enc[..slice_enc.len().min(48)])); + if slice_enc.len() <= 256 { + println!(" full = {}", hex(&slice_enc)); + } + + // Three-leaf tree (12288 B) + middle slice stream decode + println!("\n=== three-leaf 12288 ==="); + let data = patterned(12288); + let (ib, root) = inboard_encode(&data, 4); + let (ob, root2) = outboard_encode(&data, 4); + assert_eq!(root, root2); + println!( + "in_c4_len12288 root={} total={} outboard_len={}", + hex(&root), + ib.len(), + ob.len() + ); + let key = verification_key(4); + let tree = BaoTree::new(12288, BAO_BLOCK_SIZE); + let mut sidecar = Vec::new(); + let root_h = keyed_outboard_post_order(Cursor::new(&data), tree, &mut sidecar, &key).unwrap(); + let pom = PostOrderMemOutboard { + root: root_h, + tree, + data: sidecar, + }; + // second leaf group: blake3 chunks 4..8 + let ranges = ChunkRanges::from(ChunkNum(4)..ChunkNum(8)); + let mut mid = Vec::new(); + keyed_encode_ranges_validated(&data[..], &pom, &ranges, &mut mid, &key).unwrap(); + println!( + "slice_c4_12288_leaf1 enc_len={} head48={}", + mid.len(), + hex(&mid[..mid.len().min(48)]) + ); + // stream decode without trusting full plaintext buffer contents beforehand + let mut decoded = vec![0u8; 12288]; + let mut eob = EmptyOutboard { + tree, + root: root_h, + }; + keyed_decode_ranges(Cursor::new(&mid), &ranges, &mut decoded[..], &mut eob, &key).unwrap(); + assert_eq!(&decoded[4096..8192], &data[4096..8192]); + println!("slice_c4_12288_leaf1 stream_decode ok"); + + println!("\nok"); +} diff --git a/ref/parity-harness/drivers/etm-vectors/Cargo.lock b/ref/parity-harness/drivers/etm-vectors/Cargo.lock new file mode 100644 index 0000000..e22da4e --- /dev/null +++ b/ref/parity-harness/drivers/etm-vectors/Cargo.lock @@ -0,0 +1,163 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "etm-vectors" +version = "0.1.0" +dependencies = [ + "aes", + "ctr", + "hex", + "hmac", + "sha2", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[patch.unused]] +name = "bitcoinpqc" +version = "0.4.0" +source = "git+https://github.com/cryptoquick/libbitcoinpqc-bindings.git?rev=7936b56#7936b56f15e86b6764947c9298215ecfe38b712b" diff --git a/ref/parity-harness/drivers/etm-vectors/Cargo.toml b/ref/parity-harness/drivers/etm-vectors/Cargo.toml new file mode 100644 index 0000000..5794851 --- /dev/null +++ b/ref/parity-harness/drivers/etm-vectors/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "etm-vectors" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "etm-vectors" +path = "src/main.rs" + +[dependencies] +aes = "0.8.4" +ctr = "0.9.2" +hmac = "0.12.1" +sha2 = "0.10.9" +hex = "0.4" diff --git a/ref/parity-harness/drivers/etm-vectors/src/main.rs b/ref/parity-harness/drivers/etm-vectors/src/main.rs new file mode 100644 index 0000000..c525656 --- /dev/null +++ b/ref/parity-harness/drivers/etm-vectors/src/main.rs @@ -0,0 +1,132 @@ +//! Golden vectors for Carbonado v2 EtM (matches src/crypto.rs semantics). +use aes::cipher::{KeyIvInit, StreamCipher}; +use aes::Aes256; +use ctr::Ctr128BE; +use hmac::{Hmac, Mac}; +use sha2::{Digest, Sha512}; + +type HmacSha512 = Hmac; + +fn derive_subkey(master: &[u8], label: &str) -> [u8; 64] { + let mut mac = HmacSha512::new_from_slice(master).unwrap(); + mac.update(b"carbonado-v2/"); + mac.update(label.as_bytes()); + let result = mac.finalize().into_bytes(); + let mut out = [0u8; 64]; + out.copy_from_slice(&result); + out +} + +fn aes_ctr(key: &[u8; 32], nonce: &[u8; 16], data: &[u8]) -> Vec { + let mut cipher = Ctr128BE::::new(key.into(), nonce.into()); + let mut out = data.to_vec(); + cipher.apply_keystream(&mut out); + out +} + +fn etm_encrypt(master: &[u8], nonce: [u8; 16], pt: &[u8]) -> Vec { + let enc = derive_subkey(master, "aes-ctr"); + let mac_key = derive_subkey(master, "etm-hmac"); + let aes_key: [u8; 32] = enc[..32].try_into().unwrap(); + let ct = aes_ctr(&aes_key, &nonce, pt); + let mut mac = HmacSha512::new_from_slice(&mac_key).unwrap(); + mac.update(b"carbonado-v2-etm"); + mac.update(&nonce); + mac.update(&ct); + let tag = mac.finalize().into_bytes(); + let mut out = Vec::with_capacity(64 + ct.len()); + out.extend_from_slice(&tag); + out.extend_from_slice(&ct); + out +} + +fn header_mac(master: &[u8], auth_data: &[u8]) -> [u8; 64] { + let key = derive_subkey(master, "header-auth"); + let mut mac = HmacSha512::new_from_slice(&key).unwrap(); + mac.update(auth_data); + let r = mac.finalize().into_bytes(); + let mut out = [0u8; 64]; + out.copy_from_slice(&r); + out +} + +fn hex(b: &[u8]) -> String { + hex::encode(b) +} + +fn main() { + println!("=== SHA-512 ==="); + for msg in [&b""[..], &b"abc"[..], &b"The quick brown fox jumps over the lazy dog"[..]] { + let d = Sha512::digest(msg); + println!("sha512 len={} = {}", msg.len(), hex(&d)); + } + + println!("\n=== HMAC-SHA512 RFC4231-1 ==="); + let key = [0x0bu8; 20]; + let data = b"Hi There"; + let mut mac = HmacSha512::new_from_slice(&key).unwrap(); + mac.update(data); + println!("hmac = {}", hex(&mac.finalize().into_bytes())); + + println!("\n=== AES-256-CTR NIST ==="); + let key: [u8; 32] = [ + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4, + ]; + let counter: [u8; 16] = [ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, + ]; + let pt: [u8; 64] = [ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10, + ]; + let ct = aes_ctr(&key, &counter, &pt); + println!("nist_ct = {}", hex(&ct)); + + let master = [0x42u8; 32]; + let nonce = [0x11u8; 16]; + println!("\n=== Carbonado subkeys ==="); + for label in ["aes-ctr", "etm-hmac", "header-auth"] { + let sk = derive_subkey(&master, label); + println!("subkey({}) = {}", label, hex(&sk)); + } + + println!("\n=== Carbonado EtM header-path [tag|ct] ==="); + for (name, pt) in [ + ("empty", &b""[..]), + ("hello", &b"hello"[..]), + ("block16", &[0u8; 16][..]), + ("block32", &[0xABu8; 32][..]), + ("multi", &b"The quick brown fox jumps over the lazy dog"[..]), + ("block64", &[0x5Au8; 64][..]), + ] { + let out = etm_encrypt(&master, nonce, pt); + println!("{} pt_len={} blob={}", name, pt.len(), hex(&out)); + } + + let out = etm_encrypt(&master, nonce, b"hello"); + let mut low = Vec::new(); + low.extend_from_slice(&nonce); + low.extend_from_slice(&out); + println!("\nlow_level_hello = {}", hex(&low)); + + let magic = b"CARBONADO20\n"; + let hm = header_mac(&master, magic); + println!("\nheader_mac(MAGIC) = {}", hex(&hm)); + + let mut auth = Vec::new(); + auth.extend_from_slice(magic); + auth.extend_from_slice(&nonce); + auth.extend_from_slice(&[0xCDu8; 32]); + auth.extend_from_slice(&[0x00u8; 32]); + auth.push(0x05); + auth.extend_from_slice(&0u32.to_le_bytes()); + auth.extend_from_slice(&100u32.to_le_bytes()); + auth.extend_from_slice(&0u32.to_le_bytes()); + auth.extend_from_slice(&[0u8; 8]); + assert_eq!(auth.len(), 12 + 16 + 32 + 32 + 1 + 4 + 4 + 4 + 8); + let hm2 = header_mac(&master, &auth); + println!("header_mac(sample_auth) len={} = {}", auth.len(), hex(&hm2)); +} diff --git a/ref/parity-harness/drivers/rs-vectors/Cargo.lock b/ref/parity-harness/drivers/rs-vectors/Cargo.lock new file mode 100644 index 0000000..8900123 --- /dev/null +++ b/ref/parity-harness/drivers/rs-vectors/Cargo.lock @@ -0,0 +1,148 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reed-solomon-erasure" +version = "5.0.3" +dependencies = [ + "libm", + "parking_lot", + "smallvec", + "spin", +] + +[[package]] +name = "rs-vectors" +version = "0.1.0" +dependencies = [ + "hex", + "reed-solomon-erasure", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[patch.unused]] +name = "bitcoinpqc" +version = "0.4.0" +source = "git+https://github.com/cryptoquick/libbitcoinpqc-bindings.git?rev=7936b56#7936b56f15e86b6764947c9298215ecfe38b712b" diff --git a/ref/parity-harness/drivers/rs-vectors/Cargo.toml b/ref/parity-harness/drivers/rs-vectors/Cargo.toml new file mode 100644 index 0000000..e37f079 --- /dev/null +++ b/ref/parity-harness/drivers/rs-vectors/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "rs-vectors" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "rs-vectors" +path = "src/main.rs" + +[dependencies] +reed-solomon-erasure = { path = "../../../reed-solomon-erasure" } +hex = "0.4" diff --git a/ref/parity-harness/drivers/rs-vectors/src/main.rs b/ref/parity-harness/drivers/rs-vectors/src/main.rs new file mode 100644 index 0000000..ee05ff9 --- /dev/null +++ b/ref/parity-harness/drivers/rs-vectors/src/main.rs @@ -0,0 +1,100 @@ +//! Golden vectors for Carbonado RS 4/8 (matches stream/fec + reed-solomon-erasure 5.0.3). +use reed_solomon_erasure::galois_8::{self, Field}; +use reed_solomon_erasure::ReedSolomon; + +fn hex(b: &[u8]) -> String { + hex::encode(b) +} + +fn calc_padding_len(input_len: usize) -> (u32, u32) { + if input_len == 0 { + return (0, 0); + } + let stripe = 4096usize * 4; + let target = input_len.div_ceil(stripe) * stripe; + let padding_len = (target - input_len) as u32; + let chunk_size = (target / 4) as u32; + (padding_len, chunk_size) +} + +fn encode_inboard(input: &[u8]) -> (Vec, u32, u32) { + if input.is_empty() { + return (vec![], 0, 0); + } + let (pad, chunk) = calc_padding_len(input.len()); + let chunk = chunk as usize; + let mut data = input.to_vec(); + data.resize(input.len() + pad as usize, 0); + let r = ReedSolomon::::new(4, 4).unwrap(); + let mut shards: Vec> = (0..4) + .map(|i| data[i * chunk..(i + 1) * chunk].to_vec()) + .collect(); + for _ in 0..4 { + shards.push(vec![0u8; chunk]); + } + r.encode(&mut shards).unwrap(); + let mut out = Vec::with_capacity(8 * chunk); + for s in &shards { + out.extend_from_slice(s); + } + (out, pad, chunk as u32) +} + +fn main() { + println!("=== GF(2^8) samples ==="); + for (a, b) in [(0x53u8, 0xcau8), (2, 3), (7, 11), (0xff, 1)] { + println!("mul({:02x},{:02x}) = {:02x}", a, b, galois_8::mul(a, b)); + if b != 0 { + println!("div({:02x},{:02x}) = {:02x}", a, b, galois_8::div(a, b)); + } + println!("exp({:02x},3) = {:02x}", a, galois_8::exp(a, 3)); + } + + println!("\n=== padding geometry ==="); + for n in [0usize, 1, 100, 4096, 16384, 16385] { + let (p, c) = calc_padding_len(n); + println!("pad({}) = ({}, {})", n, p, c); + } + + let r = ReedSolomon::::new(4, 4).unwrap(); + println!("\n=== RS 4/8 len1 ==="); + let mut s = vec![ + vec![1u8], + vec![2], + vec![3], + vec![4], + vec![0], + vec![0], + vec![0], + vec![0], + ]; + r.encode(&mut s).unwrap(); + println!( + "parity = {:02x}{:02x}{:02x}{:02x}", + s[4][0], s[5][0], s[6][0], s[7][0] + ); + + println!("\n=== RS 4/8 seq8 ==="); + let mut s8: Vec> = (0..4) + .map(|i| (0..8).map(|j| (i * 8 + j) as u8).collect()) + .collect(); + for _ in 0..4 { + s8.push(vec![0u8; 8]); + } + r.encode(&mut s8).unwrap(); + for (i, sh) in s8.iter().enumerate() { + println!("s{} = {}", i, hex(sh)); + } + + println!("\n=== Carbonado inboard hello ==="); + let (body, pad, chunk) = encode_inboard(b"hello"); + println!("pad={} chunk={} len={}", pad, chunk, body.len()); + println!("head = {}", hex(&body[..16])); + + println!("\n=== Carbonado inboard pat100 ==="); + let pat: Vec = (0..100).map(|i| (i % 251) as u8).collect(); + let (body, pad, chunk) = encode_inboard(&pat); + println!("pad={} chunk={} len={}", pad, chunk, body.len()); + let shard_len = body.len() / 8; + println!("parity0_head8 = {}", hex(&body[4 * shard_len..4 * shard_len + 8])); +} diff --git a/ref/reed-solomon-erasure b/ref/reed-solomon-erasure new file mode 160000 index 0000000..9f97491 --- /dev/null +++ b/ref/reed-solomon-erasure @@ -0,0 +1 @@ +Subproject commit 9f974918f8c598eee351406c36fa0295f4bb4d69 diff --git a/ref/rustcrypto-block-ciphers b/ref/rustcrypto-block-ciphers new file mode 160000 index 0000000..f2dbee5 --- /dev/null +++ b/ref/rustcrypto-block-ciphers @@ -0,0 +1 @@ +Subproject commit f2dbee516b4d0cf4cb4f3045d09e35b5fd80087b diff --git a/ref/rustcrypto-hashes b/ref/rustcrypto-hashes new file mode 160000 index 0000000..82c36a4 --- /dev/null +++ b/ref/rustcrypto-hashes @@ -0,0 +1 @@ +Subproject commit 82c36a428f8d6f05f3bfccdedb243e9d1f85359d diff --git a/ref/rustcrypto-macs b/ref/rustcrypto-macs new file mode 160000 index 0000000..46797e3 --- /dev/null +++ b/ref/rustcrypto-macs @@ -0,0 +1 @@ +Subproject commit 46797e3b44973a30edb9d7f3a3ebb41810061d90 diff --git a/ref/zstd b/ref/zstd new file mode 160000 index 0000000..f8745da --- /dev/null +++ b/ref/zstd @@ -0,0 +1 @@ +Subproject commit f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/src/backend/mod.rs b/src/backend/mod.rs new file mode 100644 index 0000000..62e20c6 --- /dev/null +++ b/src/backend/mod.rs @@ -0,0 +1,124 @@ +//! Dual-backend dispatch (docs/TEST_CONTRACT.md, docs/ABI.md). +//! +//! - `backend-rust` (default): pure Rust implementation in this crate. +//! - `backend-lean`: Lean AOT `libcarbonado` via `carbonado-sys` (G8 dual-backend). +//! +//! Both features must not be enabled together for a single build that links both +//! engines into conflicting paths; prefer one engine per `cargo test` invocation. + +#[cfg(all(feature = "backend-lean", feature = "backend-rust"))] +compile_error!( + "enable only one of `backend-lean` or `backend-rust` (dual-backend CI runs them separately)" +); + +#[cfg(not(any(feature = "backend-lean", feature = "backend-rust")))] +compile_error!("enable `backend-lean` or `backend-rust` (see docs/TEST_CONTRACT.md)"); + +#[cfg(feature = "backend-rust")] +#[allow(dead_code)] // dispatch hooks land as encode/decode call sites migrate +pub mod rust_engine { + //! Marker: pure Rust paths are the default implementation modules (`encoding`, `decoding`, …). + pub const NAME: &str = "rust"; +} + +#[cfg(feature = "backend-lean")] +pub mod lean { + //! Lean AOT backend via C ABI (`carbonado-sys` / `libcarbonado`). + use crate::error::CarbonadoError; + use carbonado_sys as sys; + + pub const NAME: &str = "lean"; + + /// ABI version from the linked libcarbonado (requires `CARBONADO_LEAN_LIB`). + pub fn abi_version() -> u32 { + unsafe { sys::carbonado_abi_version() } + } + + /// Map C ABI codes to `CarbonadoError` (docs/ABI.md). Refined as mapping matures. + pub fn map_err(code: i32) -> CarbonadoError { + match code { + sys::CARBONADO_ERR_INVALID_ARGUMENT => CarbonadoError::InvalidHeaderLength, + sys::CARBONADO_ERR_INVALID_KEY_LENGTH => { + CarbonadoError::HashDecodeError(32, 0) // refine when dedicated key variant exists + } + sys::CARBONADO_ERR_AUTHENTICATION => CarbonadoError::AuthenticationFailed, + sys::CARBONADO_ERR_INVALID_MAGIC => { + CarbonadoError::InvalidMagicNumber("lean-backend".into()) + } + sys::CARBONADO_ERR_INVALID_HEADER => CarbonadoError::InvalidHeaderLength, + sys::CARBONADO_ERR_FEC => CarbonadoError::UnevenFecChunks, + sys::CARBONADO_ERR_BAO => CarbonadoError::InvalidScrubbedHash, + sys::CARBONADO_ERR_ZSTD => CarbonadoError::ZstdError("lean-backend zstd".into()), + sys::CARBONADO_ERR_SCRUB_UNNECESSARY => CarbonadoError::UnnecessaryScrub, + sys::CARBONADO_ERR_SCRUB_FAILED => CarbonadoError::InvalidScrubbedHash, + sys::CARBONADO_ERR_NOT_IMPLEMENTED => CarbonadoError::ZstdError( + "lean-backend: C ABI encode/decode not fully wired (Phase 1; stubs return NOT_IMPLEMENTED)" + .into(), + ), + sys::CARBONADO_ERR_INTERNAL => { + CarbonadoError::ZstdError("lean-backend internal error".into()) + } + _ => CarbonadoError::ZstdError(format!("lean-backend unknown error {code}")), + } + } + + /// Headered encode via Lean AOT (explicit 16-byte nonce when encrypted). + pub fn encode_headered( + master: &[u8], + plaintext: &[u8], + format: u8, + nonce: Option<&[u8; 16]>, + ) -> Result, CarbonadoError> { + let (nonce_ptr, nonce_len) = match nonce { + Some(n) => (n.as_ptr(), 16usize), + None => (std::ptr::null(), 0usize), + }; + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_encode_headered( + master.as_ptr(), + master.len(), + plaintext.as_ptr(), + plaintext.len(), + format, + nonce_ptr, + nonce_len, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + if out.is_null() { + return Err(map_err(sys::CARBONADO_ERR_INTERNAL)); + } + let v = unsafe { Vec::from_raw_parts(out, out_len, out_len) }; + // from_raw_parts takes ownership; do not free via carbonado_free. + Ok(v) + } + + /// Headered decode via Lean AOT. + pub fn decode_headered(master: &[u8], archive: &[u8]) -> Result, CarbonadoError> { + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_decode_headered( + master.as_ptr(), + master.len(), + archive.as_ptr(), + archive.len(), + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + if out.is_null() { + return Err(map_err(sys::CARBONADO_ERR_INTERNAL)); + } + Ok(unsafe { Vec::from_raw_parts(out, out_len, out_len) }) + } +} diff --git a/src/lib.rs b/src/lib.rs index fd472b7..926c916 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,9 @@ //////////////////////////////////////////////////////////////////////////////// +/// Dual-backend dispatch (`backend-rust` / `backend-lean`). See docs/TEST_CONTRACT.md. +pub mod backend; + /// For details on Carbonado formats and their uses, see the [Carbonado Format bitmask constant](constants::Format). pub mod constants; /// Symmetric cryptographic primitives for the v2 design. From b8c070ae64da98a9a2743dfdb48e8f75322b4f93 Mon Sep 17 00:00:00 2001 From: Hunter Beast Date: Thu, 13 Aug 2026 15:26:11 -0600 Subject: [PATCH 2/6] wip --- .cargo/config.toml.example | 2 +- .github/workflows/rust.yaml | 79 +- AGENTS.md | 23 +- Carbonado.lean | 1 + Carbonado/Bao/Product.lean | 7 +- Carbonado/Bao/Tree.lean | 240 ++++- Carbonado/Cli.lean | 25 +- Carbonado/Directory.lean | 22 +- Carbonado/Ffi.lean | 456 +++++++++- Carbonado/Filepack.lean | 39 +- Carbonado/Main.lean | 208 ++++- Carbonado/Outboard.lean | 106 ++- Carbonado/Pipeline.lean | 11 +- Carbonado/RkyvFilepack.lean | 451 ++++++++++ Carbonado/Scrub.lean | 166 ++++ Carbonado/Shard.lean | 2 +- Carbonado/Slh.lean | 161 +++- CarbonadoTest/Bao.lean | 49 ++ CarbonadoTest/Pipeline.lean | 33 +- CarbonadoTest/Slh.lean | 11 +- Cargo.toml | 26 +- README.md | 7 +- build.rs | 22 + carbonado-sys/Cargo.toml | 5 + carbonado-sys/build.rs | 61 +- carbonado-sys/src/lib.rs | 126 +++ doc/STREAMING_PARALLELISM.md | 56 +- doc/TEST_STRATEGY.md | 24 +- docs/ABI.md | 256 ++++-- docs/GAPS.md | 230 ++++- docs/LIMITS.md | 109 ++- docs/PARITY.md | 32 +- docs/SPEC-MATRIX.md | 8 +- docs/TEST_CONTRACT.md | 233 ++++- examples/dump_rkyv_r9.rs | 44 + flake.nix | 111 ++- include/carbonado.h | 158 +++- justfile | 176 +++- nix/native/carbonado_abi.c | 710 ++++++++++++++- nix/native/carbonado_slh.c | 199 +++++ nix/native/default.nix | 68 +- nix/tooling-purity.nix | 15 +- ref/README.md | 11 +- src/backend/mod.rs | 723 ++++++++++++++- src/bin/carbonado/key_store.rs | 23 +- src/bin/carbonado/main.rs | 7 +- src/crypto.rs | 3 + src/decoding.rs | 129 ++- src/encoding.rs | 66 +- src/file.rs | 349 ++++++-- src/lib.rs | 5 +- src/stream/compress.rs | 51 +- src/stream/crypto_stream.rs | 30 +- src/stream/decode.rs | 289 +++++- src/stream/decode_async.rs | 72 +- src/stream/encode.rs | 524 ++++++++++- src/stream/mod.rs | 6 +- src/stream/slice.rs | 99 ++- src/stream/spool.rs | 34 + tests/common/inboard_parity.rs | 1 + tests/determinism_roundtrip.rs | 831 ++++++++++++++++++ tests/directory_archive.rs | 9 +- tests/filepack_interop.rs | 7 + tests/fixtures/g9/README.md | 90 ++ tests/fixtures/g9/lean/body_c0.bin | 1 + tests/fixtures/g9/lean/body_c0.meta.json | 9 + tests/fixtures/g9/lean/body_c1.bin | 3 + tests/fixtures/g9/lean/body_c1.meta.json | 10 + tests/fixtures/g9/lean/body_c12.bin | Bin 0 -> 33224 bytes tests/fixtures/g9/lean/body_c12.meta.json | 9 + tests/fixtures/g9/lean/body_c13.bin | Bin 0 -> 33224 bytes tests/fixtures/g9/lean/body_c13.meta.json | 10 + tests/fixtures/g9/lean/body_c4.bin | Bin 0 -> 34 bytes tests/fixtures/g9/lean/body_c4.meta.json | 9 + tests/fixtures/g9/lean/body_c5.bin | Bin 0 -> 114 bytes tests/fixtures/g9/lean/body_c5.meta.json | 10 + tests/fixtures/g9/lean/body_c8.bin | Bin 0 -> 32768 bytes tests/fixtures/g9/lean/body_c8.meta.json | 9 + tests/fixtures/g9/lean/body_c9.bin | Bin 0 -> 32768 bytes tests/fixtures/g9/lean/body_c9.meta.json | 10 + tests/fixtures/g9/lean/headered_c12.bin | Bin 0 -> 33401 bytes tests/fixtures/g9/lean/headered_c12.meta.json | 9 + tests/fixtures/g9/lean/headered_c13.bin | Bin 0 -> 33401 bytes tests/fixtures/g9/lean/headered_c13.meta.json | 10 + tests/fixtures/g9/lean/headered_c4.bin | Bin 0 -> 211 bytes tests/fixtures/g9/lean/headered_c4.meta.json | 9 + tests/fixtures/g9/lean/headered_c5.bin | Bin 0 -> 275 bytes tests/fixtures/g9/lean/headered_c5.meta.json | 10 + tests/fixtures/g9/lean/outboard_c12/main.bin | 1 + tests/fixtures/g9/lean/outboard_c12/meta.json | 13 + tests/fixtures/g9/lean/outboard_c12/out.bin | 0 tests/fixtures/g9/lean/outboard_c12/par.bin | Bin 0 -> 16384 bytes .../fixtures/g9/lean/outboard_c13/header.bin | Bin 0 -> 177 bytes tests/fixtures/g9/lean/outboard_c13/main.bin | 2 + tests/fixtures/g9/lean/outboard_c13/meta.json | 14 + tests/fixtures/g9/lean/outboard_c13/out.bin | 0 tests/fixtures/g9/lean/outboard_c13/par.bin | Bin 0 -> 16384 bytes tests/fixtures/g9/lean/outboard_c14/main.bin | Bin 0 -> 35 bytes tests/fixtures/g9/lean/outboard_c14/meta.json | 13 + tests/fixtures/g9/lean/outboard_c14/out.bin | 0 tests/fixtures/g9/lean/outboard_c14/par.bin | Bin 0 -> 16384 bytes tests/fixtures/g9/lean/outboard_c4/main.bin | 1 + tests/fixtures/g9/lean/outboard_c4/meta.json | 13 + tests/fixtures/g9/lean/outboard_c4/out.bin | 0 tests/fixtures/g9/lean/outboard_c5/header.bin | Bin 0 -> 177 bytes tests/fixtures/g9/lean/outboard_c5/main.bin | 2 + tests/fixtures/g9/lean/outboard_c5/meta.json | 14 + tests/fixtures/g9/lean/outboard_c5/out.bin | 0 tests/fixtures/g9/rust/body_c0.bin | 1 + tests/fixtures/g9/rust/body_c0.meta.json | 9 + tests/fixtures/g9/rust/body_c1.bin | 3 + tests/fixtures/g9/rust/body_c1.meta.json | 10 + tests/fixtures/g9/rust/body_c12.bin | Bin 0 -> 33224 bytes tests/fixtures/g9/rust/body_c12.meta.json | 9 + tests/fixtures/g9/rust/body_c13.bin | Bin 0 -> 33224 bytes tests/fixtures/g9/rust/body_c13.meta.json | 10 + tests/fixtures/g9/rust/body_c4.bin | Bin 0 -> 34 bytes tests/fixtures/g9/rust/body_c4.meta.json | 9 + tests/fixtures/g9/rust/body_c5.bin | Bin 0 -> 114 bytes tests/fixtures/g9/rust/body_c5.meta.json | 10 + tests/fixtures/g9/rust/body_c8.bin | Bin 0 -> 32768 bytes tests/fixtures/g9/rust/body_c8.meta.json | 9 + tests/fixtures/g9/rust/body_c9.bin | Bin 0 -> 32768 bytes tests/fixtures/g9/rust/body_c9.meta.json | 10 + tests/fixtures/g9/rust/headered_c12.bin | Bin 0 -> 33401 bytes tests/fixtures/g9/rust/headered_c12.meta.json | 9 + tests/fixtures/g9/rust/headered_c13.bin | Bin 0 -> 33401 bytes tests/fixtures/g9/rust/headered_c13.meta.json | 10 + tests/fixtures/g9/rust/headered_c4.bin | Bin 0 -> 211 bytes tests/fixtures/g9/rust/headered_c4.meta.json | 9 + tests/fixtures/g9/rust/headered_c5.bin | Bin 0 -> 275 bytes tests/fixtures/g9/rust/headered_c5.meta.json | 10 + tests/fixtures/g9/rust/outboard_c12/main.bin | 1 + tests/fixtures/g9/rust/outboard_c12/meta.json | 13 + tests/fixtures/g9/rust/outboard_c12/out.bin | 0 tests/fixtures/g9/rust/outboard_c12/par.bin | Bin 0 -> 16384 bytes .../fixtures/g9/rust/outboard_c13/header.bin | Bin 0 -> 177 bytes tests/fixtures/g9/rust/outboard_c13/main.bin | 2 + tests/fixtures/g9/rust/outboard_c13/meta.json | 14 + tests/fixtures/g9/rust/outboard_c13/out.bin | 0 tests/fixtures/g9/rust/outboard_c13/par.bin | Bin 0 -> 16384 bytes tests/fixtures/g9/rust/outboard_c14/main.bin | Bin 0 -> 35 bytes tests/fixtures/g9/rust/outboard_c14/meta.json | 13 + tests/fixtures/g9/rust/outboard_c14/out.bin | 0 tests/fixtures/g9/rust/outboard_c14/par.bin | Bin 0 -> 16384 bytes tests/fixtures/g9/rust/outboard_c4/main.bin | 1 + tests/fixtures/g9/rust/outboard_c4/meta.json | 13 + tests/fixtures/g9/rust/outboard_c4/out.bin | 0 tests/fixtures/g9/rust/outboard_c5/header.bin | Bin 0 -> 177 bytes tests/fixtures/g9/rust/outboard_c5/main.bin | 2 + tests/fixtures/g9/rust/outboard_c5/meta.json | 14 + tests/fixtures/g9/rust/outboard_c5/out.bin | 0 ...03f681cd60c01f7605ee11acd5423024f.adam.c14 | Bin 0 -> 33401 bytes ...4cb680f9d7a795002e17a295b904d2b2265466.c14 | Bin 0 -> 24 bytes tests/fixtures/phase3_g9_directory/README.txt | 21 + ...10823c14caa4723dd3e301528f1e3c534400e6.c14 | Bin 0 -> 20 bytes tests/fixtures/rkyv/README.md | 30 + tests/fixtures/rkyv/empty_manifest.bin | Bin 0 -> 13 bytes tests/fixtures/rkyv/multi_entry_ots.bin | Bin 0 -> 275 bytes tests/fixtures/rkyv/ots_first_only.bin | Bin 0 -> 251 bytes tests/fixtures/rkyv/path_inline_8.bin | Bin 0 -> 131 bytes tests/fixtures/rkyv/path_ool_9.bin | Bin 0 -> 140 bytes tests/fixtures/rkyv/rkyv_cfp2_prefix.bin | Bin 0 -> 131 bytes tests/fixtures/rkyv/single_entry.bin | Bin 0 -> 131 bytes tests/fixtures/rkyv/two_segments.bin | Bin 0 -> 191 bytes tests/g9_cross_backend.rs | 807 +++++++++++++++++ tests/lean_backend_phase2.rs | 512 +++++++++++ tests/lean_backend_phase3.rs | 447 ++++++++++ tests/lean_backend_phase4.rs | 676 ++++++++++++++ tests/lean_backend_smoke.rs | 277 ++++++ tests/parallel_determinism.rs | 4 +- tests/rkyv_golden_lock.rs | 233 +++++ tests/seekable_slices.rs | 69 +- tests/serial_fec_path.rs | 4 +- tests/slh_outboard.rs | 6 +- tests/streaming.rs | 153 +++- tests/streaming_async.rs | 23 +- tests/streaming_limits.rs | 54 ++ 178 files changed, 11072 insertions(+), 753 deletions(-) create mode 100644 Carbonado/RkyvFilepack.lean create mode 100644 build.rs create mode 100644 examples/dump_rkyv_r9.rs create mode 100644 nix/native/carbonado_slh.c create mode 100644 tests/determinism_roundtrip.rs create mode 100644 tests/fixtures/g9/README.md create mode 100644 tests/fixtures/g9/lean/body_c0.bin create mode 100644 tests/fixtures/g9/lean/body_c0.meta.json create mode 100644 tests/fixtures/g9/lean/body_c1.bin create mode 100644 tests/fixtures/g9/lean/body_c1.meta.json create mode 100644 tests/fixtures/g9/lean/body_c12.bin create mode 100644 tests/fixtures/g9/lean/body_c12.meta.json create mode 100644 tests/fixtures/g9/lean/body_c13.bin create mode 100644 tests/fixtures/g9/lean/body_c13.meta.json create mode 100644 tests/fixtures/g9/lean/body_c4.bin create mode 100644 tests/fixtures/g9/lean/body_c4.meta.json create mode 100644 tests/fixtures/g9/lean/body_c5.bin create mode 100644 tests/fixtures/g9/lean/body_c5.meta.json create mode 100644 tests/fixtures/g9/lean/body_c8.bin create mode 100644 tests/fixtures/g9/lean/body_c8.meta.json create mode 100644 tests/fixtures/g9/lean/body_c9.bin create mode 100644 tests/fixtures/g9/lean/body_c9.meta.json create mode 100644 tests/fixtures/g9/lean/headered_c12.bin create mode 100644 tests/fixtures/g9/lean/headered_c12.meta.json create mode 100644 tests/fixtures/g9/lean/headered_c13.bin create mode 100644 tests/fixtures/g9/lean/headered_c13.meta.json create mode 100644 tests/fixtures/g9/lean/headered_c4.bin create mode 100644 tests/fixtures/g9/lean/headered_c4.meta.json create mode 100644 tests/fixtures/g9/lean/headered_c5.bin create mode 100644 tests/fixtures/g9/lean/headered_c5.meta.json create mode 100644 tests/fixtures/g9/lean/outboard_c12/main.bin create mode 100644 tests/fixtures/g9/lean/outboard_c12/meta.json create mode 100644 tests/fixtures/g9/lean/outboard_c12/out.bin create mode 100644 tests/fixtures/g9/lean/outboard_c12/par.bin create mode 100644 tests/fixtures/g9/lean/outboard_c13/header.bin create mode 100644 tests/fixtures/g9/lean/outboard_c13/main.bin create mode 100644 tests/fixtures/g9/lean/outboard_c13/meta.json create mode 100644 tests/fixtures/g9/lean/outboard_c13/out.bin create mode 100644 tests/fixtures/g9/lean/outboard_c13/par.bin create mode 100644 tests/fixtures/g9/lean/outboard_c14/main.bin create mode 100644 tests/fixtures/g9/lean/outboard_c14/meta.json create mode 100644 tests/fixtures/g9/lean/outboard_c14/out.bin create mode 100644 tests/fixtures/g9/lean/outboard_c14/par.bin create mode 100644 tests/fixtures/g9/lean/outboard_c4/main.bin create mode 100644 tests/fixtures/g9/lean/outboard_c4/meta.json create mode 100644 tests/fixtures/g9/lean/outboard_c4/out.bin create mode 100644 tests/fixtures/g9/lean/outboard_c5/header.bin create mode 100644 tests/fixtures/g9/lean/outboard_c5/main.bin create mode 100644 tests/fixtures/g9/lean/outboard_c5/meta.json create mode 100644 tests/fixtures/g9/lean/outboard_c5/out.bin create mode 100644 tests/fixtures/g9/rust/body_c0.bin create mode 100644 tests/fixtures/g9/rust/body_c0.meta.json create mode 100644 tests/fixtures/g9/rust/body_c1.bin create mode 100644 tests/fixtures/g9/rust/body_c1.meta.json create mode 100644 tests/fixtures/g9/rust/body_c12.bin create mode 100644 tests/fixtures/g9/rust/body_c12.meta.json create mode 100644 tests/fixtures/g9/rust/body_c13.bin create mode 100644 tests/fixtures/g9/rust/body_c13.meta.json create mode 100644 tests/fixtures/g9/rust/body_c4.bin create mode 100644 tests/fixtures/g9/rust/body_c4.meta.json create mode 100644 tests/fixtures/g9/rust/body_c5.bin create mode 100644 tests/fixtures/g9/rust/body_c5.meta.json create mode 100644 tests/fixtures/g9/rust/body_c8.bin create mode 100644 tests/fixtures/g9/rust/body_c8.meta.json create mode 100644 tests/fixtures/g9/rust/body_c9.bin create mode 100644 tests/fixtures/g9/rust/body_c9.meta.json create mode 100644 tests/fixtures/g9/rust/headered_c12.bin create mode 100644 tests/fixtures/g9/rust/headered_c12.meta.json create mode 100644 tests/fixtures/g9/rust/headered_c13.bin create mode 100644 tests/fixtures/g9/rust/headered_c13.meta.json create mode 100644 tests/fixtures/g9/rust/headered_c4.bin create mode 100644 tests/fixtures/g9/rust/headered_c4.meta.json create mode 100644 tests/fixtures/g9/rust/headered_c5.bin create mode 100644 tests/fixtures/g9/rust/headered_c5.meta.json create mode 100644 tests/fixtures/g9/rust/outboard_c12/main.bin create mode 100644 tests/fixtures/g9/rust/outboard_c12/meta.json create mode 100644 tests/fixtures/g9/rust/outboard_c12/out.bin create mode 100644 tests/fixtures/g9/rust/outboard_c12/par.bin create mode 100644 tests/fixtures/g9/rust/outboard_c13/header.bin create mode 100644 tests/fixtures/g9/rust/outboard_c13/main.bin create mode 100644 tests/fixtures/g9/rust/outboard_c13/meta.json create mode 100644 tests/fixtures/g9/rust/outboard_c13/out.bin create mode 100644 tests/fixtures/g9/rust/outboard_c13/par.bin create mode 100644 tests/fixtures/g9/rust/outboard_c14/main.bin create mode 100644 tests/fixtures/g9/rust/outboard_c14/meta.json create mode 100644 tests/fixtures/g9/rust/outboard_c14/out.bin create mode 100644 tests/fixtures/g9/rust/outboard_c14/par.bin create mode 100644 tests/fixtures/g9/rust/outboard_c4/main.bin create mode 100644 tests/fixtures/g9/rust/outboard_c4/meta.json create mode 100644 tests/fixtures/g9/rust/outboard_c4/out.bin create mode 100644 tests/fixtures/g9/rust/outboard_c5/header.bin create mode 100644 tests/fixtures/g9/rust/outboard_c5/main.bin create mode 100644 tests/fixtures/g9/rust/outboard_c5/meta.json create mode 100644 tests/fixtures/g9/rust/outboard_c5/out.bin create mode 100644 tests/fixtures/phase3_g9_directory/16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14 create mode 100644 tests/fixtures/phase3_g9_directory/90ccff39cc098af6242e5ead3c4cb680f9d7a795002e17a295b904d2b2265466.c14 create mode 100644 tests/fixtures/phase3_g9_directory/README.txt create mode 100644 tests/fixtures/phase3_g9_directory/c710a801faf330c99d2ce6aeb210823c14caa4723dd3e301528f1e3c534400e6.c14 create mode 100644 tests/fixtures/rkyv/README.md create mode 100644 tests/fixtures/rkyv/empty_manifest.bin create mode 100644 tests/fixtures/rkyv/multi_entry_ots.bin create mode 100644 tests/fixtures/rkyv/ots_first_only.bin create mode 100644 tests/fixtures/rkyv/path_inline_8.bin create mode 100644 tests/fixtures/rkyv/path_ool_9.bin create mode 100644 tests/fixtures/rkyv/rkyv_cfp2_prefix.bin create mode 100644 tests/fixtures/rkyv/single_entry.bin create mode 100644 tests/fixtures/rkyv/two_segments.bin create mode 100644 tests/g9_cross_backend.rs create mode 100644 tests/lean_backend_phase2.rs create mode 100644 tests/lean_backend_phase3.rs create mode 100644 tests/lean_backend_phase4.rs create mode 100644 tests/lean_backend_smoke.rs create mode 100644 tests/rkyv_golden_lock.rs diff --git a/.cargo/config.toml.example b/.cargo/config.toml.example index 706253d..8e4efb3 100644 --- a/.cargo/config.toml.example +++ b/.cargo/config.toml.example @@ -6,7 +6,7 @@ # just setup-bao-tree # cp .cargo/config.toml.example .cargo/config.toml -[patch."https://github.com/SurmountSystems/bao-tree.git"] +[patch."https://github.com/n0-computer/bao-tree.git"] bao-tree = { path = "../bao-tree" } # `bitcoinpqc` 0.4 mirror lag: the repo ships `.cargo/config.toml` with a temporary diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 5ef1410..3117469 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -30,7 +30,7 @@ jobs: - name: Format check run: just fmt - - name: Lint (native, all targets/features) + - name: Lint (native; backend-rust + optional features, never --all-features) run: just lint lint-wasm: @@ -83,12 +83,14 @@ jobs: RUST_BACKTRACE: 1 - name: Test (serial FEC path without parallel) - run: cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path + # Mutual exclusion: must name backend-rust under --no-default-features. + run: cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path env: RUST_BACKTRACE: 1 - - name: Test (all features) - run: cargo test --all-features + - name: Test (backend-rust + optional features; never --all-features) + # --all-features enables both backend-rust and backend-lean → compile_error!. + run: cargo test --features "async,async-tokio,man-gen" env: RUST_BACKTRACE: 1 @@ -132,11 +134,12 @@ jobs: - uses: Swatinem/rust-cache@v2 - - name: Check ${{ matrix.target }} (with pqc) - run: cargo check --target ${{ matrix.target }} --all-features + - name: Check ${{ matrix.target }} (backend-rust + pqc/optional; never --all-features) + # --all-features enables both backends → compile_error!. + run: cargo check --target ${{ matrix.target }} --features "async,async-tokio,man-gen" - - name: Check ${{ matrix.target }} (no pqc) - run: cargo check --target ${{ matrix.target }} --no-default-features + - name: Check ${{ matrix.target }} (backend-rust only, no pqc) + run: cargo check --target ${{ matrix.target }} --no-default-features --features "backend-rust" # WASM browser testing is limited because libbitcoinpqc uses C code + cmake. # Full wasm-pack test can be run locally with: @@ -161,5 +164,61 @@ jobs: - uses: Swatinem/rust-cache@v2 - - name: Check wasm32 (no pqc) - run: cargo check --target wasm32-unknown-unknown --no-default-features \ No newline at end of file + - name: Check wasm32 (backend-rust only, no pqc) + run: cargo check --target wasm32-unknown-unknown --no-default-features --features "backend-rust" + + # Dual-backend Phase 5 / G11 + R7 G8 full close: Linux lean full dual suite freeze. + # Normative command: `just test-lean-ci` = unfiltered cargo test under lean features + # (builds libcarbonado via nix if needed; fail-closed if .so missing). + # backend-rust full suite remains the `desktop` job above — never regress. + dual-backend-lean: + runs-on: ubuntu-latest + needs: lint + timeout-minutes: 180 + + steps: + - uses: actions/checkout@v4 + + - name: Checkout bao-tree keyed fork (sibling for path dep) + uses: actions/checkout@v4 + with: + repository: SurmountSystems/bao-tree + ref: 76-keyed-bao + path: ../bao-tree + + - uses: extractions/setup-just@v2 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - name: Free disk for nix + cargo + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + df -h + + - name: Install Nix + uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Build libcarbonado (Lean AOT) + run: nix build .#libcarbonado -o result-libcarbonado + + - name: Fail-closed if libcarbonado missing + run: | + set -euo pipefail + test -f result-libcarbonado/lib/libcarbonado.so + test -d result-libcarbonado/include + ls -la result-libcarbonado/lib/ + ls -la result-libcarbonado/include/ + + - name: Dual-backend lean freeze matrix (just test-lean-ci) + run: just test-lean-ci + env: + RUST_BACKTRACE: 1 + CARBONADO_LEAN_LIB: ${{ github.workspace }}/result-libcarbonado/lib + CARBONADO_LEAN_INCLUDE: ${{ github.workspace }}/result-libcarbonado/include + LD_LIBRARY_PATH: ${{ github.workspace }}/result-libcarbonado/lib \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index fafe029..c816e6a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,16 @@ | **Nix flakes** | Build Lean AOT, purity/no-sorry, package `libcarbonado` | | **`ref/`** | Pinned oracles (bao-tree, RustCrypto, zstd, …) | -**Parity bar (G8):** `cargo test` with `backend-rust` (default) and `backend-lean` (links Lean AOT C). Same tests; not separate Lean-only demos. See [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md), [docs/ABI.md](docs/ABI.md), [docs/PARITY.md](docs/PARITY.md), [docs/GAPS.md](docs/GAPS.md). +**Parity bar (G8):** same tests on both engines — not Lean-only demos. See [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md), [docs/ABI.md](docs/ABI.md), [docs/PARITY.md](docs/PARITY.md), [docs/GAPS.md](docs/GAPS.md). + +```bash +cargo test # backend-rust (default features) +just test-lean-ci # backend-lean full dual suite (G8 closed at R7 / G11) +# Equivalent unfiltered lean suite: +# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +``` + +Default `Cargo.toml` features already enable `backend-rust`. Adding `--features backend-lean` without `--no-default-features` enables **both** engines and hits `compile_error!` in `src/backend/mod.rs`. | Concern | Allowed | |---------|---------| @@ -24,11 +33,11 @@ **Prove everything:** machine-checked Lean theorems **and** bit-match via the Rust suite on `backend-lean`. -**Agents:** implement and verify (`nix build`, `nix flake check`, `cargo test`). Do **not** create commits or push; agents do not own git history. **Never regress `backend-rust` `cargo test`.** Do **not** implement Phase 1+ C ABI exports unless that phase is the assigned task. +**Agents:** implement and verify (`nix build`, `nix flake check`, `cargo test`, `just test-lean-ci` when touching dual-backend). Do **not** create commits or push; agents do not own git history. **Never regress `backend-rust` `cargo test`.** Phase 0–**5 closed** for dual-backend **allowlist + CI freeze** (G11). **Full-suite G8 closed at R7** (measured green; freeze = full dual suite). **G9 closed at R8** — no-compress body/headered/outboard both directions (`tests/g9_cross_backend.rs` + `tests/fixtures/g9/`). **W2 closed:** **W2d** codecode/decodec shipped (`tests/determinism_roundtrip.rs`); **W2a/W2b permanent** cross-engine Compression / directory encode residuals (same-engine re-encode green; decode interop SSOT). **R9 pure Lean depth closed** for G10 SLH FFI + seekable outboard slice C + rkyv dual-decode; **W3 closed** pure Lean rkyv encode + Directory/CLI (dual-suite composition may still use Rust bitcoinpqc / Rust rkyv as product SSOT — never claim dual-suite *requires* pure Lean). **R10 async dual policy closed** — dual freeze **never** requires `async`; optional `stream_decode_async` is dual-aware under `backend-lean`+`async` via R5 E1 `stream_decode` (disk O(encoded) staging; lean peak RAM **O(encoded + logical)**; not E2). **W5a/G1 closed** — permanent no `ref/carbonado-rust` product pin; live `src/`/`tests/` dual-suite SSOT; third-party `ref/` oracles only. Post-G8 residuals remaining: feature-gated `streaming_async` / `parallel_determinism` (permanent freeze exclusion); ~~W1a+W1b dual honesty~~ **closed** (public outboard stream E2 = S4 composition under lean; pure Lean chunked C residual); ~~W2d codecode/decodec~~ **closed**; ~~W2a/W2b~~ **permanent** (cross-engine compress/dir encode); ~~Lean rkyv encode/CLI CFP2~~ **W3 closed**; ~~W4a inboard O(slice) retain~~ **closed**; **W4b** permanent full-buffer C outboard slice; **W4c** permanent buffer-only zstd under lean; **W4d** permanent FEC O(body) + async encoded spool. -**Gates:** `nix flake check` (Lean); `cargo test` (Rust default); dual-backend CI as G8 lands (Phase 0 docs closed; Phase 1+ engineering open — see GAPS P0–P5). +**Gates:** `nix flake check` (Lean); `cargo test` (Rust default / CI `desktop`); `just test-lean-ci` (Lean full dual suite / CI `dual-backend-lean`); `just test-g9` for cross-backend matrix only. Phase 0–**5** + **R7 G8 full** + **R8 G9 matrix** + **R9 pure Lean depth** + **R10 async dual policy** + **W2 determinism** + **W4 memory** + **W5a G1** closed (permanent no `ref/carbonado-rust` product pin; live `src/`/`tests/` SSOT; W4b–d permanent residuals) — see GAPS R7–R10 + W2 + W4 + W5a + post-G8 residuals. -**C ABI:** normative surface in `include/carbonado.h` + [docs/ABI.md](docs/ABI.md). Phase 0–1 honesty: encode/decode may still return `NOT_IMPLEMENTED` until Phase 1 wiring. +**C ABI:** normative surface in `include/carbonado.h` + [docs/ABI.md](docs/ABI.md). Phase 2: body/headered/outboard/scrub/verify_slice live via Lean AOT `libcarbonado`. Phase 3 directory: **composition** (no new directory C symbols) — `just test-lean-phase3`. Phase 4: SLH/OTS dual-suite composition (Rust bitcoinpqc + CBOTS; no new SLH C symbols); CLI dual-engine for **directory** + buffer APIs — `just test-lean-phase4`. **R5 stream E1:** `stream_encode_inboard` / `stream_decode` / **encrypted** `stream_*_outboard` spool→Lean under `backend-lean` (O(logical); not E2 chunked). **W1a:** `file::decode_stream` MAC-before-body then Lean `decode_headered`. **W1b:** public **non-Compression** outboard stream S4 O(chunk/stripe) composition under lean (c4/c12 MVP; Compression under lean O(logical) bulk zstd); encrypted/inboard remain Lean E1. **R6:** outboard FEC erasure for truncated main (`decodeOutboardFec` ≡ Rust `fec_with_parity`). **R7:** full G8 close — `just test-lean-ci` = unfiltered lean suite (includes `bin_*`) / job `dual-backend-lean`. **R8:** G9 full cross-backend matrix — `just test-g9` / fixtures under `tests/fixtures/g9/`. **R10:** async dual policy — freeze excludes `async`; lean+async `stream_decode_async` → dual-aware `stream_decode` (E1). --- @@ -38,7 +47,7 @@ | Axis | Status | |------|--------| | **Streaming / memory** | Phase 1 fused sync path shipped (`SeekableSpool`, streaming EtM, stripe FEC). **M1:** non-FEC verification (c6) uses `SeekWriteAt` (O(chunk) RAM); FEC verification retains O(FEC body) shard buffers under segment-wide RS geometry (`finish_into` avoids a second full logical `Vec`). Residuals: FEC O(segment body), O(sidecar) outboard verify, async encoded-body spool. See [doc/STREAMING_PARALLELISM.md](doc/STREAMING_PARALLELISM.md). **Not** the same as Bao slice/stream verification. | -| **Concurrency** | Phase 2 optional `async` / `stream_decode_async` (disk spool bridge; WASM `NotImplemented`). | +| **Concurrency** | Phase 2 optional `async` / `stream_decode_async` (disk spool bridge; **R10:** dual-aware via R5 E1 under lean+async; freeze never requires `async`; WASM `NotImplemented`). | | **Parallelism** | Phase 3 `parallel` feature (default on): `std::thread::scope` RS parity; WASM serial at runtime. No rayon; Tokio is not the CPU-parallel story. | **PQC:** `bitcoinpqc` 0.4, SLH-DSA-**SHA2**-128s sidecars only (`SLH_DSA_SHA2_128S`). Dev SHAKE-128s sidecars are incompatible — re-sign. @@ -104,7 +113,7 @@ These rules were added because the same misunderstandings have caused significan - Full documentation of every security-relevant decision (nonce scope, subkey labels, single-nonce behavior, sidecar signing rules, CTR counter management, etc.). - Real benchmarks proving hardware acceleration claims. - WASM support either works cleanly or has precise documented limitations. - - CI is strict (full clippy --all-targets --all-features -D warnings, relevant targets tested). + - CI is strict: `cargo clippy --all-targets --features "async,async-tokio,man-gen" -D warnings` (never `--all-features` — that enables both `backend-rust` and `backend-lean` and hits `compile_error!`). Dual-backend lean gate: `just test-lean-ci`. - Error handling is complete and specific; no lossy or generic errors hiding crypto failures. - Zeroization of secret material where practical. - Test coverage includes adversarial, large-payload, and cross-layer cases. @@ -947,7 +956,7 @@ This tension is acknowledged but not resolved in the current design. Carbonado i Remaining open (documented; active work called out): - **Pipeline memory residual (hard-break track):** fused encode/decode is O(chunk) spool + O(stripe) FEC encode; non-FEC verification decode is O(chunk) via `SeekWriteAt`; FEC verification decode retains O(FEC body) shard buffers (`FecInboardWriteAt`); outboard verify uses `PostOrderOutboard` + `ReadAt` (O(hash pair) per node); `stream_decode_async` fully spools encoded body to disk. Distinct from Bao **slice** verification (already O(slice) memory). See [doc/STREAMING_PARALLELISM.md](doc/STREAMING_PARALLELISM.md). -- **WASM:** `cargo clippy --target wasm32-unknown-unknown --no-default-features` is green (CI `lint-wasm`). **wasm32 + `pqc` probe (2026-07-08):** pointing global `CC_wasm32-unknown-unknown` at `libbitcoinpqc-bindings/wasm/clang-wasm32.sh` breaks **`zstd-sys`** (it tries to assemble `huf_decompress_amd64.S` with the wasm clang). Residual is build-env / dep CC scoping — not Carbonado crypto logic. Keep CI wasm lint **no-pqc** until bitcoinpqc (or zstd) wasm build is isolated. +- **WASM:** `cargo clippy --target wasm32-unknown-unknown --no-default-features --features "backend-rust"` is green (CI `lint-wasm`). **wasm32 + `pqc` probe (2026-07-08):** pointing global `CC_wasm32-unknown-unknown` at `libbitcoinpqc-bindings/wasm/clang-wasm32.sh` breaks **`zstd-sys`** (it tries to assemble `huf_decompress_amd64.S` with the wasm clang). Residual is build-env / dep CC scoping — not Carbonado crypto logic. Keep CI wasm lint **no-pqc** until bitcoinpqc (or zstd) wasm build is isolated. - Bao crate: Surmount keyed bao-tree fork (`76-keyed-bao`), 4 KiB groups, `default-features = false` (no tokio/fs on wasm). Temporary until upstream. - reed-solomon-erasure: upstream "looking for maintainers"; periodic re-eval (no runtime issues). - (Perf: inboard `verify_slice` is O(slice) memory but O(N) encoded-byte I/O; outboard slice verify is O(slice) time+memory; scrub pre-check uses `verify_inboard_keyed` with O(1) retained decode memory (S5).) diff --git a/Carbonado.lean b/Carbonado.lean index 1dcab5f..63251b0 100644 --- a/Carbonado.lean +++ b/Carbonado.lean @@ -17,6 +17,7 @@ import Carbonado.Scrub import Carbonado.Shard import Carbonado.Adamantine import Carbonado.Filepack +import Carbonado.RkyvFilepack import Carbonado.Outboard import Carbonado.Directory import Carbonado.Cli diff --git a/Carbonado/Bao/Product.lean b/Carbonado/Bao/Product.lean index 8c1b6a5..d2fc283 100644 --- a/Carbonado/Bao/Product.lean +++ b/Carbonado/Bao/Product.lean @@ -62,11 +62,16 @@ def decodeSliceForFormat (format : UInt8) (root : ByteArray) (contentLen index c (response : ByteArray) : Except BaoError ByteArray := decodeSliceResponse (carbonadoVerificationKey format) root contentLen index count response -/-- Extract/verify slice from full inboard under format (auth-first). -/ +/-- Extract/verify slice from full inboard under format (W4a auth-first O(slice) retain). -/ def verifySliceInboardForFormat (format : UInt8) (root input : ByteArray) (index count : Nat) : Except BaoError ByteArray := verifySliceInboard (carbonadoVerificationKey format) root input index count +/-- Seekable outboard slice verify under format (O(slice + height) hash; offset walk W4b). -/ +def verifySliceOutboardForFormat (format : UInt8) (root bare outboard : ByteArray) + (index count : Nat) : Except BaoError ByteArray := + verifySliceOutboard (carbonadoVerificationKey format) root bare outboard index count + /-- Different format bytes yield different verification keys. -/ theorem verification_key_format_domain : toHex (carbonadoVerificationKey 4) ≠ toHex (carbonadoVerificationKey 6) := by diff --git a/Carbonado/Bao/Tree.lean b/Carbonado/Bao/Tree.lean index 25d03c7..6e25c45 100644 --- a/Carbonado/Bao/Tree.lean +++ b/Carbonado/Bao/Tree.lean @@ -215,6 +215,66 @@ partial def decodeRec (startLeaf : Nat) (contentLen : Nat) (isRoot : Bool) decodeRec midLeaf rightLen false rQ key rightH input pos pure (appendBA leftData rightData, pos) +/-- Retain only the overlap of `[logicalOff, logicalOff + leafLen)` with + `[retainStart, retainEnd)` from leaf bytes (W4a SliceRegionWriter analogue). -/ +def retainLeafOverlap (leaf : ByteArray) (logicalOff leafLen retainStart retainEnd : Nat) : + ByteArray := + let leafEnd := logicalOff + leafLen + let copyStart := max logicalOff retainStart + let copyEnd := min leafEnd retainEnd + if copyStart ≥ copyEnd then + ByteArray.empty + else + leaf.extract (copyStart - logicalOff) (copyEnd - logicalOff) + +/-- Full-layout inboard response walk that authenticates every leaf/parent but retains + only bytes in `[retainStart, retainEnd)`. + + Callers pass the full inboard artifact (`[u64le|response]`) with `pos` starting at + **8** so the walk does **not** allocate a second full-response copy (W4a review #1). + Leaf hashing may temporarily extract up to one 4 KiB group (`O(leaf)` temps). + + **Memory (retained output):** O(range) — not full logical body. + **Time / I/O:** O(N) over the embedded full-range bao response (inboard embeds + `ChunkRanges::all()`; partial range decode would desync the sequential stream). + Matches Rust `verify_slice_inboard_seekable` / `SliceRegionWriter` contract (W4a). +-/ +partial def decodeRecRetainRange (startLeaf : Nat) (contentLen : Nat) (isRoot : Bool) + (key : ByteArray) (expected : ByteArray) (input : ByteArray) (pos : Nat) + (logicalOff retainStart retainEnd : Nat) : Except BaoError (ByteArray × Nat) := do + let nLeaves := leafGroupCount contentLen + if nLeaves ≤ 1 then + if pos + contentLen > input.size then + throw .truncatedResponse + let data := input.extract pos (pos + contentLen) + let startChunk := startLeaf * chunksPerSlice + let h := hashLeafGroup startChunk data isRoot key + if !ctEq h expected then + throw .authenticationFailed + let kept := retainLeafOverlap data logicalOff contentLen retainStart retainEnd + pure (kept, pos + contentLen) + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let leftLen := min midBytes contentLen + let rightLen := contentLen - leftLen + if pos + 64 > input.size then + throw .truncatedResponse + let leftH := input.extract pos (pos + 32) + let rightH := input.extract (pos + 32) (pos + 64) + let parentH := keyedParentCV leftH rightH isRoot key + if !ctEq parentH expected then + throw .authenticationFailed + let pos := pos + 64 + let (leftKept, pos) ← + decodeRecRetainRange startLeaf leftLen false key leftH input pos + logicalOff retainStart retainEnd + let (rightKept, pos) ← + decodeRecRetainRange (startLeaf + mid) rightLen false key rightH input pos + (logicalOff + leftLen) retainStart retainEnd + pure (appendBA leftKept rightKept, pos) + /-- Verify and decode full inboard `[u64le|response]` under `key` and expected root. -/ def decodeInboard (key root input : ByteArray) : Except BaoError ByteArray := do if root.size != outLen then @@ -241,10 +301,34 @@ def decodeInboard (key root input : ByteArray) : Except BaoError ByteArray := do throw .authenticationFailed pure data +/-- Authenticate full inboard without retaining logical body (W4a auth-only walk). + + Walks `input` from offset 8 (no second full-response `extract` copy). +-/ +def verifyInboardAuth (key root input : ByteArray) : Except BaoError Unit := do + if root.size != outLen then + throw .invalidRootLength + let contentLen ← contentLenPrefix input + if contentLen == 0 then + let expect := keyedRoot key ByteArray.empty + if !ctEq expect root then + throw .authenticationFailed + if input.size > 8 then + throw .trailingData + pure () + else + -- pos=8: sequential walk over inboard artifact without copying response tail. + let (_kept, endPos) ← + decodeRecRetainRange 0 contentLen true key root input 8 0 0 0 + if endPos < input.size then + throw .trailingData + if endPos > input.size then + throw .truncatedResponse + pure () + /-- Verify inboard without retaining body. -/ -def verifyInboard (key root input : ByteArray) : Except BaoError Unit := do - let _ ← decodeInboard key root input - pure () +def verifyInboard (key root input : ByteArray) : Except BaoError Unit := + verifyInboardAuth key root input /-- Verify bare main + post-order outboard against root. -/ def verifyOutboard (key root bare outboard : ByteArray) : Except BaoError Unit := do @@ -304,23 +388,45 @@ def sliceResponseMatchesEncode (key data : ByteArray) (index count : Nat) let (_root, enc) := encodeSliceResponse key data index count ctEq enc response -/-- Decode full inboard (always authenticates), then extract `count` slices at `index`. +/-- Authenticate full inboard response; retain only `count` slices at `index` (W4a). + + Integrity runs **before** the `count = 0` empty return — corrupt inboard never succeeds + on this pure-Lean / C path. (Dual Rust API `lean::verify_slice` short-circuits empty + success on `count==0` without auth — parity with pure-Rust seekable.) - Integrity runs **before** the `count = 0` empty return — corrupt inboard never succeeds. - `count = 0` after successful decode returns empty (extract semantics, not skip-auth). + **Retained memory:** O(slice) output; walk starts at offset 8 (no second full-response + copy). Leaf hashing may use O(leaf) temporary extracts. **Time:** O(N) over embedded + full-range response. C ABI still passes the full inboard body as input. -/ def extractSliceFromInboard (key root input : ByteArray) (index count : Nat) : Except BaoError ByteArray := do - let data ← decodeInboard key root input + if root.size != outLen then + throw .invalidRootLength + let contentLen ← contentLenPrefix input if count == 0 then + -- Auth-first empty extract (Lean C product): walk without retaining body. + verifyInboardAuth key root input return ByteArray.empty + if contentLen == 0 then + throw .invalidSliceIndex let sliceStart := index * leafBytes - if sliceStart ≥ data.size then + if sliceStart ≥ contentLen then throw .invalidSliceIndex - let sliceEnd := min data.size (sliceStart + count * leafBytes) - pure (data.extract sliceStart sliceEnd) + let sliceEnd := min contentLen (sliceStart + count * leafBytes) + let expectLen := sliceEnd - sliceStart + -- pos=8: walk full inboard artifact; do not copy response tail into a second buffer. + let (kept, endPos) ← + decodeRecRetainRange 0 contentLen true key root input 8 0 sliceStart sliceEnd + if endPos < input.size then + throw .trailingData + if endPos > input.size then + throw .truncatedResponse + -- Fail-closed if retain geometry under-produced (regression guard; W4 review #5). + if kept.size != expectLen then + throw .authenticationFailed + pure kept -/-- Strict verify of slice inside full inboard: full decode then extract. +/-- Strict verify of slice inside full inboard (W4a seekable retain). Same auth-first contract as `extractSliceFromInboard`. -/ @@ -328,6 +434,118 @@ def verifySliceInboard (key root input : ByteArray) (index count : Nat) : Except BaoError ByteArray := extractSliceFromInboard key root input index count +/-- Post-order outboard byte length for bare main of size `dataLen` (matches `createOutboard`). -/ +partial def outboardLenFor (dataLen : Nat) : Nat := + let nLeaves := leafGroupCount dataLen + if nLeaves ≤ 1 then + 0 + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let leftLen := min midBytes dataLen + let rightLen := dataLen - leftLen + outboardLenFor leftLen + outboardLenFor rightLen + 64 + +/-- + Seekable outboard slice verify over a sub-range of bare main (offset + len). + + Avoids recursive `ByteArray.extract` of whole unqueried halves (W4b lean-side). + Unqueried subtrees consume only outboard geometry length (no main leaf hashes). + + **Time:** O(slice + tree height) leaf hashing (not full re-encode of unqueried sides). + **Retained output:** O(slice). Input bare main + full outboard buffers still provided + at the C boundary (permanent full-buffer residual — no streaming ReadAt C ABI). + + Sibling hashes for unqueried subtrees are taken from the outboard parent pairs + (bao-tree range-verify semantics). Full outboard length must match geometry. +-/ +partial def verifyOutboardSliceRecAt (startLeaf : Nat) (bare : ByteArray) + (dataOff dataLen : Nat) (isRoot : Bool) (query : LeafQuery) (key expected : ByteArray) + (outboard : ByteArray) (obPos : Nat) : Except BaoError (ByteArray × Nat) := do + if query.isEmpty then + -- Unqueried subtree: parent already bound `expected` via outboard pair; no bytes. + pure (ByteArray.empty, obPos) + else + let nLeaves := leafGroupCount dataLen + if nLeaves ≤ 1 then + if dataOff + dataLen > bare.size then + throw .truncatedResponse + let data := bare.extract dataOff (dataOff + dataLen) + let startChunk := startLeaf * chunksPerSlice + let h := hashLeafGroup startChunk data isRoot key + if !ctEq h expected then + throw .authenticationFailed + pure (data, obPos) + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let midLeaf := startLeaf + mid + let leftLen := min midBytes dataLen + let rightLen := dataLen - leftLen + let leftObLen := outboardLenFor leftLen + let rightObLen := outboardLenFor rightLen + let pairPos := obPos + leftObLen + rightObLen + if pairPos + 64 > outboard.size then + throw .truncatedResponse + let leftH := outboard.extract pairPos (pairPos + 32) + let rightH := outboard.extract (pairPos + 32) (pairPos + 64) + let parentH := keyedParentCV leftH rightH isRoot key + if !ctEq parentH expected then + throw .authenticationFailed + let (lQ, rQ) := query.split startLeaf midLeaf + let (leftSlice, _) ← + verifyOutboardSliceRecAt startLeaf bare dataOff leftLen false lQ key leftH + outboard obPos + let (rightSlice, _) ← + verifyOutboardSliceRecAt midLeaf bare (dataOff + leftLen) rightLen false rQ + key rightH outboard (obPos + leftObLen) + pure (appendBA leftSlice rightSlice, pairPos + 64) + +/-- Back-compat wrapper: whole bare buffer as data region. -/ +partial def verifyOutboardSliceRec (startLeaf : Nat) (data : ByteArray) (isRoot : Bool) + (query : LeafQuery) (key expected : ByteArray) (outboard : ByteArray) (obPos : Nat) : + Except BaoError (ByteArray × Nat) := + verifyOutboardSliceRecAt startLeaf data 0 data.size isRoot query key expected outboard obPos + +/-- + Verified read of `count` contiguous 4 KiB slices at `index` from bare main + + post-order outboard. + + * `count = 0` → empty success **immediately** (no root/geometry/OOB/auth checks) — + matches Rust `verify_slice_outboard` / `stream/slice.rs` extract semantics. + * When `count > 0`: wrong root / tampered outboard / tampered slice → + `authenticationFailed`; OOB index → `invalidSliceIndex`; outboard length + mismatch → `truncatedResponse` / `trailingData`. +-/ +def verifySliceOutboard (key root bare outboard : ByteArray) (index count : Nat) : + Except BaoError ByteArray := do + if count == 0 then + -- Match Rust `verify_slice_outboard`: empty success before any geometry/auth. + return ByteArray.empty + if root.size != outLen then + throw .invalidRootLength + if bare.size == 0 then + throw .invalidSliceIndex + let expectObLen := outboardLenFor bare.size + if outboard.size < expectObLen then + throw .truncatedResponse + if outboard.size > expectObLen then + throw .trailingData + let sliceStart := index * leafBytes + if sliceStart ≥ bare.size then + throw .invalidSliceIndex + let sliceEnd := min bare.size (sliceStart + count * leafBytes) + let expectLen := sliceEnd - sliceStart + let (slice, _) ← + verifyOutboardSliceRecAt 0 bare 0 bare.size true (sliceLeafQuery index count) key + root outboard 0 + -- Fail-closed if range walk under-produced (regression guard; W4 review #5). + if slice.size != expectLen then + throw .authenticationFailed + pure slice + /-- Root equals keyed_hash (determinism / multi-dimensional naming basis). -/ theorem root_eq_keyed_hash (key data : ByteArray) : keyedRoot key data = keyedHash key data := rfl diff --git a/Carbonado/Cli.lean b/Carbonado/Cli.lean index 433ce4c..f731861 100644 --- a/Carbonado/Cli.lean +++ b/Carbonado/Cli.lean @@ -18,6 +18,7 @@ import Carbonado.Pipeline import Carbonado.Outboard import Carbonado.Adamantine import Carbonado.Filepack +import Carbonado.RkyvFilepack import Carbonado.Directory import Carbonado.Slh import Carbonado.Bao.Blake3 @@ -28,6 +29,7 @@ open Carbonado.Constants open Carbonado.Crypto.Util open Carbonado.Header open Carbonado.Pipeline +open Carbonado.RkyvFilepack open Carbonado.Outboard open Carbonado.Adamantine open Carbonado.Filepack @@ -70,7 +72,9 @@ def helpText : String := " -o output file or directory\n" ++ " carbonado slh parse Validate SLH1 sidecar wire (7860 B)\n" ++ " carbonado slh verify --root --pk \n" ++ - " Cryptographic bind-to-root (fails closed exit 1 until SLH-DSA FFI — LIMITS)\n" + " Live SLH-DSA-SHA2-128s verify via libbitcoinpqc (carbonado_slh_*);\n" ++ + " exit 0 accept / exit 1 reject or bad wire. Dual-suite may still use\n" ++ + " Rust bitcoinpqc composition (W3c / LIMITS) — pure Lean path is optional.\n" /-- Parse 64 hex chars → 32 bytes. -/ def parseMaster (s : String) : Except CliError ByteArray := @@ -202,7 +206,7 @@ def encodeFile (inputPath : System.FilePath) (outputPath : Option System.FilePat match encodeHeadered master nonce data format 0 (replicate slhPublicKeyLen 0) (replicate 8 0) with | .error e => pure (.error (.pipeline e)) - | .ok (hdr, archive) => + | .ok (hdr, archive, _info) => let out := match outputPath with | some p => p @@ -342,7 +346,7 @@ def decodeDir (catalogPath outputDir : System.FilePath) (master : ByteArray) : match splitPayload payload with | .error ae => pure (.error (.directory (ofAdamantineError ae))) | .ok (manBytes, baoBundle) => - match FilepackManifest.fromWireBytes manBytes expectedRoot with + match decodeCatalogBody manBytes expectedRoot with | .error pe => pure (.error (.directory (ofFilepackError pe))) | .ok manifest => if manifest.formatLevel != catalogFmt then @@ -382,7 +386,7 @@ def decodeDir (catalogPath outputDir : System.FilePath) (master : ByteArray) : else pure ByteArray.empty let pad := paddingForMainLen main.size fmtBits.fec match decodeOutboardBody master sref.segmentBaoRoot main verOb - fecPar pad fmtBits with + fecPar pad fmtBits false ByteArray.empty with | .error pe => return .error (.pipeline pe) | .ok part => recovered := appendBA recovered part match checkContentBlake3 recovered entry.contentBlake3 with @@ -514,17 +518,16 @@ def runCommand (cmd : String) (args : List String) : IO UInt32 := do match parseSidecar bytes with | .error e => IO.eprintln (formatCliError (.slh e)); pure 1 | .ok sig => - -- Fail-closed: never exit 0 without real SLH-DSA verify. - -- Until FFI is linked, always-false oracle → verificationFailed → exit 1. - match verifyBound (fun _ _ _ => false) pk root sig with - | .error .verificationFailed + -- G10: live SLH-DSA verify via libbitcoinpqc (fail-closed on reject). + match verifyBound liveVerifyOracle pk root sig with + | .error .verificationFailed => + IO.eprintln "slh verify: signature rejected" + pure 1 | .error .signatureUnavailable => - IO.eprintln "slh verify: cryptographic verification unavailable (LIMITS: no SLH-DSA FFI)" - IO.eprintln s!" wire parse ok (sigLen={sig.size}); NOT verified — exit 1" + IO.eprintln "slh verify: cryptographic verification unavailable" pure 1 | .error e => IO.eprintln (formatCliError (.slh e)); pure 1 | .ok () => - -- Only reachable once real oracle is linked and accepts. IO.println s!"slh verify ok root={toHex root}" pure 0 catch e => diff --git a/Carbonado/Directory.lean b/Carbonado/Directory.lean index e5ec38f..8944943 100644 --- a/Carbonado/Directory.lean +++ b/Carbonado/Directory.lean @@ -3,10 +3,13 @@ Layout (AGENTS §7.1): * Catalog: inboard headered `{catalog_root}.adam.c14` / `.adam.c15` - body = Adamantine10 envelope (CFP2 FilepackManifest + centralized Bao bundle) + body = Adamantine10 envelope (**rkyv** FilepackManifestWire v2 + centralized Bao bundle) * Segments: bare mains `{seg_root}.c12|c13|c14|c15` (no .out/.par on disk) * Bundle: per-segment [verification_outboard][fec_parity] indexed by SegmentRef + **W3:** pure Lean encode emits rkyv (bit-compatible with Rust dual-suite decode). + Decode accepts rkyv **or** legacy CFP2 via `decodeCatalogBody`. + Path rules fail-closed via Filepack.validateRelPath. Content integrity: BLAKE3 of recovered plaintext vs entry.content_blake3. -/ @@ -17,6 +20,7 @@ import Carbonado.Pipeline import Carbonado.Outboard import Carbonado.Adamantine import Carbonado.Filepack +import Carbonado.RkyvFilepack import Carbonado.Bao.Blake3 import Carbonado.Fec.Inboard @@ -29,6 +33,7 @@ open Carbonado.Pipeline open Carbonado.Outboard open Carbonado.Adamantine open Carbonado.Filepack +open Carbonado.RkyvFilepack open Carbonado.Bao.Blake3 open Carbonado.Fec.Inboard @@ -351,7 +356,9 @@ def encodeDirectory (master : ByteArray) (files : Array DirFile) err := some .insufficientNonces else if fmtBits.encrypted then nonceIdx := nonceIdx + 1 - match encodeOutboardBody master nonce chunk fmtBits with + -- Directory segments use low-level embedded-nonce layout (matches Rust + -- encoding::encode_outboard), not header-path file::encode_outboard. + match encodeOutboardBody master nonce chunk fmtBits false with | .error pe => err := some (ofPipelineError pe) | .ok oenc => -- Single-leaf trees may have empty post-order outboard (no parent pairs). @@ -418,7 +425,7 @@ def encodeDirectory (master : ByteArray) (files : Array DirFile) match err with | some e => pure (.error e) | none => - -- Build CFP2 manifest (catalog root placeholder zeros until headered encode). + -- Build rkyv FilepackManifestWire v2 (catalog root bound via filename, not wire). let placeholderRoot := replicate hashLen 0 let manifest : FilepackManifest := { version := filepackManifestVersion @@ -426,7 +433,7 @@ def encodeDirectory (master : ByteArray) (files : Array DirFile) catalogBaoRoot := placeholderRoot entries := entries } - match manifest.toWireBytes with + match encodeCatalogBody manifest with | .error pe => pure (.error (ofFilepackError pe)) | .ok manBytes => match buildPayload manBytes bundle.bytes with @@ -449,7 +456,7 @@ def encodeDirectory (master : ByteArray) (files : Array DirFile) match encodeHeadered master catNonce adamBody catFmtBits 0 (replicate slhPublicKeyLen 0) (replicate 8 0) with | .error pe => pure (.error (ofPipelineError pe)) - | .ok (hdr, catalogBytes) => + | .ok (hdr, catalogBytes, _info) => -- Rebind catalog root into manifest is not on wire; filename binds root. let root := hdr.hash match catalogFilename root catalogFmt with @@ -521,7 +528,7 @@ def decodeDirectory (master : ByteArray) (archive : DirectoryArchive) : match splitPayload payload with | .error ae => .error (ofAdamantineError ae) | .ok (manBytes, baoBundle) => - match FilepackManifest.fromWireBytes manBytes expectedRoot with + match decodeCatalogBody manBytes expectedRoot with | .error pe => .error (ofFilepackError pe) | .ok manifest => if manifest.formatLevel != catalogFmt then @@ -567,7 +574,8 @@ def decodeDirectory (master : ByteArray) (archive : DirectoryArchive) : | .ok fecPar => let pad := paddingForMainLen art.main.size fmtBits.fec match decodeOutboardBody master sref.segmentBaoRoot - art.main verOb fecPar pad fmtBits with + art.main verOb fecPar pad fmtBits false + ByteArray.empty with | .error pe => err := some (ofPipelineError pe) | .ok part => recovered := appendBA recovered part diff --git a/Carbonado/Ffi.lean b/Carbonado/Ffi.lean index 3ec6b87..c7db416 100644 --- a/Carbonado/Ffi.lean +++ b/Carbonado/Ffi.lean @@ -2,13 +2,38 @@ C ABI surface for dual-backend parity (docs/ABI.md). Pure helpers map Pipeline results to ABI error codes. `@[export]` entry points - are thin wrappers for the Lean AOT static library (`libcarbonado`). + return packed ByteArrays for the C glue in `nix/native/carbonado_abi.c`: + + status-prefixed: [u32 LE status][payload…] + encode body: [u32 LE status][u32 LE padding][u32 LE chunk_len] + [u32 LE bytes_ecc][u32 LE verifiable_slice_count] + [u32 LE bytes_compressed][u32 LE bytes_encrypted] + [32-byte hash][body…] + (success prefix 60 bytes; error = [status:4] only) + encode headered: [u32 LE status][u32 LE padding][u32 LE chunk_len] + [u32 LE bytes_ecc][u32 LE verifiable_slice_count] + [u32 LE bytes_compressed][u32 LE bytes_encrypted] + [archive…] + (success prefix 28 bytes; error = [status:4] only) + encode outboard: [u32 LE status][u32 LE padding][u32 LE chunk_len] + [u32 LE bytes_compressed][u32 LE bytes_encrypted] + [32 hash][u32 main_len][main][u32 ob_len][ob] + [u32 par_len][par] + (fixed prefix before segments: 52 bytes) + + C wrappers initialize the Lean runtime, convert buffers ↔ ByteArray, and expose + the stable `carbonado_*` symbols in `include/carbonado.h`. + + Phase 2 adds outboard / scrub / slice exports (additive on ABI version 1). + R3 adds compress/encrypt stage counters on encode packs (ABI version stays 1). -/ import Carbonado.Constants import Carbonado.Crypto.Util import Carbonado.Bao.Product import Carbonado.Header import Carbonado.Pipeline +import Carbonado.Outboard +import Carbonado.Scrub namespace Carbonado.Ffi @@ -17,13 +42,12 @@ open Carbonado.Crypto.Util open Carbonado.Bao.Product open Carbonado.Header open Carbonado.Pipeline +open Carbonado.Outboard +open Carbonado.Scrub /-- ABI version (must match `include/carbonado.h` / docs/ABI.md). -/ def abiVersion : UInt32 := 1 -@[export carbonado_abi_version] -def carbonado_abi_version : UInt32 := abiVersion - /-- Stable C error codes (docs/ABI.md). -/ def ok : UInt32 := 0 def errInvalidArgument : UInt32 := 1 @@ -38,39 +62,201 @@ def errScrubUnnecessary : UInt32 := 9 def errScrubFailed : UInt32 := 10 def errNotImplemented : UInt32 := 11 def errInternal : UInt32 := 12 +/-- Distinct from scrub recovery failure (docs/ABI.md Phase 2). -/ +def errScrubRequiresVerification : UInt32 := 13 + +/-- Collapse `PipelineError` into ABI codes (exhaustive; docs/ABI.md). -/-- Collapse `PipelineError` into ABI codes (exhaustive; docs/ABI.md). -/ + R4 fidelity: Bao auth / short inboard prefix must not collapse into a single + `errBao` diagnostic — dual-suite `matches!` expects `AuthenticationFailed` and + `InvalidHeaderLength` respectively (same as pure Rust). `invalidSliceIndex` + stays `errBao` at the C boundary; Rust `lean::verify_slice` applies geometry + pre-checks to surface `InvalidSliceIndex { index, content_len }`. +-/ def ofPipelineError : PipelineError → UInt32 | .invalidKeyLength => errInvalidKeyLength - | .payloadAuthenticationFailed | .headerAuthenticationFailed => errAuthentication + | .payloadAuthenticationFailed | .headerAuthenticationFailed + | .baoAuthenticationFailed => errAuthentication | .badMagic => errInvalidMagic - | .invalidHeaderLength | .truncatedBody | .invalidFieldLength => errInvalidHeader + | .invalidHeaderLength | .truncatedBody | .invalidFieldLength + | .invalidPrefix => errInvalidHeader | .unevenShards | .tooFewShards | .emptyShard | .incorrectShardSize | .badGeometry | .paddingTooLarge | .singularMatrix => errFec - | .baoAuthenticationFailed | .truncatedResponse | .trailingData - | .invalidPrefix | .invalidRootLength | .invalidSliceIndex | .invalidSliceCount => errBao + | .truncatedResponse | .trailingData + | .invalidRootLength | .invalidSliceIndex | .invalidSliceCount => errBao | .compressionFailed | .decompressionFailed | .decompressOutputTooLarge | .zstdInvalidInput => errZstd | .unnecessaryScrub => errScrubUnnecessary - | .scrubRequiresVerification | .invalidScrubbedHash => errScrubFailed + | .invalidScrubbedHash => errScrubFailed + | .scrubRequiresVerification => errScrubRequiresVerification | .invalidCiphertextLength | .invalidNonceLength | .insufficientNonces => errInvalidArgument | .invalidChunkSequence | .emptySegment => errInvalidArgument def masterOk (master : ByteArray) : Bool := master.size == 32 || master.size == 64 -/-- Pure headered encode for FFI (explicit nonce; zero SLH/meta). -/ -def encodeHeaderedBytes (master nonce plaintext : ByteArray) (format : UInt8) : - Except UInt32 ByteArray := +/-- Append u32 little-endian. -/ +def pushU32LE (out : ByteArray) (x : UInt32) : ByteArray := + out.push (UInt8.ofNat (UInt32.toNat x % 256)) + |>.push (UInt8.ofNat (UInt32.toNat (x >>> 8) % 256)) + |>.push (UInt8.ofNat (UInt32.toNat (x >>> 16) % 256)) + |>.push (UInt8.ofNat (UInt32.toNat (x >>> 24) % 256)) + +/-- Pack `[u32 LE status][payload]`. -/ +def packStatus (code : UInt32) (payload : ByteArray) : ByteArray := + appendBA (pushU32LE ByteArray.empty code) payload + +/-- Pack encode body error: `[u32 LE status]` only (C parses status-first; see carbonado_abi.c). -/ +def packEncodeErr (code : UInt32) : ByteArray := + pushU32LE ByteArray.empty code + +/-- Encode metadata fields returned with body/headered/outboard success packs. -/ +structure EncodeMeta where + padding : UInt32 + chunkLen : UInt32 + bytesEcc : UInt32 + verifiableSliceCount : UInt32 + bytesCompressed : UInt32 + bytesEncrypted : UInt32 + deriving DecidableEq + +/-- Convert pipeline `EncodeInfo` length fields to u32 `EncodeMeta` (fail-closed on overflow). -/ +def encodeMetaOf (info : EncodeInfo) : Except UInt32 EncodeMeta := + match natToU32Field info.paddingLen with + | .error e => .error (ofPipelineError e) + | .ok pad => + match natToU32Field info.chunkLen with + | .error e => .error (ofPipelineError e) + | .ok cl => + match natToU32Field info.bytesEcc with + | .error e => .error (ofPipelineError e) + | .ok be => + match natToU32Field info.verifiableSliceCount with + | .error e => .error (ofPipelineError e) + | .ok vsc => + match natToU32Field info.bytesCompressed with + | .error e => .error (ofPipelineError e) + | .ok bc => + match natToU32Field info.bytesEncrypted with + | .error e => .error (ofPipelineError e) + | .ok be2 => + .ok { + padding := pad + chunkLen := cl + bytesEcc := be + verifiableSliceCount := vsc + bytesCompressed := bc + bytesEncrypted := be2 + } + +/-- Pack six u32 EncodeMeta fields after status (24 bytes). -/ +def pushEncodeMeta (out : ByteArray) (em : EncodeMeta) : ByteArray := + pushU32LE + (pushU32LE + (pushU32LE + (pushU32LE + (pushU32LE + (pushU32LE out em.padding) + em.chunkLen) + em.bytesEcc) + em.verifiableSliceCount) + em.bytesCompressed) + em.bytesEncrypted + +/-- Pack encode body success: + `[u32 LE status=0][padding:4][chunk_len:4][bytes_ecc:4][vsc:4] + [bytes_compressed:4][bytes_encrypted:4][32 hash][body]`. + + Requires `hash.size = 32`; otherwise packs `errInternal` (defensive layout guard). + Header size after status: 24 + 32 = 56; total prefix 60 bytes. +-/ +def packEncodeOk (em : EncodeMeta) (hash body : ByteArray) : ByteArray := + if hash.size != 32 then + packEncodeErr errInternal + else + let hdr := pushEncodeMeta (pushU32LE ByteArray.empty ok) em + appendBA (appendBA hdr hash) body + +/-- Pack headered encode success: + `[status:4][pad:4][chunk:4][ecc:4][vsc:4][comp:4][enc:4][archive…]`. + + Total meta prefix 28 bytes (status + EncodeMeta). +-/ +def packHeaderedOk (em : EncodeMeta) (archive : ByteArray) : ByteArray := + appendBA (pushEncodeMeta (pushU32LE ByteArray.empty ok) em) archive + +/-- Pack length-prefixed segment: `[u32 LE len][bytes]`, or `none` if len overflows u32. -/ +def packLenPrefixed? (payload : ByteArray) : Option ByteArray := + match natToU32Field payload.size with + | .error _ => none + | .ok n => some (appendBA (pushU32LE ByteArray.empty n) payload) + +/-- Pack outboard encode success: + `[status:4][padding:4][chunk_len:4][bytes_compressed:4][bytes_encrypted:4][hash:32] + [u32 main_len][main][u32 ob_len][ob][u32 par_len][par]`. + + Fixed prefix before segments: 52 bytes. Oversized segments → `errInternal`. +-/ +def packOutboardOk (padding chunkLen bytesCompressed bytesEncrypted : UInt32) + (hash main ob par : ByteArray) : ByteArray := + if hash.size != 32 then + packEncodeErr errInternal + else + match packLenPrefixed? main with + | none => packEncodeErr errInternal + | some m => + match packLenPrefixed? ob with + | none => packEncodeErr errInternal + | some o => + match packLenPrefixed? par with + | none => packEncodeErr errInternal + | some p => + let hdr := + pushU32LE + (pushU32LE + (pushU32LE + (pushU32LE (pushU32LE ByteArray.empty ok) padding) + chunkLen) + bytesCompressed) + bytesEncrypted + let withHash := appendBA hdr hash + appendBA (appendBA (appendBA withHash m) o) p + +/-- Pure headered encode for FFI (explicit nonce; optional SLH pk + 8-byte metadata). + + Header always carries a 16-byte `payload_nonce` (Rust `file::encode`). Encrypted + formats require a caller-supplied 16-byte nonce; public formats use zeros when + nonce is empty/absent. + + `slhPublicKey` must be empty or 32 bytes (empty → zeros). `metadata` must be empty + or 8 bytes (empty → zeros). Wrong lengths → `errInvalidArgument`. + + Returns full archive + stage-counter `EncodeMeta` (R3). +-/ +def encodeHeaderedBytes (master nonce plaintext slhPublicKey metadataBytes : ByteArray) + (format : UInt8) : Except UInt32 (ByteArray × EncodeMeta) := if !masterOk master then .error errInvalidKeyLength - else if (FormatBits.ofUInt8 format).encrypted && nonce.size != nonceLen then + else if !(slhPublicKey.size == 0 || slhPublicKey.size == 32) then + .error errInvalidArgument + else if !(metadataBytes.size == 0 || metadataBytes.size == 8) then .error errInvalidArgument else let fmt := FormatBits.ofUInt8 format - let n := if fmt.encrypted then nonce else ByteArray.mkEmpty 0 - match encodeHeadered master n plaintext fmt 0 (ByteArray.mkEmpty 32) (ByteArray.mkEmpty 8) with - | .error e => .error (ofPipelineError e) - | .ok (_hdr, archive) => .ok archive + let n := + if fmt.encrypted then nonce + else if nonce.size == nonceLen then nonce + else replicate nonceLen 0 + let slh := if slhPublicKey.size == 32 then slhPublicKey else replicate 32 0 + let metaBytes := if metadataBytes.size == 8 then metadataBytes else replicate 8 0 + if n.size != nonceLen then + .error errInvalidArgument + else + match encodeHeadered master n plaintext fmt 0 slh metaBytes with + | .error e => .error (ofPipelineError e) + | .ok (_hdr, archive, info) => + match encodeMetaOf info with + | .error e => .error e + | .ok em => .ok (archive, em) /-- Pure headered decode for FFI. -/ def decodeHeaderedBytes (master archive : ByteArray) : Except UInt32 ByteArray := @@ -80,23 +266,249 @@ def decodeHeaderedBytes (master archive : ByteArray) : Except UInt32 ByteArray : | .error e => .error (ofPipelineError e) | .ok pt => .ok pt +/-- Low-level body encode (embedded-nonce encrypt path; `headerPathEncrypt = false`). -/ +def encodeBodyBytes (master nonce plaintext : ByteArray) (format : UInt8) : + Except UInt32 (ByteArray × ByteArray × EncodeMeta) := + if !masterOk master then .error errInvalidKeyLength + else + let fmt := FormatBits.ofUInt8 format + if fmt.encrypted && nonce.size != nonceLen then + .error errInvalidArgument + else + let n := if fmt.encrypted then nonce else ByteArray.empty + match encodeBody master n plaintext fmt false with + | .error e => .error (ofPipelineError e) + | .ok enc => + match encodeMetaOf enc.info with + | .error e => .error e + | .ok em => .ok (enc.body, enc.baoHash, em) + +/-- Low-level body decode. -/ +def decodeBodyBytes (master hash body : ByteArray) (padding : UInt32) (format : UInt8) : + Except UInt32 ByteArray := + if !masterOk master then .error errInvalidKeyLength + else if hash.size != 32 then .error errInvalidArgument + else + let fmt := FormatBits.ofUInt8 format + -- Nonce unused for public / embedded-nonce decrypt (passed empty). + match decodeBody master ByteArray.empty hash body padding.toNat fmt false with + | .error e => .error (ofPipelineError e) + | .ok pt => .ok pt + +/-- Outboard encode. + + `headerPath ≠ 0` → encrypted bare main is `[tag|ct]` (nonce out-of-band). + `headerPath = 0` → encrypted bare main is `[nonce|tag|ct]` (embedded). + + Returns main/ob/par/hash + pad/chunk + compress/encrypt stage counters (R3). +-/ +def encodeOutboardBytes (master nonce plaintext : ByteArray) (format headerPath : UInt8) : + Except UInt32 + (ByteArray × ByteArray × ByteArray × ByteArray × UInt32 × UInt32 × UInt32 × UInt32) := + if !masterOk master then .error errInvalidKeyLength + else + let fmt := FormatBits.ofUInt8 format + if fmt.encrypted && nonce.size != nonceLen then + .error errInvalidArgument + else + let n := if fmt.encrypted then nonce else ByteArray.empty + let hp := headerPath != 0 + match encodeOutboardBody master n plaintext fmt hp with + | .error e => .error (ofPipelineError e) + | .ok enc => + match natToU32Field enc.paddingLen with + | .error e => .error (ofPipelineError e) + | .ok pad => + match natToU32Field enc.chunkLen with + | .error e => .error (ofPipelineError e) + | .ok cl => + match natToU32Field enc.bytesCompressed with + | .error e => .error (ofPipelineError e) + | .ok bc => + match natToU32Field enc.bytesEncrypted with + | .error e => .error (ofPipelineError e) + | .ok be => + .ok (enc.main, enc.verificationOutboard, enc.fecParity, enc.baoHash, + pad, cl, bc, be) + +/-- Outboard decode. `headerPath`/`nonce` must match encode-time layout. -/ +def decodeOutboardBytes (master hash main verOutboard fecParity : ByteArray) + (padding : UInt32) (format headerPath : UInt8) (nonce : ByteArray) : + Except UInt32 ByteArray := + if !masterOk master then .error errInvalidKeyLength + else if hash.size != 32 then .error errInvalidArgument + else + let fmt := FormatBits.ofUInt8 format + let hp := headerPath != 0 + if hp && fmt.encrypted && nonce.size != nonceLen then + .error errInvalidArgument + else + let n := if hp && fmt.encrypted then nonce else ByteArray.empty + match decodeOutboardBody master hash main verOutboard fecParity padding.toNat fmt hp n with + | .error e => .error (ofPipelineError e) + | .ok pt => .ok pt + +/-- Inboard scrub (returns recovered body bytes). -/ +def scrubBytes (body hash : ByteArray) (padding : UInt32) (format : UInt8) : + Except UInt32 ByteArray := + if hash.size != 32 then .error errInvalidArgument + else + let fmt := FormatBits.ofUInt8 format + match scrubInboard body hash padding.toNat fmt with + | .error e => .error (ofPipelineError e) + | .ok recovered => .ok recovered + +/-- Outboard scrub (returns recovered bare main). -/ +def scrubOutboardBytes (main verOutboard fecParity hash : ByteArray) + (padding chunkLen : UInt32) (format : UInt8) : Except UInt32 ByteArray := + if hash.size != 32 then .error errInvalidArgument + else + let fmt := FormatBits.ofUInt8 format + match scrubOutboard main verOutboard fecParity hash padding.toNat chunkLen.toNat fmt with + | .error e => .error (ofPipelineError e) + | .ok bare => .ok bare + +/-- Inboard verify_slice / extract_slice (W4a: auth-first O(slice) retain; full body at C). + + Product path walks the full inboard artifact from offset 8 (no second response copy; + O(N) time) but retains only the requested slice bytes (O(slice) output). C ABI still + takes the full inboard body buffer as input. `count == 0` still authenticates first + (Lean C auth-first contract), unlike the Rust lean wrapper which short-circuits empty + success before C (parity with pure-Rust `verify_slice_inboard_seekable`). +-/ +def verifySliceBytes (body hash : ByteArray) (index count : UInt32) (format : UInt8) : + Except UInt32 ByteArray := + if hash.size != 32 then .error errInvalidArgument + else if count.toNat == 0 then + match verifySliceInboardForFormat format hash body 0 0 with + | .error e => .error (ofPipelineError (ofBaoError e)) + | .ok data => .ok data + else + match verifySliceInboardForFormat format hash body index.toNat count.toNat with + | .error e => .error (ofPipelineError (ofBaoError e)) + | .ok data => .ok data + +/-- Seekable outboard verify_slice (O(slice + height) hash; full buffers at C ABI — W4b permanent). -/ +def verifySliceOutboardBytes (main outboard hash : ByteArray) + (index count : UInt32) (format : UInt8) : Except UInt32 ByteArray := + if hash.size != 32 then .error errInvalidArgument + else + match verifySliceOutboardForFormat format hash main outboard index.toNat count.toNat with + | .error e => .error (ofPipelineError (ofBaoError e)) + | .ok data => .ok data + /-- Format verification key (32 bytes). -/ def verificationKeyBytes (format : UInt8) : ByteArray := carbonadoVerificationKey format /-- Round-trip self-check (public or encrypted with given nonce). -/ def roundtripHeaderedOk (master nonce plaintext : ByteArray) (format : UInt8) : Bool := - match encodeHeaderedBytes master nonce plaintext format with + match encodeHeaderedBytes master nonce plaintext ByteArray.empty ByteArray.empty format with | .error _ => false - | .ok arch => + | .ok (arch, _em) => match decodeHeaderedBytes master arch with | .error _ => false | .ok pt => ctEq pt plaintext +--------------------------------------------------------------------------- +-- @[export] surface for C glue (namespaced `l_` to avoid clashing with C ABI) +--------------------------------------------------------------------------- + +/-- Packed: always 32-byte key (status implied OK). -/ +@[export l_carbonado_verification_key] +def l_carbonado_verification_key (format : UInt8) : ByteArray := + verificationKeyBytes format + +/-- Packed success: EncodeMeta + archive; error: `[status:4]` only. + Optional `slhPublicKey` (0 or 32 B) and `metadataBytes` (0 or 8 B). -/ +@[export l_carbonado_encode_headered] +def l_carbonado_encode_headered (master nonce plaintext slhPublicKey metadataBytes : ByteArray) + (format : UInt8) : ByteArray := + match encodeHeaderedBytes master nonce plaintext slhPublicKey metadataBytes format with + | .error e => packEncodeErr e + | .ok (arch, em) => packHeaderedOk em arch + +/-- Packed: `[status:4][plaintext…]`. -/ +@[export l_carbonado_decode_headered] +def l_carbonado_decode_headered (master archive : ByteArray) : ByteArray := + match decodeHeaderedBytes master archive with + | .error e => packStatus e ByteArray.empty + | .ok pt => packStatus ok pt + +/-- Packed success: encode meta + hash + body; error: `[status:4]` only. -/ +@[export l_carbonado_encode] +def l_carbonado_encode (master nonce plaintext : ByteArray) (format : UInt8) : ByteArray := + match encodeBodyBytes master nonce plaintext format with + | .error e => packEncodeErr e + | .ok (body, hash, em) => + if hash.size != 32 then packEncodeErr errInternal + else packEncodeOk em hash body + +/-- Packed: `[status:4][plaintext…]`. -/ +@[export l_carbonado_decode] +def l_carbonado_decode (master hash body : ByteArray) (padding : UInt32) (format : UInt8) : ByteArray := + match decodeBodyBytes master hash body padding format with + | .error e => packStatus e ByteArray.empty + | .ok pt => packStatus ok pt + +/-- Packed outboard encode success layout; error `[status:4]`. + + `headerPath ≠ 0` → header-path `[tag|ct]` encrypt; `0` → embedded-nonce. +-/ +@[export l_carbonado_encode_outboard] +def l_carbonado_encode_outboard (master nonce plaintext : ByteArray) + (format headerPath : UInt8) : ByteArray := + match encodeOutboardBytes master nonce plaintext format headerPath with + | .error e => packEncodeErr e + | .ok (main, ob, par, hash, pad, cl, bc, be) => + packOutboardOk pad cl bc be hash main ob par + +/-- Packed: `[status:4][plaintext…]`. + + `headerPath ≠ 0` requires 16-byte `nonce` for encrypted formats. +-/ +@[export l_carbonado_decode_outboard] +def l_carbonado_decode_outboard (master hash main verOutboard fecParity : ByteArray) + (padding : UInt32) (format headerPath : UInt8) (nonce : ByteArray) : ByteArray := + match decodeOutboardBytes master hash main verOutboard fecParity padding format headerPath nonce with + | .error e => packStatus e ByteArray.empty + | .ok pt => packStatus ok pt + +/-- Packed: `[status:4][recovered…]`. -/ +@[export l_carbonado_scrub] +def l_carbonado_scrub (body hash : ByteArray) (padding : UInt32) (format : UInt8) : ByteArray := + match scrubBytes body hash padding format with + | .error e => packStatus e ByteArray.empty + | .ok rec => packStatus ok rec + +/-- Packed: `[status:4][recovered bare…]`. -/ +@[export l_carbonado_scrub_outboard] +def l_carbonado_scrub_outboard (main verOutboard fecParity hash : ByteArray) + (padding chunkLen : UInt32) (format : UInt8) : ByteArray := + match scrubOutboardBytes main verOutboard fecParity hash padding chunkLen format with + | .error e => packStatus e ByteArray.empty + | .ok bare => packStatus ok bare + +/-- Packed: `[status:4][slice bytes…]`. -/ +@[export l_carbonado_verify_slice] +def l_carbonado_verify_slice (body hash : ByteArray) (index count : UInt32) (format : UInt8) : + ByteArray := + match verifySliceBytes body hash index count format with + | .error e => packStatus e ByteArray.empty + | .ok data => packStatus ok data + +/-- Packed: `[status:4][slice bytes…]` from bare main + post-order outboard. -/ +@[export l_carbonado_verify_slice_outboard] +def l_carbonado_verify_slice_outboard (main outboard hash : ByteArray) + (index count : UInt32) (format : UInt8) : ByteArray := + match verifySliceOutboardBytes main outboard hash index count format with + | .error e => packStatus e ByteArray.empty + | .ok data => packStatus ok data + theorem abiVersion_eq : abiVersion = 1 := by native_decide -theorem masterOk_32 : masterOk (ByteArray.mkArray 32 0) = true := by native_decide +theorem masterOk_32 : masterOk (replicate 32 0) = true := by native_decide -theorem masterOk_31 : masterOk (ByteArray.mkArray 31 0) = false := by native_decide +theorem masterOk_31 : masterOk (replicate 31 0) = false := by native_decide end Carbonado.Ffi diff --git a/Carbonado/Filepack.lean b/Carbonado/Filepack.lean index 16a8ae2..781ba6a 100644 --- a/Carbonado/Filepack.lean +++ b/Carbonado/Filepack.lean @@ -1,14 +1,26 @@ /- FilepackManifest v2 for Adamantine catalogs (Program G). - **Wire note (LIMITS):** Rust uses rkyv `FilepackManifestWire`. Lean ships a - deterministic **CFP2** native codec with the same *logical* fields (version, - format_level, entries with SegmentRef + content_blake3). Adamantine envelope - framing matches Rust (`manifest_len` + body + `bundle_len` + bundle); the - *manifest body* is Lean-native CFP2, not rkyv — interop with Rust-produced - catalogs requires a converter (tracked LIMITS). Product CLI uses CFP2 end-to-end. - - Path rules: fail-closed (no `..`, no absolute, no backslash, length caps). + **Wire note (LIMITS / dual-suite):** Dual-suite directory archives use Rust + **rkyv** `FilepackManifestWire` as the normative Adamantine payload body. Under + `backend-lean`, `file::encode_directory` / `decode_directory` keep rkyv in Rust + and dispatch segment/catalog *crypto* through Lean C ABI (composition). Dual-suite + does **not** require pure Lean rkyv. + + **W3 pure Lean product wire:** `Carbonado/RkyvFilepack.lean` provides bit-exact + rkyv **encode** (`encodeRkyvManifest` / `encodeCatalogBody`) and **decode** + (`decodeRkyvManifest` / `decodeCatalogBody`) matching goldens in + `tests/fixtures/rkyv/`. Pure Lean `Directory.encodeDirectory` / CLI emit rkyv + so Rust dual-suite `decode_directory` can consume Lean-made catalogs. + Dual-decode **prefers rkyv first** (avoids CFP2 sniffer hijack of roots starting + with ASCII `CFP2`). + + This module also ships a deterministic **CFP2** Lean-native codec with the same + *logical* fields for legacy demos / dual-decode fallback. CFP2 is **not** + byte-identical to rkyv; product pure-Lean encode prefers rkyv. + + Path rules: fail-closed (no `..`, no absolute, no backslash, **UTF-8 byte** + length cap). Lean is stricter than Rust on empty components / NUL (LIMITS). -/ import Carbonado.Constants import Carbonado.Crypto.Util @@ -26,7 +38,7 @@ def filepackManifestVersion : Nat := 2 /-- Max entries (DoS). -/ def maxFilepackEntries : Nat := 100000 -/-- Max rel_path bytes. -/ +/-- Max rel_path **UTF-8 bytes** (Rust `MAX_REL_PATH_LEN`; not Unicode scalar count). -/ def maxRelPathLen : Nat := 4096 /-- Max OTS proof blob. -/ @@ -128,12 +140,17 @@ structure FilepackManifest where /-- Fail-closed relative path validation (AGENTS / Rust `validate_rel_path` + extras). - Rejects: empty, too long, `\`, absolute `/`, `..`, empty components, NUL. + Rejects: empty, too long (**UTF-8 byte** length > `maxRelPathLen`, matching Rust + `rel.len()` / `MAX_REL_PATH_LEN`), `\`, absolute `/`, `..`, empty components, NUL. + + **Stricter than Rust (LIMITS):** Lean also rejects empty path components (`a//b`) + and embedded NUL. Handcrafted Rust rkyv with `//` can fail Lean decode validate; + normal `encode_directory` FS walks do not emit those paths. -/ def validateRelPath (rel : String) : Except FilepackError Unit := if rel.isEmpty then .error .emptyRelPath - else if rel.length > maxRelPathLen then + else if (utf8 rel).size > maxRelPathLen then .error .relPathTooLong else if rel.contains '\\' then .error .relPathBackslash diff --git a/Carbonado/Main.lean b/Carbonado/Main.lean index df3374b..8972689 100644 --- a/Carbonado/Main.lean +++ b/Carbonado/Main.lean @@ -11,6 +11,7 @@ import Carbonado.Scrub import Carbonado.Shard import Carbonado.Adamantine import Carbonado.Filepack +import Carbonado.RkyvFilepack import Carbonado.Outboard import Carbonado.Directory import Carbonado.Cli @@ -39,6 +40,7 @@ open Carbonado.Scrub open Carbonado.Shard open Carbonado.Adamantine open Carbonado.Filepack +open Carbonado.RkyvFilepack open Carbonado.Outboard open Carbonado.Directory open Carbonado.Cli @@ -546,6 +548,27 @@ def runDemo : IO Unit := do | .ok _ => pure () | .error e => fail s!"outboard verify: {repr e}" IO.println "outboard encode/verify ok" + -- R9 seekable outboard slice (O(slice) path) + match verifySliceOutboardForFormat 4 rOb bao5k ob 0 1 with + | .ok s => + expectTrue "ob slice size" (s.size == 4096) + expectTrue "ob slice bytes" (toHex s == toHex (bao5k.extract 0 4096)) + | .error e => fail s!"ob slice: {repr e}" + match verifySliceOutboardForFormat 4 rOb bao5k ob 1 1 with + | .ok s => + expectTrue "ob slice1 size" (s.size == 904) + expectTrue "ob slice1 bytes" (toHex s == toHex (bao5k.extract 4096 5000)) + | .error e => fail s!"ob slice1: {repr e}" + match verifySliceOutboardForFormat 4 rOb bao5k ob 0 0 with + | .ok s => expectTrue "ob count0" (s.size == 0) + | .error e => fail s!"ob count0: {repr e}" + let mut badOb := ob + badOb := badOb.set! 0 (badOb.get! 0 ^^^ 1) + match verifySliceOutboardForFormat 4 rOb bao5k badOb 0 1 with + | Except.error .authenticationFailed => pure () + | Except.error e => fail s!"ob tamper: expected authenticationFailed, got {repr e}" + | Except.ok _ => fail "ob tamper: ok" + IO.println "outboard slice verify ok" -- Slice first group of 5000: stream decode (no plaintext oracle) let (rSlice, sliceEnc) := encodeSliceForFormat 4 bao5k 0 1 @@ -769,7 +792,7 @@ def runDemo : IO Unit := do match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) 0 zeroSlhPk zeroMeta with | .error e => fail s!"enc c0 for len: {repr e}" - | .ok (_h, arch) => + | .ok (_h, arch, _info) => -- trailer after body still recovers let withTrailer := appendBA arch (ofList [0xaa, 0xbb, 0xcc]) match decodeHeadered master42 withTrailer with @@ -785,7 +808,7 @@ def runDemo : IO Unit := do match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 5) 0 zeroSlhPk zeroMeta with | .error e => fail s!"enc c5 for trailer: {repr e}" - | .ok (_h, arch) => + | .ok (_h, arch, _info) => let withTrailer := appendBA arch (ofList [0xde, 0xad]) match decodeHeadered master42 withTrailer with | .ok pt => expectTrue "c5 trailer ignore" (toHex pt == toHex (utf8 "hello")) @@ -1140,19 +1163,38 @@ def runDemo : IO Unit := do | .error .verificationFailed => pure () | .error e => fail s!"bad sig: expected verificationFailed, got {repr e}" | .ok _ => fail "bad sig: ok" - match signRoot (replicate 128 0x42) rootA with - | .error .signatureUnavailable => pure () - | .error e => fail s!"sign: expected signatureUnavailable, got {repr e}" - | .ok _ => fail "sign: ok" match signRoot (replicate 128 0x42) (ofList [1]) with | .error .invalidRootLength => pure () | .error e => fail s!"sign root len: expected invalidRootLength, got {repr e}" | .ok _ => fail "sign root len: ok" + match signRoot (replicate 16 0x42) rootA with + | .error .invalidEntropyLength => pure () + | .error e => fail s!"sign entropy: expected invalidEntropyLength, got {repr e}" + | .ok _ => fail "sign entropy: ok" match mkBinding (ofList [1]) rootA goodSig with | .error .invalidPublicKeyLength => pure () | .error e => fail s!"pk len: expected invalidPublicKeyLength, got {repr e}" | .ok _ => fail "pk len: ok" - IO.println "SLH bind-to-root + unavailable sign ok" + -- G10 live SLH-DSA: fixed entropy keygen → sign root → verify; fail-closed on tamper + match keygenAndSignRoot (replicate 128 0x42) rootA with + | .error e => fail s!"live keygen/sign: {repr e}" + | .ok (pkLive, _sk, sigLive) => + expectTrue "live sig len" (sigLive.size == slh1SignatureLen) + expectTrue "live pk len" (pkLive.size == slhPublicKeyLen) + match verifyRoot pkLive rootA sigLive with + | .ok () => pure () + | .error e => fail s!"live verify good: {repr e}" + match verifyRoot pkLive rootB sigLive with + | .error .verificationFailed => pure () + | .error e => fail s!"live wrong root: expected verificationFailed, got {repr e}" + | .ok () => fail "live wrong root: ok" + let mut badSig := sigLive + badSig := badSig.set! 0 (badSig.get! 0 ^^^ 1) + match verifyRoot pkLive rootA badSig with + | .error .verificationFailed => pure () + | .error e => fail s!"live bad sig: expected verificationFailed, got {repr e}" + | .ok () => fail "live bad sig: ok" + IO.println "SLH live sign/verify ok" IO.println "program F stack ok" @@ -1214,6 +1256,158 @@ def runDemo : IO Unit := do | _ => fail "rel ok" IO.println "filepack path rules ok" + -- R9/W3 pure Lean rkyv encode + dual-decode goldens (Rust FilepackManifestWire v2) + match fromHex? goldenEmptyHex with + | none => fail "rkyv empty hex" + | some emptyBytes => + expectTrue "rkyv empty len" (emptyBytes.size == 13) + match decodeRkyvManifest emptyBytes (replicate hashLen 0) with + | .error e => fail s!"rkyv empty decode: {repr e}" + | .ok m => + expectTrue "rkyv empty ver" (m.version == 2) + expectTrue "rkyv empty fmt" (m.formatLevel == 0x0e) + expectTrue "rkyv empty entries" (m.entries.size == 0) + -- W3a: encode empty must bit-match golden + match encodeRkyvManifest m with + | .error e => fail s!"rkyv empty encode: {repr e}" + | .ok enc => + expectTrue "rkyv empty encode golden" (ctEq enc emptyBytes) + match encodeRkyvManifest m with + | .ok enc2 => expectTrue "rkyv empty codecode" (ctEq enc enc2) + | .error e => fail s!"rkyv empty encode2: {repr e}" + match fromHex? goldenSingleHex with + | none => fail "rkyv single hex" + | some singleBytes => + expectTrue "rkyv single len" (singleBytes.size == 131) + match decodeRkyvManifest singleBytes (replicate hashLen 0x33) with + | .error e => fail s!"rkyv single decode: {repr e}" + | .ok m => + expectTrue "rkyv single ver" (m.version == 2) + expectTrue "rkyv single fmt" (m.formatLevel == 0x0e) + expectTrue "rkyv single n" (m.entries.size == 1) + let e := m.entries[0]! + expectTrue "rkyv path" (e.relPath == "a.txt") + expectTrue "rkyv segfmt" (e.segmentFormat == 0x0e) + expectTrue "rkyv segs" (e.segments.size == 1) + expectTrue "rkyv main_len" (e.segments[0]!.mainLen == 100) + expectTrue "rkyv blake" (ctEq e.contentBlake3 (replicate hashLen 0x22)) + match encodeRkyvManifest m with + | .error e => fail s!"rkyv single encode: {repr e}" + | .ok enc => + expectTrue "rkyv single encode golden" (ctEq enc singleBytes) + match decodeRkyvManifest enc (replicate hashLen 0x33) with + | .error e => fail s!"rkyv single redecode: {repr e}" + | .ok m2 => + expectTrue "rkyv single roundtrip path" (m2.entries[0]!.relPath == "a.txt") + match decodeCatalogBody singleBytes (replicate hashLen 0x33) with + | .ok _ => pure () + | .error e => fail s!"rkyv dual-decode: {repr e}" + -- Multi-entry + out-of-line path + OTS Some golden + match fromHex? goldenMultiOtsHex with + | none => fail "rkyv multi hex" + | some multiBytes => + expectTrue "rkyv multi len" (multiBytes.size == 275) + match decodeRkyvManifest multiBytes (replicate hashLen 0x55) with + | .error e => fail s!"rkyv multi decode: {repr e}" + | .ok m => + expectTrue "rkyv multi n" (m.entries.size == 2) + expectTrue "rkyv multi a" (m.entries[0]!.relPath == "a.txt") + expectTrue "rkyv multi b" (m.entries[1]!.relPath == "b/longer-path-name.txt") + match m.entries[1]!.otsProof with + | some p => expectTrue "rkyv multi ots" (p.size == 4 && p.get! 0 == 0xab) + | none => fail "rkyv multi ots missing" + match encodeRkyvManifest m with + | .error e => fail s!"rkyv multi encode: {repr e}" + | .ok enc => + expectTrue "rkyv multi encode golden" (ctEq enc multiBytes) + match encodeRkyvManifest m with + | .ok enc2 => expectTrue "rkyv multi codecode" (ctEq enc enc2) + | .error e => fail s!"rkyv multi encode2: {repr e}" + -- Truncated buffer fail-closed (exact invalidWire) + match decodeRkyvManifest (multiBytes.extract 0 12) (replicate hashLen 0x55) with + | .error .invalidWire => pure () + | .error e => fail s!"rkyv trunc: expected invalidWire, got {repr e}" + | .ok _ => fail "rkyv trunc: ok" + -- Extra free-form goldens (path 8/9 boundary, two segs, OTS mix) + match fromHex? goldenPathInline8Hex with + | none => fail "rkyv path8 hex" + | some b => + match decodeRkyvManifest b (replicate hashLen 0) with + | .error e => fail s!"rkyv path8 decode: {repr e}" + | .ok m => + expectTrue "rkyv path8" (m.entries[0]!.relPath == "12345678") + match encodeRkyvManifest m with + | .ok enc => expectTrue "rkyv path8 encode" (ctEq enc b) + | .error e => fail s!"rkyv path8 encode: {repr e}" + match fromHex? goldenPathOol9Hex with + | none => fail "rkyv path9 hex" + | some b => + match decodeRkyvManifest b (replicate hashLen 0) with + | .error e => fail s!"rkyv path9 decode: {repr e}" + | .ok m => + expectTrue "rkyv path9" (m.entries[0]!.relPath == "123456789") + match encodeRkyvManifest m with + | .ok enc => expectTrue "rkyv path9 encode" (ctEq enc b) + | .error e => fail s!"rkyv path9 encode: {repr e}" + match fromHex? goldenTwoSegmentsHex with + | none => fail "rkyv two_seg hex" + | some b => + match decodeRkyvManifest b (replicate hashLen 0) with + | .error e => fail s!"rkyv two_seg decode: {repr e}" + | .ok m => + expectTrue "rkyv two_seg n" (m.entries[0]!.segments.size == 2) + expectTrue "rkyv two_seg idx1" (m.entries[0]!.segments[1]!.chunkIndex == 1) + match encodeRkyvManifest m with + | .ok enc => expectTrue "rkyv two_seg encode" (ctEq enc b) + | .error e => fail s!"rkyv two_seg encode: {repr e}" + match fromHex? goldenOtsFirstOnlyHex with + | none => fail "rkyv ots_mix hex" + | some b => + match decodeRkyvManifest b (replicate hashLen 0) with + | .error e => fail s!"rkyv ots_mix decode: {repr e}" + | .ok m => + expectTrue "rkyv ots_mix n" (m.entries.size == 2) + match m.entries[0]!.otsProof, m.entries[1]!.otsProof with + | some p, none => expectTrue "rkyv ots_mix first" (p.size == 2) + | _, _ => fail "rkyv ots_mix shape" + match encodeRkyvManifest m with + | .ok enc => expectTrue "rkyv ots_mix encode" (ctEq enc b) + | .error e => fail s!"rkyv ots_mix encode: {repr e}" + -- Sniffer regression: rkyv body starting with ASCII "CFP2" must dual-decode as rkyv + match fromHex? goldenRkyvCfp2PrefixHex with + | none => fail "rkyv cfp2_prefix hex" + | some b => + expectTrue "rkyv cfp2_prefix magic" (isCfp2 b) + match decodeCatalogBody b (replicate hashLen 0) with + | .error e => fail s!"rkyv cfp2_prefix dual-decode hijacked: {repr e}" + | .ok m => + expectTrue "rkyv cfp2_prefix path" (m.entries[0]!.relPath == "a.txt") + expectTrue "rkyv cfp2_prefix root0" (m.entries[0]!.segments[0]!.segmentBaoRoot.get! 0 == 0x43) + match encodeRkyvManifest m with + | .ok enc => expectTrue "rkyv cfp2_prefix encode" (ctEq enc b) + | .error e => fail s!"rkyv cfp2_prefix encode: {repr e}" + -- CFP2 still dual-decodes when rkyv fails (genuine CFP2 empty-ish) + let cfp2Empty : FilepackManifest := { + version := 2 + formatLevel := 0x0e + catalogBaoRoot := replicate hashLen 0 + entries := #[] + } + match cfp2Empty.toWireBytes with + | .error e => fail s!"cfp2 empty wire: {repr e}" + | .ok cfp2Bytes => + expectTrue "cfp2 magic" (isCfp2 cfp2Bytes) + match decodeCatalogBody cfp2Bytes (replicate hashLen 0) with + | .error e => fail s!"cfp2 dual-decode: {repr e}" + | .ok m => expectTrue "cfp2 dual n" (m.entries.size == 0) + -- UTF-8 byte length path cap (Issue 2): multi-byte chars count as bytes not codepoints + let multiByte := String.mk (List.replicate 1366 '你') -- 1366 * 3 = 4098 > 4096 bytes + match validateRelPath multiByte with + | .error .relPathTooLong => pure () + | .error e => fail s!"utf8 path cap: expected relPathTooLong, got {repr e}" + | .ok () => fail "utf8 path cap: accepted oversize multi-byte path" + IO.println "rkyv FilepackManifestWire encode/decode goldens ok" + -- Outboard c12/c14 roundtrip let pubMaster := replicate 32 0 match roundtripOutboard pubMaster nonce11 (utf8 "hello-outboard") (FormatBits.ofUInt8 12) with diff --git a/Carbonado/Outboard.lean b/Carbonado/Outboard.lean index 5510280..6acfbf2 100644 --- a/Carbonado/Outboard.lean +++ b/Carbonado/Outboard.lean @@ -37,6 +37,10 @@ structure OutboardEncoded where baoHash : ByteArray paddingLen : Nat chunkLen : Nat + /-- Post-compress size when Compression bit set; else 0 (matches Rust `EncodeInfo`). -/ + bytesCompressed : Nat + /-- Post-encrypt size when Encrypted bit set; else 0. -/ + bytesEncrypted : Nat deriving DecidableEq /-- @@ -59,16 +63,31 @@ def encodeOutboardParity (input : ByteArray) : Except PipelineError (ByteArray .ok (body.extract parityStart body.size, pad, chunk) /-- - Decode with main + parity sidecars (undamaged path: all data present in main). + Empty-archive policy for outboard FEC (matches Rust `fec_with_parity`). - Pads main to stripe geometry, rebuilds k data + m-k parity shards, reconstructs, - strips padding. + Both main and parity empty → empty logical payload, **padding ignored**. + Shared by `decodeOutboardFec` and (via that helper) `decodeOutboardBody`. +-/ +def emptyOutboardFecArchive : Except PipelineError ByteArray := + .ok ByteArray.empty + +/-- + Decode with main + parity sidecars (matches Rust `decoding::fec_with_parity`). + + Stripe geometry comes from the parity sidecar (`shard_len = parity_len / (m-k)`), + not from truncated main length. Data shards that are fully present in `main` + (end ≤ min(main.size, logical_len)) are kept; any partial, missing, or + padding-boundary data column is an **erasure** (`none`) so RS can reconstruct + from intact parity. Zero-padding truncated main and treating all data shards + as present would silently feed zeroed columns into RS and corrupt recovery. + + Empty main + empty parity: always empty (see `emptyOutboardFecArchive`); padding + is not validated in that branch (Rust ignores it too). -/ def decodeOutboardFec (main parity : ByteArray) (padding : Nat) : Except PipelineError ByteArray := - if main.size == 0 && padding == 0 then - if parity.size == 0 then .ok ByteArray.empty - else .error .unevenShards + if main.size == 0 && parity.size == 0 then + emptyOutboardFecArchive else if parity.size == 0 then .error .emptyShard else if parity.size % (fecM - fecK) != 0 then @@ -83,17 +102,18 @@ def decodeOutboardFec (main parity : ByteArray) (padding : Nat) : .error .paddingTooLarge else let logicalLen := paddedTotal - padding - -- Copy main into padded buffer (zeros after main). - let padded := - if main.size ≥ paddedTotal then - main.extract 0 paddedTotal - else - padWithZeros main paddedTotal + -- Present prefix of logical body only (not zero-filled pad tail). + let copyLen := min main.size logicalLen Id.run do let mut opts : Array (Option ByteArray) := Array.mkEmpty fecM for i in [:fecK] do let start := i * shardLen - opts := opts.push (some (padded.extract start (start + shardLen))) + let stop := start + shardLen + if stop ≤ copyLen then + opts := opts.push (some (main.extract start stop)) + else + -- Truncated / partial / padding-region data column — erasure. + opts := opts.push none for j in [:fecM - fecK] do let start := j * shardLen opts := opts.push (some (parity.extract start (start + shardLen))) @@ -102,25 +122,30 @@ def decodeOutboardFec (main parity : ByteArray) (padding : Nat) : | .ok data => pure (.ok data) /-- - Outboard encode body (embedded-nonce encrypt when Encrypted). + Outboard encode body. + + `headerPath = true` → encrypted bare main is `[tag|ct]` (nonce out-of-band; matches + `file::encode_outboard` / `stream_encode_outboard_buffer(..., Some(nonce))`). + `headerPath = false` → encrypted bare main is `[nonce|tag|ct]` (matches low-level + `encoding::encode_outboard`). `nonce` is required when `format.encrypted` (pure model has no CSPRNG). -/ -def encodeOutboardBody (master nonce plaintext : ByteArray) (format : FormatBits) : - Except PipelineError OutboardEncoded := +def encodeOutboardBody (master nonce plaintext : ByteArray) (format : FormatBits) + (headerPath : Bool) : Except PipelineError OutboardEncoded := let formatByte := format.toUInt8 match compressStep plaintext format.compression with | .error e => .error e - | .ok (afterComp, _) => + | .ok (afterComp, bytesCompressed) => let encRes : Except PipelineError ByteArray := if format.encrypted then - -- Embedded layout for bare mains (matches encoding::encode_outboard). - encryptStep master nonce afterComp false + encryptStep master nonce afterComp headerPath else .ok afterComp match encRes with | .error e => .error e | .ok bareMain => + let bytesEncrypted := if format.encrypted then bareMain.size else 0 match (if format.fec then encodeOutboardParity bareMain else .ok (ByteArray.empty, 0, 0)) @@ -136,6 +161,8 @@ def encodeOutboardBody (master nonce plaintext : ByteArray) (format : FormatBits baoHash := root paddingLen := paddingLen chunkLen := chunkLen + bytesCompressed := bytesCompressed + bytesEncrypted := bytesEncrypted } else .ok { @@ -145,15 +172,22 @@ def encodeOutboardBody (master nonce plaintext : ByteArray) (format : FormatBits baoHash := zeroHash paddingLen := paddingLen chunkLen := chunkLen + bytesCompressed := bytesCompressed + bytesEncrypted := bytesEncrypted } /-- - Outboard decode: Bao verify → FEC reconstruct → decrypt embedded → decompress. + Outboard decode: Bao verify → FEC reconstruct → decrypt → decompress. + + `headerPath` / `nonce` must match encode-time layout: + * headerPath → decrypt with explicit `nonce` over `[tag|ct]` + * !headerPath → embedded-nonce decrypt (nonce arg unused) `padding` must match encode-time padding (directory uses `calcPaddingLen main_len`). -/ def decodeOutboardBody (master root main verOutboard fecParity : ByteArray) - (padding : Nat) (format : FormatBits) : Except PipelineError ByteArray := + (padding : Nat) (format : FormatBits) (headerPath : Bool) (nonce : ByteArray) : + Except PipelineError ByteArray := let formatByte := format.toUInt8 -- Bao verify first when verification bit set (empty post-order outboard is valid for single-leaf). let afterBao : Except PipelineError ByteArray := @@ -168,31 +202,39 @@ def decodeOutboardBody (master root main verOutboard fecParity : ByteArray) | .ok main' => let afterFec : Except PipelineError ByteArray := if format.fec then - if main'.size == 0 then - .ok ByteArray.empty - else if fecParity.size == 0 then - .error .emptyShard - else - decodeOutboardFec main' fecParity padding + -- Single policy: empty+empty → empty; empty main + parity → reconstruct; + -- truncated/partial data shards → erasures (see `decodeOutboardFec`). + decodeOutboardFec main' fecParity padding else .ok main' match afterFec with | .error e => .error e | .ok afterF => - -- Embedded-nonce decrypt when encrypted. - match decryptStep master ByteArray.empty afterF format.encrypted false with + let decNonce := if headerPath then nonce else ByteArray.empty + match decryptStep master decNonce afterF format.encrypted headerPath with | .error e => .error e | .ok afterDec => decompressStep afterDec format.compression -/-- Round-trip outboard for a format. -/ +/-- Round-trip outboard for a format (embedded-nonce layout; low-level parity). -/ def roundtripOutboard (master nonce plaintext : ByteArray) (format : FormatBits) : Except PipelineError Bool := - match encodeOutboardBody master nonce plaintext format with + match encodeOutboardBody master nonce plaintext format false with + | .error e => .error e + | .ok enc => + match decodeOutboardBody master enc.baoHash enc.main enc.verificationOutboard + enc.fecParity enc.paddingLen format false ByteArray.empty with + | .error e => .error e + | .ok pt => .ok (ctEq pt plaintext) + +/-- Round-trip outboard with header-path encrypt (nonce out-of-band). -/ +def roundtripOutboardHeaderPath (master nonce plaintext : ByteArray) (format : FormatBits) : + Except PipelineError Bool := + match encodeOutboardBody master nonce plaintext format true with | .error e => .error e | .ok enc => match decodeOutboardBody master enc.baoHash enc.main enc.verificationOutboard - enc.fecParity enc.paddingLen format with + enc.fecParity enc.paddingLen format true nonce with | .error e => .error e | .ok pt => .ok (ctEq pt plaintext) diff --git a/Carbonado/Pipeline.lean b/Carbonado/Pipeline.lean index 9dc77db..231bb14 100644 --- a/Carbonado/Pipeline.lean +++ b/Carbonado/Pipeline.lean @@ -323,10 +323,13 @@ def natToU32Field (n : Nat) : Except PipelineError UInt32 := if n > u32Max then .error .invalidFieldLength else .ok (UInt32.ofNat n) -/-- Headered encode: body + authenticated 177-byte Header (header-path encrypt). -/ +/-- Headered encode: body + authenticated 177-byte Header (header-path encrypt). + + Third component is pipeline `EncodeInfo` (stage counters for C ABI / dual-backend). +-/ def encodeHeadered (master nonce plaintext : ByteArray) (format : FormatBits) (chunkIndex : UInt32) (slhPublicKey metadata : ByteArray) : - Except PipelineError (Header × ByteArray) := + Except PipelineError (Header × ByteArray × EncodeInfo) := match encodeBody master nonce plaintext format true with | .error e => .error e | .ok enc => @@ -341,7 +344,7 @@ def encodeHeadered (master nonce plaintext : ByteArray) (format : FormatBits) match hdr.toBytes with | .error e => .error (ofHeaderError e) | .ok hdrBytes => - .ok (hdr, appendBA hdrBytes enc.body) + .ok (hdr, appendBA hdrBytes enc.body, enc.info) /-- Headered decode: **header MAC verified first**, then body with `payload_nonce`. @@ -408,7 +411,7 @@ def roundtripHeadered (master nonce plaintext : ByteArray) (format : FormatBits) Except PipelineError Bool := match encodeHeadered master nonce plaintext format 0 zeroSlhPk zeroMeta with | .error e => .error e - | .ok (_hdr, archive) => + | .ok (_hdr, archive, _info) => match decodeHeadered master archive with | .error e => .error e | .ok pt => .ok (ctEq pt plaintext) diff --git a/Carbonado/RkyvFilepack.lean b/Carbonado/RkyvFilepack.lean new file mode 100644 index 0000000..4d0446e --- /dev/null +++ b/Carbonado/RkyvFilepack.lean @@ -0,0 +1,451 @@ +/- + Pure Lean codec for rkyv `FilepackManifestWire` v2 (R9 decode + **W3a encode**). + + **Layout contract (rkyv 0.8.16 + `unaligned`, little-endian FixedUsize=u32):** + matches Rust `src/filepack_manifest.rs` under the same crate features. + + Root (13 B) at **end** of buffer: + version:u32 | format_level:u8 | entries:ArchivedVec (RelPtr i32 + len u32) + + FilepackEntry stride 58 B (inline path header): + rel_path: ArchivedStringRepr (8 B) + content_blake3: [u8; 32] + segment_format: u8 + segments: ArchivedVec (8 B) + ots_proof: ArchivedOption slot (9 B) + + SegmentRef: 60 B fixed. + + **Encode algorithm (matches rkyv HighSerializer two-phase):** + 1. For each entry in order: write nested pointed-to data in field order + (ool path bytes, then SegmentRef array, then OTS proof bytes if Some). + 2. Write contiguous ArchivedFilepackEntry records. + 3. Write root at end. + + Dual-suite directory wire remains **Rust rkyv SSOT** via composition for product + encode under `backend-lean`. Pure Lean directory/CLI (W3b) emits this rkyv body + so Rust dual-suite `decode_directory` can consume Lean-made catalogs. + CFP2 remains available for pure-Lean demos only (`FilepackManifest.toWireBytes`); + it is **not** byte-identical to rkyv. +-/ +import Carbonado.Constants +import Carbonado.Crypto.Util +import Carbonado.Filepack + +namespace Carbonado.RkyvFilepack + +open Carbonado.Constants +open Carbonado.Crypto.Util +open Carbonado.Filepack + +def maxRkyvPayloadLen : Nat := 16 * 1024 * 1024 +def stringInlineCap : Nat := 8 +def optionVecSlot : Nat := 9 +def segmentRefSize : Nat := 60 +def rootSize : Nat := 13 +def entryMetaSize : Nat := stringInlineCap + 32 + 1 + 8 + optionVecSlot + +def readU32LE (buf : ByteArray) (off : Nat) : Except FilepackError UInt32 := + if off + 4 > buf.size then .error .invalidWire + else .ok (getUInt32LE buf off) + +def readI32LE (buf : ByteArray) (off : Nat) : Except FilepackError Int := + if off + 4 > buf.size then .error .invalidWire + else + let n := UInt32.toNat (getUInt32LE buf off) + if n ≥ 0x80000000 then + .ok (Int.ofNat n - (Int.ofNat 0x100000000)) + else + .ok (Int.ofNat n) + +def readU64LE (buf : ByteArray) (off : Nat) : Except FilepackError UInt64 := + if off + 8 > buf.size then .error .invalidWire + else .ok (getUInt64LE buf off) + +/-- Little-endian signed i32 (two's complement). -/ +def putI32LE (x : Int) : ByteArray := + let n : Nat := + if x ≥ 0 then + x.toNat % 0x100000000 + else + (0x100000000 - ((-x).toNat % 0x100000000)) % 0x100000000 + putUInt32LE (UInt32.ofNat n) + +/-- Relative pointer: offset is from `fromPos` (start of RelPtr field) to `toPos`. -/ +def putRelPtr (fromPos toPos : Nat) : ByteArray := + putI32LE (Int.ofNat toPos - Int.ofNat fromPos) + +/-- + rkyv little-endian out-of-line string length packing: + insert `10` as bits 7–6 of the low byte; high length bits start at bit 8 + (`(len & !0x3f) << 2` ≡ `(len / 64) << 8`). +-/ +def packOolStringLen (len : Nat) : UInt32 := + let low := len &&& 0x3f + let packed := low ||| 0x80 ||| ((len / 64) <<< 8) + UInt32.ofNat (packed % 0x100000000) + +/-- Inline ArchivedStringRepr (len ≤ 8): fill 0xff then overlay UTF-8. -/ +def encodeInlineString (pathBytes : ByteArray) : ByteArray := + Id.run do + let mut out := replicate stringInlineCap 0xff + let n := min pathBytes.size stringInlineCap + for i in [:n] do + out := out.set! i (pathBytes.get! i) + pure out + +/-- + Out-of-line string header (8 B). Relative offset is from the **start of the + 8-byte header** (rkyv `ArchivedStringRepr::emplace_out_of_line`), not from + the offset field at +4. +-/ +def encodeOolStringHeader (headerPos dataPos len : Nat) : ByteArray := + appendBA (putUInt32LE (packOolStringLen len)) (putI32LE (Int.ofNat dataPos - Int.ofNat headerPos)) + +def resolveRelPtr (buf : ByteArray) (relPtrPos : Nat) : Except FilepackError Nat := do + let off ← readI32LE buf relPtrPos + let target := Int.ofNat relPtrPos + off + if target < 0 then throw .invalidWire + let t := target.toNat + if t > buf.size then throw .invalidWire + pure t + +/-- Decode ArchivedStringRepr at `pos` (8-byte header). -/ +def decodeArchivedString (buf : ByteArray) (pos : Nat) : Except FilepackError String := do + if pos + stringInlineCap > buf.size then throw .invalidWire + let b0 := buf.get! pos + if b0 &&& 0xc0 == 0x80 then + -- out-of-line (rkyv little-endian len packing) + let rawLenNat := UInt32.toNat (← readU32LE buf pos) + let low6 := rawLenNat % 64 + let high := rawLenNat / 256 + let len := low6 + high * 64 + let off ← readI32LE buf (pos + 4) + let target := Int.ofNat pos + off + if target < 0 then throw .invalidWire + let t := target.toNat + if t + len > buf.size then throw .invalidWire + let bytes := buf.extract t (t + len) + match String.fromUTF8? bytes with + | some s => pure s + | none => throw .invalidWire + else + let mut len := 0 + let mut done := false + for i in [:stringInlineCap] do + if !done then + if buf.get! (pos + i) == 0xff then + done := true + else + len := len + 1 + let bytes := buf.extract pos (pos + len) + match String.fromUTF8? bytes with + | some s => pure s + | none => throw .invalidWire + +def decodeSegmentRef (buf : ByteArray) (pos : Nat) : Except FilepackError SegmentRef := do + if pos + segmentRefSize > buf.size then throw .invalidWire + pure { + segmentBaoRoot := buf.extract pos (pos + 32) + chunkIndex := ← readU32LE buf (pos + 32) + mainLen := ← readU64LE buf (pos + 36) + verificationOutboardOffset := ← readU32LE buf (pos + 44) + verificationOutboardLen := ← readU32LE buf (pos + 48) + fecParityOffset := ← readU32LE buf (pos + 52) + fecParityLen := ← readU32LE buf (pos + 56) + } + +def decodeSegmentVec (buf : ByteArray) (vecPos : Nat) : + Except FilepackError (Array SegmentRef) := do + if vecPos + 8 > buf.size then throw .invalidWire + let target ← resolveRelPtr buf vecPos + let lenNat := UInt32.toNat (← readU32LE buf (vecPos + 4)) + if lenNat > maxSegmentsPerEntry then throw .tooManySegments + let mut segs : Array SegmentRef := #[] + for i in [:lenNat] do + segs := segs.push (← decodeSegmentRef buf (target + i * segmentRefSize)) + pure segs + +def decodeOptionalBytes (buf : ByteArray) (pos : Nat) : + Except FilepackError (Option ByteArray) := do + if pos + optionVecSlot > buf.size then throw .invalidWire + let tag := buf.get! pos + if tag == 0 then + pure none + else if tag == 1 then + let target ← resolveRelPtr buf (pos + 1) + let lenNat := UInt32.toNat (← readU32LE buf (pos + 5)) + if lenNat > maxOtsProofLen then throw .otsProofTooLarge + if target + lenNat > buf.size then throw .invalidWire + pure (some (buf.extract target (target + lenNat))) + else + throw .invalidWire + +def decodeEntry (buf : ByteArray) (entryPos : Nat) : Except FilepackError FilepackEntry := do + if entryPos + entryMetaSize > buf.size then throw .invalidWire + let relPath ← decodeArchivedString buf entryPos + let blakePos := entryPos + stringInlineCap + let contentBlake3 := buf.extract blakePos (blakePos + 32) + let fmtPos := blakePos + 32 + let segmentFormat := buf.get! fmtPos + let segVecPos := fmtPos + 1 + let segments ← decodeSegmentVec buf segVecPos + let otsProof ← decodeOptionalBytes buf (segVecPos + 8) + pure { + relPath := relPath + contentBlake3 := contentBlake3 + segmentFormat := segmentFormat + segments := segments + otsProof := otsProof + } + +/-- Decode rkyv FilepackManifestWire v2 (`catalogBaoRoot` from `.adam.cXX` filename). -/ +def decodeRkyvManifest (bytes : ByteArray) (catalogBaoRoot : ByteArray) : + Except FilepackError FilepackManifest := do + if bytes.size == 0 then throw .invalidWire + if bytes.size > maxRkyvPayloadLen then throw .tooManyEntries + if bytes.size < rootSize then throw .invalidWire + if catalogBaoRoot.size != hashLen then throw .invalidHashLength + let rootPos := bytes.size - rootSize + let version := UInt32.toNat (← readU32LE bytes rootPos) + let formatLevel := bytes.get! (rootPos + 4) + let entriesVecPos := rootPos + 5 + let entriesTarget ← resolveRelPtr bytes entriesVecPos + let entryCount := UInt32.toNat (← readU32LE bytes (entriesVecPos + 4)) + if entryCount > maxFilepackEntries then throw .tooManyEntries + let mut entries : Array FilepackEntry := #[] + for i in [:entryCount] do + entries := entries.push (← decodeEntry bytes (entriesTarget + i * entryMetaSize)) + let m : FilepackManifest := { + version := version + formatLevel := formatLevel + catalogBaoRoot := catalogBaoRoot + entries := entries + } + m.validate + pure m + +/-- Nested-data resolver for one entry (rkyv serialize phase). -/ +structure EntryResolver where + pathBytes : ByteArray + pathOol : Bool + pathDataPos : Nat + segsPos : Nat + segsLen : Nat + /-- `none` = Option::None; `some (pos, len)` = Some(vec) with data at pos. -/ + ots : Option (Nat × Nat) + entry : FilepackEntry + deriving Inhabited + +/-- + Encode `FilepackManifestWire` v2 as rkyv 0.8.16 + unaligned bytes. + + Fail-closed: runs `FilepackManifest.validate` first (paths, version, + format_level, segment geometry). Payload size capped at `maxRkyvPayloadLen`. +-/ +def encodeRkyvManifest (m : FilepackManifest) : Except FilepackError ByteArray := + match m.validate with + | .error e => .error e + | .ok () => + Id.run do + -- Phase 1: nested pointed-to data in entry / field order (rkyv serialize). + let mut buf := ByteArray.empty + let mut resolvers : Array EntryResolver := Array.mkEmpty m.entries.size + let mut err : Option FilepackError := none + for i in [:m.entries.size] do + if err.isNone then + let e := m.entries[i]! + let pathBytes := utf8 e.relPath + if pathBytes.size > maxRelPathLen then + err := some .relPathTooLong + else + let mut pathOol := false + let mut pathDataPos : Nat := 0 + if pathBytes.size > stringInlineCap then + pathOol := true + pathDataPos := buf.size + buf := appendBA buf pathBytes + -- SegmentRef array (60 B each; POD layout matches ArchivedSegmentRef). + let segsPos := buf.size + for j in [:e.segments.size] do + if err.isNone then + match (e.segments[j]!).toBytes with + | .error se => err := some se + | .ok sb => + if sb.size != segmentRefSize then + err := some .invalidWire + else + buf := appendBA buf sb + if err.isNone then + let mut ots : Option (Nat × Nat) := none + match e.otsProof with + | none => pure () + | some proof => + if proof.size > maxOtsProofLen then + err := some .otsProofTooLarge + else + let p := buf.size + buf := appendBA buf proof + ots := some (p, proof.size) + if err.isNone then + resolvers := resolvers.push { + pathBytes := pathBytes + pathOol := pathOol + pathDataPos := pathDataPos + segsPos := segsPos + segsLen := e.segments.size + ots := ots + entry := e + } + match err with + | some e => pure (.error e) + | none => + -- Phase 2: contiguous ArchivedFilepackEntry records (resolve_aligned, zeroed). + let entriesPos := buf.size + for i in [:resolvers.size] do + let r := resolvers[i]! + let entryPos := buf.size + if r.pathOol then + buf := appendBA buf (encodeOolStringHeader entryPos r.pathDataPos r.pathBytes.size) + else + buf := appendBA buf (encodeInlineString r.pathBytes) + buf := appendBA buf r.entry.contentBlake3 + buf := buf.push r.entry.segmentFormat + let segVecPos := buf.size + buf := appendBA buf (putRelPtr segVecPos r.segsPos) + buf := appendBA buf (putUInt32LE (UInt32.ofNat r.segsLen)) + match r.ots with + | none => + -- ArchivedOption::None: tag 0 + zero padding (resolve_aligned zeros). + buf := appendBA buf (replicate optionVecSlot 0) + | some (op, olen) => + buf := buf.push 1 + let relPtrPos := buf.size + buf := appendBA buf (putRelPtr relPtrPos op) + buf := appendBA buf (putUInt32LE (UInt32.ofNat olen)) + -- Phase 3: root at end. + buf := appendBA buf (putUInt32LE (UInt32.ofNat m.version)) + buf := buf.push m.formatLevel + let entriesVecPos := buf.size + buf := appendBA buf (putRelPtr entriesVecPos entriesPos) + buf := appendBA buf (putUInt32LE (UInt32.ofNat m.entries.size)) + if buf.size > maxRkyvPayloadLen then + pure (.error .tooManyEntries) + else + pure (.ok buf) + +/-- Product catalog body encode: **rkyv** FilepackManifestWire v2 (W3). -/ +def encodeCatalogBody (m : FilepackManifest) : Except FilepackError ByteArray := + encodeRkyvManifest m + +def isCfp2 (bytes : ByteArray) : Bool := + bytes.size ≥ 4 && + bytes.get! 0 == 0x43 && bytes.get! 1 == 0x46 && + bytes.get! 2 == 0x50 && bytes.get! 3 == 0x32 + +/-- + Dual-decode catalog body: **prefer rkyv** (product wire / W3), fall back to CFP2. + + Do **not** sniffer-dispatch solely on the first four bytes equaling `CFP2`: a valid + rkyv body can begin with nested data whose first four bytes are `0x43 0x46 0x50 0x32` + (e.g. a segment Bao root). Trying rkyv first keeps those catalogs decodable. + Genuine CFP2 fails rkyv validate and then succeeds via `fromWireBytes`. +-/ +def decodeCatalogBody (bytes : ByteArray) (catalogBaoRoot : ByteArray) : + Except FilepackError FilepackManifest := + match decodeRkyvManifest bytes catalogBaoRoot with + | .ok m => .ok m + | .error rkyvErr => + if isCfp2 bytes then + FilepackManifest.fromWireBytes bytes catalogBaoRoot + else + .error rkyvErr + +/-- Empty-entries golden (Rust FilepackManifest v2, format c14). -/ +def goldenEmptyHex : String := "020000000efbffffff00000000" + +/-- Single-entry golden: `a.txt`, one SegmentRef, no OTS. -/ +def goldenSingleHex : String := + "1111111111111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "612e747874ffffff" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e9bffffff01000000000000000000000000" ++ + "020000000ec1ffffff01000000" + +/-- Multi-entry golden: `a.txt` + out-of-line long path + OTS Some (275 B). + + Regenerated via `cargo run --example dump_rkyv_r9` → `tests/fixtures/rkyv/multi_entry_ots.bin`. +-/ +def goldenMultiOtsHex : String := + "1111111111111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "622f6c6f6e6765722d706174682d6e616d652e747874" ++ + "4444444444444444444444444444444444444444444444444444444444444444" ++ + "00000000c80000000000000000000000400000004000000080000000" ++ + "abcdef01" ++ + "612e747874ffffff" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e45ffffff01000000000000000000000000" ++ + "9600000070ffffff" ++ + "3333333333333333333333333333333333333333333333333333333333333333" ++ + "0e5dffffff010000000190ffffff04000000" ++ + "020000000e87ffffff02000000" + +/-- Exactly 8-byte path (inline boundary). -/ +def goldenPathInline8Hex : String := + "1111111111111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "3132333435363738" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e9bffffff01000000000000000000000000" ++ + "020000000ec1ffffff01000000" + +/-- Exactly 9-byte path (out-of-line boundary). -/ +def goldenPathOol9Hex : String := + "313233343536373839" ++ + "1111111111111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "89000000bbffffff" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e9bffffff01000000000000000000000000" ++ + "020000000ec1ffffff01000000" + +/-- One entry, two SegmentRefs (contiguous 0..1). -/ +def goldenTwoSegmentsHex : String := + "1111111111111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "1212121212121212121212121212121212121212121212121212121212121212" ++ + "010000003200000000000000c0000000400000000001000080000000" ++ + "612e747874ffffff" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e5fffffff02000000000000000000000000" ++ + "020000000ec1ffffff01000000" + +/-- OTS Some on first entry only; second None. -/ +def goldenOtsFirstOnlyHex : String := + "1111111111111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "dead" ++ + "4444444444444444444444444444444444444444444444444444444444444444" ++ + "00000000c80000000000000000000000400000004000000080000000" ++ + "612e747874ffffff" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e5dffffff010000000190ffffff02000000" ++ + "622e747874ffffff" ++ + "3333333333333333333333333333333333333333333333333333333333333333" ++ + "0e61ffffff01000000000000000000000000" ++ + "020000000e87ffffff02000000" + +/-- rkyv body beginning with ASCII `CFP2` (segment root prefix) — sniffer regression. -/ +def goldenRkyvCfp2PrefixHex : String := + "4346503211111111111111111111111111111111111111111111111111111111" ++ + "00000000640000000000000000000000400000004000000080000000" ++ + "612e747874ffffff" ++ + "2222222222222222222222222222222222222222222222222222222222222222" ++ + "0e9bffffff01000000000000000000000000" ++ + "020000000ec1ffffff01000000" + +theorem golden_empty_hex_len : goldenEmptyHex.length = 26 := by native_decide + +end Carbonado.RkyvFilepack diff --git a/Carbonado/Scrub.lean b/Carbonado/Scrub.lean index 1938c13..2a768ac 100644 --- a/Carbonado/Scrub.lean +++ b/Carbonado/Scrub.lean @@ -33,6 +33,7 @@ open Carbonado.Constants open Carbonado.Crypto.Util open Carbonado.Fec.RS open Carbonado.Fec.Inboard +open Carbonado.Bao.Tree open Carbonado.Bao.Product open Carbonado.Pipeline @@ -191,4 +192,169 @@ def scrubWithMissing (fecBody wantRoot : ByteArray) (padding : Nat) | some art => pure (.ok art) | none => pure (.error .invalidScrubbedHash) +/-- + Geometry-only leaf extract from a full-range pre-order Bao response (no hash auth). + + Used by product scrub entry so RS combinatorial recovery can see FEC shards even when + some leaf groups are bit-corrupted. Parent pairs (64 B) are skipped by geometry only. + Truncation → `truncatedResponse` (mapped by caller to scrub failure when irrecoverable). +-/ +partial def extractLeavesUnauth (contentLen : Nat) (input : ByteArray) (pos : Nat) : + Except BaoError (ByteArray × Nat) := do + let nLeaves := leafGroupCount contentLen + if nLeaves ≤ 1 then + if pos + contentLen > input.size then + throw .truncatedResponse + pure (input.extract pos (pos + contentLen), pos + contentLen) + else + let groups := nextPow2 nLeaves + let mid := groups / 2 + let midBytes := mid * leafBytes + let leftLen := min midBytes contentLen + let rightLen := contentLen - leftLen + if pos + 64 > input.size then + throw .truncatedResponse + let pos := pos + 64 + let (leftData, pos) ← extractLeavesUnauth leftLen input pos + let (rightData, pos) ← extractLeavesUnauth rightLen input pos + pure (appendBA leftData rightData, pos) + +/-- Peel Bao-inboard to logical (FEC) body without authentication. -/ +def peelInboardBody (body : ByteArray) : Except PipelineError ByteArray := + match contentLenPrefix body with + | .error e => .error (ofBaoError e) + | .ok contentLen => + if contentLen == 0 then + .ok ByteArray.empty + else + let response := body.extract 8 body.size + match extractLeavesUnauth contentLen response 0 with + | .error e => .error (ofBaoError e) + | .ok (data, _) => + if data.size != contentLen then + .error .truncatedResponse + else + .ok data + +/-- + Product scrub for inboard archives (Rust `decoding::scrub` spirit). + + * No Verification bit → `scrubRequiresVerification` + * Bao verifies → `unnecessaryScrub` + * No FEC → `invalidScrubbedHash` (cannot recover without RS) + * FEC: geometry-peel → `scrubFecThenBao` combinatorial search +-/ +def scrubInboard (body wantRoot : ByteArray) (padding : Nat) (format : FormatBits) : + Except PipelineError ByteArray := + if !format.verification then + .error .scrubRequiresVerification + else + match decodeInboardForFormat format.toUInt8 wantRoot body with + | .ok _ => .error .unnecessaryScrub + | .error _ => + if !format.fec then + .error .invalidScrubbedHash + else + match peelInboardBody body with + | .error _ => .error .invalidScrubbedHash + | .ok fecBody => + scrubFecThenBao fecBody wantRoot padding format.toUInt8 + +/-- + Try reconstruct bare main from selected shard mask (outboard geometry). + + `shards` must have size `fecM`. Data shards 0..k-1 are padded-main strips; + parity shards k..m-1 come from the parity sidecar. On success returns logical + bare main (padding stripped) that re-encodes to `wantRoot` under outboard Bao. +-/ +def tryOutboardMask (shards : Array ByteArray) (padding : Nat) (formatByte : UInt8) + (wantRoot : ByteArray) (mask : Nat) : Option ByteArray := + if shards.size != fecM then + none + else if popcount8 mask < fecK then + none + else + Id.run do + let mut opts : Array (Option ByteArray) := Array.mkEmpty fecM + for i in [:fecM] do + if (mask >>> i) % 2 == 1 then + opts := opts.push (some (shards[i]!)) + else + opts := opts.push none + match reconstructLogical opts padding with + | .error _ => pure none + | .ok logical => + let (root, _) := encodeOutboardForFormat formatByte logical + if ctEq root wantRoot then + some logical + else + none + +/-- Search masks for outboard bare-main recovery. -/ +def searchOutboardMasks (shards : Array ByteArray) (padding : Nat) (formatByte : UInt8) + (wantRoot : ByteArray) : Option ByteArray := + if shards.size != fecM then + none + else + Id.run do + let mut found : Option ByteArray := none + for mask in [:256] do + if found.isNone then + match tryOutboardMask shards padding formatByte wantRoot mask with + | some bare => found := some bare + | none => pure () + pure found + +/-- + Product scrub for outboard (Rust `scrub_outboard`). + + * No Verification → `scrubRequiresVerification` + * Outboard verifies → `unnecessaryScrub` + * No FEC / empty parity → `invalidScrubbedHash` (or empty-shard when parity required) + * Else: combinatorial RS over padded main strips + parity shards; accept bare whose + re-encoded outboard root matches `wantRoot`. +-/ +def scrubOutboard (main verOutboard fecParity wantRoot : ByteArray) + (padding chunkLen : Nat) (format : FormatBits) : Except PipelineError ByteArray := + if !format.verification then + .error .scrubRequiresVerification + else + match verifyOutboardForFormat format.toUInt8 wantRoot main verOutboard with + | .ok () => .error .unnecessaryScrub + | .error _ => + if !format.fec then + .error .invalidScrubbedHash + else if chunkLen == 0 then + .error .badGeometry + else if fecParity.size == 0 then + .error .emptyShard + else if fecParity.size % (fecM - fecK) != 0 then + .error .unevenShards + else + let parityShardLen := fecParity.size / (fecM - fecK) + if parityShardLen != chunkLen then + .error .incorrectShardSize + else + let paddedTotal := chunkLen * fecK + if padding > paddedTotal then + .error .paddingTooLarge + else + let logicalLen := paddedTotal - padding + let copyLen := min main.size logicalLen + let padded := + if main.size ≥ paddedTotal then + main.extract 0 paddedTotal + else + padWithZeros (main.extract 0 copyLen) paddedTotal + Id.run do + let mut shards : Array ByteArray := Array.mkEmpty fecM + for i in [:fecK] do + shards := shards.push (padded.extract (i * chunkLen) ((i + 1) * chunkLen)) + for j in [:fecM - fecK] do + shards := shards.push + (fecParity.extract (j * chunkLen) ((j + 1) * chunkLen)) + match searchOutboardMasks shards padding format.toUInt8 wantRoot with + | some bare => pure (.ok bare) + | none => pure (.error .invalidScrubbedHash) + end Carbonado.Scrub diff --git a/Carbonado/Shard.lean b/Carbonado/Shard.lean index e77ba93..2c6a38f 100644 --- a/Carbonado/Shard.lean +++ b/Carbonado/Shard.lean @@ -82,7 +82,7 @@ def encodeShards (master plaintext : ByteArray) (format : FormatBits) let seg := segments[i]! match encodeHeadered master nonce seg format (UInt32.ofNat i) slhPublicKey metadata with | .error e => err := some e - | .ok (hdr, archive) => + | .ok (hdr, archive, _info) => out := out.push { chunkIndex := UInt32.ofNat i header := hdr diff --git a/Carbonado/Slh.lean b/Carbonado/Slh.lean index 9903c2b..582baea 100644 --- a/Carbonado/Slh.lean +++ b/Carbonado/Slh.lean @@ -1,18 +1,18 @@ /- - SLH-DSA-SHA2-128s sidecar wire format + Bao-root binding (Program F). + SLH-DSA-SHA2-128s sidecar wire format + Bao-root binding (Program F / R9 G10). Normative (AGENTS §2.3): * Sidecar: `SLH1` (4) + raw signature (7856) = 7860 bytes * Public key (32 B) lives in Header.slh_public_key, not the sidecar * Signature is over the 32-byte Bao root of the target container - Real SPHINCS+/libbitcoinpqc sign-verify is **not** linked in this program - (libbitcoinpqc submodule empty / heavy cmake — see LIMITS). This module - provides fail-closed wire codec, binding model, and theorems. Optional FFI - can replace the oracle later without changing the wire API. + **G10 (R9):** real SLH-DSA via `@[extern]` into libbitcoinpqc objects linked + in `libcarbonado_native.a` (`nix/native/carbonado_slh.c`). Lean elaborator + bodies are fail-closed fallbacks (do **not** `native_decide` over live crypto). + AOT `Main` / C ABI / dual-suite composition exercise the real oracle. - Large-array roundtrips (7856 B sig) are gated in AOT Main, not `native_decide` - (elaboration cost). + Dual-suite product SLH may still use Rust `bitcoinpqc` composition; pure Lean + is for `libcarbonado` purity. Composition remains SSOT for dual-suite wire. -/ import Carbonado.Constants import Carbonado.Crypto.Util @@ -36,8 +36,10 @@ inductive SlhError where | invalidRootLength /-- Signature verification failed (oracle returned false). -/ | verificationFailed - /-- Oracle/sign path refused (no real crypto linked, bad params, etc.). -/ + /-- Keygen/sign refused (bad params, crypto failure). -/ | signatureUnavailable + /-- Entropy shorter than 128 bytes for keygen. -/ + | invalidEntropyLength deriving DecidableEq, Repr /-- Detached SLH1 sidecar contents (signature only; pk is out-of-band). -/ @@ -150,14 +152,133 @@ def mockOracleFor (acceptedRoot acceptedSig : ByteArray) (_pk message sig : ByteArray) : Bool := ctEq message acceptedRoot && ctEq sig acceptedSig -/-- Placeholder sign: always `signatureUnavailable` until libbitcoinpqc is linked. -/ -def signRoot (_secretKeyEntropy root : ByteArray) : Except SlhError ByteArray := +/-! ## Live SLH-DSA via libbitcoinpqc (G10) + + Status-prefixed blobs match `carbonado_slh.c` / zstd pattern. + Elaborator bodies are fail-closed (status **3** / verify 0) — not for `native_decide`. + Status 1 is reserved for short entropy only (see `decodeSlhStatusPayload`). +-/ + +/-- Status-prefix helper for elaborator `@[extern]` bodies (keygen/sign). + + Uses status **3** (crypto / unavailable), not 1 (short entropy), so elaborator + fallbacks map to `signatureUnavailable` rather than `invalidEntropyLength`. +-/ +def slhStatusFail : ByteArray := + ByteArray.mk #[3] + +/-- + Raw keygen: entropy (≥128) → `[status][pk 32 | sk 64]`. + AOT: real `slh_dsa_sha2_128s_keygen`. Elaborator: status 3 fail. +-/ +@[extern "carbonado_slh_keygen_raw"] +def keygenRaw (entropy : @& ByteArray) : ByteArray := + slhStatusFail + +/-- + Raw sign: sk (64) + message → `[status][sig 7856]`. + AOT: real deterministic `slh_dsa_sha2_128s_sign`. Elaborator: status 3 fail. +-/ +@[extern "carbonado_slh_sign_raw"] +def signRaw (sk : @& ByteArray) (message : @& ByteArray) : ByteArray := + slhStatusFail + +/-- + Raw verify: pk + message + sig → `1` accept / `0` reject. + AOT: real `slh_dsa_sha2_128s_verify`. Elaborator: always reject (fail-closed). +-/ +@[extern "carbonado_slh_verify_raw"] +def verifyRaw (pk : @& ByteArray) (message : @& ByteArray) (signature : @& ByteArray) : UInt8 := + 0 + +/-- Decode status-prefixed keygen/sign blob (SLH-specific; not Compress zstd). + + Status codes from `carbonado_slh.c`: + * 0 → ok payload + * 1 → short entropy (`invalidEntropyLength`) + * 2 → other bad argument (`signatureUnavailable`) + * 3+ → crypto failure (`signatureUnavailable`) +-/ +def decodeSlhStatusPayload (raw : ByteArray) : Except SlhError ByteArray := + if raw.size == 0 then + .error .signatureUnavailable + else + let code := raw.get! 0 + if code == 0 then + .ok (raw.extract 1 raw.size) + else if code == 1 then + .error .invalidEntropyLength + else + .error .signatureUnavailable + +/-- Keygen from ≥128 bytes entropy → `(publicKey, secretKey)`. -/ +def keygen (entropy : ByteArray) : Except SlhError (ByteArray × ByteArray) := + if entropy.size < 128 then + .error .invalidEntropyLength + else + match decodeSlhStatusPayload (keygenRaw entropy) with + | .error e => .error e + | .ok payload => + if payload.size != slhPublicKeyLen + 64 then + .error .signatureUnavailable + else + .ok (payload.extract 0 slhPublicKeyLen, payload.extract slhPublicKeyLen payload.size) + +/-- Live SLH-DSA verify predicate for `verifyBound` / CLI. -/ +def liveVerifyOracle (pk message sig : ByteArray) : Bool := + verifyRaw pk message sig == 1 + +/-- Sign a 32-byte Bao root with a 64-byte secret key. -/ +def signWithSk (secretKey root : ByteArray) : Except SlhError ByteArray := if root.size != hashLen then .error .invalidRootLength - else + else if secretKey.size != 64 then .error .signatureUnavailable + else + match decodeSlhStatusPayload (signRaw secretKey root) with + | .error e => .error e + | .ok sig => + if sig.size != slh1SignatureLen then + .error .invalidSignatureLength + else + .ok sig -/-! ## Theorems: wire framing + bind-to-root (small native_decide cases) -/ +/-- + Product sign path: ≥128-byte entropy → keygen → sign 32-byte Bao root. + Returns the 7856-byte raw signature (not SLH1 sidecar framing). +-/ +def signRoot (secretKeyEntropy root : ByteArray) : Except SlhError ByteArray := + if root.size != hashLen then + .error .invalidRootLength + else if secretKeyEntropy.size < 128 then + .error .invalidEntropyLength + else + match keygen secretKeyEntropy with + | .error e => .error e + | .ok (_pk, sk) => signWithSk sk root + +/-- Keygen + sign, returning `(pk, sk, sig)` for roundtrip tests. -/ +def keygenAndSignRoot (entropy root : ByteArray) : + Except SlhError (ByteArray × ByteArray × ByteArray) := + if root.size != hashLen then + .error .invalidRootLength + else + match keygen entropy with + | .error e => .error e + | .ok (pk, sk) => + match signWithSk sk root with + | .error e => .error e + | .ok sig => .ok (pk, sk, sig) + +/-- Verify raw signature over Bao root with live oracle. -/ +def verifyRoot (publicKey root signature : ByteArray) : Except SlhError Unit := + verifyBound liveVerifyOracle publicKey root signature + +/-! ## Theorems: wire framing + bind-to-root (small native_decide cases) + + Live crypto is **not** theorem-gated (`native_decide` cannot link externs). + Length gates + mock-oracle binding remain pure. +-/ theorem slh1_sidecar_len : slh1SidecarLen = 7860 := slh1SidecarLen_eq @@ -255,20 +376,20 @@ theorem wrong_root_fails : | _ => false) = true := by native_decide -/-- signRoot is unavailable without linked PQC (fail-closed). -/ -theorem sign_unavailable : - (match signRoot (replicate 128 0x42) (replicate hashLen 0) with - | .error .signatureUnavailable => true - | _ => false) = true := by - native_decide - -/-- signRoot rejects bad root length before unavailability. -/ +/-- signRoot rejects bad root length before crypto. -/ theorem sign_bad_root : (match signRoot (replicate 128 0x42) (ofList [1]) with | .error .invalidRootLength => true | _ => false) = true := by native_decide +/-- signRoot rejects short entropy before crypto. -/ +theorem sign_bad_entropy : + (match signRoot (replicate 16 0x42) (replicate hashLen 0) with + | .error .invalidEntropyLength => true + | _ => false) = true := by + native_decide + /-- Magic list matches ASCII SLH1. -/ theorem slh1_magic_bytes : slh1Magic = [0x53, 0x4c, 0x48, 0x31] := slh1Magic_eq_literal diff --git a/CarbonadoTest/Bao.lean b/CarbonadoTest/Bao.lean index 4982997..171010f 100644 --- a/CarbonadoTest/Bao.lean +++ b/CarbonadoTest/Bao.lean @@ -232,4 +232,53 @@ theorem stream_slice_trailing : isTrailing (decodeSliceForFormat 4 root 100 0 1 long)) = true := by native_decide +/-! ## W4a / W4b multi-leaf seekable slice (O(slice) retain) -/ + +/-- ~3 × 4 KiB patterned payload (multi-leaf inboard + non-empty outboard). -/ +private def pat12k : ByteArray := + Id.run do + let mut out := ByteArray.empty + for i in [:12288] do + out := out.push (UInt8.ofNat ((i / 4096 + i % 256) % 251)) + pure out + +/-- W4a: middle leaf from inboard matches plaintext window. -/ +theorem w4a_inboard_mid_slice_matches : + (let (root, art) := encodeInboardForFormat 4 pat12k + match verifySliceInboardForFormat 4 root art 1 1 with + | .ok s => ctEq s (pat12k.extract 4096 8192) + | .error _ => false) = true := by + native_decide + +/-- W4a: multi-leaf count=2 span matches extract. -/ +theorem w4a_inboard_two_slices : + (let (root, art) := encodeInboardForFormat 4 pat12k + match verifySliceInboardForFormat 4 root art 0 2 with + | .ok s => ctEq s (pat12k.extract 0 8192) + | .error _ => false) = true := by + native_decide + +/-- W4a short-file single-leaf inboard slice = full body. -/ +theorem w4a_inboard_short_file : + (let (root, art) := encodeInboardForFormat 4 pat100 + match verifySliceInboardForFormat 4 root art 0 1 with + | .ok s => ctEq s pat100 + | .error _ => false) = true := by + native_decide + +/-- W4b: multi-leaf outboard mid slice matches bare window. -/ +theorem w4b_outboard_mid_slice_matches : + (let (root, ob) := encodeOutboardForFormat 4 pat12k + match verifySliceOutboardForFormat 4 root pat12k ob 1 1 with + | .ok s => ctEq s (pat12k.extract 4096 8192) + | .error _ => false) = true := by + native_decide + +/-- W4a: tampered multi-leaf inboard fails auth on mid slice. -/ +theorem w4a_tamper_mid_auth_fail : + (let (root, art) := encodeInboardForFormat 4 pat12k + let bad := art.set! (min 20 (art.size - 1)) (art.get! (min 20 (art.size - 1)) ^^^ 0x5a) + isAuthFail (verifySliceInboardForFormat 4 root bad 1 1)) = true := by + native_decide + end CarbonadoTest.Bao diff --git a/CarbonadoTest/Pipeline.lean b/CarbonadoTest/Pipeline.lean index 679217f..eb6081e 100644 --- a/CarbonadoTest/Pipeline.lean +++ b/CarbonadoTest/Pipeline.lean @@ -20,6 +20,7 @@ import Carbonado.Scrub import Carbonado.Shard import Carbonado.Fec.Inboard import Carbonado.Bao.Product +import Carbonado.Ffi import CarbonadoTest.Scaffold namespace CarbonadoTest.Pipeline @@ -34,6 +35,7 @@ open Carbonado.Scrub open Carbonado.Shard open Carbonado.Fec.Inboard open Carbonado.Bao.Product +open Carbonado.Ffi private def master42 : ByteArray := replicate 32 0x42 private def nonce11 : ByteArray := replicate 16 0x11 @@ -301,7 +303,7 @@ theorem truncated_body_path : (match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) 0 zeroSlhPk zeroMeta with | .error _ => false - | .ok (_h, arch) => + | .ok (_h, arch, _info) => if arch.size ≤ headerLen + 1 then false else match decodeHeadered master42 (arch.extract 0 (headerLen + 1)) with @@ -314,7 +316,7 @@ theorem trailer_ignored_c0 : (match encodeHeadered master42 nonce11 (utf8 "hello") (FormatBits.ofUInt8 0) 0 zeroSlhPk zeroMeta with | .error _ => false - | .ok (_h, arch) => + | .ok (_h, arch, _info) => match decodeHeadered master42 (appendBA arch (ofList [0xaa, 0xbb])) with | .ok pt => ctEq pt (utf8 "hello") | .error _ => false) = true := by @@ -407,4 +409,31 @@ theorem c15_odd : formatC15.toUInt8 % 2 = 1 := by native_decide theorem c14_even : formatC14.toUInt8 % 2 = 0 := by native_decide +/-! ## R2 headered FFI: wrong-length SLH/meta fail-closed (encodeHeaderedBytes) + + C ABI is length-implicit (null → empty BA; non-null always copies fixed 32/8). + Wrong sizes are only expressible on the Lean pure/export ByteArray surface. +-/ + +/-- SLH pk size ∉ {0, 32} → errInvalidArgument (before encode). -/ +theorem encode_headered_bytes_bad_slh_len : + (match encodeHeaderedBytes master42 nonce11 (utf8 "x") (replicate 16 0) ByteArray.empty 0 with + | .error e => e == errInvalidArgument + | .ok _ => false) = true := by + native_decide + +/-- Metadata size ∉ {0, 8} → errInvalidArgument (before encode). -/ +theorem encode_headered_bytes_bad_meta_len : + (match encodeHeaderedBytes master42 nonce11 (utf8 "x") ByteArray.empty (replicate 4 0) 0 with + | .error e => e == errInvalidArgument + | .ok _ => false) = true := by + native_decide + +/-- Empty SLH + empty meta accepted (zeros on wire) — control for length gates. -/ +theorem encode_headered_bytes_empty_slh_meta_ok : + (match encodeHeaderedBytes master42 nonce11 (utf8 "x") ByteArray.empty ByteArray.empty 0 with + | .ok _ => true + | .error _ => false) = true := by + native_decide + end CarbonadoTest.Pipeline diff --git a/CarbonadoTest/Slh.lean b/CarbonadoTest/Slh.lean index 00c641e..7a6148f 100644 --- a/CarbonadoTest/Slh.lean +++ b/CarbonadoTest/Slh.lean @@ -2,6 +2,7 @@ Program F — SLH1 wire + Bao-root binding theorems. Large 7856-byte signature roundtrips are AOT Main only (not native_decide). + Live SLH-DSA is AOT-only (extern); pure theorems cover wire + length gates. -/ import Carbonado.Constants import Carbonado.Crypto.Util @@ -71,16 +72,16 @@ theorem wrong_root_path : | .error .verificationFailed => true | _ => false) = true := wrong_root_fails -theorem sign_unavail : - (match signRoot (replicate 128 0x42) (replicate hashLen 0) with - | .error .signatureUnavailable => true - | _ => false) = true := sign_unavailable - theorem sign_bad_root_len : (match signRoot (replicate 128 0x42) (ofList [1]) with | .error .invalidRootLength => true | _ => false) = true := sign_bad_root +theorem sign_short_entropy : + (match signRoot (replicate 16 0x42) (replicate hashLen 0) with + | .error .invalidEntropyLength => true + | _ => false) = true := sign_bad_entropy + /-- bindingFromSidecar: short file → invalidSidecarLength. -/ theorem binding_short_sidecar : (match bindingFromSidecar (replicate slhPublicKeyLen 0) (replicate hashLen 0) diff --git a/Cargo.toml b/Cargo.toml index cbe164d..246f435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,21 +17,14 @@ include = ["src/**/*", "LICENSE", "README.md", "doc/man/*.1", "doc/man/README.md # All Bao logic now uses the local keyed fork for 4KB chunk groups + format-keyed roots. bao = "0.13" -# Local keyed fork (SurmountSystems/bao-tree) — default 4KB groups (BlockSize::from_chunk_log(2)) -# via BAO_BLOCK_SIZE. Keyed mode makes root = keyed_hash(key_from_format, data) so the -# Bao hash commits to the exact format pipeline chosen (multi-dimensional naming). -# See AGENTS.md and constants::BAO_BLOCK_SIZE. -# Temporary until upstreamed to n0-computer/bao-tree. -# To fetch the fork (for agents): -# git clone -b 76-keyed-bao git@github.com:SurmountSystems/bao-tree.git ../bao-tree -# (or: git clone -b 76-keyed-bao https://github.com/SurmountSystems/bao-tree.git ../bao-tree) -# Keyed fork (SurmountSystems/bao-tree branch 76-keyed-bao). Git dep supports `cargo install` -# from crates.io and `cargo install --path .` without a sibling checkout. -# For faster local iteration, copy `.cargo/config.toml.example` → `.cargo/config.toml` -# after `just setup-bao-tree` to patch in `../bao-tree`. -# We set default-features = false because the fork's defaults pull in tokio/fs/etc -# which are not needed (we only use sync keyed) and break some cross/wasm targets. -bao_tree = { package = "bao-tree", git = "https://github.com/SurmountSystems/bao-tree.git", branch = "76-keyed-bao", default-features = false, features = ["validate"] } +# Keyed Bao from n0-computer/bao-tree PR 78 (rklaehn `keyed-bao`, builds on #77). +# Default 4KB groups (BlockSize::from_chunk_log(2)) via BAO_BLOCK_SIZE. Keyed mode +# makes root = keyed_hash(key_from_format, data) so the Bao hash commits to the +# exact format pipeline (multi-dimensional naming). See AGENTS.md and +# constants::BAO_BLOCK_SIZE. +# default-features = false: crate defaults pull in tokio/fs and break some +# cross/wasm targets. We only need sync keyed + validate. +bao_tree = { package = "bao-tree", git = "https://github.com/n0-computer/bao-tree.git", branch = "keyed-bao", default-features = false, features = ["validate"] } futures-lite = { version = "2", optional = true, default-features = false, features = ["std"] } tokio = { version = "1", features = ["rt"], optional = true } bitmask-enum = "2.1.0" @@ -91,7 +84,8 @@ default = ["backend-rust", "pqc", "ots", "cli", "parallel"] # Pure Rust implementation (default engine). backend-rust = [] # Lean AOT engine via carbonado-sys / libcarbonado (G8). Requires CARBONADO_LEAN_LIB. -backend-lean = ["dep:carbonado-sys"] +# `require-lib` fail-closes the build when the AOT shared library is missing. +backend-lean = ["dep:carbonado-sys", "carbonado-sys/require-lib"] pqc = ["dep:bitcoinpqc"] # OpenTimestamps stub stamping (offline/testable; no network calendar in default build). ots = [] diff --git a/README.md b/README.md index a799be3..f98379d 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ Non-blocking inboard decode for P2P fetch, HTTP range reads, and UDP assembly ad ```bash cargo test --test parallel_determinism # Phase 3 determinism (default) -cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path # serial FEC path +cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path # serial FEC (must name backend) cargo test --features async --test streaming_async ``` @@ -357,10 +357,13 @@ just all # everything (fmt, lint, tests, release build, source grep |--------|----------------| | `just fmt` | Formatting | | `just lint` | Clippy **and** source checks (no v1 ECIES, prod `unwrap`, magic string, etc.) | -| `just test` | Full test suite | +| `just test` | Full test suite (`backend-rust` default) | | `just test-smoke` | Slice/streaming/sharding/bao contract tests | +| `just test-lean-ci` | Dual-backend lean **full suite** freeze (G8 closed at R7; needs Nix + `libcarbonado`; CI `dual-backend-lean`) | | `just build` + `just test-cli` | Release binary + CLI tests | +**Dual-backend (Rust + Lean AOT):** default `cargo test` is pure Rust. Lean engine tests require `nix build .#libcarbonado -o result-libcarbonado`, then `CARBONADO_LEAN_LIB` / `CARBONADO_LEAN_INCLUDE` / `LD_LIBRARY_PATH` (or just `just test-lean-ci`, which builds and fail-closes if the shared library is missing). CI runs both: job `desktop` (`backend-rust`) and job `dual-backend-lean` (`just test-lean-ci`). Full lean suite parity (**G8**) is **closed at R7** — freeze = full dual suite under lean features; see [docs/GAPS.md](docs/GAPS.md) and [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md). + CI runs the same recipes — see `.github/workflows/rust.yaml`. ## Benchmarks diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..5c5de10 --- /dev/null +++ b/build.rs @@ -0,0 +1,22 @@ +//! Propagate libcarbonado rpath onto final binaries/tests (backend-lean). +//! +//! `carbonado-sys` sets `rustc-link-search` / `rustc-link-lib`, but `rustc-link-arg` +//! rpath from a dependency build script is not applied to dependents' final links. + +use std::env; +use std::path::PathBuf; + +fn main() { + println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_LIB"); + println!("cargo:rerun-if-cfg=feature=\"backend-lean\""); + + let lean = env::var("CARGO_FEATURE_BACKEND_LEAN").is_ok(); + if !lean { + return; + } + if let Ok(lib) = env::var("CARBONADO_LEAN_LIB") { + let lib_dir = PathBuf::from(lib); + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + } +} diff --git a/carbonado-sys/Cargo.toml b/carbonado-sys/Cargo.toml index 5bfcb00..6e53b21 100644 --- a/carbonado-sys/Cargo.toml +++ b/carbonado-sys/Cargo.toml @@ -8,4 +8,9 @@ publish = false [dependencies] +# When enabled (by carbonado `backend-lean`), fail the build if CARBONADO_LEAN_LIB is unset +# or does not contain libcarbonado shared/static artifacts. +[features] +require-lib = [] + [build-dependencies] diff --git a/carbonado-sys/build.rs b/carbonado-sys/build.rs index 3d75e46..533d99d 100644 --- a/carbonado-sys/build.rs +++ b/carbonado-sys/build.rs @@ -1,12 +1,21 @@ //! Link against Nix-built `libcarbonado` when `CARBONADO_LEAN_LIB` / `CARBONADO_LEAN_INCLUDE` -//! are set (or `OUT_DIR` sibling after `nix build .#libcarbonado` + env). +//! are set (or after `nix build .#libcarbonado` + env). //! //! ```bash -//! nix build .#libcarbonado -//! export CARBONADO_LEAN_LIB=$PWD/result/lib -//! export CARBONADO_LEAN_INCLUDE=$PWD/result/include -//! cargo test -p carbonado --features backend-lean +//! nix build .#libcarbonado -o result-libcarbonado +//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +//! # Freeze allowlist (Phase 5 / G11): just test-lean-ci +//! cargo test -p carbonado --no-default-features --features "backend-lean,pqc,ots,cli" //! ``` +//! +//! Prefers the shared library (`libcarbonado.so`) produced by leanc (Lean runtime +//! already linked). Falls back to static `libcarbonado.a` when only the archive +//! is present (requires a full Lean link line — not the default path). +//! +//! With the `require-lib` feature (enabled by carbonado `backend-lean`), a missing +//! `CARBONADO_LEAN_LIB` or missing library file is a **hard build error** (fail-closed). use std::env; use std::path::PathBuf; @@ -15,6 +24,7 @@ fn main() { println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_LIB"); println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_INCLUDE"); + let require_lib = env::var_os("CARGO_FEATURE_REQUIRE_LIB").is_some(); let lib = env::var_os("CARBONADO_LEAN_LIB").map(PathBuf::from); let include = env::var_os("CARBONADO_LEAN_INCLUDE").map(PathBuf::from); @@ -23,14 +33,47 @@ fn main() { } if let Some(lib_dir) = lib { + let so = lib_dir.join("libcarbonado.so"); + let dylib = lib_dir.join("libcarbonado.dylib"); + let archive = lib_dir.join("libcarbonado.a"); + if !so.exists() && !dylib.exists() && !archive.exists() { + let msg = format!( + "CARBONADO_LEAN_LIB={} has no libcarbonado.so/.dylib/.a — run: nix build .#libcarbonado -o result-libcarbonado", + lib_dir.display() + ); + if require_lib { + panic!("{msg}"); + } + println!("cargo:warning={msg}"); + return; + } + println!("cargo:rustc-link-search=native={}", lib_dir.display()); - println!("cargo:rustc-link-lib=static=carbonado"); - // Lean/zstd static archive may need system libs when full Lean objects are linked later. + + if so.exists() || dylib.exists() { + println!("cargo:rustc-link-lib=dylib=carbonado"); + // Runtime resolution for tests without installing into system paths. + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + } else { + println!("cargo:rustc-link-lib=static=carbonado"); + println!( + "cargo:warning=libcarbonado shared object missing; linking static (may need Lean runtime libs)" + ); + } println!("cargo:rustc-link-lib=pthread"); println!("cargo:rustc-link-lib=m"); println!("cargo:rustc-link-lib=dl"); + } else if require_lib { + panic!( + "CARBONADO_LEAN_LIB unset while carbonado-sys/require-lib is enabled (backend-lean). \ + Build: nix build .#libcarbonado -o result-libcarbonado && \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib \ + CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include" + ); } else { - // Allow crate to compile docs/check without the AOT lib; link fails at use if missing. - println!("cargo:warning=CARBONADO_LEAN_LIB unset; carbonado-sys will not link libcarbonado"); + // Allow standalone carbonado-sys docs/check without the AOT lib. + println!( + "cargo:warning=CARBONADO_LEAN_LIB unset; carbonado-sys will not link libcarbonado" + ); } } diff --git a/carbonado-sys/src/lib.rs b/carbonado-sys/src/lib.rs index 316d126..cfb1475 100644 --- a/carbonado-sys/src/lib.rs +++ b/carbonado-sys/src/lib.rs @@ -19,6 +19,7 @@ pub const CARBONADO_ERR_SCRUB_UNNECESSARY: c_int = 9; pub const CARBONADO_ERR_SCRUB_FAILED: c_int = 10; pub const CARBONADO_ERR_NOT_IMPLEMENTED: c_int = 11; pub const CARBONADO_ERR_INTERNAL: c_int = 12; +pub const CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION: c_int = 13; extern "C" { pub fn carbonado_abi_version() -> u32; @@ -34,6 +35,12 @@ extern "C" { out: *mut *mut u8, out_len: *mut usize, hash_out: *mut u8, + padding_out: *mut u32, + chunk_len_out: *mut u32, + bytes_ecc_out: *mut u32, + verifiable_slice_count_out: *mut u32, + bytes_compressed_out: *mut u32, + bytes_encrypted_out: *mut u32, ) -> c_int; pub fn carbonado_decode( master: *const u8, @@ -55,8 +62,16 @@ extern "C" { format: u8, nonce: *const u8, nonce_len: usize, + slh_pk: *const u8, + metadata: *const u8, out: *mut *mut u8, out_len: *mut usize, + padding_out: *mut u32, + chunk_len_out: *mut u32, + bytes_ecc_out: *mut u32, + verifiable_slice_count_out: *mut u32, + bytes_compressed_out: *mut u32, + bytes_encrypted_out: *mut u32, ) -> c_int; pub fn carbonado_decode_headered( master: *const u8, @@ -67,6 +82,117 @@ extern "C" { out_len: *mut usize, ) -> c_int; pub fn carbonado_verification_key(format: u8, key_out: *mut u8) -> c_int; + pub fn carbonado_encode_outboard( + master: *const u8, + master_len: usize, + plaintext: *const u8, + plaintext_len: usize, + format: u8, + nonce: *const u8, + nonce_len: usize, + header_path: u8, + main_out: *mut *mut u8, + main_len: *mut usize, + outboard_out: *mut *mut u8, + outboard_len: *mut usize, + parity_out: *mut *mut u8, + parity_len: *mut usize, + hash_out: *mut u8, + padding_out: *mut u32, + chunk_len_out: *mut u32, + bytes_compressed_out: *mut u32, + bytes_encrypted_out: *mut u32, + ) -> c_int; + pub fn carbonado_decode_outboard( + master: *const u8, + master_len: usize, + hash: *const u8, + hash_len: usize, + main: *const u8, + main_len: usize, + outboard: *const u8, + outboard_len: usize, + parity: *const u8, + parity_len: usize, + padding: u32, + format: u8, + header_path: u8, + nonce: *const u8, + nonce_len: usize, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_scrub( + body: *const u8, + body_len: usize, + hash: *const u8, + hash_len: usize, + padding: u32, + format: u8, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_scrub_outboard( + main: *const u8, + main_len: usize, + outboard: *const u8, + outboard_len: usize, + parity: *const u8, + parity_len: usize, + hash: *const u8, + hash_len: usize, + padding: u32, + chunk_len: u32, + format: u8, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_verify_slice( + body: *const u8, + body_len: usize, + hash: *const u8, + hash_len: usize, + index: u32, + count: u32, + format: u8, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_verify_slice_outboard( + main: *const u8, + main_len: usize, + outboard: *const u8, + outboard_len: usize, + hash: *const u8, + hash_len: usize, + index: u32, + count: u32, + format: u8, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_slh_keygen( + entropy: *const u8, + entropy_len: usize, + pk_out: *mut u8, + sk_out: *mut u8, + ) -> c_int; + pub fn carbonado_slh_sign( + secret_key: *const u8, + secret_key_len: usize, + message: *const u8, + message_len: usize, + out: *mut *mut u8, + out_len: *mut usize, + ) -> c_int; + pub fn carbonado_slh_verify( + public_key: *const u8, + public_key_len: usize, + message: *const u8, + message_len: usize, + signature: *const u8, + signature_len: usize, + ) -> c_int; } /// Safe wrapper: free a buffer returned by libcarbonado. diff --git a/doc/STREAMING_PARALLELISM.md b/doc/STREAMING_PARALLELISM.md index 460e3cc..3c66b54 100644 --- a/doc/STREAMING_PARALLELISM.md +++ b/doc/STREAMING_PARALLELISM.md @@ -28,14 +28,16 @@ Decode: | `stream_encode_inboard_body` (FEC) | Yes (4 KiB feed) | **O(8 × chunk_len)** — S2 eliminates pre-FEC `bare_len` staging | | `stream_encode_inboard_body` (Bao verify) | Yes (leaf-at-a-time) | **O(leaf + outboard)** — S3: `FecStripeReadAt` / `SeekReadAt`, no body staging `Vec` | | `encode_stream` / `encode_shard_stream` | Yes | O(chunk) preprocess spool + O(stripe) FEC | -| `decode_stream` / `file::decode` | Yes (bounded `encoded_len`) | O(chunk) spool staging + streaming EtM on header path | -| `stream_decode` (Verification c6) | Yes (incremental Read) | **(A)** Bao → `SeekWriteAt` on post-preprocess spool (**O(chunk)** RAM); **(B)** streaming EtM/decompress | -| `stream_decode` (Verification+FEC c12/c14/c15) | Yes (incremental Read) | **(A)** Bao → `FecInboardWriteAt` O(FEC body) shards (segment-wide stripe); `finish_into` streams logical without second full `Vec`; **(B)** O(chunk) spool + EtM | -| `stream_decode` (c4/c8) | Bounded when `encoded_body_len` set | O(stripe) FEC or O(compressed); rejects trailing bytes | -| `stream_decode_outboard` | Incremental main/parity copy | Bao verify via `PostOrderOutboard` + `ReadAt` (O(hash pair) per node; spool or slice); main via spool | +| `decode_stream` / `file::decode` | Yes (bounded `encoded_len`) | **backend-rust:** O(chunk) spool + streaming EtM. **backend-lean (W1a):** MAC-before-body, then materialize header+`encoded_len` → Lean `decode_headered`; peak **O(archive+plaintext)** E1. See [docs/LIMITS.md](../docs/LIMITS.md). | +| `stream_decode` (Verification c6) | Yes (incremental Read) | **backend-rust:** Bao → `SeekWriteAt` (**O(chunk)**). **backend-lean:** E1 spool→Lean body decode (O(encoded+logical)). | +| `stream_decode` (Verification+FEC c12/c14/c15) | Yes (incremental Read) | **backend-rust:** Bao → `FecInboardWriteAt` O(FEC body) + EtM. **backend-lean:** E1 Lean buffer. | +| `stream_decode` (c4/c8) | Bounded when `encoded_body_len` set | rust O(stripe)/O(compressed); lean E1 | +| `stream_*_outboard` **public non-Compression** (c0/c4/c8/c12) | Yes | **W1b E2 both backends:** S4 O(chunk/stripe) geometric (lean: **composition**, not pure Lean stream). | +| `stream_*_outboard` **public + Compression** (c2/c6/c10/c14) | Yes | **backend-rust:** O(chunk) streaming zstd. **backend-lean:** **O(logical)** bulk zstd for Lean frame parity — **not E2**. | +| `stream_*_outboard` **encrypted** | Yes | **backend-rust:** S4 + EtM spool. **backend-lean:** E1 Lean buffer (crypto dual). | | `scrub` | Seekable slices | O(1) Bao verify sink + combinatorial shard search (S5); `scrub_outboard` verify retains O(sidecar) | -**Bottom line:** Phase 1 fused encode/decode uses `SeekableSpool` for preprocess and post-Bao/FEC staging with streaming MAC-then-decrypt. **M1:** c6 `SeekWriteAt` O(chunk); FEC verify O(FEC body) shards + `finish_into`. **M2:** outboard verify uses `PostOrderOutboard` + `ReadAt` (no full sidecar `Vec` copy). Residual: FEC O(segment body) under single segment-wide RS stripe; async encoded-body spool. Scrub uses discard `WriteAt` (S5). +**Bottom line:** Phase 1 fused encode/decode uses `SeekableSpool` for preprocess and post-Bao/FEC staging with streaming MAC-then-decrypt. **M1 (pipeline):** c6 `SeekWriteAt` O(chunk); FEC verify O(FEC body) shards + `finish_into`. **M2:** outboard verify uses `PostOrderOutboard` + `ReadAt`. **W1b:** public **non-compress** outboard stream is O(chunk/stripe) under both backends (lean = geometric composition); Compression under lean is O(logical). **W4a:** inboard `verify_slice` Lean retains O(slice) (time O(N); full body input). **W4b permanent:** outboard slice C still full main+outboard buffers. **W4c permanent:** lean compress buffer-only. **W4d permanent:** FEC O(segment body); async encoded-body disk spool. Residual: pure Lean chunked C ABI; inboard/encrypted lean stream E1. Tests documenting this: `tests/streaming_limits.rs`. @@ -48,6 +50,10 @@ Tests documenting this: `tests/streaming_limits.rs`. | **M1** | ~~Non-FEC c6 → `SeekWriteAt`~~ **Shipped**; FEC `finish_into` **Shipped**. Further FEC O(segment) needs multi-stripe geometry (format-level) or accept segment-bounded residual | c6 O(chunk); c12–c15 O(FEC body) shards | | **M2** | ~~Outboard `PostOrderOutboard` + `ReadAt`~~ **Shipped** (slice or spool; O(hash pair) per node) | Dropped O(sidecar) mem copy | | **M3** | Async: wire `bao_tree::io::fsm` (or equivalent); drop full encoded-body spool | Remove ~2× encoded disk I/O on async path | +| **W4a** | ~~Inboard seekable O(slice) retain (Lean)~~ **Shipped** | O(slice) output; O(N) time; full body input at C | +| **W4b** | Outboard ReadAt C callback ABI | **Permanent residual** — full main+outboard buffers at C | +| **W4c** | Dual-safe streaming zstd | **Permanent residual** — buffer-only under lean (W2a) | +| **W4d** | FEC multi-stripe / async no-spool FSM | **Permanent residual** this wave — metrics below | ### Design principles @@ -102,6 +108,18 @@ Tests documenting this: `tests/streaming_limits.rs`. 4. **Outboard sidecar ordering:** `.out` and `.par` are derived from the same logical body; parallel write is fine after body is known. +### W4d metrics (permanent residual — 2026-07) + +| Metric | `backend-rust` | `backend-lean` | +|--------|----------------|----------------| +| FEC verify peak RAM | O(FEC body) = 8 × `chunk_len` shard buffers for segment-wide stripe (`FecInboardWriteAt`) | E1 full body `Vec` + decode (pipeline E1) or same O(FEC body) when on rust S4 composition paths | +| Async `stream_decode_async` disk | O(encoded) staging spool + O(logical) plaintext spool | same disk staging | +| Async peak RAM | spool/chunk + FEC residual O(FEC body) on sync path | **O(encoded + logical)** after E1 `read_encoded_body` | +| Async without encoded spool | **not shipped** (full FSM out of scope W4d) | **not shipped** | +| Double buffering | staging spool then sync path may re-read; not free earlier without API break | same | + +Honesty: do **not** claim async is O(chunk) under lean+async. Freeze never requires `async` (R10). + ### Deferred parallel work (after memory M1–M3) Pipeline **memory** elimination outranks these: @@ -148,8 +166,9 @@ Phase 2 adds **concurrency** (non-blocking fetch / range-read / UDP assembly ada 2. **Async is an adapter layer** — same internal stage graph; different trait bounds on sources/sinks. 3. **Runtime-agnostic traits** — `futures_lite::AsyncRead` / `AsyncWrite` (stdlib-compatible async I/O surface), not a hard Tokio dependency in library core. 4. **Optional `async` Cargo feature** — enables `stream_decode_async` and `stream::io` async helpers; default build stays sync-only. `bao_tree` keeps `default-features = false, features = ["validate"]`; with `async`, `bao_tree/tokio_fsm` is enabled but **not referenced by crate code yet** (reserved; no runtime effect on decode today). -5. **Bao bridge choice (Phase 2)** — async decode **does not** call `bao_tree::io::fsm`. Encoded input is fully staged to a disk-backed [`SeekableSpool`](../src/stream/spool.rs) via [`async_copy_bounded`](../src/stream/io.rs), then the existing sync keyed Bao + FEC + decrypt stages run unchanged. This preserves MAC-before-decrypt and bounded-read contracts without duplicating crypto logic, at the cost of **O(encoded_body)** disk I/O per decode (explicit tradeoff vs sync S4 incremental read). Phase 3 milestone: wire FSM or incremental async Bao to skip encoded spool. -6. **Executor blocking** — sync pipeline runs inside `async fn`; enable `async-tokio` for `spawn_blocking` offload or call from a dedicated thread pool. +5. **Bao bridge choice (Phase 2)** — async decode **does not** call `bao_tree::io::fsm`. Encoded input is fully staged to a disk-backed [`SeekableSpool`](../src/stream/spool.rs) via [`async_copy_bounded`](../src/stream/io.rs), then dual-aware sync [`stream_decode`](../src/stream/decode.rs) runs (R10: same engine dispatch as R5 E1 under `backend-lean`; S4 pipeline under `backend-rust`). This preserves MAC-before-decrypt and bounded-read contracts without duplicating crypto logic, at the cost of **O(encoded_body)** disk I/O per decode (explicit tradeoff vs sync S4 incremental read). Not stream E2 / true chunked. +6. **Executor blocking** — dual-aware sync path runs inside `async fn`; enable `async-tokio` for `spawn_blocking` offload or call from a dedicated thread pool. +7. **Dual freeze (R10 permanent)** — `just test-lean-ci` never enables `async`; optional lean+async is dual-engine product path, not freeze. ### Feature flag matrix @@ -170,18 +189,25 @@ Phase 2 adds **concurrency** (non-blocking fetch / range-read / UDP assembly ada ### WASM -Keep **`async` off** on `wasm32` deployments: `SeekableSpool` uses host temp files. `stream_decode_async` is exported when `async` is enabled but returns `NotImplemented` on `wasm32` at runtime. CI may compile `--all-features` on `wasm32-unknown-unknown`; that means "compiles, unsupported at runtime" — not a deployment target for async decode. +Keep **`async` off** on `wasm32` deployments: `SeekableSpool` uses host temp files. `stream_decode_async` is exported when `async` is enabled but returns `NotImplemented` on `wasm32` at runtime. CI may compile `backend-rust` + optional features on `wasm32-unknown-unknown` (never `--all-features` — dual-backend mutual exclusion); that means "compiles, unsupported at runtime" — not a deployment target for async decode. ### Disk I/O overhead (Phase 2) -Per decode on native targets with `async`: up to three temp spools (encoded staging, pipeline `post_preprocess`, plaintext staging). Encoded bytes are written once on ingress and read again by the sync pipeline — roughly **2× encoded-body disk traffic** vs sync incremental `Read`. Error cleanup uses `SeekableSpool::drop` (`0600` on Unix). Track elimination of encoded spool as a Phase 3 metric. +Per decode on native targets with `async` (R10 dual-aware after staging): + +| Engine | Temp spools | Peak RAM (honest) | +|--------|-------------|-------------------| +| **`backend-rust` (S4)** | Up to **three** disk spools: encoded staging, pipeline `post_preprocess`, plaintext staging | Spool/chunk-oriented; FEC verification may retain O(FEC body) shard buffers | +| **`backend-lean` (E1)** | **Two** disk spools: encoded staging + plaintext staging (no rust `post_preprocess` spool) | **O(encoded + logical)** — E1 materializes body `Vec` then Lean plaintext | + +Encoded bytes are written once on ingress and read again by the sync path — roughly **2× encoded-body disk traffic** vs rust sync incremental `Read`. Error cleanup uses `SeekableSpool::drop` (`0600` on Unix). Track elimination of encoded spool as a future metric (not stream E2 yet). ### Public API (async) -- [`stream_decode_async`](../src/stream/decode_async.rs) — `AsyncPipelineSource` → `AsyncPipelineSink` inboard decode; delegates to sync `stream_decode_inboard_pipeline` (format matrix tested in `tests/streaming_async.rs`: c4/c6/c8/c12/c14/c15). Verification-format truncated bounded reads surface staging `UnexpectedEof` on async vs `BaoResponseTruncated` on sync (documented in rustdoc). +- [`stream_decode_async`](../src/stream/decode_async.rs) — `AsyncPipelineSource` → `AsyncPipelineSink` inboard decode; after staging, dual-aware [`stream_decode`](../src/stream/decode.rs) (R10; format matrix in `tests/streaming_async.rs`: c4/c6/c8/c12/c14/c15). Verification-format truncated bounded reads under `backend-rust` surface staging `UnexpectedEof` on async vs `BaoResponseTruncated` on sync (documented in rustdoc). - [`stream::io`](../src/stream/io.rs) — `PipelineSource` / `PipelineSink` (sync), `AsyncPipelineSource` / `AsyncPipelineSink` + `async_copy_bounded` / `async_copy_all` (feature `async`). -Tests: `cargo test --features async --test streaming_async`. +Tests: `cargo test --features async --test streaming_async` (rust). Optional dual: lean features + `async` / `async-tokio` (not freeze). ## Phase 3: Parallelism (shipped) @@ -209,11 +235,11 @@ Implementation: [`src/stream/parallel.rs`](../src/stream/parallel.rs) — `Paral | Feature | Default | Enables | |---------|---------|---------| | `parallel` | yes | Scoped parallel RS parity in `FecInboardEncoder::take_stripe`; std only (serial at runtime on `wasm32`) | -| `--no-default-features` + `pqc,ots,cli` | — | Serial RS parity (`reed_solomon_erasure::encode`); CI-gated via `serial_fec_path` | +| `--no-default-features` + `backend-rust,pqc,ots,cli` | — | Serial RS parity (`reed_solomon_erasure::encode`); CI-gated via `serial_fec_path` | | `async` | no | Phase 2 async adapters (orthogonal) | | `async-tokio` | no | Phase 2 + `spawn_blocking` offload | -Combine flags independently: `cargo test --all-features` exercises compile-time matrix; determinism tests gate on `parallel` only. +Combine flags carefully: never `cargo test --all-features` (enables both backends → `compile_error!`). Prefer default + `--features "async,async-tokio,man-gen"` for the optional rust matrix; determinism tests gate on `parallel` only. ### WASM @@ -234,7 +260,7 @@ On `wasm32`, `parallel` compiles but [`should_parallelize_rs_parity`](../src/str | Multi-file directory segments (independent processes) | CTR stream under one archive nonce | | | Bao root finalize after all leaves | -Determinism contract: parallel encode produces **bit-identical** body bytes and keyed Bao roots vs the serial `encode_sep` / `rs.encode` reference (`encode_rs_parity_serial`, unit-tested against `rs.encode`). Serial FEC path (no `parallel`) is CI-gated separately (`cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path`). Scrub roundtrip under parallel encode: `parallel_determinism::parallel_encode_inboard_scrub_roundtrip_c12_c14`. Tests: `cargo test --test parallel_determinism`. +Determinism contract: parallel encode produces **bit-identical** body bytes and keyed Bao roots vs the serial `encode_sep` / `rs.encode` reference (`encode_rs_parity_serial`, unit-tested against `rs.encode`). Serial FEC path (no `parallel`) is CI-gated separately (`cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path`). Scrub roundtrip under parallel encode: `parallel_determinism::parallel_encode_inboard_scrub_roundtrip_c12_c14`. Tests: `cargo test --test parallel_determinism`. ## References diff --git a/doc/TEST_STRATEGY.md b/doc/TEST_STRATEGY.md index 277e615..686562a 100644 --- a/doc/TEST_STRATEGY.md +++ b/doc/TEST_STRATEGY.md @@ -132,10 +132,16 @@ Carbonado uses **reed-solomon-erasure 4/8**: any **4 of 8** shards reconstruct t ## Running tests ```bash -# Full native gate (serial FEC path + full matrix) -cargo test --features "pqc,ots,cli" -cargo test --all-features -cargo clippy --all-targets --all-features -- -D warnings +# Full native gate (default + serial FEC + optional features) +# Never --all-features: enables both backend-rust and backend-lean → compile_error!. +cargo test +cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path +cargo test --features "async,async-tokio,man-gen" +cargo clippy --all-targets --features "async,async-tokio,man-gen" -- -D warnings + +# Dual-backend lean freeze (G11 + R7 G8 full): just test-lean-ci +# = unfiltered cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +# Feature-gated async/parallel suites are 0 tests under this feature set (not dual residual). # FEC-focused cargo test --test fec_chaos --test fec_scrub_matrix --test shard_fec_scrub @@ -154,9 +160,9 @@ cargo test --features async-tokio --test streaming_async cargo test --test parallel_determinism # Serial FEC path without `parallel` (exercises fec.rs rs.encode branch) -cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path +cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path -# WASM lint (no pqc) +# WASM lint (backend-rust only, no pqc) just lint-wasm ``` @@ -165,7 +171,9 @@ just lint-wasm - Keep chaos tests on native Linux (may be slow at 256 KiB × 4 public levels) - Shard FEC scrub tests parallel-safe (unique temp dirs per test) - **Default gate:** `cargo test` (includes `parallel` and `parallel_determinism`) -- **Serial FEC gate:** `cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path` — must run before or alongside `--all-features` +- **Serial FEC gate:** `cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path` — must name `backend-rust` under `--no-default-features` +- **Optional rust matrix:** `cargo test --features "async,async-tokio,man-gen"` (never `--all-features`) - **Phase 3 determinism:** covered by default `cargo test --test parallel_determinism` (RS parity vs `encode_rs_parity_serial`, c12/c14 bytes + Bao root, scrub roundtrip) -- **WASM `parallel`:** compile-only in `test-matrix` (`cargo check --target wasm32-unknown-unknown --all-features`); runtime serial fallback documented in `STREAMING_PARALLELISM.md` § Phase 3 WASM +- **WASM `parallel`:** compile-only in `test-matrix` (`cargo check --target wasm32-unknown-unknown --features "async,async-tokio,man-gen"` and no-pqc `backend-rust` only); runtime serial fallback documented in `STREAMING_PARALLELISM.md` § Phase 3 WASM +- **Lean dual freeze (G11 + R7 G8 full closed):** job `dual-backend-lean` / `just test-lean-ci` = unfiltered full lean suite; `streaming_async` needs `async`, `parallel_determinism` needs `parallel` (not in dual feature set) - Proptest cases capped at 32 for `fec_chaos` (raise when stable) \ No newline at end of file diff --git a/docs/ABI.md b/docs/ABI.md index 705acd2..162858f 100644 --- a/docs/ABI.md +++ b/docs/ABI.md @@ -6,10 +6,10 @@ Stable C interface for the **Lean AOT engine** (`libcarbonado`). Rust `backend-l | Artifact | Role | |----------|------| -| [`include/carbonado.h`](../include/carbonado.h) | C declarations (v0 surface) | +| [`include/carbonado.h`](../include/carbonado.h) | C declarations (v0 core + Phase 2 additive) | | [`carbonado-sys/src/lib.rs`](../carbonado-sys/src/lib.rs) | Rust FFI bindings + error constants | -| [`nix/native/carbonado_abi.c`](../nix/native/carbonado_abi.c) | C stubs / weak symbols in the native archive | -| [`Carbonado/Ffi.lean`](../Carbonado/Ffi.lean) | Lean pure helpers + planned `@[export]` surface | +| [`nix/native/carbonado_abi.c`](../nix/native/carbonado_abi.c) | Strong C exports calling Lean `@[export] l_carbonado_*` | +| [`Carbonado/Ffi.lean`](../Carbonado/Ffi.lean) | Lean pure helpers + live `@[export]` surface | | This document | Ownership, versioning, error codes, link instructions | **ABI version:** `1` (`CARBONADO_ABI_VERSION`). Bump major on breaking changes (symbol rename, error-code reuse, semantic change of successful outputs). @@ -21,7 +21,7 @@ Stable C interface for the **Lean AOT engine** (`libcarbonado`). Rust `backend-l | Pattern | Rule | |---------|------| | Input buffers | Caller owns; not freed by libcarbonado | -| Output buffers | Returned via `uint8_t **out` + `size_t *out_len`; allocated with the same allocator family as `carbonado_free` (malloc); **caller frees with `carbonado_free`** (or takes ownership via `Vec::from_raw_parts` on the Rust side — do not double-free) | +| Output buffers | Returned via `uint8_t **out` + `size_t *out_len`; allocated with **`malloc`**. C callers free with **`carbonado_free`**. Rust `backend-lean` **copies** into a `Vec` then calls **`carbonado_free`** (allocator-agnostic; safe with jemalloc/mimalloc GlobalAlloc) | | Errors | Integer codes only on the hot path; no heap error strings in v0 | | Null | Null input pointers with non-zero lengths → `CARBONADO_ERR_INVALID_ARGUMENT` (when implemented) | @@ -44,33 +44,41 @@ Lean: `Carbonado.Ffi.abiVersion` / `@[export carbonado_abi_version]`. ## Error codes (v0) -Stable integers shared by `include/carbonado.h`, `carbonado-sys`, and `Carbonado.Ffi`. Map to `CarbonadoError` in `src/backend/mod.rs` (`lean::map_err`). Unknown codes → generic failure. +Stable integers shared by `include/carbonado.h`, `carbonado-sys`, and `Carbonado.Ffi`. Converted to `CarbonadoError` in `src/backend/mod.rs` (`lean::map_err`). Unknown codes → generic failure. -| Code | Name | Meaning | Approximate Rust mapping | -|-----:|------|---------|--------------------------| -| 0 | `CARBONADO_OK` | Success | `Ok` | -| 1 | `CARBONADO_ERR_INVALID_ARGUMENT` | Null/lengths/nonce size/sequence | bad args; nonce length; empty segment | -| 2 | `CARBONADO_ERR_INVALID_KEY_LENGTH` | Master not 32 or 64 bytes | `InvalidKeyLength` (or current stand-in until dedicated variant) | -| 3 | `CARBONADO_ERR_AUTHENTICATION` | Header MAC / payload EtM / Bao auth | `AuthenticationFailed` (+ header MAC fails) | -| 4 | `CARBONADO_ERR_INVALID_MAGIC` | Bad `CARBONADO20\n` (or related magic) | `InvalidMagicNumber` | -| 5 | `CARBONADO_ERR_INVALID_HEADER` | Truncated/malformed header or body bounds | `InvalidHeaderLength` / truncated body | -| 6 | `CARBONADO_ERR_FEC` | RS geometry / shard errors | `UnevenFecChunks` / FEC failures | -| 7 | `CARBONADO_ERR_BAO` | Keyed Bao verify / slice stream errors | Bao / verification failures | -| 8 | `CARBONADO_ERR_ZSTD` | Compress/decompress failures | `ZstdError` | -| 9 | `CARBONADO_ERR_SCRUB_UNNECESSARY` | Scrub not needed | `UnnecessaryScrub` | -| 10 | `CARBONADO_ERR_SCRUB_FAILED` | Scrub cannot recover / requires verification | `InvalidScrubbedHash` / `ScrubRequiresVerification` | -| 11 | `CARBONADO_ERR_NOT_IMPLEMENTED` | Surface not exported or still stubbed | fail closed (Phase 0–1 stubs) | -| 12 | `CARBONADO_ERR_INTERNAL` | Unexpected / allocator / invariant | internal | +Two columns matter for dual-backend work: -**Collapse rule:** Fine-grained Lean `PipelineError` variants map through `Carbonado.Ffi.ofPipelineError` into these codes at the C boundary. Distinct failure modes that tests assert via `matches!` must either keep distinct codes or get refined Rust-side mapping before those tests are on the lean allowlist. Do **not** map unrelated failures to a single diagnostic variant permanently. +- **Target mapping** — intended 1:1 (or documented multi-source) diagnostics for failure-mode `matches!` tests. +- **Current `map_err` (Phase 2)** — live mapping; scrub codes 9/10/13 are distinct. Residual: InvalidArgument still generic; FEC modes collapse to `UnevenFecChunks` (with targeted MissingFecParity remap on outboard scrub). -**Phase 0–1 honesty:** Until real exports are linked, all encode/decode/verification_key C entry points return `CARBONADO_ERR_NOT_IMPLEMENTED` (weak stubs in `carbonado_abi.c`). +| Code | Name | Meaning | Target Rust mapping | Current `lean::map_err` (Phase 2 + R4) | +|-----:|------|---------|---------------------|--------------------------------------| +| 0 | `CARBONADO_OK` | Success | `Ok` | `Ok` | +| 1 | `CARBONADO_ERR_INVALID_ARGUMENT` | Null/lengths/nonce size/sequence | dedicated arg/nonce/segment variants as needed | `InternalStateError("…invalid argument…")` (**P2 residual:** add `InvalidArgument` / reuse nonce variants before allowlist expands to nonce/sequence fails) | +| 2 | `CARBONADO_ERR_INVALID_KEY_LENGTH` | Master not 32 or 64 bytes | `InvalidKeyLength` | **`InvalidKeyLength`** | +| 3 | `CARBONADO_ERR_AUTHENTICATION` | Header MAC / payload EtM / **Bao auth** | `AuthenticationFailed` | **`AuthenticationFailed`** (R4: `baoAuthenticationFailed` joins header/payload auth) | +| 4 | `CARBONADO_ERR_INVALID_MAGIC` | Bad `CARBONADO20\n` (or related magic) | `InvalidMagicNumber` | `InvalidMagicNumber("lean-backend")` | +| 5 | `CARBONADO_ERR_INVALID_HEADER` | Truncated/malformed header, body bounds, **short inboard Bao prefix** | `InvalidHeaderLength` | **`InvalidHeaderLength`** (R4: Lean `invalidPrefix` maps here) | +| 6 | `CARBONADO_ERR_FEC` | RS geometry / shard errors | `UnevenFecChunks` / FEC failures | `UnevenFecChunks` | +| 7 | `CARBONADO_ERR_BAO` | Bao stream truncation / trailing / residual slice geometry | `BaoResponseTruncated` (not auth) | **`BaoResponseTruncated`** (R4: auth no longer collapses here) | +| 8 | `CARBONADO_ERR_ZSTD` | Compress/decompress failures | `ZstdError` | `ZstdError("lean-backend zstd")` | +| 9 | `CARBONADO_ERR_SCRUB_UNNECESSARY` | Scrub not needed | `UnnecessaryScrub` | `UnnecessaryScrub` | +| 10 | `CARBONADO_ERR_SCRUB_FAILED` | Scrub cannot recover | `InvalidScrubbedHash` | `InvalidScrubbedHash` | +| 11 | `CARBONADO_ERR_NOT_IMPLEMENTED` | Surface not exported or still stubbed | `NotImplemented` | **`NotImplemented`** | +| 12 | `CARBONADO_ERR_INTERNAL` | Unexpected / allocator / invariant | internal / dedicated | `InternalStateError` | +| 13 | `CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION` | Scrub without Verification bit | `ScrubRequiresVerification` | **`ScrubRequiresVerification`** (P2) | + +**Phase 2 + R4 mapping:** scrub requires-verification is distinct (code 13). **R4:** `ofPipelineError` sends `baoAuthenticationFailed` → code 3 and `invalidPrefix` → code 5 (no longer collapsed into code 7). `InvalidSliceIndex { index, content_len }` is produced by Rust-side geometry pre-checks in `lean::verify_slice` (C ABI carries no structured fields). Residual: no dedicated InvalidArgument; MissingFecParity may still surface via FEC path when parity absent after verify fail. + +**Collapse rule (C boundary):** Fine-grained Lean `PipelineError` variants map through `Carbonado.Ffi.ofPipelineError` into these **integer codes**. Distinct failure modes that tests assert via `matches!` must either keep distinct codes or get refined Rust-side mapping before those tests are on the lean allowlist. Do **not** permanently map unrelated failures to a single diagnostic variant. + +**Phase 2 status:** v0 body/headered **plus** outboard/scrub/verify_slice are **live**. Encode packs include chunk/ecc/vsc metadata; **R3** adds `bytes_compressed` / `bytes_encrypted` (nullable C out-params; ABI version remains 1 additive). --- -## Core v0 functions (in `include/carbonado.h`) +## Core functions (in `include/carbonado.h`) -These are the **only** product symbols declared in the header today. Signatures must match the header byte-for-byte in meaning. +Signatures must match the header byte-for-byte in meaning. ### Lifecycle @@ -84,14 +92,24 @@ void carbonado_free(void *p); Low-level layout: when encrypted, the body uses the embedded-nonce blob shape Rust low-level paths use (`[nonce|tag|ct]` inside the encrypt stage as applicable). For **public** formats, `nonce` may be null / `nonce_len == 0`. For **encrypted** formats, `nonce` must be 16 bytes (tests use fixed nonces for determinism). ```c -/* out: verifiable body only (no Carbonado Header). hash_out: 32-byte Bao root. */ +/* out: verifiable body only (no Carbonado Header). hash_out: 32-byte Bao root. + * Meta out-params optional (nullable). Skipped compress/encrypt stages report 0 (R3). + * Lean pack: success = [status:4][pad:4][chunk:4][ecc:4][vsc:4] + * [bytes_compressed:4][bytes_encrypted:4][hash:32][body…] (prefix 60); + * error = [status:4] only. C parses status first so encode failures return real ABI codes. */ int carbonado_encode( const uint8_t *master, size_t master_len, /* 32 or 64 */ const uint8_t *plaintext, size_t plaintext_len, uint8_t format, const uint8_t *nonce, size_t nonce_len, /* 16 if encrypted; else 0/null */ uint8_t **out, size_t *out_len, - uint8_t hash_out[32] + uint8_t hash_out[32], + uint32_t *padding_out, /* nullable */ + uint32_t *chunk_len_out, /* nullable */ + uint32_t *bytes_ecc_out, /* nullable */ + uint32_t *verifiable_slice_count_out, /* nullable */ + uint32_t *bytes_compressed_out, /* nullable (R3) */ + uint32_t *bytes_encrypted_out /* nullable (R3) */ ); ``` @@ -111,13 +129,29 @@ int carbonado_decode( ### Headered encode/decode (≈ Rust `file::encode` / `file::decode`) ```c -/* Full file: Header (177 B) || body. Bao root lives in the header. */ +/* Full file: Header (177 B) || body. Bao root lives in the header. + * slh_pk: NULL → zero-filled field; non-NULL must point to exactly 32 valid bytes. + * metadata: NULL → zero-filled field; non-NULL must point to exactly 8 valid bytes. + * Additive params (R2 SLH/meta + R3 stage counters): ABI version stays 1. + * Lean pack: success = [status:4][pad:4][chunk:4][ecc:4][vsc:4] + * [bytes_compressed:4][bytes_encrypted:4][archive…] (prefix 28); + * error = [status:4] only. + * C is length-implicit (non-null always copies fixed width). Wrong lengths are + * Lean/export ByteArray-only (encodeHeaderedBytes → errInvalidArgument). */ int carbonado_encode_headered( const uint8_t *master, size_t master_len, const uint8_t *plaintext, size_t plaintext_len, uint8_t format, const uint8_t *nonce, size_t nonce_len, /* 16 when Encrypted bit set */ - uint8_t **out, size_t *out_len + const uint8_t *slh_pk, /* nullable 32 B */ + const uint8_t *metadata, /* nullable 8 B */ + uint8_t **out, size_t *out_len, + uint32_t *padding_out, /* nullable (R3) */ + uint32_t *chunk_len_out, /* nullable */ + uint32_t *bytes_ecc_out, /* nullable */ + uint32_t *verifiable_slice_count_out, /* nullable */ + uint32_t *bytes_compressed_out, /* nullable */ + uint32_t *bytes_encrypted_out /* nullable */ ); int carbonado_decode_headered( @@ -127,7 +161,19 @@ int carbonado_decode_headered( ); ``` -Lean pure analogues (not yet live C malloc wrappers): `Carbonado.Ffi.encodeHeaderedBytes` / `decodeHeaderedBytes`. +Lean pure + live C: `Carbonado.Ffi.encodeHeaderedBytes` / `decodeHeaderedBytes` via +`l_carbonado_encode_headered` / `l_carbonado_decode_headered`. +Lean takes `slhPublicKey` / `metadataBytes` as `ByteArray` (empty or exact length 32 / 8; +empty → zeros; other sizes → `errInvalidArgument`). Coverage: `CarbonadoTest.Pipeline` +`encode_headered_bytes_bad_*_len` theorems. Rust `lean::encode_headered` returns +`(archive, EncodeInfo)`; `file::encode` threads that info (R3). High-level `file::encode` +still passes `slh_public_key = None` (zeros); dual-suite SLH sets the field via `Header` +APIs / sidecars (G10-A). + +**R2 residual (not a dedicated `InvalidArgument` variant):** if code 1 reaches Rust +`map_err`, callers see `InternalStateError("lean-backend invalid argument")` — same +P2 residual as other `CARBONADO_ERR_INVALID_ARGUMENT` sources (table above). Typed +headered encode never surfaces wrong SLH/meta lengths through C. ### Verification key @@ -140,54 +186,146 @@ Lean pure: `Carbonado.Ffi.verificationKeyBytes`. --- -## Future C surface (not in `include/carbonado.h` v0) +### Outboard / scrub / slice (Phase 2 — live) + +```c +/* header_path != 0 → encrypted bare main [tag|ct] (file::encode_outboard); + * header_path == 0 → embedded [nonce|tag|ct] (encoding::encode_outboard). */ +/* Lean pack prefix 52: status+pad+chunk+comp+enc+hash; then len-prefixed main/ob/par (R3). */ +int carbonado_encode_outboard(/* master, pt, format, nonce, header_path → main/outboard/parity + hash + pad/chunk + compress/encrypt */); +int carbonado_decode_outboard(/* master, hash, main, outboard, parity, padding, format, header_path, nonce → plaintext */); +int carbonado_scrub(/* body, hash, padding, format → recovered body or SCRUB_* error */); +int carbonado_scrub_outboard(/* main, outboard, parity, hash, padding, chunk_len, format → bare */); +int carbonado_verify_slice(/* body, hash, index, count, format → slice bytes */); +/* R9: seekable outboard slice (O(slice+height) hash; full main+outboard buffers). */ +int carbonado_verify_slice_outboard(/* main, outboard, hash, index, count, format → slice */); +/* R9 / G10: SLH-DSA-SHA2-128s (libbitcoinpqc objects in libcarbonado_native.a). */ +int carbonado_slh_keygen(/* entropy≥128 → pk[32], sk[64] */); +int carbonado_slh_sign(/* sk[64], message → malloc 7856 B sig */); +int carbonado_slh_verify(/* pk[32], message, sig[7856] → OK or AUTHENTICATION */); +``` + +See `include/carbonado.h` for full signatures. `extract_slice` is verify_slice with `count == 1` (Rust-side). + +**`verify_slice` (inboard) — W4a closed (retained output):** Lean walks the full inboard bao response for authentication (O(N) time; inboard embeds full-range response) starting at offset 8 (no second full-response copy) but retains only the requested slice bytes (O(slice) output) via `decodeRecRetainRange` — same class as Rust `SliceRegionWriter` / `verify_slice_inboard_seekable`. Leaf hashing may use O(leaf) temps. C ABI still takes the **full inboard body buffer as input** (no streaming ReadAt). Do **not** claim O(slice) peak RSS when the caller already holds the full body `Vec`. **`count == 0` split:** Lean C / pure `verifySliceInboard` is **auth-first**; dual Rust API / `lean::verify_slice` short-circuits empty success before C (parity with pure-Rust seekable). + +**`verify_slice_outboard` (R9 + W4b permanent full-buffer):** Lean walks only the requested leaf-group range (O(slice + height) hash work; W4b offset walk avoids recursive full half-extracts). C ABI still takes full main + full outboard buffers in memory — **permanent residual** (no additive `carbonado_verify_slice_outboard_at` / `ReadAt` callback ABI this wave). Under `backend-lean` the Rust dispatcher materializes `data_len` once when `data` is a generic `ReadAt`. + +**`count == 0` (outboard slice):** empty success **immediately** — no root/geometry/OOB/auth checks (matches Rust `stream/slice.rs::verify_slice_outboard`). Authentication and OOB apply only when `count > 0`. + +**SLH C error mapping (R9):** `carbonado_slh_keygen` / `_sign` library failure → `CARBONADO_ERR_INTERNAL`; `carbonado_slh_verify` reject → `CARBONADO_ERR_AUTHENTICATION`; bad args → `CARBONADO_ERR_INVALID_ARGUMENT`. Empty message (`NULL`, len 0) is accepted (non-NULL empty buffer passed to libbitcoinpqc). + +## Phase 3 directory (composition — no new C symbols) -The following are **planned** for later ABI revisions when Phase 2+ test classes need them. They are **not** declared in the current header; do not document them as exported. +Directory dual-backend does **not** add `carbonado_encode_directory` / `decode_directory` C exports. +`file::encode_directory` / `decode_directory` under `backend-lean` compose existing ABI: -| Future symbol (illustrative) | Rust analogue | Target phase | -|------------------------------|---------------|--------------| -| `carbonado_encode_outboard` / `carbonado_decode_outboard` | `encode_outboard` / `decode_outboard` | Phase 2 | -| `carbonado_scrub` / `carbonado_scrub_outboard` | `scrub` / `scrub_outboard` | Phase 2 | -| `carbonado_verify_slice` / `carbonado_extract_slice` | `verify_slice` / `extract_slice` | Phase 2 | -| Directory / Adamantine catalog helpers | `encode_directory` / `decode_directory` | Phase 3 (rkyv) | -| SLH sign/verify | `crypto::slh_dsa_*` | Phase 4 (G10) | +| Directory stage | Lean C ABI used | +|-----------------|-----------------| +| Bare segment mains | `carbonado_encode_outboard` / `carbonado_decode_outboard` (embedded-nonce) | +| Catalog inboard c14/c15 | `carbonado_encode_headered` / `carbonado_decode_headered` | +| rkyv FilepackManifest v2 + Adamantine envelope/payload + FS | **Dual-suite composition:** Rust rkyv+FS (SSOT). **Pure Lean Directory/CLI** also emit rkyv (**W3**); CFP2 dual-decode fallback only | -Until exported, Lean or Rust may implement these **above** the v0 body/headered ABI without new C symbols, but the dual-backend bar for those tests is green only when both backends produce matching results. +Dual-suite catalogs are **rkyv** (same as `backend-rust`; Rust composition SSOT for dual encode). Pure Lean Directory/CLI emit rkyv (**W3**); CFP2 is dual-decode fallback only (LIMITS). +Allowlist: `tests/lean_backend_phase3.rs` (`just test-lean-phase3`). + +**Bao error mapping (R4):** Lean `ofPipelineError` no longer collapses all Bao failures to +`CARBONADO_ERR_BAO`. Current fidelity: + +| Lean failure | ABI code | Rust mapping | +|--------------|----------|--------------| +| Bao auth (wrong key / root / leaf-parent mismatch) | 3 `AUTHENTICATION` | `AuthenticationFailed` | +| Short inboard prefix (`invalidPrefix`) | 5 `INVALID_HEADER` | `InvalidHeaderLength` | +| Truncation / trailing / residual geometry (no Rust pre-check) | 7 `BAO` | `BaoResponseTruncated` | +| OOB slice index | n/a (Rust pre-check in `lean::verify_slice`) | `InvalidSliceIndex { index, content_len }` | + +Directory catalog body-tamper under both backends surfaces `AuthenticationFailed` for keyed Bao +auth failure (see `tests/directory_archive.rs`). Residual: pure-Rust outboard/inboard entry +points may still use `OutboardVerificationFailed` in other paths — do not re-collapse auth to +code 7. + +## Phase 4: SLH / CLI / OTS (composition for dual-suite) + +**G10 strategy A (dual-suite, still valid):** product SLH under `backend-lean` may use Rust +`crypto::slh_dsa_*` + `bitcoinpqc` composition. Dual-suite does **not** require pure Lean SLH. + +**R9 / G10 full (pure Lean depth):** libbitcoinpqc SLH-DSA-SHA2-128s objects are linked into +`libcarbonado_native.a` (pinned `b309f444…`). Live symbols: + +| Symbol | Role | +|--------|------| +| `carbonado_slh_keygen` | entropy ≥128 → pk 32 + sk 64 | +| `carbonado_slh_sign` | sk 64 + message → malloc 7856 B signature | +| `carbonado_slh_verify` | pk + message + sig → OK / AUTHENTICATION | +| Lean `@[extern]` | `carbonado_slh_{keygen,sign,verify}_raw` → `Carbonado/Slh.lean` `signRoot` / `verifyRoot` | + +Dual-suite may keep Rust bitcoinpqc composition as product SSOT; pure Lean is for +`libcarbonado` purity. Fail-closed on bad signatures. + +Allowlist: `tests/lean_backend_phase4.rs` + `tests/slh_outboard.rs` (`just test-lean-phase4`). + +## R9 additive C surface (ABI version stays 1) + +| Symbol | Rust analogue | Status | +|--------|---------------|--------| +| `carbonado_verify_slice_outboard` | `verify_slice_outboard` | **live** (R9) | +| `carbonado_slh_keygen` / `_sign` / `_verify` | `crypto::slh_dsa_*` | **live** (R9 / G10) | +| Optional pure-buffer directory helpers | composition | optional (not required) | --- -## Implementation status (Phase 0 close) +## Implementation status (Phase 2–4 close) | Symbol | Lean pure | C in `libcarbonado` | `carbonado-sys` | Rust `backend-lean` dispatch | |--------|-----------|---------------------|-----------------|------------------------------| -| `carbonado_abi_version` | `@[export]` in `Ffi.lean` | **implemented** (`carbonado_abi.c`) | bound | `lean::abi_version` | -| `carbonado_free` | — | **implemented** | bound | used on free paths | -| `carbonado_encode` | planned / helpers partial | **weak stub → NOT_IMPLEMENTED** | bound | not yet on crate-root `encode` | -| `carbonado_decode` | planned | **weak stub → NOT_IMPLEMENTED** | bound | not yet on crate-root `decode` | -| `carbonado_encode_headered` | pure `encodeHeaderedBytes` | **weak stub → NOT_IMPLEMENTED** | bound | `lean::encode_headered` wrapper exists; needs live lib | -| `carbonado_decode_headered` | pure `decodeHeaderedBytes` | **weak stub → NOT_IMPLEMENTED** | bound | `lean::decode_headered` wrapper exists; needs live lib | -| `carbonado_verification_key` | pure `verificationKeyBytes` | **weak stub → NOT_IMPLEMENTED** | bound | not yet wired to crate root | -| outboard / scrub / slice C | partial Lean modules | **not in header** | — | later | - -**Phase 1 definition of done (engineering, not this doc pass):** real non-stub implementations for the v0 encode/decode/verification_key symbols in the linked archive; Phase 1 test allowlist green on `backend-lean`; `backend-rust` still full green. - -Update this table as Phase 1 lands. +| `carbonado_abi_version` | `abiVersion` (C-owned) | **live** | bound | `lean::abi_version` | +| `carbonado_free` | — | **live** | bound | C callers: `carbonado_free`. Rust `backend-lean`: copy via `take_buf` then `carbonado_free` (not `Vec::from_raw_parts`) | +| `carbonado_encode` | `l_carbonado_encode` | **live** (+ meta; R3 compress/encrypt) | bound | `encoding::encode` → `lean::encode` | +| `carbonado_decode` | `l_carbonado_decode` | **live** | bound | `decoding::decode` → `lean::decode` | +| `carbonado_encode_headered` | `l_carbonado_encode_headered` | **live** (+ SLH/meta R2; EncodeMeta R3) | bound | `file::encode` → `lean::encode_headered` | +| `carbonado_decode_headered` | `l_carbonado_decode_headered` | **live** | bound | `file::decode` → `lean::decode_headered` | +| `carbonado_verification_key` | `l_carbonado_verification_key` | **live** | bound | `crypto::carbonado_verification_key` | +| `carbonado_encode_outboard` | `l_carbonado_encode_outboard` | **live** (+ R3 compress/encrypt) | bound | `encoding::encode_outboard` | +| `carbonado_decode_outboard` | `l_carbonado_decode_outboard` | **live** | bound | `decoding::decode_outboard` | +| `carbonado_scrub` | `l_carbonado_scrub` | **live** | bound | `decoding::scrub` | +| `carbonado_scrub_outboard` | `l_carbonado_scrub_outboard` | **live** | bound | `decoding::scrub_outboard` | +| `carbonado_verify_slice` | `l_carbonado_verify_slice` | **live** | bound | `decoding::verify_slice` | +| `carbonado_verify_slice_outboard` | `l_carbonado_verify_slice_outboard` | **live** (R9) | bound | `stream::verify_slice_outboard` → `lean::verify_slice_outboard` | +| `carbonado_slh_keygen` | `@[extern]` raw | **live** (R9) | bound | optional; dual-suite may use Rust `bitcoinpqc` | +| `carbonado_slh_sign` | `@[extern]` raw | **live** (R9) | bound | optional; dual-suite may use Rust `bitcoinpqc` | +| `carbonado_slh_verify` | `@[extern]` raw | **live** (R9) | bound | optional; dual-suite may use Rust `bitcoinpqc` | + +**Phase 2 allowlist:** `tests/lean_backend_smoke.rs` + `tests/lean_backend_phase2.rs` (`just test-lean-phase2`). + +**Phase 3 allowlist:** + `tests/lean_backend_phase3.rs` + `tests/format_policy.rs` (`just test-lean-phase3`). +Directory composition (rkyv catalog, no new C symbols). + +**Phase 4 allowlist:** + `tests/lean_backend_phase4.rs` + `tests/slh_outboard.rs` (`just test-lean-phase4`; features include `cli` for subprocess smoke). +SLH/CLI/OTS composition (no new C symbols). + +**Phase 5 freeze (G11) + R7 G8 full close:** `just test-lean-ci` — full dual suite under lean features (unfiltered `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"`). See [GAPS.md](./GAPS.md) R7 / [TEST_CONTRACT.md](./TEST_CONTRACT.md) Phase 5. Full-suite G8 **closed** at R7; post-G8 residuals are purity/feature-policy only. --- ## Linking ```text -# After: nix build .#libcarbonado -export CARBONADO_LEAN_LIB=$PWD/result/lib -export CARBONADO_LEAN_INCLUDE=$PWD/result/include -cargo test --no-default-features --features "backend-lean,pqc,ots" -# typical link line: -L $CARBONADO_LEAN_LIB -lcarbonado -lpthread -ldl -lm +# After: nix build .#libcarbonado -o result-libcarbonado +export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +# Phase 5 freeze (CI + humans): +just test-lean-ci +# Phase-scoped: +# just test-lean-phase4 +# link: -L $CARBONADO_LEAN_LIB -lcarbonado (+ rpath); NEEDED libleanshared from nix store ``` -Exact `cargo` `rustc-link-*` flags live in [`carbonado-sys/build.rs`](../carbonado-sys/build.rs). If `CARBONADO_LEAN_LIB` is unset, `carbonado-sys` warns and does not link — encode/decode cannot succeed. +Exact `cargo` `rustc-link-*` flags live in [`carbonado-sys/build.rs`](../carbonado-sys/build.rs). With feature `require-lib` (enabled by carbonado `backend-lean`), a missing `CARBONADO_LEAN_LIB` or missing library file is a **hard build error**. Without `require-lib`, unset env only warns (docs/check). CI / `just test-lean-ci` **fail-closed** if `libcarbonado.so` (or `.dylib`) is missing under `CARBONADO_LEAN_LIB`. + +**Packaging:** `nix build .#libcarbonado -o result-libcarbonado` produces `lib/libcarbonado.so` (leanc + whole-archive Lean AOT + zstd/ABI glue + NEEDED absolute nix-store `libleanshared`) and `lib/libcarbonado.a`. Prefer the shared object from `carbonado-sys` (rpath set from `CARBONADO_LEAN_LIB`). Redistribution is nix-store-coupled until a bundled runtime story lands. -**Known residual (not Phase 0):** flake/`libcarbonado` packaging and full Lean export linkage may still need Phase 1 work; Phase 0 does not require `nix build .#libcarbonado` green for doc closure. +**`EncodeInfo` on lean (R3):** full stage counters from Lean pack — `padding_len`, `chunk_len`, `bytes_ecc`, `verifiable_slice_count`, `bytes_compressed`, `bytes_encrypted`, body lengths. Skipped compress/encrypt stages report **0** (matches Rust stream path). `compression_factor` / `amplification_factor` computed in Rust from those fields. --- diff --git a/docs/GAPS.md b/docs/GAPS.md index 89ff047..4c08fc0 100644 --- a/docs/GAPS.md +++ b/docs/GAPS.md @@ -12,35 +12,237 @@ Living inventory. IDs are durable; close only when theorems and/or parity gates | Build / proofs | Nix flakes (`nix flake check`, `libcarbonado` package) | | Oracles | `ref/` pins + parity drivers | -**Parity bar (G8):** `cargo test` with `backend-rust` and with `backend-lean` (links Lean AOT C). Same tests; not separate Lean-only demos. +**Parity bar (G8):** same `tests/` on both engines (not Lean-only demos). Default features enable `backend-rust` only — do **not** pass `--features backend-lean` alone (both engines → `compile_error!`). + +```bash +cargo test # backend-rust (default) +just test-lean-ci # G8 freeze = full dual suite +# Equivalent unfiltered lean suite (includes bin_* via cli feature): +# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +``` See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), AGENTS.md dual-backend block. | ID | Gap | Status | |----|-----|--------| | G0 | Lean+Nix scaffold | **closed** (Program A) | -| G1 | `ref/` pins + dual-backend SSOT clarity | **partial** (pins present; dual-backend docs closed at P0; optional `ref/carbonado-rust` freeze still open) | +| G1 | `ref/` pins + dual-backend SSOT clarity | **closed** (2026-07 **W5a**: third-party `ref/` pins present; dual-backend SSOT docs closed at P0; **permanent policy — no** `ref/carbonado-rust` product pin — live `src/` + `tests/` are dual-suite SSOT; see [PARITY.md](./PARITY.md)) | | G2 | EtM Lean | **closed** (Program B) | | G3 | RS 4/8 Lean | **closed** (Program C) | | G4 | Keyed Bao Lean | **closed** (Program D) | | G5 | Pipeline / stream / scrub / shard | **closed** (Program E) | -| G6 | zstd link + SLH product | **partial** (zstd closed; SLH FFI open — see G10) | -| G7 | Adamantine + CLI | **partial** (Lean CFP2 path closed; **rkyv wire for dual-suite open** — Phase 3) | -| **G8** | **Dual-backend: C ABI + `cargo test --features backend-lean` full suite** | **open** (P0 inventory/docs **closed**; P1+ engineering open) | -| G9 | Cross-backend encode/decode matrix (Rust↔Lean) | **open** (depends on G8 Phase 2 body/headered stability) | -| G10 | libbitcoinpqc real SLH sign/verify in libcarbonado | **open** (wire+binding in Lean; FFI not linked) | -| G11 | Live CI matrix both backends | **open** (depends on G8 Phase 1+ allowlist → full; freeze at P5) | +| G6 | zstd link + SLH product | **closed** (zstd AOT closed; dual-suite SLH composition **P4**; pure Lean SLH FFI **R9/G10** — dual-suite may still use Rust `bitcoinpqc` composition by design) | +| G7 | Adamantine + CLI | **partial** (dual directory + stream E1 + **W1a/W1b** closed; **W3** pure Lean rkyv encode/CLI closed; dual-suite rkyv encode remains Rust composition SSOT; pure Lean chunked stream residual) | +| **G8** | **Dual-backend: C ABI + full `cargo test --no-default-features --features "backend-lean,pqc,ots[,cli]"` suite** | **closed** (2026-07 R7: full suite green under lean; freeze = full suite via `just test-lean-ci`; post-G8 purity/feature residuals below) | +| G9 | Cross-backend encode/decode matrix (Rust↔Lean) | **closed** (2026-07 R8: no-compress body/headered/outboard both directions + fixed-nonce encrypted; `tests/g9_cross_backend.rs` + `tests/fixtures/g9/`; **W2d** codecode/decodec shipped; **W2a/W2b** permanent cross-engine compress/dir encode residuals) | +| G10 | libbitcoinpqc real SLH sign/verify in libcarbonado | **closed** (R9: pin `b309f444…` into `libcarbonado_native.a`; `carbonado_slh_*` C ABI + Lean `@[extern]`; AOT `signRoot`/`verifyRoot` live; dual-suite may still use Rust composition) | +| G11 | Live CI matrix both backends | **closed** (2026-07 P5: Linux job `dual-backend-lean` runs `just test-lean-ci`; `desktop` keeps `backend-rust` full suite) | ## Dual-backend phases (G8 breakdown) | Phase | Work | Status | |-------|------|--------| | **P0** | Test contract inventory, ABI.md, GAPS/AGENTS dual-backend, cross-doc consistency | **closed** (2026-07; docs-only) | -| P1 | C ABI v0 live exports + libcarbonado link + `backend-lean` core allowlist green | **open** (stubs still `NOT_IMPLEMENTED`) | -| P2 | Format matrix + scrub/outboard/stream + cross-backend buffer (G9 start) | **open** | -| P3 | rkyv-compatible catalog + directory suite (G7 residual) | **open** | -| P4 | PQC (G10) + CLI dual path | **open** | -| P5 | CI freeze both backends; G8 + G11 closed | **open** | +| **P1** | C ABI v0 live exports + libcarbonado link + `backend-lean` core allowlist green | **closed** (2026-07; live `carbonado_*` via Lean AOT + `tests/lean_backend_smoke.rs`) | +| **P2** | Format matrix + scrub/outboard/stream + cross-backend buffer (G9 start) | **closed** (2026-07; additive outboard/scrub/slice C ABI; lean dispatch; `lean_backend_phase2.rs`; residual: full `format`/`codec`/`fec_scrub` suites, directory, seekable outboard slice, CI freeze → P3–P5) | +| **P3** | rkyv-compatible catalog + directory suite (G7 residual) | **closed** (2026-07; composition: Rust rkyv + Lean segment/catalog crypto; `lean_backend_phase3.rs`; G9 dir fixture) | +| **P4** | PQC (G10) + CLI dual path + directory OTS cases | **closed** (2026-07; G10 strategy A: Rust `bitcoinpqc` SLH + Lean container; CLI dual = directory library/subprocess dual-engine + lean-linked binary smoke; single-file stream E1 closed at **R5**; **W1a/W1b** dual honesty closed later; directory OTS CBOTS; `lean_backend_phase4` + `slh_outboard`; pure Lean SLH FFI residual **closed later at R9**) | +| **P5** | CI freeze both backends; G11 closed; G8 allowlist freeze | **closed** (2026-07; allowlist freeze; **full-suite G8 closed at R7**) | + +### P3 deliverables (evidence of close) + +| Deliverable | Location | +|-------------|----------| +| Dual-suite catalog wire = **rkyv** FilepackManifest v2 (not CFP2) | `src/filepack_manifest.rs` + `encode_directory` under `backend-lean` | +| Directory composition dispatch | segment `encode_outboard`/`decode_outboard` + catalog `file::encode`/`decode` → Lean C ABI; FS/rkyv/Adamantine framing stay Rust (`src/backend/mod.rs`, `src/file.rs`) | +| Allowlist | `tests/lean_backend_phase3.rs` (+ `format_policy`; smoke+phase2 still required) — `just test-lean-phase3` | +| G9 directory seed | `tests/fixtures/phase3_g9_directory/` rust-encoded → lean `decode_directory` | +| Docs | this file, [ABI.md](./ABI.md), [TEST_CONTRACT.md](./TEST_CONTRACT.md), [LIMITS.md](./LIMITS.md), AGENTS dual-backend | + +**P3 honest residuals (historical; P4 closed OTS/CLI dual-suite):** pure Lean CLI still CFP2 (no Lean rkyv codec); no new directory C ABI symbols (composition only); rust-root checksum goldens (`filepack_interop` golden) are `backend-rust`-only until G9 encode bit-match; full format/codec suite → G8/P5. + +### P4 deliverables (evidence of close) + +| Deliverable | Location | +|-------------|----------| +| G10 dual-suite SLH (strategy A composition) | Rust `crypto::slh_*` + `bitcoinpqc` under both backends; Lean `Carbonado/Slh.lean` wire+bind model; pure Lean `carbonado_slh_*` C ABI added later at **R9** (optional purity) | +| SLH allowlist | `tests/slh_outboard.rs` + `tests/lean_backend_phase4.rs` SLH cases (`just test-lean-phase4`) | +| CLI dual path **as of P4 close** | **Directory** encode/decode library + subprocess hit Lean composition; **buffer** single-file APIs (`file::encode` / `encode_outboard`) hit Lean. **Stream dual at R5 (after P4):** `stream_encode_*` / `stream_decode_*` spool→Lean E1 (O(logical); not E2). At P4 close **`file::decode_stream` remained pure-Rust** — dual later at **W1a** (see post-G8 residual table). Default binary remains rust-engine. | +| Directory OTS | Offline CBOTS stubs (`ots` feature) compose over Lean catalog/segment crypto; phase4 OTS roundtrip + fail-closed cases | +| Allowlist gate | `lean_backend_smoke` + `phase2` + `phase3` + `format_policy` + `slh_outboard` + `lean_backend_phase4` | +| Docs | this file, [ABI.md](./ABI.md), [TEST_CONTRACT.md](./TEST_CONTRACT.md), [LIMITS.md](./LIMITS.md), AGENTS dual-backend | + +**Live CLI dual (post-P4 supersession — not P4 evidence):** R5 E1 stream I/O + **W1a** `file::decode_stream` → Lean `decode_headered` + **W1b** public non-compress outboard S4 E2 composition. See residual table / LIMITS E1/E2 matrix. + +**P4 honest residuals (historical; superseded in part at R9/R10/W1a):** ~~pure Lean `signRoot` / libbitcoinpqc~~ **closed R9**; ~~seekable outboard slice C~~ **closed R9**; rkyv dual-decode **closed R9** (encode residual remains); stream dual E1 is **spool-to-buffer** (not true chunked stream — O(logical) RAM); ~~`file::decode_stream` pure-Rust~~ **W1a closed**; ~~residual full files `sharding` / `fec_chaos`~~ **green at R6**; ~~async dual~~ **closed R10**. + +### P5 deliverables (evidence of close) + +| Deliverable | Location | +|-------------|----------| +| Frozen dual suite command (CI + humans) | `just test-lean-ci` (P5 name was “allowlist”; **live since R7** = full unfiltered dual suite) | +| Linux CI job (G11) | `.github/workflows/rust.yaml` job **`dual-backend-lean`**: `cachix/install-nix-action` → `nix build .#libcarbonado -o result-libcarbonado` → fail-closed `.so` check → `just test-lean-ci` | +| `backend-rust` CI | existing **`desktop`** job: `cargo test` (+ serial FEC with `backend-rust,pqc,ots,cli` + optional `async,async-tokio,man-gen` + smoke + CLI) — never `--all-features` (dual-backend mutual exclusion) | +| Env contract | `CARBONADO_LEAN_LIB`, `CARBONADO_LEAN_INCLUDE`, `LD_LIBRARY_PATH` (see [ABI.md](./ABI.md) Linking + [TEST_CONTRACT.md](./TEST_CONTRACT.md) Phase 5) | +| Docs | this file, [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [LIMITS.md](./LIMITS.md), AGENTS dual-backend | + +**P5 freeze allowlist (historical; superseded at R7):** P1–P4 gates + measured-green full files (P5 + R1–R6 growth). Kept for archaeology; live freeze is the full suite. + +| Class | Tests (P5-era explicit `--test` list) | +|-------|--------| +| Phase gates | `lean_backend_smoke`, `lean_backend_phase2`, `lean_backend_phase3`, `lean_backend_phase4` | +| P3–P4 companions | `format_policy`, `slh_outboard` | +| Measured-green full files (P5 + R1–R6) | `bao_keyed_contract`, `directory_archive`, `filepack_interop`, `deprecation_aliases`, `fec_scrub_matrix`, `shard_fec_scrub`, `serial_fec_path`, `adversarial_proptest`, `udp_fec_sim`, `apocalypse`, **`format`**, **`header_tamper`**, **`format_amplification`**, **`codec`**, **`seekable_slices`**, **`streaming`**, **`streaming_limits`**, **`sharding`**, **`fec_chaos`** | + +### R7 — Full G8 close + freeze expansion (2026-07) + +**G8 full suite closed** under lean features with live `libcarbonado`. Freeze equals full dual suite: + +```bash +# R7 freeze = full dual suite (lib + integration, including bin_*). +just test-lean-ci +# Equivalent: +# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +``` + +| Former residual (now freeze-green) | Closed at | +|------------------------------------|-----------| +| `tests/format.rs` | R1 | +| `tests/codec.rs` | R4 | +| `tests/header_tamper.rs` | R2 | +| `tests/format_amplification.rs` | R3 | +| `tests/streaming.rs` / `streaming_limits.rs` | R5 E1 | +| `tests/seekable_slices.rs` | R4 | +| `tests/sharding.rs` | R6 | +| `tests/fec_chaos.rs` | R6 | +| lib unit missing verification outboard | R1 | +| Full `bin_*` CLI matrix under lean-linked binary | R7 (in unfiltered freeze) | + +**Post-G8 residuals** (not dual-suite red; purity / feature-policy / composition honesty). +**Post-R10 work program (waves):** ~~W0 hygiene~~ **closed** · ~~W1 dual honesty~~ **closed** (~~W1a~~ `decode_stream` dual; ~~W1b~~ public outboard stream E2 MVP) · ~~W2 G9 bit-match + **codecode/decodec** determinism~~ **closed** (W2d shipped; W2a/W2b permanent cross-engine residuals) · ~~W3 pure Lean rkyv encode~~ **closed** · ~~W4 memory/streaming quality~~ **closed** (W4a closed; W4b–d permanent residuals) · ~~W5a G1 pin~~ **closed** (permanent no product pin) · W5b CHIPs (external; out of product-tree scope). + +| Residual | Notes | Wave | +|----------|--------|------| +| Pure Lean SLH FFI | **closed R9** (G10) — dual suite may still use Rust `bitcoinpqc` composition (not required to switch) | — | +| Pure Lean rkyv full encode + directory CLI | **W3 closed** — `encodeRkyvManifest` + Directory/CLI emit rkyv; goldens empty/single/multi+OTS; dual-suite encode remains Rust rkyv composition SSOT | **closed** | +| Seekable outboard slice C ABI | **R9 closed** for O(slice+height) hash; **W4b permanent:** full main+outboard buffers at C (no ReadAt callback ABI) | **W4b permanent** | +| Inboard `verify_slice` O(body) retain | **W4a closed** — O(slice) retained output + O(N) full-response walk; C still takes full body input buffer | **W4a closed** | +| Streaming zstd under lean | **W4c permanent** — buffer-only bulk zstd for Lean frame parity (W2a); no dual-safe streaming frames | **W4c permanent** | +| FEC O(body) / async encoded spool | **W4d permanent** — segment-wide RS O(FEC body); async always disk-stages O(encoded); no full async FSM this wave | **W4d permanent** | +| Async dual policy | **closed R10** — freeze never requires `async` | — | +| `streaming_async` / `parallel_determinism` | **permanent feature-gate** off freeze | — | +| ~~Stream E2 / dual honesty~~ | **W1a+W1b closed** — see below | **closed** | +| Pure Lean chunked stream C ABI | No streaming C symbols; inboard/encrypted stream remain E1; public outboard E2 is **composition** | residual after W1b | +| ~~codecode / decodec determinism suite~~ | **W2d closed** — `tests/determinism_roundtrip.rs` (no-compress body/headered/outboard; same-engine compress + directory) | **closed** | +| Compression encode bit-match (cross-engine) | **permanent residual (W2a)** — Lean AOT zstd frames ≠ Rust `zstd`; decode interop only; same-engine codecode green | permanent | +| Directory encode bit-match (cross-engine) | **permanent residual (W2b)** — live rust `0b119f12…` ≠ live lean `f67b6f49…` (pinned); `phase3_g9_directory` decode-only SSOT (not re-encode golden); same-engine codecode green | permanent | + +### W1 — Dual-suite honesty (closed 2026-07) + +| Item | Status | Detail | +|------|--------|--------| +| **W1a** `file::decode_stream` | **closed** | Under lean: header+`encoded_len` body → Lean `decode_headered` (E1 RAM). Smoke: `decode_stream_codecode_decodec_public_c14`. | +| **W1b** stream E2 MVP | **closed** | **Public non-Compression** `stream_*_outboard` under lean use rust **S4 O(chunk/stripe)** geometric composition (G9 **no-compress** wire bit-match; c4/c12 evidenced; **not** pure-Lean stream). Public **+ Compression** under lean is **O(logical)** bulk zstd (not E2). **Encrypted** outboard + all inboard stream + buffer APIs remain Lean **E1**. Smoke: `stream_outboard_public_e2_codecode_decodec_c4_c12` (2 MiB). Matrix: [LIMITS.md](./LIMITS.md) Stream E1/E2. | +| Residual | pure Lean chunked C ABI | Future — no false “true stream” claims for E1 paths | + +### R10 — Async dual policy (closed 2026-07) + +**DoD:** Either permanent freeze exclusion of `async`, **or** dual-aware lean+async after R5 E1 stable. **Both:** freeze excludes async permanently; product path dual-aware when both features are on. + +| Policy | Detail | +|--------|--------| +| Dual freeze | `just test-lean-ci` / `"backend-lean,pqc,ots,cli"` — **never** includes `async` / `async-tokio` (permanent) | +| Contract tests | `tests/streaming_async.rs` is `#![cfg(feature = "async")]` → **0 tests** under freeze (not dual-suite red) | +| Sync stream dual | R5 E1 + **W1b** public outboard S4 composition (see W1 table above) | +| `stream_decode_async` honesty | After O(encoded) disk staging, calls dual-aware `stream_decode` — `backend-lean` → Lean E1 buffer decode; `backend-rust` → S4 pipeline. No silent pure-Rust when lean+async. | +| Costs (honest) | Disk O(encoded) staging always; under lean peak RAM **O(encoded + logical)** (E1 body `Vec` + plaintext); rust peak spool/chunk (FEC residual O(FEC body)); **not** pure Lean stream E2 | +| WASM | `NotImplemented` (host temp spool) | +| Optional smoke | `cargo test --no-default-features --features "backend-lean,pqc,ots,async,async-tokio" --test streaming_async` (+ `CARBONADO_LEAN_LIB`) — **not** freeze | + +**Out of R10 scope (post-R10; ~~W1a+W1b~~ **closed**; ~~W2~~ **closed**; ~~W3~~ **closed**; ~~W4~~ **closed** with permanent residuals W4b–d; ~~W5a G1~~ **closed** permanent no product pin):** pure Lean chunked stream C residual; W5b CHIPs (external). + +### W5a — G1 `ref/carbonado-rust` pin bookkeeping (closed 2026-07) + +| Item | Status | Detail | +|------|--------|--------| +| **W5a** G1 product pin | **closed** (permanent policy) | **No** `ref/carbonado-rust` submodule or freeze SHA. Dual-suite SSOT = live `src/` + `tests/` (G8). `ref/` pins third-party oracles only. Rationale + procedure: [PARITY.md](./PARITY.md) freeze strategy. Docs: GAPS G1, LIMITS, SPEC-MATRIX, `ref/README.md` intro+row, AGENTS status+gates, `nix/tooling-purity.nix` comments. | +| **W5b** CHIPs | out of product-tree scope | External CHIPs normative drafting — not closed by W5a | + +### W4 — Memory / streaming quality (closed 2026-07) + +| Item | Status | Detail | +|------|--------|--------| +| **W4a** Inboard seekable slice | **closed** | Lean `decodeRecRetainRange` / `verifySliceInboard`: walk from offset 8 (no second response copy), **O(slice) retained** output (Rust `SliceRegionWriter` class); O(leaf) temps. Time O(N). C still takes full inboard body **input**. `count==0`: Lean C auth-first; dual Rust short-circuit. Tests: `seekable_slices`, `CarbonadoTest/Bao` multi-leaf. | +| **W4b** Outboard streaming C | **permanent residual** | Lean offset walk (`verifyOutboardSliceRecAt`) avoids recursive full half-extracts; hash O(slice+height). **No** additive `carbonado_verify_slice_outboard_at` / ReadAt callback — full main+outboard buffers at C remain permanent. Old full-buffer symbol kept. | +| **W4c** Streaming zstd | **permanent residual** | Prefer buffer-only under lean for Lean AOT frame parity (W2a cross-engine compress residual). Do not claim dual-safe streaming frames or E2 for Compression formats under lean. See LIMITS Stream E1/E2 matrix. | +| **W4d** FEC / async spool | **permanent residual** | FEC verification retains O(FEC body) shard buffers (segment-wide RS). `stream_decode_async` always disk-stages O(encoded); lean+async peak RAM O(encoded+logical). Full async FSM without encoded spool out of scope. Metrics in [STREAMING_PARALLELISM.md](../doc/STREAMING_PARALLELISM.md). | + +### W2 — G9 bit-match + determinism (closed 2026-07) + +| Item | Status | Detail | +|------|--------|--------| +| **W2d** codecode / decodec | **closed** | `tests/determinism_roundtrip.rs` — EDE + DED under G9 MASTER/NONCE/`g9_matrix_v1`. Matrix: body c0/c1/c4/c5/c8/c9/c12/c13; headered c4/c5/c12/c13; outboard c4/c5/c12/c13. Both engines (default rust + lean freeze). Asserts `pt' == pt` and `A' == A` / `B == A`. | +| **W2a** Compression cross-engine | **permanent residual** | Same-engine body/headered/outboard compress codecode/decodec green. Cross-engine: G9 `outboard_c14` mains differ (frame descriptor `00` vs `20`; roots `0abe5781…` vs `129b4518…`); hard-asserted fail-closed. Decode interop only. See [LIMITS.md](./LIMITS.md). | +| **W2b** Directory cross-engine | **permanent residual** | Same-engine public directory codecode/decodec green. Cross-engine residual is **live rust `0b119f12…` ≠ live lean `f67b6f49…`** (pinned hard asserts). `phase3_g9_directory` (`16e2369f…`) is **decode-only SSOT**, not a live re-encode golden. | +| **W2c** Full c0–c15 G9 matrix | **skipped** (optional) | Not required after W2a permanent residual. | + +### W3 — Pure Lean product wire (closed 2026-07) + +| Item | Status | Detail | +|------|--------|--------| +| **W3a** rkyv encode | **closed** | Pure Lean `encodeRkyvManifest` bit-matches Rust goldens empty/single/multi+OTS (`tests/fixtures/rkyv/`); encode→decode roundtrip; encode twice → same bytes (codecode). AOT: `rkyv FilepackManifestWire encode/decode goldens ok`. | +| **W3b** Directory / CLI | **closed** | `Directory.encodeDirectory` + CLI emit rkyv catalog body; decode via `decodeCatalogBody` (rkyv **or** CFP2 fallback). Pure Lean directory roundtrip AOT green. Dual-suite encode remains Rust rkyv composition SSOT. | +| **W3c** dual SLH via C | **wontfix** (docs honesty) | Dual-suite keeps Rust `bitcoinpqc` composition; pure Lean `carbonado_slh_*` for AOT/CLI only. Never claim dual-suite requires pure Lean SLH. | + +### R9 — Pure Lean depth (optional post-G8; 2026-07) + +| Track | Status | Evidence | +|-------|--------|----------| +| **G10 full** SLH FFI | **closed** | `nix/native/carbonado_slh.c` + libbitcoinpqc pin; `carbonado_slh_*`; Lean `signRoot`/`verifyRoot`; demo greps `SLH live sign/verify ok` | +| Seekable outboard slice C | **closed** | `carbonado_verify_slice_outboard` + Lean `verifySliceOutboard`; `backend-lean` dispatch; demo `outboard slice verify ok`; single-leaf short-main regression in `seekable_slices` | +| Pure Lean rkyv | **encode+decode closed (W3)** | `Carbonado/RkyvFilepack.lean` encode/decode goldens empty + single + multi/OTS (`tests/fixtures/rkyv/`); Directory/CLI emit rkyv; dual-suite composition still Rust rkyv SSOT for product encode | + +**R9 freeze evidence:** rebuild `nix build .#libcarbonado` + `just test-lean-ci` after R9 review fixes (docs + SLH status taxonomy + count=0 docs + single-leaf test). Dual-suite composition SSOT unchanged. + +**Never claim dual-suite requires pure Lean** when composition remains SSOT for product wire. + +**P5 bar (historical):** G11 closed + dual allowlist frozen in CI. **R7 bar:** full G8 suite closed; `just test-lean-ci` runs unfiltered full dual suite. + +### R8 — G9 full cross-backend matrix (2026-07) + +**G9 closed** for the no-compress body/headered/outboard matrix (public + encrypted fixed-nonce), **both directions**: + +| Deliverable | Location | +|-------------|----------| +| Fixtures | `tests/fixtures/g9/{rust,lean}/` — body c0/c1/c4/c5/c8/c9/c12/c13; headered c4/c5/c12/c13; outboard c4/c5/c12/c13/c14 | +| Contract tests | `tests/g9_cross_backend.rs` — lean→rust (default features), rust→lean (lean features + lib), re-encode bit-match | +| Pins | MASTER/NONCE from Phase 2; plaintext `g9_matrix_v1`; regen via `just g9-gen-fixtures` / `G9_WRITE_FIXTURES=1` | +| Human gate | `just test-g9` (both directions); auto-included in `just test-lean-ci` (R7 freeze) | +| Fixed-nonce APIs | `encode_with_nonce`, `file::encode_with_nonce`, `stream_encode_buffer_with_nonce` (production defaults still CSPRNG) | + +**G9 residuals (honest; settled at W2):** + +| Residual | Notes | +|----------|--------| +| Compression encode bit-match | **permanent (W2a)** — Lean AOT zstd ≠ Rust `zstd`; c14 outboard **decode** interop only; same-engine codecode green in `determinism_roundtrip` | +| Directory encode bit-match | **permanent (W2b)** — live rust≠lean catalog roots (pinned); `phase3_g9_directory` is **decode-only SSOT** (not live re-encode golden); same-engine directory codecode green | +| Full c0–c15 with Compression | deferred / not required (W2c optional skipped) | +| ~~codecode/decodec suite~~ | **W2d closed** — `tests/determinism_roundtrip.rs` | + +**P5 bar (historical):** G11 closed + dual allowlist frozen in CI. **R7 bar:** full G8 suite closed; `just test-lean-ci` runs unfiltered full dual suite. + +### P2 deliverables (evidence of close) + +| Deliverable | Location | +|-------------|----------| +| C ABI outboard / scrub / verify_slice (+ encode meta fields) | `include/carbonado.h`, `nix/native/carbonado_abi.c`, `Carbonado/Ffi.lean` | +| Lean scrub geometry peel + outboard scrub | `Carbonado/Scrub.lean` | +| Rust `backend-lean` dispatch | `src/backend/mod.rs`, `encoding`/`decoding`/`stream::{encode,decode}` | +| Allowlist smoke | `tests/lean_backend_smoke.rs` + `tests/lean_backend_phase2.rs` (`just test-lean-phase2`) | +| G9 start | rust golden body/headered → lean decode; lean re-encode bit-match | +| Docs | this file, [ABI.md](./ABI.md), [TEST_CONTRACT.md](./TEST_CONTRACT.md), [LIMITS.md](./LIMITS.md), AGENTS dual-backend | + +**P2 honest residuals (historical; P3–P5 closed directory/OTS/CLI dual + CI freeze):** full `tests/format.rs` / `codec.rs` later green at R7; `fec_scrub_matrix` measured green and included in P5 freeze; seekable outboard slice API **closed R9**; ~~async dual~~ **closed R10**; CI freeze closed at P5. ### P0 deliverables (evidence of close) @@ -49,6 +251,6 @@ See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [PARITY.md](./PA | Full `tests/*.rs` classification + API map + Phase 1 allowlist | [TEST_CONTRACT.md](./TEST_CONTRACT.md) | | C ABI ownership, error codes, v0 symbols, stub honesty, link notes | [ABI.md](./ABI.md) + `include/carbonado.h` | | Dual-backend SSOT rules | AGENTS.md (top block); this file | -| Cross-doc model (no “Lean replaces Rust” product rule) | [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), [PROOFS.md](./PROOFS.md) | +| Cross-doc model (no “Lean replaces Rust” product rule) | [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), [PROOFS.md](./PROOFS.md), [VISION.md](./VISION.md) | P0 does **not** require live encode/decode through `backend-lean` or full `nix build .#libcarbonado` product export maturity. diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 27cddb6..2f46a24 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -7,11 +7,11 @@ | Backend | Status | |---------|--------| | **Rust** (`src/`, default `backend-rust`) | First-class production library + CLI; full `cargo test` | -| **Lean AOT** (`Carbonado/`, `libcarbonado`, optional `backend-lean`) | Second engine: proofs + wire/C ABI; dual-suite phased (G8) | +| **Lean AOT** (`Carbonado/`, `libcarbonado`, optional `backend-lean`) | Second engine: proofs + wire/C ABI; dual-suite G8 closed at R7 | | **Rust `tests/`** | Normative behavioral contract for **both** engines | - AOT CLI (`packages.carbonado` / `nix run`) runs **Programs A–G**: constants, EtM, FEC, keyed Bao, full pipeline (c0–c15), Header wire, scrub, stream bounds, multi-segment shards, **zstd-20 compression (linked)**, **SLH1 sidecar wire + bind-to-root model**, **Adamantine 1.0 directories**, **encode/decode/slh CLI**. -- Rust tree (`src/`, `tests/`, …) **stays** first-class (AGENTS dual-backend). Optional historical pin under `ref/carbonado-rust` is G1 residual — not a license to delete `src/` or `tests/`. +- Rust tree (`src/`, `tests/`, …) **stays** first-class (AGENTS dual-backend). **G1/W5a closed:** permanent policy — **no** `ref/carbonado-rust` product pin; live tree is dual-suite SSOT. Not a license to delete `src/` or `tests/`. - Lean theorem/test tree is **`CarbonadoTest/`** (not `Tests/`) so it does not collide with Rust `tests/` on case-insensitive filesystems (Darwin APFS). - Dependency direction is **CarbonadoTest → Carbonado** only. @@ -35,10 +35,12 @@ - Full Lean BLAKE3 reference (hash / keyed_hash / derive_key + hazmat subtree/parent CVs) ported from the BLAKE3 reference algorithm; parity vs `ref/blake3` 1.8.5 portable semantics and bao-vectors. - Keyed Bao product paths: format verification key, root, inboard `[u64le|response]`, post-order outboard, **stream slice decode** (`decodeSliceResponse` / `decodeSliceForFormat`) against `(key, root, contentLen)` via `decodeRec` — returns authenticated bytes from the response, **not** a re-encode oracle over trusted plaintext. -- Inboard slice extract (`verifySliceInboard*`) always runs full `decodeInboard` **before** any `count = 0` empty return (auth-first). Stream decode rejects `count = 0` with `invalidSliceCount`. +- **W4a inboard slice** (`verifySliceInboard*`): full-response walk over the inboard artifact from offset 8 (`decodeRecRetainRange` — no second full-response copy); **O(slice) retained output**; O(leaf) temporary leaf extracts for hashing; O(N) time (same class as Rust `verify_slice_inboard_seekable`). +- **`count = 0` split (honest):** pure Lean / C `carbonado_verify_slice` is **auth-first** (corrupt fails, then empty). Dual product API under `backend-lean` (`lean::verify_slice` / `carbonado::verify_slice`) **short-circuits** `Ok([])` without auth — parity with pure-Rust `verify_slice_inboard_seekable`. Stream decode rejects `count = 0` with `invalidSliceCount`. +- **W4b outboard slice:** O(slice + height) hash via offset walk; full main+outboard **input** buffers at C ABI remain **permanent** (no ReadAt callback symbol). - Error taxonomy: short stream → `truncatedResponse`; overlong stream → `trailingData` (distinct). - Tree model uses **leaf-group** recursion (4096 B) matching bao-tree `BlockSize::from_chunk_log(2)` IO. -- **Not claimed:** SIMD BLAKE3 throughput; O(slice) memory for **inboard** slice extract (full inboard materialize then extract — same class as some Rust inboard paths); standalone slice responses are O(response) for stream decode; async/tokio bao-tree APIs; pre-order outboard layout. +- **Not claimed:** SIMD BLAKE3 throughput; O(slice) **peak RSS** when C/caller already holds full body/main buffers (caller body is still O(body)); standalone slice responses are O(response) for stream decode; async/tokio bao-tree APIs; pre-order outboard layout; streaming ReadAt C ABI for outboard slice. - **Constant-time:** logical `ctEq` only on hash compares; not a CT proof. - Parity is bit-match goldens vs `ref/bao-tree` @ lock + `bao-vectors` driver, not a live Nix `diff` harness yet. @@ -48,11 +50,11 @@ - **Header:** 177 B wire codec; `header_mac` verified before body (`decodeHeadered`). Authenticated `encoded_len` bounds the body (`truncatedBody` if short; trailers after `encoded_len` ignored). Public metadata only. - **Nonce layouts:** header-path `[tag|ct]` vs low-level `[nonce|tag|ct]` as in EtM; pure model takes caller-supplied nonce (no CSPRNG). - **MAC-before-decrypt:** EtM still refuses keystream until MAC ok; pipeline only decrypts after Bao/FEC reverse. -- **Scrub:** pure RS combinatorial search on FEC body + re-encode + Bao root compare. Does **not** implement Rust seekable slice extract entry; tests use `scrubWithMissing` / `scrubFecThenBao` after FEC body is known. Opaque Bao-only damage without FEC extract → `invalidScrubbedHash`. +- **Scrub:** product `scrubInboard` peels Bao via geometry-only leaf walk then RS combinatorial search + re-encode Bao root oracle; `scrubOutboard` recovers bare main from main+parity. Pristine → `unnecessaryScrub`; no Verification → `scrubRequiresVerification`. Opaque Bao-only (no FEC) damage → `invalidScrubbedHash`. Memory residual: full peel materializes logical FEC body (not O(slice) scrub entry like Rust S5 seekable extract). - **Stream model:** pure stripe transducer + proved O(stripe) retain bounds; product `encodeBody` still uses segment-wide RS geometry (same as Rust residual). Multi-stripe encode model is documented alternative, not default parity path. - **Sharding:** pure multi-segment headered encode/decode with contiguous `chunk_index` validation. -- **Outboard product pipeline** (`.out`/`.par` high-level file APIs) not fully composed as a separate encode/decode surface in Lean yet (Bao outboard primitives exist from D). -- Parity: format-matrix roundtrips in Lean AOT; live Nix vs `ref/carbonado-rust` product-matrix still open (G8). +- **Outboard product pipeline:** Lean `Outboard.encodeOutboardBody` / `decodeOutboardBody` live via C ABI (Phase 2). Directory high-level dual-suite APIs closed at P3 via composition (Rust rkyv + Lean outboard/headered); pure Lean CLI/directory emit **rkyv** catalog bodies (**W3**). +- Parity: format-matrix roundtrips in Lean AOT; dual-suite product-matrix is live `src/` + `tests/` (G8) — **no** frozen `ref/carbonado-rust` pin (**G1/W5a** permanent policy); G9 closed at R8 for no-compress body/headered/outboard both directions (`tests/fixtures/g9/`). ## Program F zstd + SLH (shipped with declared residuals) @@ -62,22 +64,23 @@ - **Pipeline:** Compression bit → `compressLevel20` / `decompress` in `compressStep` / `decompressStep`; errors map 1:1 via `ofZstdError` → `compressionFailed` | `decompressionFailed` | `decompressOutputTooLarge` | `zstdInvalidInput` (no lumped catch-all). - **DoS cap:** decompressed output ≤ 256 MiB (`maxDecompressedLen`, matches Rust `MAX_SEGMENT_MAIN_LEN`). - **Interpreter / `native_decide`:** `@[extern]` bodies are identity fallbacks; **do not** `native_decide` compression formats (extern needs native symbols). Pure tests: status decode + bit-clear paths + non-compression format matrix. **Real** zstd + c2/c6/c14/c15 gated by AOT `demo` (`ZSTD_compress` API goldens empty/hello). -- **Not claimed:** streaming zstd (buffer API only); multi-threaded zstd; dictionary compression. +- **W4c permanent residual — streaming zstd:** product AOT and lean dual path use **buffer** zstd only (`ZSTD_compress` / `zstd::bulk` under lean; Rust streaming `copy_encode` under `backend-rust` only). Streaming frames are **not** dual-safe / bit-match Lean (W2a). Do **not** claim E2 for public Compression outboard under lean. Multi-threaded zstd and dictionary compression not claimed. -### SLH-DSA sidecars (wire + binding; no real PQC FFI yet) +### SLH-DSA sidecars (wire + binding + live FFI at R9 / G10) - **Wire:** `Carbonado.Slh` — `SLH1` + 7856 B sig = 7860 B; parse/build fail-closed (`invalidSidecarLength` vs `badSlhMagic` vs `invalidSignatureLength` distinct). - **Binding:** `verifyBound` / `verifyBoundToExpected` — signature is over the 32-byte Bao root; wrong root → `verificationFailed`; pk size / root size / sig size have distinct errors. -- **Sign:** `signRoot` returns `signatureUnavailable` (fail-closed) until libbitcoinpqc is linked. -- **Why no real SLH yet:** `ref/bitcoinpqc` pin is present but nested `libbitcoinpqc` submodule is empty; full cmake+secp+SLH link is deferred. Product integrates **wire + theorems + Header.slh_public_key slot**; real sign/verify oracle is the next deepen step. -- **Mock oracles** in tests only; never used as production crypto. +- **Sign/verify (R9 / G10 closed):** live SLH-DSA-SHA2-128s via `@[extern]` into libbitcoinpqc objects linked in `libcarbonado_native.a` (flake pin `b309f444…`; `nix/native/carbonado_slh.c`). Product APIs: `signRoot` / `keygen` / `verifyRoot` / `liveVerifyOracle`; C ABI `carbonado_slh_{keygen,sign,verify}`. +- **Elaborator residual:** `@[extern]` bodies are fail-closed (status fail / verify reject) for `native_decide`; real crypto only in AOT / linked `libcarbonado`. Do **not** `native_decide` live sign/verify. +- **Dual-suite:** product SLH under `backend-lean` may still use Rust `bitcoinpqc` composition (G10 strategy A); pure Lean is optional purity for `libcarbonado` — dual-suite does **not** require pure Lean SLH. +- **Mock oracles** in pure theorems only; never used as production crypto. ## External C (declared) | Component | Status | |-----------|--------| | zstd | **Linked** static via `nix/native` + flake `zstdPinned` (commit `f8745da6…` / same as `ref/zstd` v1.5.7); no shared libzstd | -| SLH-DSA-SHA2-128s | Wire + binding in Lean; **FFI not linked** (libbitcoinpqc residual) | +| SLH-DSA-SHA2-128s | **Linked R9** — SLH-only objects from flake pin `b309f444…` + `carbonado_slh_*` C ABI; dual-suite may keep Rust composition | ## Program G Adamantine + CLI (shipped with declared residuals) @@ -87,16 +90,22 @@ - Payload framing matches Rust: `[u32 LE man_len][man][u32 LE bun_len][bun]`. - Dev `ADAMANTINE1\n` / `ADAMANTINE2\n` rejected with `unsupportedVersion`. -### Filepack manifest — **CFP2 Lean-native, not rkyv** +### Filepack manifest — dual path -- Logical fields match FilepackManifest v2 (version, format_level, entries, SegmentRef, content_blake3, optional OTS). -- Wire body magic `CFP2` + deterministic LE layout (`Carbonado.Filepack`). -- **Not** byte-identical to Rust rkyv `FilepackManifestWire`. Adamantine *envelope* framing is shared; manifest body interop with Rust-produced catalogs needs a converter (future). Product CLI is CFP2 end-to-end. +| Path | Manifest body | Status | +|------|---------------|--------| +| **Dual-suite** (`backend-lean` via Rust API / `tests/`) | **rkyv** `FilepackManifestWire` v2 (normative Adamantine 1.0) | **P3 closed** — composition: Rust rkyv + Lean segment/catalog crypto (SSOT for dual encode) | +| **Pure Lean CLI / Program G** | **rkyv** via `Carbonado.RkyvFilepack.encodeCatalogBody` (**W3**) | Wire-compatible with dual-suite decode; goldens in `tests/fixtures/rkyv/` | + +- Logical fields match (version, format_level, entries, SegmentRef, content_blake3, optional OTS). +- Adamantine *envelope* framing is shared (`ADAMANTINE10\n`, payload `[man_len][man][bun_len][bun]`). +- Pure Lean **rkyv encode+decode closed at W3** (`encodeRkyvManifest` / `decodeRkyvManifest` + goldens empty/single/multi+OTS). Dual-suite catalog **encode** remains Rust rkyv composition SSOT (do **not** claim dual-suite requires pure Lean). CFP2 remains dual-decode fallback only (`decodeCatalogBody`). ### Directory model - Catalog: inboard headered `{root}.adam.c14`/`.adam.c15`; segments: bare mains `{root}.c12`–`.c15`; centralized Bao+FEC bundle in Adamantine payload. -- Path rules fail-closed: empty, `..`, absolute `/`, `\`, empty components, NUL, length cap. +- Path rules fail-closed: empty, `..`, absolute `/`, `\`, empty components, NUL, length cap (**UTF-8 bytes**, matching Rust `MAX_REL_PATH_LEN`). +- **Pure Lean stricter than Rust path validate:** Lean also rejects empty components (`a//b`) and embedded NUL. Rust `validate_rel_path` does not; handcrafted rkyv with `//` can pass dual-suite Rust validate and fail Lean `decodeRkyvManifest`→`validate`. Normal FS `encode_directory` walks do not emit those paths. - Content BLAKE3 checked after segment recovery. - Segment policy Auto/ForceRaw/ForceCompressed/ForceC12–C15; legacy c4–c7 rejected. - OTS: `REQUIRE_OTS` flag → fail-closed `otsFeatureRequired` (no OTS stamps in Lean path). @@ -106,22 +115,31 @@ - `demo` / no-args: full A–G self-test (flake `checks.demo`). - `encode`/`decode` single-file (headered) and directory; single-file default name `{bao_root_hex}.c{fmt:02x}` (AGENTS hex). -- `slh parse`: wire only (exit 0 on valid frame). `slh verify`: **exit 1** until real SLH-DSA FFI (never soft-success). +- `slh parse`: wire only (exit 0 on valid frame). `slh verify`: live SLH-DSA via `liveVerifyOracle` (exit 0 on accept; exit 1 on reject / bad wire — never soft-success). - Nonces: `/dev/urandom` for encrypted encode. - Directory default outdir `{input}-archive/`. - Encode rejects `requireOts` (`otsFeatureRequired`); does not mint undecodeable archives. - CLI encode rejects symlink source entries (`symlinkNotAllowed`); decode refuses write-through symlinks when detectible. -## Dual-backend (G8 — P0 closed, engineering open) +## Dual-backend (G8 full closed at R7; P0–P5 closed for allowlist + CI freeze) | Backend | Status | |---------|--------| -| `backend-rust` (default) | Full Rust engine; full `cargo test` (must never regress) | -| `backend-lean` | **Scaffolding** — header, `carbonado-sys`, feature flags, weak C stubs (`NOT_IMPLEMENTED`); pure Lean FFI helpers exist; **not** full suite | -| Cross encode/decode Rust↔Lean | Not yet (G9; after Phase 2) | +| `backend-rust` (default) | Full Rust engine; full `cargo test` (must never regress); CI job **`desktop`** | +| `backend-lean` | **G8 full closed (R7)** — body/headered/outboard/scrub/verify_slice C ABI + directory composition + SLH/OTS composition; CLI dual-engine for **directory** (+ buffer APIs) + **single-file stream E1** (inboard/encrypted spool→Lean; O(logical) RAM) + **W1b public non-compress outboard S4 composition E2** (c0/c4/c8/c12; Compression under lean O(logical) bulk zstd; not pure Lean stream); freeze = **full dual suite** via `just test-lean-ci` / CI job **`dual-backend-lean`** (G11 **closed**) | +| Cross encode/decode Rust↔Lean | **G9 closed (R8)** — no-compress body/headered/outboard both directions (`tests/g9_cross_backend.rs` + `tests/fixtures/g9/`); **W2d** codecode/decodec shipped (`tests/determinism_roundtrip.rs`); **W2a/W2b permanent:** cross-engine Compression / directory encode not bit-identical (decode interop + same-engine re-encode only); directory decode seed remains `phase3_g9_directory` | | Docs / inventory (Phase 0) | **Closed** — [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md) | -Rust-only for now: `async`, `async-tokio`, and (by default) multi-thread `parallel` paths. Lean uses serial RS. +**R7 freeze command (measured green):** `just test-lean-ci` runs unfiltered `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"` (lib units + all integration tests including `bin_*`). Never `--features backend-lean` alone — defaults already enable `backend-rust`. + +**Permanent feature-gated exclusions from dual freeze** (lean features stay `"backend-lean,pqc,ots,cli"` — **never** add `async` / `async-tokio` / `parallel` to freeze): + +| Item | Why | +|------|-----| +| `tests/streaming_async.rs` | `#![cfg(feature = "async")]` — dual freeze does not enable `async` → **0 tests** under freeze (**R10 permanent policy**). Optional product path: under `backend-lean`+`async`, `stream_decode_async` is dual-engine via R5 E1 `stream_decode` (disk O(encoded) staging; lean peak RAM **O(encoded + logical)** body+plaintext; not E2). Default desktop CI often enables `async` on **rust** only. | +| `tests/parallel_determinism.rs` | `#![cfg(feature = "parallel")]` — dual freeze does not enable `parallel`; Lean RS path is serial | + +**R10 closed:** freeze never requires async; optional lean+async dual-aware. Multi-thread `parallel` product paths remain rust-engine / feature-gated; Lean uses serial RS. ## Not claimed yet @@ -129,9 +147,40 @@ Rust-only for now: `async`, `async-tokio`, and (by default) multi-thread `parall - Secret zeroization proofs / automatic zeroize of master keys - WASM product target - Throughput parity with AES-NI / SIMD RS / SIMD BLAKE3 Rust paths (optimize after correctness) -- Real SLH-DSA sign/verify via libbitcoinpqc (G10) -- Byte-identical rkyv FilepackManifest interop with Rust directories (required for directory dual-suite — Phase 3 / G7) -- Live C ABI encode/decode (stubs return `NOT_IMPLEMENTED` until Phase 1) -- Full `cargo test --features backend-lean` (G8 open; Phase 1 allowlist first) -- Live CI matrix both backends (G11) -- Live Nix product-matrix vs optional frozen `ref/carbonado-rust` +- ~~Pure Lean SLH-DSA sign/verify via libbitcoinpqc~~ **closed R9 / G10** (`carbonado_slh_*` + Lean `@[extern]`; dual-suite may still use Rust `bitcoinpqc` composition) +- ~~Pure Lean rkyv encode + directory CLI CFP2~~ **W3 closed** — pure Lean rkyv encode/decode + directory/CLI emit rkyv; dual-suite catalog encode remains Rust rkyv composition SSOT (not required pure Lean) +- **W3c SLH:** dual-suite keeps Rust `bitcoinpqc` composition for SLH; pure Lean `carbonado_slh_*` available for AOT/CLI only (honest composition SSOT) +- ~~Seekable outboard slice C~~ **closed R9** (`carbonado_verify_slice_outboard`); **W4b permanent:** full main+outboard buffers at C (no streaming ReadAt / callback ABI) +- ~~Inboard `verify_slice` O(body) retain~~ **W4a closed** — O(slice) retained output; O(N) time; C full body **input** remains +- ~~Stream dual under lean is E1-only~~ **W1b closed (MVP):** see **Stream E1/E2 API matrix** below. Pure Lean chunked stream C ABI residual remains (no streaming C symbols). +- ~~`file::decode_stream` pure-Rust residual~~ **W1a closed** — under lean, spools header+`encoded_len` body → Lean `decode_headered` (peak O(archive+plaintext); not E2) +- ~~Full **codecode** / **decodec** matrix~~ **W2d closed** — `tests/determinism_roundtrip.rs` (no-compress + same-engine compress/directory) +- **W2a permanent residual — Compression cross-engine encode:** Lean AOT zstd frames are **not** bit-identical to Rust `zstd` even at level 20 / same pin rev. Measured evidence (G9 `outboard_c14`): mains both 35 B; frame descriptor byte differs (`28b5 2ffd **00**…` rust vs `28b5 2ffd **20**…` lean); Bao roots and FEC parity diverge. **Policy:** decode interop only across engines; re-encode not bit-identical; same-engine codecode/decodec still requires `A' == A` (green). Do not claim rust↔lean compress wire identity. +- **W2b permanent residual — Directory cross-engine encode:** compare **live rust vs live lean** catalog roots under identical pins (phase3 seed tree, zero master, default options) — **not** lean-vs-stale-seed. Pinned in `tests/determinism_roundtrip.rs`: live rust `0b119f12…`, live lean `f67b6f49…` (hard `assert_ne!`). `phase3_g9_directory` catalog `16e2369f…` is **decode-only SSOT** (lags live rust catalog packaging while segment mains may still match). Same-engine directory codecode/decodec green. +- **W4c permanent residual — streaming zstd under lean:** buffer-only bulk zstd for Lean frame parity; public Compression outboard under lean stays O(logical) — not E2 (see matrix). +- **W4d permanent residual — FEC / async spool:** FEC verify O(FEC body) shards (segment-wide RS); async always disk-stages O(encoded); lean+async peak RAM O(encoded+logical). See [STREAMING_PARALLELISM.md](../doc/STREAMING_PARALLELISM.md). +- ~~Live Nix product-matrix vs optional frozen `ref/carbonado-rust`~~ **W5a / G1 closed** — permanent no product pin; live `src/` + `tests/` SSOT; third-party `ref/` pins only + +### Slice memory honesty (W4a / W4b) + +| Path | Retained hash/output work | Input buffers at C / dual lean | Claim | +|------|---------------------------|--------------------------------|-------| +| Inboard `carbonado_verify_slice` / Lean `verifySliceInboard` | **O(slice)** retained (W4a); walk from offset 8 (no second response copy); O(leaf) temps | Full inboard body (caller / C) | O(slice) **output**; peak still includes full body input when resident | +| Outboard `carbonado_verify_slice_outboard` | **O(slice+height)** hash (R9) | Full main + full outboard | **Permanent** full-buffer input (W4b) | +| Rust `verify_slice_inboard_seekable` | O(slice) via `SliceRegionWriter` | Full inboard body (`&[u8]`) | O(slice) **output**; peak O(body) when blob resident | +| Rust `verify_slice_outboard` + `ReadAt` (`backend-rust`) | O(slice) | Streaming ReadAt | True O(slice) RSS when data is file-backed | +| Rust `verify_slice_outboard` + `ReadAt` (`backend-lean`) | O(slice) hash after materialize | Materializes `data_len` once → C | Full main copy once (honest) | + +### Stream E1/E2 API matrix (W1b — dual honesty) + +| API | `backend-rust` peak | `backend-lean` peak | Lean dual engine? | +|-----|---------------------|---------------------|-------------------| +| `stream_encode_buffer` / `stream_decode_buffer` (+ outboard buffer) | O(logical) | O(logical) | **Yes** — Lean buffer C ABI | +| `file::decode_stream` / `file::decode` | O(chunk) spool (rust S4) | O(archive+plaintext) **E1** (W1a; MAC-before-body) | **Yes** — Lean `decode_headered` | +| `stream_encode_inboard` / `stream_decode` | O(chunk/stripe) S4 | O(logical) **E1** (disk-spool ingest + Lean buffer) | **Yes** — Lean body encode/decode | +| `stream_*_outboard` **public non-Compression** (c0/c4/c8/c12) | **O(chunk/stripe) E2** | **O(chunk/stripe) E2** (c4/c12 MVP smoke) | **Composition** — rust S4 geometric; G9 **no-compress** wire bit-match (Compression residual **W2a**); **not** pure-Lean stream | +| `stream_*_outboard` **public + Compression** (c2/c6/c10/c14) | O(chunk) streaming zstd | **O(logical)** bulk zstd (`stream_compress` buffer API; **W4c permanent** buffer-only under lean) | Composition S4, **not E2** under lean; no dual-safe streaming frames | +| `stream_*_outboard` **encrypted** | O(chunk) S4 + EtM spool | O(logical) **E1** | **Yes** — Lean `encode_outboard` / `decode_outboard` (crypto dual) | +| `stream_decode_async` (optional `async`) | disk O(encoded) + S4 | disk O(encoded) + E1 | Dual-aware via `stream_decode` (R10); freeze never requires `async` | + +**Honesty rules:** never claim “true stream” / O(chunk) for Lean **E1** paths or for public **Compression** outboard under lean. W1b E2 MVP = public **non-Compression** outboard composition only. Pure Lean chunked stream requires a future streaming C ABI. diff --git a/docs/PARITY.md b/docs/PARITY.md index 5470f37..1eae627 100644 --- a/docs/PARITY.md +++ b/docs/PARITY.md @@ -2,9 +2,9 @@ ## Method (dual-backend) -1. **Primary parity bar (G8):** the same Rust tests under `tests/` pass on **`backend-rust`** and **`backend-lean`** (Lean AOT `libcarbonado` via C ABI). See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md). +1. **Primary parity bar (G8):** the same Rust tests under `tests/` pass on **`backend-rust`** and **`backend-lean`** (Lean AOT `libcarbonado` via C ABI). See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md). **G8 full closed at R7** — `just test-lean-ci` / CI `dual-backend-lean` runs the **full** dual suite under lean features; permanent feature-gated / purity residuals only (`streaming_async` / `parallel_determinism` feature-gated off freeze — **R10** closed async dual policy; ~~pure Lean rkyv encode~~ **W3 closed** — dual-suite encode remains Rust rkyv composition SSOT; ~~W1a+W1b~~ dual honesty closed — public outboard stream E2 is S4 composition under lean; pure Lean chunked C residual). 2. **Component oracles:** pin reference trees under `ref/`; keep offline drivers (`etm-vectors`, `rs-vectors`, `bao-vectors`) for fast regression against Lean goldens / AOT demos. -3. **Cross-backend tests (G9):** Rust encode → Lean decode and reverse once ABI encode/decode are stable (after G8 Phase 2). +3. **Cross-backend tests (G9):** **closed at R8** — Rust encode → Lean decode and reverse for no-compress body/headered/outboard (public + fixed-nonce encrypted); fixtures under `tests/fixtures/g9/`; contract `tests/g9_cross_backend.rs`. **W2d** same-engine codecode/decodec in `tests/determinism_roundtrip.rs`. **Permanent residuals (W2a/W2b):** cross-engine Compression / directory encode bit-match (decode interop only). **SSOT roles (do not invert):** @@ -15,7 +15,7 @@ | Lean `Carbonado/` + AOT `libcarbonado` | Second engine: proofs + wire/C-ABI compatible implementation | | `ref/` | Pinned third-party oracles and vector drivers | -Pin the exact trees the Rust product used; Lean AOT must remain wire-compatible with that contract. Rust is **not** demoted to “oracle only” while dual-backend work is in progress (G1 optional freeze is a pin, not a product deletion). +Pin the exact **third-party** trees the Rust product used (Bao, RS, crypto crates, zstd, bitcoinpqc, …); Lean AOT must remain wire-compatible with that contract. Rust product under `src/` + `tests/` is **first-class SSOT** — not demoted to “oracle only.” **G1 closed (W5a):** no `ref/carbonado-rust` product pin (permanent policy; see freeze strategy below). ## Pins (from Carbonado `Cargo.lock` / Surmount) @@ -30,8 +30,8 @@ Pin the exact trees the Rust product used; Lean AOT must remain wire-compatible | `ref/zstd` | `https://github.com/facebook/zstd.git` | tag **`v1.5.7`** → **`f8745da6ff1ad1e7bab384bd1f9d742439278e99`** — **product SSOT** for static libzstd in `nix/native` (not nixpkgs.src); Rust crate was zstd 0.13.3 | | `ref/bitcoinpqc` | `https://github.com/cryptoquick/libbitcoinpqc-bindings.git` | **`7936b56f15e86b6764947c9298215ecfe38b712b`** | | `ref/crates/ctr-0.9.2` | crates.io `ctr` 0.9.2 | checksum `0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835` — **vendored (Program B)** | -| `ref/carbonado-rust` | optional pin of live tree (`src/`, `tests/`, …) | freeze commit **pending** (G1); live tree remains first-class | | `ref/parity-harness` | in-repo | `drivers/etm-vectors` (B); `drivers/rs-vectors` (C); `drivers/bao-vectors` (D); directory/CFP2 vectors deferred (G residual) | +| ~~`ref/carbonado-rust`~~ | **not used** | **G1/W5a permanent policy:** no product pin submodule — live `src/` + `tests/` are dual-suite SSOT | ## Submodules @@ -107,8 +107,8 @@ cd ref/parity-harness/drivers/bao-vectors && cargo run --quiet 2. **Goldens (AOT `demo`):** API frames for empty (`28b52ffd2000010000`) and `hello` (`28b52ffd200529000068656c6c6f`); corrupt frame → `decompressionFailed`; tight `maxOut` → `outputTooLarge`; zeros shrink; pipeline c2/c6 + **headered c3/c7**; full format matrix incl. compression at runtime. 3. **Interpreter residual:** Lean `@[extern]` bodies are identity for elaborator; real frames only in AOT (LIMITS). CarbonadoTest `native_decide` covers non-compression formats + status maps. 4. **SLH1 wire:** magic `SLH1`, signature 7856 B, sidecar 7860 B — matches `src/crypto.rs` `SLH1_*` and AGENTS §2.3. Pure suite: length errors + `parseMagicAtExactLen` / `badSlhMagic` gate theorems; full-length build/parse + all-zero magic in `demo`. -5. **Bind-to-root:** Lean model requires signature message = Bao root (`verifyBoundToExpected`); wrong root → `verificationFailed`. Real SLH-DSA verify via libbitcoinpqc **not** linked (nested submodule empty). -6. **Optional future drivers:** `ref/parity-harness/drivers/zstd-vectors`, `slh-vectors` once PQC FFI lands. +5. **Bind-to-root + live SLH (R9 / G10):** Lean model requires signature message = Bao root (`verifyBoundToExpected`); wrong root → `verificationFailed`. Real SLH-DSA-SHA2-128s sign/verify is **linked** via flake pin `b309f444…` (same SSOT as `ref/bitcoinpqc` submodule target) into `libcarbonado_native.a` — C `carbonado_slh_*` + Lean `@[extern]`. Nested worktree emptiness is not a product residual when the flake fetch pin is present. Dual-suite product SLH may still use Rust `bitcoinpqc` composition. +6. **Optional drivers:** `ref/parity-harness/drivers/zstd-vectors`, `slh-vectors` for cross-oracle goldens (live FFI already in product AOT / libcarbonado). ## Adding a gate @@ -126,11 +126,19 @@ git -C ref/bao-tree checkout 02916e784bb0afe0fd5a73c291c8c5335865e166 CI must checkout recursively once submodules are recorded on the default branch. -## carbonado-rust freeze strategy (optional pin — G1 residual) +## carbonado-rust freeze strategy (G1 / W5a — permanent policy: no product pin) -Dual-backend model keeps **live** Rust under `src/` and `tests/` as first-class. An optional historical pin is separate: +**Decision (closed 2026-07 W5a): do not add `ref/carbonado-rust`.** -1. Keep production Rust under `src/`, `tests/`, etc. (do not delete for “Lean purity”). -2. Optionally add submodule or subtree `ref/carbonado-rust` at a named freeze commit for long-lived oracle/goldens isolation. -3. **Do not** treat Lean as a replacement that removes the Rust engine: G8 requires both backends against the same `tests/`. -4. Record any freeze SHA here and in [GAPS.md](GAPS.md) G1 when created. +Dual-backend model keeps **live** Rust under `src/` and `tests/` as first-class production + normative contract. A separate frozen product tree is **not required** and is **not** part of the dual-backend freeze. + +| Policy | Detail | +|--------|--------| +| Product SSOT | Live `src/` + `tests/` (G8: both backends against the same suite) | +| What `ref/` pins | Third-party oracles only (bao-tree, zstd, bitcoinpqc, RustCrypto, blake3, reed-solomon-erasure, parity-harness, vendored crates) | +| `ref/carbonado-rust` | **Absent by policy** — do not invent a submodule or SHA | +| Why no product pin | (1) Dual-suite SSOT is the live tree, not a historical snapshot. (2) A product pin duplicates `src/`/`tests/`, confuses which tree is normative, and is unused by CI `dual-backend-lean`. (3) Third-party pins already freeze what Lean/goldens compare against. | +| Future archaeology | A release-specific historical checkout remains *possible* outside this policy if needed; it is **non-required** and must not replace live `src/`/`tests/` or demote Rust to “oracle only.” | +| Do not | Delete or demote live `src/` / `tests/` for “Lean purity”; treat Lean as a replacement that removes the Rust engine (G8 requires both). | + +G1 status: **closed** with this permanent no-pin policy — see [GAPS.md](GAPS.md). diff --git a/docs/SPEC-MATRIX.md b/docs/SPEC-MATRIX.md index 5bca1d1..3d573d8 100644 --- a/docs/SPEC-MATRIX.md +++ b/docs/SPEC-MATRIX.md @@ -15,8 +15,8 @@ Every product capability maps to Lean module(s), parity gate(s), and proof statu | Streaming bounds | `Carbonado.Stream` | demo greps + theorems | **Program E:** O(stripe) FEC retain theorems (`maxFecStripeRetain`); pure stripe transducer model | | Sharding | `Carbonado.Shard` | demo multi-segment roundtrip | **Program E:** budget split + `chunk_index` sequence + headered segments | | Zstd-20 compress | `Carbonado.Compress`, `CarbonadoTest.Compress` | `demo` API goldens (empty/hello); pipeline c2/c6 | **Program F closed**: linked zstd; status taxonomy; interpreter identity fallback (LIMITS) | -| SLH1 sidecars | `Carbonado.Slh`, `CarbonadoTest.Slh` | `demo` wire + bind-to-root | **Program F closed** for wire/binding theorems; real SLH-DSA FFI residual (LIMITS) | -| Adamantine directory | `Carbonado.Adamantine`, `Filepack`, `Outboard`, `Directory`, `CarbonadoTest.Directory` | `demo` Program G greps; pure roundtrip AOT | **Program G closed** for product path: Adamantine10 + CFP2 manifest + outboard segments + fail-closed paths + content BLAKE3. rkyv body residual (LIMITS) | -| CLI | `Carbonado.Cli`, `Carbonado.Main` | `demo` + CLI subcommands | **Program G:** encode/decode file+dir; single-file default `{bao_root_hex}.c{fmt:02x}`; dir default `{input}-archive/`; `slh parse` wire; `slh verify` fail-closed exit 1 until FFI | +| SLH1 sidecars | `Carbonado.Slh`, `CarbonadoTest.Slh` | `demo` wire + bind-to-root + live sign/verify | **Program F + R9/G10 closed**: wire/binding theorems; live SLH-DSA-SHA2-128s via libbitcoinpqc pin + `carbonado_slh_*` (LIMITS: elaborator fail-closed; dual-suite may keep Rust composition) | +| Adamantine directory | `Carbonado.Adamantine`, `Filepack`, `RkyvFilepack`, `Outboard`, `Directory`, `CarbonadoTest.Directory` | `demo` Program G greps; pure roundtrip AOT; dual-suite `lean_backend_phase3` | **Program G + W3 closed**: pure Lean directory/CLI emit **rkyv** FilepackManifestWire v2 (goldens). **P3 dual-suite closed**: rkyv catalog via Rust composition + Lean segment/catalog crypto (dual-suite encode SSOT; not pure Lean required). CFP2 dual-decode fallback only | +| CLI | `Carbonado.Cli`, `Carbonado.Main` | `demo` + CLI subcommands | **Program G + R9:** encode/decode file+dir; single-file default `{bao_root_hex}.c{fmt:02x}`; dir default `{input}-archive/`; `slh parse` wire; `slh verify` live oracle (exit 0 accept / exit 1 reject) | -Expand rows until full product parity with dual-backend G8 (`backend-rust` + `backend-lean` on `tests/`) and optional `ref/carbonado-rust` freeze is closed. Component rows above track Lean+Nix proof/oracle gates; dual-suite status is [GAPS.md](./GAPS.md) G8 / [TEST_CONTRACT.md](./TEST_CONTRACT.md). +Expand rows until full product parity with dual-backend G8 (`backend-rust` + `backend-lean` on `tests/`). **G1/W5a closed:** no optional `ref/carbonado-rust` product pin (live tree SSOT). Component rows above track Lean+Nix proof/oracle gates; dual-suite status is [GAPS.md](./GAPS.md) G8 / [TEST_CONTRACT.md](./TEST_CONTRACT.md). diff --git a/docs/TEST_CONTRACT.md b/docs/TEST_CONTRACT.md index 7a8b76e..13c4911 100644 --- a/docs/TEST_CONTRACT.md +++ b/docs/TEST_CONTRACT.md @@ -12,16 +12,22 @@ See [ABI.md](./ABI.md), [PARITY.md](./PARITY.md), [GAPS.md](./GAPS.md) G8, [LIMI | Feature flag | Engine | Expected of this suite | |--------------|--------|------------------------| | `backend-rust` (default) | Pure Rust (`src/encoding`, `src/decoding`, `src/file`, …) | Full green (always) | -| `backend-lean` | Lean AOT via `carbonado-sys` / `libcarbonado` | Phased allowlist → full green (G8) | +| `backend-lean` | Lean AOT via `carbonado-sys` / `libcarbonado` | Full green (G8 closed at R7; freeze = unfiltered dual suite) | ```bash -# Normative default +# Normative default (backend-rust) cargo test -# Dual-backend (after Phase 1 wiring + linked lib) -# nix build .#libcarbonado -# export CARBONADO_LEAN_LIB=$PWD/result/lib CARBONADO_LEAN_INCLUDE=$PWD/result/include -cargo test --no-default-features --features "backend-lean,pqc,ots" +# Dual-backend freeze (Phase 5 / G11 + R7 G8 full close — shared CI + humans) +# Prefer the single recipe (builds libcarbonado if needed; fail-closed if .so missing): +just test-lean-ci + +# Manual equivalent (R7: freeze = full unfiltered dual suite): +# nix build .#libcarbonado -o result-libcarbonado +# export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +# export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +# export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" ``` Helpers under `tests/common/` are not separate contract files; they support the files below. @@ -46,20 +52,29 @@ Helpers under `tests/common/` are not separate contract files; they support the | **fec_scrub** | `apocalypse.rs` | Phase 2 | Large-sample encode/scrub chaos | | **stream** | `streaming.rs` | Phase 2 | Stream encode/decode buffer + outboard | | **stream** | `streaming_limits.rs` | Phase 2 | Bounds, FEC encoder, crypto stream, scrub | -| **stream** | `seekable_slices.rs` | Phase 2 | O(slice) verify inboard/outboard | +| **stream** | `seekable_slices.rs` | Phase 2 | O(slice) retain/hash; full body/main at lean C (W4a/W4b; see LIMITS slice table) | | **shard** | `sharding.rs` | Phase 2 | `encode_shard_stream` / `decode_shards_stream` | -| **directory** | `directory_archive.rs` | Phase 3 | Adamantine 1.0 + rkyv catalog + scrub_outboard | +| **directory** | `directory_archive.rs` | Phase 3 (core); OTS cases Phase 4 closed | Adamantine 1.0 + rkyv catalog + scrub_outboard; OTS via CBOTS composition | | **directory** | `filepack_interop.rs` | Phase 3 | Filepack / CBOR interop + directory decode | | **directory** | `format_policy.rs` | Phase 3 | Segment format policy (no I/O encode) | -| **cli** | `bin_cli.rs` | Phase 4 | Prebuilt `carbonado` binary CLI | +| **cli** | `bin_cli.rs` | Phase 4 | Prebuilt `carbonado` binary CLI (default rust-engine; lean when rebuilt with `backend-lean,cli`) | | **cli** | `bin_smoke.rs` | Phase 4 | CLI smoke encode/decode | | **cli** | `bin_heuristics.rs` | Phase 4 | Filename heuristics + CLI | -| **pqc** | `slh_outboard.rs` | Phase 4 | SLH-DSA sidecars + header `slh_public_key` | -| **async** | `streaming_async.rs` | **rust-only** (unless declared) | `stream_decode_async` | -| **parallel** | `parallel_determinism.rs` | rust-only or serial lean path | RS parallel vs serial determinism | +| **pqc** | `slh_outboard.rs` | Phase 4 closed | SLH-DSA sidecars + header `slh_public_key` (Rust bitcoinpqc under both backends) | +| **async** | `streaming_async.rs` | **feature-gated permanent** (needs `async`; **R10 closed**) | Freeze excludes `async` → 0 tests under dual suite; optional lean+async dual-aware via R5 E1 | +| **parallel** | `parallel_determinism.rs` | **feature-gated** (needs `parallel`; not dual residual) | RS parallel vs serial determinism; 0 tests under dual feature set | | **parallel** | `serial_fec_path.rs` | Phase 2 (serial) | Serial FEC encoder vs buffer path | +| **lean_allowlist** | `lean_backend_smoke.rs` | Phase 1 closed | Dual-backend body/headered/auth smoke (`just test-lean-smoke`) | +| **lean_allowlist** | `lean_backend_phase2.rs` | Phase 2 closed | Outboard/scrub/slice/stream + G9 seed (`just test-lean-phase2`) | +| **lean_allowlist** | `lean_backend_phase3.rs` | Phase 3 closed | Directory composition + G9 dir fixture (`just test-lean-phase3`) | +| **lean_allowlist** | `lean_backend_phase4.rs` | Phase 4 closed | SLH composition + CLI dual (directory dual-engine + buffer APIs; stream E1 dual closed at R5; **W1a** `decode_stream` dual closed; **W1b** public non-compress outboard composition E2 closed; pure Lean chunked stream C residual) + directory OTS (`just test-lean-phase4`) | +| **g9_matrix** | `g9_cross_backend.rs` | **R8 / G9 closed** | Full cross-backend no-compress matrix both directions; fixtures `tests/fixtures/g9/` (`just test-g9`) | +| **determinism** | `determinism_roundtrip.rs` | **W2d closed** | codecode (EDE) + decodec (DED) no-compress matrix; same-engine compress (body/headered/outboard) + directory; W2a/W2b hard residual asserts (live-vs-live dir roots; G9 c14 mains) | +| **lean_freeze (P5+R7)** | see Phase 5 / R7 section | Phase 5 + R7 closed | Freeze = full dual suite: `just test-lean-ci` (unfiltered lean features; includes `g9_cross_backend` + `determinism_roundtrip`) | + +**Inventory count:** 32 integration test files under `tests/*.rs` (R8: `g9_cross_backend`; W2: `determinism_roundtrip`). -**Inventory count:** 26 integration test files under `tests/*.rs` (complete as of Phase 0 close). +**Directory vs OTS:** Core Adamantine / rkyv dual-suite green is **Phase 3**. Entry/catalog OTS proof cases (feature `ots`) are **Phase 4 closed** via pure-Rust CBOTS composition over Lean container crypto (no Lean-native stamping). --- @@ -69,24 +84,29 @@ Mapped from actual `use carbonado::…` imports in `tests/*.rs`. C ABI column is | API / type | Typical tests | C ABI priority | |------------|---------------|----------------| -| `encode` / `decode` (crate root = `encoding`/`decoding`) | codec, format, header_tamper, fec_*, apocalypse, udp_fec_sim, parallel_determinism | **v0** (`carbonado_encode` / `carbonado_decode`) | -| `encode_outboard` / `decode_outboard` | format, fec_*, bao_keyed, streaming*, directory, adversarial | **v1+** (not in `include/carbonado.h` v0) | -| `scrub` / `scrub_outboard` | codec, format, fec_*, apocalypse, streaming_limits, shard_fec_scrub, directory | **v1+** | -| `verify_slice` / `extract_slice` | codec, seekable_slices, bao_keyed | **v1+** | -| `verify_slice_inboard_seekable` / `verify_slice_outboard` | bao_keyed, seekable_slices | **v1+** | +| `encode` / `decode` / `encode_with_nonce` (crate root = `encoding`/`decoding`) | codec, format, header_tamper, fec_*, apocalypse, udp_fec_sim, parallel_determinism, **g9_cross_backend** | **v0** (`carbonado_encode` / `carbonado_decode`; fixed nonce via optional C arg) | +| `encode_outboard` / `decode_outboard` | format, fec_*, bao_keyed, streaming*, directory, adversarial | **P2 live** (`carbonado_encode_outboard` / `carbonado_decode_outboard`) | +| `scrub` / `scrub_outboard` | codec, format, fec_*, apocalypse, streaming_limits, shard_fec_scrub, directory | **P2 live** (`carbonado_scrub` / `carbonado_scrub_outboard`) | +| `verify_slice` / `extract_slice` | codec, seekable_slices, bao_keyed | **P2 + W4a** (`carbonado_verify_slice`; extract = count 1; O(slice) retain; full body input) | +| `verify_slice_inboard_seekable` / `verify_slice_outboard` | bao_keyed, seekable_slices | **v1+**; **R9** outboard C live (`carbonado_verify_slice_outboard` + lean dispatch) | +| `crypto::slh_*` (pure Lean path) | AOT demo / optional | **R9** `carbonado_slh_*` live; dual-suite may keep Rust bitcoinpqc | +| rkyv catalog encode+decode (Lean) | AOT demo goldens | **W3** `Carbonado/RkyvFilepack` encode/decode; dual-suite encode still Rust rkyv composition SSOT | | `carbonado_verification_key` | bao_keyed_contract | **v0** | | `file::encode` / `file::decode` / `Header` | format, format_amplification, header_tamper, streaming_limits, slh_outboard, adversarial | **v0** (`carbonado_encode_headered` / `carbonado_decode_headered`) | -| `file::encode_stream` / `decode_stream` | streaming, streaming_limits | Phase 2 (may stay Rust-side over buffer ABI) | -| `file::encode_directory` / `encode_directory_with_options` / `decode_directory` | directory_archive, filepack_interop, udp_fec_sim | Phase 3 (+ rkyv wire) | -| `stream_encode_buffer` / `stream_decode_buffer` (+ outboard buffer variants) | streaming*, bao_keyed, parallel_determinism | Phase 2 | +| `file::encode_stream` / `decode_stream` | streaming, streaming_limits | **R5 E1** encode_stream → Lean; **W1a** `decode_stream` spool→Lean `decode_headered` (E1 RAM; not E2) | +| `file::encode_directory` / `encode_directory_with_options` / `decode_directory` | directory_archive, filepack_interop, udp_fec_sim | **P3 live** (composition: rkyv+FS Rust; segment/catalog crypto via outboard/headered Lean ABI) | +| `stream_encode_buffer` / `stream_decode_buffer` (+ outboard buffer variants) | streaming*, bao_keyed, parallel_determinism | Phase 2 + **R5** stream I/O E1 over same Lean buffer ABI | +| `stream_encode_inboard` / `stream_decode` | streaming, streaming_limits | **R5 E1** spool-to-buffer → Lean (O(logical); not E2) | +| `stream_encode_outboard` / `stream_decode_outboard` | streaming, streaming_limits | **W1b:** public non-compress → S4 O(chunk/stripe) composition E2 under lean; public+Compression under lean O(logical) bulk zstd; encrypted → Lean E1 | | `stream::fec::*` / `stream::parallel::*` | streaming_limits, serial_fec_path, parallel_determinism | rust-internal / serial lean | | `encode_shard_stream` / `decode_shards_stream` | sharding, shard_fec_scrub | Phase 2 | -| Adamantine / filepack_manifest / format_policy | directory_*, filepack_interop, format_policy | Phase 3 | -| `crypto::slh_*` / sidecar helpers | slh_outboard | Phase 4 (G10) | -| `ots::*` | directory_archive (feature `ots`) | Phase 4 | +| Adamantine / filepack_manifest / format_policy | directory_*, filepack_interop, format_policy | **P3 live** (rkyv wire; format_policy pure Rust) | +| `crypto::slh_*` / sidecar helpers (dual-suite product) | slh_outboard, lean_backend_phase4 | **P4 live** (G10-A: Rust `bitcoinpqc` composition under both backends) | +| `carbonado_slh_*` C ABI (pure Lean path) | AOT demo / optional C consumers | **R9 live** optional purity; dual-suite need not switch from composition | +| `ots::*` | directory_archive OTS + lean_backend_phase4 (feature `ots`) | **P4 live** (Rust CBOTS; no Lean OTS engine) | | Deprecation aliases (`PackIndex`, …) | deprecation_aliases | n/a (API surface only) | -| `stream_decode_async` | streaming_async | **rust-only** initially | -| CLI binary (`src/bin/carbonado.rs`) | bin_* | Phase 4 | +| `stream_decode_async` | streaming_async | **R10:** optional adapter; dual freeze never requires `async`; under lean+async → dual-aware `stream_decode` (disk O(encoded); lean peak RAM O(encoded+logical); not E2) | +| CLI binary (`src/bin/carbonado`) | bin_*, lean_backend_phase4 | **P4 + R5 + W1:** directory CLI + buffer APIs + stream encode E1 + `decode_stream` W1a + public outboard W1b composition; rebuild with `cli`+`backend-lean` | ### Error-contract note (both backends) @@ -96,7 +116,7 @@ Tests that `matches!` ultra-specific `CarbonadoError` variants require a stable ## Phase 1 allowlist (first green `backend-lean` gate) -**Honest status at Phase 0 close:** C symbols exist in `include/carbonado.h` and `carbonado-sys`, but `nix/native/carbonado_abi.c` weak stubs return `CARBONADO_ERR_NOT_IMPLEMENTED` for all encode/decode/verification_key entry points. Lean has pure helpers in `Carbonado/Ffi.lean` (`encodeHeaderedBytes`, `decodeHeaderedBytes`, `ofPipelineError`) — **not** yet linked as live C exports in the archive used by `backend-lean`. Phase 1 work is: real `@[export]` / link, Rust dispatch into `src/backend/lean`, then the allowlist below. +**Phase 1 closed:** live C ABI via Lean AOT (`l_carbonado_*` + `carbonado_abi.c`), `nix build .#libcarbonado`, Rust `backend-lean` dispatch for `encode`/`decode`/`file::{encode,decode}`/`carbonado_verification_key`, allowlist `tests/lean_backend_smoke.rs` (`just test-lean-smoke`). ### Phase 1 scope (concrete) @@ -118,7 +138,131 @@ Tests that `matches!` ultra-specific `CarbonadoError` variants require a stable - Changing normative wire format - Replacing or deleting the Rust engine -Document the live allowlist in CI / justfile as it grows. Full suite remains the G8 end state (Phase 5). +Document the live freeze in CI / justfile. Full suite is the G8 bar — **closed at R7** (see Phase 5 / R7). + +--- + +## Phase 2 allowlist (outboard / scrub / slice + G9 start) + +**Phase 2 closed:** additive C ABI for outboard encode/decode, scrub / scrub_outboard, verify_slice; richer encode metadata (`chunk_len`, `bytes_ecc`, `verifiable_slice_count`); Lean geometry-peel inboard scrub + outboard scrub; Rust dispatch under `backend-lean` for those APIs + stream buffer composition over body/outboard ABI; allowlist `tests/lean_backend_smoke.rs` + `tests/lean_backend_phase2.rs` (`just test-lean-phase2`). + +### Phase 2 scope (concrete) + +1. **Public formats** c0/c4/c12/c14 (+ fixed-nonce encrypted helpers for c5) on body, headered, outboard. +2. **New C symbols (ABI v1 additive):** + - `carbonado_encode_outboard` / `carbonado_decode_outboard` + - `carbonado_scrub` / `carbonado_scrub_outboard` + - `carbonado_verify_slice` + - `CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION` (13) — distinct from scrub recovery failure + - encode body pack extended with chunk/ecc/vsc fields (nullable C out-params) +3. **G9 start:** rust-engine golden buffers (c0/c4 body + headered c4) decoded under lean; lean re-encode bit-matches rust body. +4. **Stream buffers:** `stream_encode_buffer` / `stream_decode_buffer` / outboard buffer helpers compose over Lean under `backend-lean` (not silent pure-Rust). + +### Phase 2 residuals (historical; full suite green at R7) + +- ~~Residual sharding/fec_chaos~~ **green under lean (R6)** — **`format` (R1)** / **`header_tamper` (R2)** / **`format_amplification` (R3)** / **`codec` + `seekable_slices` (R4)** / **`streaming` + `streaming_limits` (R5)** / **`sharding` + `fec_chaos` (R6)** / `bao_keyed_contract` / `fec_scrub_matrix` in freeze +- Seekable outboard slice C **R9 live** (`carbonado_verify_slice_outboard` + lean dispatch; **W4b permanent** full buffers at C ABI — no ReadAt callback) +- **Inboard `verify_slice` under lean (W4a closed):** auth-first O(slice) retain via `decodeRecRetainRange` (O(N) time over full response; full body input at C) — parity with Rust `SliceRegionWriter` class; `seekable_slices` freeze-green +- CI freeze (Phase 5); pure Lean SLH FFI **closed at R9** (dual-suite may keep composition) +- ~~Pure Lean rkyv encode residual (**W3**)~~ **closed** — pure Lean encode/decode + Directory/CLI rkyv; dual-suite still uses Rust rkyv via composition +- Rust-root directory checksum goldens under lean encode (**W2b permanent residual** — same-engine directory codecode green in `determinism_roundtrip`; cross-engine roots may differ; `phase3_g9_directory` decode SSOT) + +### Determinism contracts (**W2d shipped**) + +| Contract | Steps | Assert | +|----------|-------|--------| +| **codecode** (EDE) | encode → decode → encode | `pt' == pt` and `A' == A` under fixed params (nonce pinned when Encrypted) | +| **decodec** (DED) | decode archive → encode → decode | `pt' == pt` and `B == A` when encode is deterministic under same pins | + +**Shipped:** `tests/determinism_roundtrip.rs` (auto-included in lean freeze). Pins: G9 MASTER/NONCE/`g9_matrix_v1`. + +| Matrix | Coverage | Wire equality | +|--------|----------|---------------| +| No-compress body | c0/c1/c4/c5/c8/c9/c12/c13 | full `A' == A` both engines | +| No-compress headered | c4/c5/c12/c13 | full `A' == A` both engines | +| No-compress outboard | c4/c5/c12/c13 | full wire (main/out/par/header) both engines | +| Compress body (same-engine) | c2/c3/c6/c7/c10/c11/c14/c15 | same-engine `A' == A`; **cross-engine permanent residual (W2a)** | +| Compress headered (same-engine) | c6/c7/c14/c15 | same-engine `A' == A` | +| Compress outboard (same-engine) | c6/c7/c14/c15 | same-engine wire equality | +| Directory public (same-engine) | phase3 seed tree, zero master | same-engine catalog+segments | +| DED from G9 body goldens | body no-compress, active-engine fixtures | re-encode matches committed golden | +| W2a residual | G9 `outboard_c14` rust vs lean mains | hard `assert_ne!` + frame descriptor (fail-closed if fixtures missing) | +| W2b residual | live rust vs live lean catalog roots | hard pins `0b119f12…` ≠ `f67b6f49…`; seed `16e2369f…` decode-only | + +Encrypted without fixed nonce: wire identity out of scope. + +--- + +## Phase 3 allowlist (directory dual-backend) + +**Phase 3 closed:** directory encode/decode under `backend-lean` via composition (Rust rkyv FilepackManifest v2 + Adamantine framing + FS; Lean outboard/headered crypto). Allowlist `tests/lean_backend_phase3.rs` (+ `format_policy`); G9 rust-encode fixture → lean decode (`tests/fixtures/phase3_g9_directory/`). `just test-lean-phase3`. + +Core `directory_archive` and non-golden `filepack_interop` pass under lean in practice; dedicated allowlist remains the gate. OTS dual-backend / CLI dual → Phase 4 (closed). + +--- + +## Phase 4 allowlist (PQC + CLI + directory OTS) + +**Phase 4 closed:** dual-suite SLH via **G10 strategy A** (Rust `bitcoinpqc` `crypto::slh_*` under both backends). **R9:** pure Lean `signRoot`/`verifyRoot` live via libbitcoinpqc in `libcarbonado` (optional purity path; dual-suite need not switch). Directory OTS: offline CBOTS composition (no Lean-native stamping). + +**CLI dual (honest scope):** +- **Dual-engine:** directory encode/decode (library + subprocess), buffer APIs (`file::encode` / `file::encode_outboard` / headered decode), inboard/encrypted stream **E1** → Lean buffer ABI, **W1a** `decode_stream` → Lean `decode_headered`. +- **W1b:** public `stream_*_outboard` under lean is S4 O(chunk/stripe) **composition** (not pure Lean stream). Pure Lean chunked C residual remains. +- **Lean-linked binary:** `cli` + `backend-lean` proves link/run; directory subprocess + stream dual paths are CLI evidence. + +```bash +just test-lean-phase4 +# or: +cargo test --no-default-features --features "backend-lean,pqc,ots,cli" \ + --test lean_backend_smoke --test lean_backend_phase2 --test lean_backend_phase3 \ + --test format_policy --test slh_outboard --test lean_backend_phase4 +``` + +Allowlist: `lean_backend_phase4.rs` + `slh_outboard.rs` (+ Phase 1–3). Full `bin_*` matrix optional under lean-linked binary. + +--- + +## Phase 5 — CI freeze both backends (G11 closed; G8 allowlist freeze → R7 full) + +**Phase 5 closed** as dual-backend CI freeze of the allowlist. **R7 expanded freeze to full dual suite and closed full-suite G8.** + +### Normative commands + +| Backend | CI job (`.github/workflows/rust.yaml`) | Local / shared command | +|---------|----------------------------------------|-------------------------| +| `backend-rust` | **`desktop`** | `cargo test`; serial: `--no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path`; optional: `--features "async,async-tokio,man-gen"` (never `--all-features`); smoke + CLI | +| `backend-lean` | **`dual-backend-lean`** | `just test-lean-ci` (full dual suite as of R7) | + +### Lean env + build (fail-closed) + +```bash +nix build .#libcarbonado -o result-libcarbonado +export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +# Fail-closed: libcarbonado.so (or .dylib) must exist under CARBONADO_LEAN_LIB +just test-lean-ci +``` + +- **Never** enable both engines: must use `--no-default-features --features "backend-lean,pqc,ots,cli"` (not `--features backend-lean` alone). +- If `CARBONADO_LEAN_LIB` is unset, `just test-lean-ci` runs `nix build .#libcarbonado -o result-libcarbonado` then exports env. +- If the shared library is still missing after build, the recipe **exits non-zero** (fail-closed). +- `carbonado-sys` with feature `require-lib` (enabled by `backend-lean`) **hard-errors** if `CARBONADO_LEAN_LIB` is unset or the library file is missing; CI always sets env after nix build. +- Lean-only integration crates (`tests/lean_backend_*.rs`) use `#![cfg(feature = "backend-lean")]`. Under default/`backend-rust` builds they compile as empty harnesses (0 tests) — expected, not a silent skip of freeze coverage (freeze always uses `backend-lean`). + +### Freeze contents (R7) + +Unfiltered full dual suite under features `"backend-lean,pqc,ots,cli"`: lib units + all integration tests (phase gates, measured-green files from P5/R1–R6, **`bin_cli` / `bin_heuristics` / `bin_smoke`**, etc.). Historical P5 explicit `--test` allowlist documented in [GAPS.md](./GAPS.md) for archaeology. + +### G8 status (honest) + +| Claim | Status | +|-------|--------| +| G11 live CI both backends (Linux) | **closed** | +| G8 **allowlist** dual-backend bar (P5 freeze) | **closed** (this phase) | +| G8 **full** `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"` | **closed** (R7 2026-07) — freeze equals full suite | + +**Post-G8 residuals (purity / feature-policy — not dual-suite red):** ~~pure Lean SLH FFI~~ **R9 closed**; ~~seekable outboard slice C~~ **R9 closed**; ~~rkyv dual-decode~~ **R9 closed**; ~~pure Lean rkyv encode + Directory/CLI~~ **W3 closed** (dual-suite Rust rkyv encode composition SSOT; never claim dual-suite *requires* pure Lean); ~~async dual policy~~ **R10 closed** (freeze never requires `async`; lean+async dual-aware via E1); `streaming_async` / `parallel_determinism` permanently feature-gated off freeze; ~~W1a+W1b dual honesty / public outboard E2~~ **closed** (pure Lean chunked C residual); ~~W2d codecode/decodec~~ **closed** (`determinism_roundtrip`); ~~W2a/W2b~~ **permanent residuals** (cross-engine compress/dir encode; same-engine green); ~~W4a inboard O(slice) retain~~ **closed**; **W4b** permanent full-buffer C outboard slice; **W4c** permanent buffer-only zstd under lean; **W4d** permanent FEC O(body) + async encoded spool. --- @@ -126,10 +270,34 @@ Document the live allowlist in CI / justfile as it grows. Full suite remains the | Phase | Test classes unlocked | Depends on | |-------|----------------------|------------| -| 2 | fec_scrub, stream, shard, remaining core | scrub/outboard/slice ABI or Rust-side composition over body ABI; format matrix | -| 3 | directory, format_policy, filepack_interop, deprecation_aliases | rkyv-compatible catalog wire (not Lean-only CFP2) | -| 4 | cli, pqc (slh_outboard), ots paths | libbitcoinpqc in libcarbonado (G10); CLI dual path | -| 5 | CI freeze both backends; G8 closed | full suite + docs freeze | +| **2** | outboard/scrub/slice smoke + G9 seed + stream buffer compose | **closed** — see Phase 2 allowlist above | +| **3** | directory (core Adamantine/rkyv), format_policy, filepack_interop (non-golden), deprecation_aliases | **closed** — rkyv dual-suite via composition | +| **4** | cli dual, pqc (slh_outboard), ots directory paths | **closed** — G10-A composition; pure Lean SLH FFI **R9 closed** | +| **5** | CI freeze both backends; G11 closed; G8 allowlist freeze | **closed** — `just test-lean-ci` + `dual-backend-lean` job | +| **R7** | Full G8 close; freeze = unfiltered lean suite (incl. `bin_*`) | **closed** (2026-07) — see [GAPS.md](./GAPS.md) R7 | +| **R8** | G9 full cross-backend matrix (body/headered/outboard, both directions) | **closed** (2026-07) — `tests/g9_cross_backend.rs` + `tests/fixtures/g9/`; see [GAPS.md](./GAPS.md) R8 | + +### R8 / G9 classification + +| Concern | Detail | +|---------|--------| +| Contract file | `tests/g9_cross_backend.rs` | +| Fixtures | `tests/fixtures/g9/{rust,lean}/` (manifest JSON + binary blobs) | +| lean→rust | default `backend-rust` decodes `lean/*` (no libcarbonado required) | +| rust→lean | `backend-lean` + `CARBONADO_LEAN_LIB` decodes `rust/*`; public + fixed-nonce encrypted body re-encode bit-match | +| Matrix | body c0/c1/c4/c5/c8/c9/c12/c13; headered c4/c5/c12/c13; outboard c4/c5/c12/c13/c14 | +| Residual | **W2a/W2b permanent:** cross-engine Compression / directory encode bit-match; same-engine codecode green; `phase3_g9_directory` decode seed remains SSOT | +| Regen | `just g9-gen-fixtures` or `G9_WRITE_FIXTURES=1` on ignored `write_fixtures` | + +### W2d / determinism classification + +| Concern | Detail | +|---------|--------| +| Contract file | `tests/determinism_roundtrip.rs` | +| Contracts | **codecode** (EDE) + **decodec** (DED) — shipped for claimed matrix | +| Pins | same MASTER/NONCE/`g9_matrix_v1` as G9 | +| Engines | default `backend-rust` + lean freeze (auto-include) | +| Cross-link | [GAPS.md](./GAPS.md) W2 table; [LIMITS.md](./LIMITS.md) W2a/W2b permanent residuals | --- @@ -138,3 +306,4 @@ Document the live allowlist in CI / justfile as it grows. Full suite remains the - New `tests/*.rs` files **must** be added to the classification table above in the same PR. - New public encode/decode surfaces used by tests must be listed in the API table and, if dual-backend-relevant, in [ABI.md](./ABI.md). - Prefer strict `matches!` on specific `CarbonadoError` variants for failure-mode tests; when ABI collapse prevents 1:1 mapping, document backend-aware expectations rather than loosening asserts permanently. +- **R7 freeze is unfiltered** under lean features (`just test-lean-ci` = full `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"`). New contract integration tests **auto-enter** CI `dual-backend-lean` — no allowlist edit required. Rust-only or feature-gated suites **must** use `#![cfg(...)]` (as `streaming_async` / `parallel_determinism` do) and be documented under post-G8 residuals in [GAPS.md](./GAPS.md) / [LIMITS.md](./LIMITS.md); otherwise lean CI will compile and run them. diff --git a/examples/dump_rkyv_r9.rs b/examples/dump_rkyv_r9.rs new file mode 100644 index 0000000..643a7af --- /dev/null +++ b/examples/dump_rkyv_r9.rs @@ -0,0 +1,44 @@ +//! R9 golden dump helper (maintainer); not part of product CLI. +use carbonado::filepack_manifest::*; + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn main() { + let seg = |root_fill: u8, main_len: u64| SegmentRef { + segment_bao_root: [root_fill; 32], + chunk_index: 0, + main_len, + verification_outboard_offset: 0, + verification_outboard_len: 64, + fec_parity_offset: 64, + fec_parity_len: 128, + }; + let e1 = FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100)], + ots_proof: None, + }; + let e2 = FilepackEntry { + rel_path: "b/longer-path-name.txt".into(), // >8 bytes → out-of-line string + content_blake3: [0x33; 32], + segment_format: 0x0E, + segments: vec![seg(0x44, 200)], + ots_proof: Some(vec![0xAB, 0xCD, 0xEF, 0x01]), + }; + let m = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0x55; 32], + catalog_ots_proof: None, + entries: vec![e1, e2], + }; + let b = m.to_bytes().expect("to_bytes"); + println!("MULTI_LEN={}", b.len()); + println!("MULTI_HEX={}", hex(&b)); + std::fs::write("tests/fixtures/rkyv/multi_entry_ots.bin", &b).expect("write"); + println!("wrote tests/fixtures/rkyv/multi_entry_ots.bin"); +} diff --git a/flake.nix b/flake.nix index 04f834f..97ca92c 100644 --- a/flake.nix +++ b/flake.nix @@ -59,10 +59,20 @@ rev = "f8745da6ff1ad1e7bab384bd1f9d742439278e99"; hash = "sha256-tNFWIT9ydfozB8dWcmTMuZLCQmQudTFJIkSr0aG7S44="; }; + # R9 / G10: libbitcoinpqc pin matching ref/bitcoinpqc submodule + # (b309f444… / branch 27-slh-dsa-sha-2-128s). SLH-DSA-SHA2-128s only + # (no secp/ML-DSA) is compiled into libcarbonado_native.a. + bitcoinpqcPinned = pkgs.fetchFromGitHub { + owner = "cryptoquick"; + repo = "libbitcoinpqc"; + rev = "b309f444e383f0e8726c8697128641c7599524f1"; + hash = "sha256-dSGYo3qmq2wSltKjRgcRyrYFZO6YGoIsIG7scEWnQqE="; + }; carbonadoNative = import ./nix/native { inherit pkgs; leanAll = pkgs.lean.lean-all; zstdSrc = zstdPinned; + bitcoinpqcSrc = bitcoinpqcPinned; carbonadoInclude = ./include; }; @@ -70,8 +80,12 @@ name = "carbonado"; # Separate roots so CarbonadoTest compiles without product → test imports. # lean4-nix only discovers modules under the root name of each entry. + # Carbonado.Ffi is a root so `@[export] l_carbonado_*` AOT objects land in + # staticLib even when Main does not import Ffi. roots = [ "Carbonado.Main" + "Carbonado.Ffi" + "Carbonado.RkyvFilepack" "CarbonadoTest.Scaffold" "CarbonadoTest.EtM" "CarbonadoTest.Fec" @@ -84,11 +98,72 @@ src = productSrc; debug = false; leancFlags = ["-O3" "-DNDEBUG"]; - # Static zstd + FFI (no shared libzstd — avoids lld shlib-undefined/pthread). + # Static zstd + C ABI glue (no shared libzstd — avoids lld shlib-undefined/pthread). staticLibDeps = [carbonadoNative]; linkFlags = []; }; + # Dual-backend product archive: Lean AOT objects + native zstd/ABI glue, + # packaged as a shared library (leanc links Lean runtime) plus a static + # archive for `nm` / partial static consumers. + libcarbonado = + pkgs.runCommand "libcarbonado" { + nativeBuildInputs = [pkgs.binutils pkgs.stdenv.cc pkgs.lean.leanc]; + } '' + set -euo pipefail + mkdir -p $out/lib $out/include + + LEAN_A="${leanPkg.staticLib}/libcarbonado.a" + NATIVE_A="${carbonadoNative}/libcarbonado_native.a" + test -f "$LEAN_A" + test -f "$NATIVE_A" + + # Shared library via leanc + Lean shared stdlib (Init/runtime). + # lean4-nix staticLib is a *thin* archive; leanc/ld accept it with whole-archive. + # --whole-archive keeps @[export] + C ABI symbols from being GC'd. + # Pass libleanshared the same way buildLeanPackage.executable does (withSharedStdlib). + ${pkgs.lean.leanc}/bin/leanc -shared -fPIC \ + -Wl,--whole-archive "$LEAN_A" "$NATIVE_A" -Wl,--no-whole-archive \ + ${pkgs.lean.leanshared}/* \ + -o $out/lib/libcarbonado.so + + # Regular static archive for `nm` / consumers: thin member paths + native objects. + WORK=$(mktemp -d) + cd "$WORK" + mapfile -t LEAN_OBJS < <(${pkgs.binutils}/bin/ar t "$LEAN_A") + ${pkgs.binutils}/bin/ar x "$NATIVE_A" + ${pkgs.binutils}/bin/ar rcs $out/lib/libcarbonado.a "''${LEAN_OBJS[@]}" ./*.o + + cp ${./include}/carbonado.h $out/include/ + echo "libcarbonado: packaged static + shared" >&2 + ''; + + leanAbiCheck = + pkgs.runCommand "carbonado-lean-abi" { + nativeBuildInputs = [pkgs.binutils]; + } '' + set -euo pipefail + test -f ${libcarbonado}/include/carbonado.h + test -f ${libcarbonado}/lib/libcarbonado.a + test -f ${libcarbonado}/lib/libcarbonado.so + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_abi_version + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_free + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_encode + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_decode + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_encode_headered + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_decode_headered + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_verification_key + nm ${libcarbonado}/lib/libcarbonado.a | grep -q l_carbonado_encode_headered + nm ${libcarbonado}/lib/libcarbonado.a | grep -q l_carbonado_verification_key + # R9 / G10 SLH + seekable outboard slice + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_slh_keygen + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_slh_sign + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_slh_verify + nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_verify_slice_outboard + nm ${libcarbonado}/lib/libcarbonado.a | grep -q l_carbonado_verify_slice_outboard + echo ok > $out + ''; + noSorry = pkgs.runCommand "carbonado-no-sorry" { src = productSrc; @@ -221,11 +296,14 @@ grep -q "zstd goldens + roundtrip + error paths ok" $out grep -q "pipeline compression formats c2/c6 + headered c3/c7 ok" $out grep -q "SLH1 wire framing ok" $out - grep -q "SLH bind-to-root + unavailable sign ok" $out + grep -q "SLH live sign/verify ok" $out grep -q "program F stack ok" $out + grep -q "outboard slice verify ok" $out # Program G grep -q "adamantine wire ok" $out grep -q "filepack path rules ok" $out + grep -q "rkyv FilepackManifestWire encode/decode goldens ok" $out + # multi-entry + OTS + trunc covered in same Main block grep -q "outboard segment roundtrip ok" $out grep -q "directory pure encode/decode ok" $out grep -q "directory exact failure modes ok" $out @@ -261,29 +339,6 @@ overlays = [(lean4-nix.readToolchainFile ./lean-toolchain)]; }; - # Dual-backend: static lib + header for Rust `backend-lean` / carbonado-sys. - libcarbonado = - pkgs.runCommand "libcarbonado" {} '' - set -euo pipefail - mkdir -p $out/lib $out/include - cp ${carbonadoNative}/libcarbonado_native.a $out/lib/libcarbonado.a - cp ${carbonadoNative}/include/carbonado.h $out/include/ - cp ${carbonadoNative}/libcarbonado_native.a $out/lib/libcarbonado_native.a - ''; - - leanAbiCheck = - pkgs.runCommand "carbonado-lean-abi" { - nativeBuildInputs = [pkgs.binutils]; - } '' - set -euo pipefail - test -f ${libcarbonado}/include/carbonado.h - test -f ${libcarbonado}/lib/libcarbonado.a - # Symbols from C ABI stubs (encode may be weak NOT_IMPLEMENTED). - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_abi_version - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_free - echo ok > $out - ''; - packages = { default = leanPkg.executable; carbonado = leanPkg.executable; @@ -322,8 +377,14 @@ echo "carbonado Lean 4 + Nix dev shell" echo "Lean toolchain pin: $(cat lean-toolchain)" echo "Build: nix build .#carbonado" + echo "Lib: nix build .#libcarbonado" echo "Check: nix flake check" echo "Run: nix run" + echo "Lean backend tests:" + echo " nix build .#libcarbonado -o result-libcarbonado" + echo " export CARBONADO_LEAN_LIB=\$PWD/result-libcarbonado/lib CARBONADO_LEAN_INCLUDE=\$PWD/result-libcarbonado/include" + echo " export LD_LIBRARY_PATH=\$CARBONADO_LEAN_LIB" + echo " cargo test --no-default-features --features \"backend-lean,pqc,ots\" --test lean_backend_smoke" fi ''; }; diff --git a/include/carbonado.h b/include/carbonado.h index 633d9e3..dc6faa3 100644 --- a/include/carbonado.h +++ b/include/carbonado.h @@ -2,7 +2,7 @@ * carbonado C ABI — Lean AOT engine (libcarbonado) * * See docs/ABI.md for ownership, error codes, and versioning. - * ABI version 1 (v0 surface). + * ABI version 1 (v0 core + Phase 2 additive outboard/scrub/slice). */ #ifndef CARBONADO_H #define CARBONADO_H @@ -29,6 +29,8 @@ extern "C" { #define CARBONADO_ERR_SCRUB_FAILED 10 #define CARBONADO_ERR_NOT_IMPLEMENTED 11 #define CARBONADO_ERR_INTERNAL 12 +/** Scrub called without Verification bit (distinct from recovery failure). */ +#define CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION 13 /** Returns CARBONADO_ABI_VERSION. */ uint32_t carbonado_abi_version(void); @@ -40,6 +42,8 @@ void carbonado_free(void *p); * Low-level encode (Rust encoding::encode body shape). * On success: *out is malloc'd body, hash_out is 32-byte Bao root. * Encrypted formats require nonce_len == 16. + * padding/chunk/ecc/vsc/compressed/encrypted out-params may be NULL. + * bytes_compressed / bytes_encrypted are 0 when those stages are skipped (R3). */ int carbonado_encode( const uint8_t *master, size_t master_len, @@ -47,7 +51,13 @@ int carbonado_encode( uint8_t format, const uint8_t *nonce, size_t nonce_len, uint8_t **out, size_t *out_len, - uint8_t hash_out[32]); + uint8_t hash_out[32], + uint32_t *padding_out, + uint32_t *chunk_len_out, + uint32_t *bytes_ecc_out, + uint32_t *verifiable_slice_count_out, + uint32_t *bytes_compressed_out, + uint32_t *bytes_encrypted_out); /** * Low-level decode of a verifiable body (hash + padding + format). @@ -62,13 +72,26 @@ int carbonado_decode( /** * Headered encode: full file Header || body (Rust file::encode shape). + * slh_pk: NULL → zero-filled 32 B field; non-NULL must point to exactly 32 valid bytes + * (C always copies 32 when non-NULL; wrong lengths are Lean ByteArray-only). + * metadata: NULL → zero-filled 8 B field; non-NULL must point to exactly 8 valid bytes + * (C always copies 8 when non-NULL; wrong lengths are Lean ByteArray-only). + * Stage-counter out-params (nullable): padding/chunk/ecc/vsc/compressed/encrypted (R3). */ int carbonado_encode_headered( const uint8_t *master, size_t master_len, const uint8_t *plaintext, size_t plaintext_len, uint8_t format, const uint8_t *nonce, size_t nonce_len, - uint8_t **out, size_t *out_len); + const uint8_t *slh_pk, + const uint8_t *metadata, + uint8_t **out, size_t *out_len, + uint32_t *padding_out, + uint32_t *chunk_len_out, + uint32_t *bytes_ecc_out, + uint32_t *verifiable_slice_count_out, + uint32_t *bytes_compressed_out, + uint32_t *bytes_encrypted_out); /** * Headered decode: full file archive → plaintext. @@ -81,6 +104,135 @@ int carbonado_decode_headered( /** Format-keyed verification key (32 bytes). */ int carbonado_verification_key(uint8_t format, uint8_t key_out[32]); +/** + * Outboard encode: bare main + optional verification outboard + FEC parity sidecars. + * Any of main_out / outboard_out / parity_out must be non-NULL with matching len ptr. + * Empty sidecars return *out=NULL, *out_len=0. + * + * header_path != 0: encrypted bare main is [tag|ct] (nonce out-of-band; file::encode_outboard). + * header_path == 0: encrypted bare main is [nonce|tag|ct] (encoding::encode_outboard). + * Encrypted formats require nonce_len == 16. + * bytes_compressed_out / bytes_encrypted_out may be NULL (0 when stage skipped; R3). + */ +int carbonado_encode_outboard( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, + uint8_t header_path, + uint8_t **main_out, size_t *main_len, + uint8_t **outboard_out, size_t *outboard_len, + uint8_t **parity_out, size_t *parity_len, + uint8_t hash_out[32], + uint32_t *padding_out, + uint32_t *chunk_len_out, + uint32_t *bytes_compressed_out, + uint32_t *bytes_encrypted_out); + +/** + * Outboard decode: bare main + optional sidecars → plaintext. + * outboard/parity may be null with len 0 when the format does not require them. + * + * header_path / nonce must match encode-time layout (see carbonado_encode_outboard). + */ +int carbonado_decode_outboard( + const uint8_t *master, size_t master_len, + const uint8_t *hash, size_t hash_len, + const uint8_t *main, size_t main_len, + const uint8_t *outboard, size_t outboard_len, + const uint8_t *parity, size_t parity_len, + uint32_t padding, + uint8_t format, + uint8_t header_path, + const uint8_t *nonce, size_t nonce_len, + uint8_t **out, size_t *out_len); + +/** + * Inboard scrub: recover damaged Bao+FEC body (or SCRUB_UNNECESSARY / REQUIRES_VERIFICATION). + */ +int carbonado_scrub( + const uint8_t *body, size_t body_len, + const uint8_t *hash, size_t hash_len, + uint32_t padding, + uint8_t format, + uint8_t **out, size_t *out_len); + +/** + * Outboard scrub: recover damaged bare main using outboard + FEC parity. + */ +int carbonado_scrub_outboard( + const uint8_t *main, size_t main_len, + const uint8_t *outboard, size_t outboard_len, + const uint8_t *parity, size_t parity_len, + const uint8_t *hash, size_t hash_len, + uint32_t padding, + uint32_t chunk_len, + uint8_t format, + uint8_t **out, size_t *out_len); + +/** + * Inboard verify_slice / extract_slice: authenticated slice bytes from inboard body. + * + * W4a: Lean retains O(slice) output while walking the full inboard response for + * auth (O(N) time). Caller still supplies the full body buffer (input). + * count==0 returns empty after full auth (Lean auth-first path). + */ +int carbonado_verify_slice( + const uint8_t *body, size_t body_len, + const uint8_t *hash, size_t hash_len, + uint32_t index, + uint32_t count, + uint8_t format, + uint8_t **out, size_t *out_len); + +/** + * Seekable outboard verify_slice: authenticated slice bytes from bare main + + * post-order outboard sidecar (keyed Bao, 4 KiB groups). + * + * Time/hash work is O(slice + tree height) over the requested ranges (not full + * re-encode). C ABI still takes full main + outboard buffers in memory — W4b + * permanent residual (no streaming ReadAt / callback ABI); see docs/LIMITS.md. + * + * count==0 → empty success immediately (no auth / geometry / OOB checks) — + * matches Rust `verify_slice_outboard` extract semantics. OOB index and + * authentication apply only when count > 0. + */ +int carbonado_verify_slice_outboard( + const uint8_t *main, size_t main_len, + const uint8_t *outboard, size_t outboard_len, + const uint8_t *hash, size_t hash_len, + uint32_t index, + uint32_t count, + uint8_t format, + uint8_t **out, size_t *out_len); + +/** + * SLH-DSA-SHA2-128s keygen (G10). entropy_len must be ≥ 128. + * pk_out: 32 bytes; sk_out: 64 bytes (caller-owned stack/heap buffers). + */ +int carbonado_slh_keygen( + const uint8_t *entropy, size_t entropy_len, + uint8_t pk_out[32], + uint8_t sk_out[64]); + +/** + * SLH-DSA-SHA2-128s sign. secret_key_len must be 64. + * On success: *out is malloc'd 7856-byte signature (free with carbonado_free). + */ +int carbonado_slh_sign( + const uint8_t *secret_key, size_t secret_key_len, + const uint8_t *message, size_t message_len, + uint8_t **out, size_t *out_len); + +/** + * SLH-DSA-SHA2-128s verify. public_key_len 32; signature_len 7856. + * Returns CARBONADO_OK on accept, CARBONADO_ERR_AUTHENTICATION on reject. + */ +int carbonado_slh_verify( + const uint8_t *public_key, size_t public_key_len, + const uint8_t *message, size_t message_len, + const uint8_t *signature, size_t signature_len); + #ifdef __cplusplus } #endif diff --git a/justfile b/justfile index efd1768..0fb025f 100644 --- a/justfile +++ b/justfile @@ -46,12 +46,23 @@ fmt: fmt-fix: cargo fmt +# rustfmt has no `-W`; `--check` is the fail-if-unformatted equivalent of `cargo fmt --all -W`. +# Never `--all-features` on clippy: that enables both `backend-rust` and `backend-lean` +# and hits `compile_error!`. This is the rust-compatible stand-in (same as `just lint` / CI). +# Rust-only gate: fmt --check, clippy (rust features), nextest. Stops on first failure. +check: + cargo fmt --all -- --check + cargo clippy --all-targets --features "async,async-tokio,man-gen" -- -D warnings + cargo nextest run + # Clippy + project-specific source checks (things clippy does not know about). lint: _clippy _lint-source +# Never use `--all-features` here: that enables both `backend-rust` and `backend-lean` → compile_error!. +# Cover optional features mutually compatible with default `backend-rust`. [private] _clippy: - cargo clippy --all-targets --all-features -- -D warnings + cargo clippy --all-targets --features "async,async-tokio,man-gen" -- -D warnings [private] _lint-source: @@ -67,17 +78,23 @@ _lint-source: failures=0 pass() { echo -e "${GREEN}PASS${NC}: $1"; } fail() { echo -e "${RED}FAIL${NC}: $1"; failures=$((failures + 1)); } + # Skip #[cfg(test)] modules (any name) and bare `mod tests { ... }` blocks. + # Important: do not clear in_test on the cfg line itself (depth starts at 0). scan_non_test_src() { local mode="$1" find src -name '*.rs' -print0 | while IFS= read -r -d '' f; do awk -v mode="$mode" ' - /#\[cfg\(test\)\]/ { in_test = 1 } - /^[[:space:]]*mod tests[[:space:]]*\{/ && !in_test { in_test = 1; depth = 1; next } + /#\[cfg\(test\)\]/ { in_test = 1; entered = 0; depth = 0; next } + /^[[:space:]]*(pub[[:space:]]+)?mod[[:space:]]+tests[[:space:]]*\{/ && !in_test { + in_test = 1; entered = 0; depth = 0 + } in_test { - nopen = gsub(/\{/, "{") - nclose = gsub(/\}/, "}") + line = $0 + nopen = gsub(/\{/, "{", line) + nclose = gsub(/\}/, "}", line) depth += nopen - nclose - if (depth <= 0) in_test = 0 + if (nopen > 0) entered = 1 + if (entered && depth <= 0) { in_test = 0; entered = 0; depth = 0 } next } { @@ -135,13 +152,37 @@ _lint-source: rg -n 'MAGICNO' src/constants.rs 2>/dev/null | sed 's/^/ /' || true fi echo "" - echo "--- 4. NotImplemented not on crypto paths ---" - notimpl_returns=$(rg -n 'CarbonadoError::NotImplemented|Err\([^)]*NotImplemented' src/ 2>/dev/null || true) - if [[ -z "$notimpl_returns" ]]; then - pass "No NotImplemented returns in src/ (enum variant may exist for future use)" + echo "--- 4. NotImplemented only on intentional residual / map sites ---" + # Allowed (documented dual-backend / platform residuals — not silent crypto stubs): + # - error.rs enum variant definition + # - backend lean ABI code → CarbonadoError map arm + # - stream_decode_async on wasm32 (documented NotImplemented residual) + # - doc comments mentioning the variant + # (R2: file::encode metadata/SLH are plumbed — no longer NotImplemented) + notimpl_hits=$(rg -n 'CarbonadoError::NotImplemented|Err\([^)]*NotImplemented' src/ 2>/dev/null || true) + notimpl_bad="" + if [[ -n "$notimpl_hits" ]]; then + notimpl_bad=$(echo "$notimpl_hits" | while IFS= read -r line; do + # comments / docs + if echo "$line" | rg -q '^\S+:\d+:[[:space:]]*(//|///|\*)'; then continue; fi + # enum variant + if echo "$line" | rg -q 'src/error\.rs:'; then continue; fi + # match-arm mapping from C ABI + if echo "$line" | rg -q '=>[[:space:]]*CarbonadoError::NotImplemented'; then continue; fi + # intentional wasm async residual + if echo "$line" | rg -q 'src/stream/decode_async\.rs:'; then continue; fi + echo "$line" + done || true) + fi + if [[ -z "$notimpl_bad" ]]; then + pass "NotImplemented only at allowlisted residual/map sites (dual-backend + wasm async)" + if [[ -n "$notimpl_hits" ]]; then + echo " Allowlisted evidence:" + echo "$notimpl_hits" | sed 's/^/ /' + fi else - fail "NotImplemented returned in src/" - echo "$notimpl_returns" | sed 's/^/ /' + fail "Unexpected NotImplemented returns in src/ (not on allowlist)" + echo "$notimpl_bad" | sed 's/^/ /' fi echo "" echo "--- 5. No todo!/unimplemented! in production src/ ---" @@ -191,17 +232,19 @@ _lint-source: exit 1 fi +# wasm32: always name backend-rust under --no-default-features (mutual exclusion). lint-wasm: - cargo clippy --target wasm32-unknown-unknown --no-default-features --features "" -- -D warnings + cargo clippy --target wasm32-unknown-unknown --no-default-features --features "backend-rust" -- -D warnings -# Default features (includes `parallel`), serial FEC regression, then full feature matrix. +# Default features (includes `parallel`), serial FEC, then backend-rust + optional features. +# Never `--all-features` (enables both backends → compile_error!). test: cargo test - cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path - cargo test --all-features + cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path + cargo test --features "async,async-tokio,man-gen" test-serial: - cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path + cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path test-parallel: cargo test --test parallel_determinism @@ -210,6 +253,105 @@ test-parallel: test-smoke: cargo test --test streaming --test seekable_slices --test sharding --test bao_keyed_contract +# Shared lean env: build libcarbonado if CARBONADO_LEAN_LIB unset; fail-closed if .so/.dylib missing. +# stdout: only `export …` lines (safe for `eval "$(just _lean-env)"`); diagnostics on stderr. +[private] +_lean-env: + #!/usr/bin/env bash + set -euo pipefail + if [[ -z "${CARBONADO_LEAN_LIB:-}" ]]; then + # Dedicated symlink so other `nix build` targets do not clobber `result/`. + nix build .#libcarbonado -o result-libcarbonado + export CARBONADO_LEAN_LIB="$PWD/result-libcarbonado/lib" + export CARBONADO_LEAN_INCLUDE="$PWD/result-libcarbonado/include" + fi + if [[ ! -f "${CARBONADO_LEAN_LIB}/libcarbonado.so" && ! -f "${CARBONADO_LEAN_LIB}/libcarbonado.dylib" ]]; then + echo "FATAL: libcarbonado shared library missing under CARBONADO_LEAN_LIB=${CARBONADO_LEAN_LIB}" >&2 + echo " Build: nix build .#libcarbonado -o result-libcarbonado" >&2 + echo " Then: export CARBONADO_LEAN_LIB=\$PWD/result-libcarbonado/lib" >&2 + echo " export CARBONADO_LEAN_INCLUDE=\$PWD/result-libcarbonado/include" >&2 + exit 1 + fi + if [[ -z "${CARBONADO_LEAN_INCLUDE:-}" ]]; then + if [[ -d "$(dirname "${CARBONADO_LEAN_LIB}")/include" ]]; then + export CARBONADO_LEAN_INCLUDE="$(dirname "${CARBONADO_LEAN_LIB}")/include" + else + echo "FATAL: CARBONADO_LEAN_INCLUDE unset and cannot infer from CARBONADO_LEAN_LIB" >&2 + exit 1 + fi + fi + export LD_LIBRARY_PATH="${CARBONADO_LEAN_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + echo "CARBONADO_LEAN_LIB=$CARBONADO_LEAN_LIB" >&2 + echo "CARBONADO_LEAN_INCLUDE=$CARBONADO_LEAN_INCLUDE" >&2 + printf 'export CARBONADO_LEAN_LIB=%q\n' "$CARBONADO_LEAN_LIB" + printf 'export CARBONADO_LEAN_INCLUDE=%q\n' "$CARBONADO_LEAN_INCLUDE" + printf 'export LD_LIBRARY_PATH=%q\n' "$LD_LIBRARY_PATH" + +# Dual-backend Phase 1: build libcarbonado and run lean allowlist smoke. +test-lean-smoke: + #!/usr/bin/env bash + set -euo pipefail + eval "$(just _lean-env)" + cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_smoke + +# Dual-backend Phase 2: outboard/scrub/slice + G9 buffer seeds (+ Phase 1 smoke). +test-lean-phase2: + #!/usr/bin/env bash + set -euo pipefail + eval "$(just _lean-env)" + cargo test --no-default-features --features "backend-lean,pqc,ots" \ + --test lean_backend_smoke --test lean_backend_phase2 + +# Dual-backend Phase 3: directory composition (rkyv catalog + Lean segment/catalog crypto). +test-lean-phase3: + #!/usr/bin/env bash + set -euo pipefail + eval "$(just _lean-env)" + cargo test --no-default-features --features "backend-lean,pqc,ots" \ + --test lean_backend_smoke --test lean_backend_phase2 --test lean_backend_phase3 \ + --test format_policy + +# Dual-backend Phase 4: SLH composition (G10-A) + CLI dual path + directory OTS. +# `cli` enables lean-linked binary: directory subprocess = dual-engine; single-file stream = link smoke. +test-lean-phase4: + #!/usr/bin/env bash + set -euo pipefail + eval "$(just _lean-env)" + cargo test --no-default-features --features "backend-lean,pqc,ots,cli" \ + --test lean_backend_smoke --test lean_backend_phase2 --test lean_backend_phase3 \ + --test format_policy --test slh_outboard --test lean_backend_phase4 + +# Dual-backend Phase 5 / G11 + R7 G8 full close: shared CI + human gate. +# Freeze = full dual suite under lean features (G8 closed 2026-07 R7). See docs/GAPS.md. +# Permanent feature-gated exclusions under this feature set (0 tests, not dual residual): +# streaming_async needs `async` (R10 closed: freeze never requires async; lean+async dual-aware); +# parallel_determinism needs `parallel` (Lean RS serial). +# Post-G8 residuals (not dual-suite failures): stream E2, file::decode_stream pure-Rust, +# pure Lean rkyv encode residual — composition paths remain SSOT for those layers. +test-lean-ci: + #!/usr/bin/env bash + set -euo pipefail + eval "$(just _lean-env)" + # Full dual suite (lib units + all integration tests, including bin_*). Never add async. + cargo test --no-default-features --features "backend-lean,pqc,ots,cli" + +# G9 / R8: cross-backend matrix both directions (lean fixtures → rust; rust fixtures → lean). +test-g9: + #!/usr/bin/env bash + set -euo pipefail + cargo test --test g9_cross_backend + eval "$(just _lean-env)" + cargo test --no-default-features --features "backend-lean,pqc,ots" --test g9_cross_backend + +# Regenerate G9 goldens under tests/fixtures/g9/{rust,lean}/ (requires libcarbonado for lean). +g9-gen-fixtures: + #!/usr/bin/env bash + set -euo pipefail + G9_WRITE_FIXTURES=1 cargo test --test g9_cross_backend write_fixtures -- --ignored --nocapture + eval "$(just _lean-env)" + G9_WRITE_FIXTURES=1 cargo test --no-default-features --features "backend-lean,pqc,ots" \ + --test g9_cross_backend write_fixtures -- --ignored --nocapture + build: cargo build --bin carbonado --release diff --git a/nix/native/carbonado_abi.c b/nix/native/carbonado_abi.c index 3b7d713..f8042e4 100644 --- a/nix/native/carbonado_abi.c +++ b/nix/native/carbonado_abi.c @@ -1,17 +1,92 @@ /** * C ABI surface for libcarbonado (docs/ABI.md, include/carbonado.h). * - * Phase 1: version + free + thin wrappers. Full encode/decode path is driven from - * Lean `@[export]` symbols when linked into the AOT image; until those symbols are - * part of the shared static archive used by carbonado-sys, encode/decode return - * CARBONADO_ERR_NOT_IMPLEMENTED so Rust can fail closed instead of linking garbage. + * Phase 1+2 + R3: strong symbols for encode/decode/headered/verification_key plus + * outboard/scrub/slice that call Lean `@[export]` helpers (`l_carbonado_*` + * from Carbonado/Ffi.lean). Lean runtime is initialized once on first use. + * + * Lean pack layouts are internal to libcarbonado (co-versioned with this C glue); + * the public C API stays additive at ABI version 1 (nullable out-params). + * + * Packed Lean success layouts (errors are status-first: [u32 LE status] only): + * status payload: [u32 LE status][bytes…] + * encode body: [u32 LE status][pad:4][chunk:4][ecc:4][vsc:4] + * [comp:4][enc:4][32 hash][body…] (prefix 60) + * encode headered: [u32 LE status][pad:4][chunk:4][ecc:4][vsc:4] + * [comp:4][enc:4][archive…] (prefix 28) + * encode outboard: [u32 LE status][pad:4][chunk:4][comp:4][enc:4] + * [32 hash][u32 main_len][main][u32 ob_len][ob] + * [u32 par_len][par] (fixed prefix 52) */ +#include +#include #include #include #include #include "carbonado.h" +/* Lean runtime (symbols in Lean Init / leanrt). */ +extern void lean_initialize_runtime_module(void); +extern void lean_io_mark_end_initialization(void); +extern bool lean_io_result_is_ok(b_lean_obj_arg r); +extern void lean_io_result_show_error(b_lean_obj_arg r); + +/* Module initializer generated for Carbonado.Ffi (chains Pipeline deps). */ +extern lean_obj_res initialize_Carbonado_Ffi(uint8_t builtin); + +/* @[export] helpers from Carbonado/Ffi.lean — callee owns arguments. */ +extern lean_obj_res l_carbonado_verification_key(uint8_t format); +extern lean_obj_res l_carbonado_encode_headered(lean_obj_arg master, lean_obj_arg nonce, + lean_obj_arg plaintext, lean_obj_arg slh_pk, + lean_obj_arg metadata, uint8_t format); +extern lean_obj_res l_carbonado_decode_headered(lean_obj_arg master, lean_obj_arg archive); +extern lean_obj_res l_carbonado_encode(lean_obj_arg master, lean_obj_arg nonce, + lean_obj_arg plaintext, uint8_t format); +extern lean_obj_res l_carbonado_decode(lean_obj_arg master, lean_obj_arg hash, + lean_obj_arg body, uint32_t padding, uint8_t format); +extern lean_obj_res l_carbonado_encode_outboard(lean_obj_arg master, lean_obj_arg nonce, + lean_obj_arg plaintext, uint8_t format, + uint8_t header_path); +extern lean_obj_res l_carbonado_decode_outboard(lean_obj_arg master, lean_obj_arg hash, + lean_obj_arg main, lean_obj_arg ver_outboard, + lean_obj_arg fec_parity, uint32_t padding, + uint8_t format, uint8_t header_path, + lean_obj_arg nonce); +extern lean_obj_res l_carbonado_scrub(lean_obj_arg body, lean_obj_arg hash, uint32_t padding, + uint8_t format); +extern lean_obj_res l_carbonado_scrub_outboard(lean_obj_arg main, lean_obj_arg ver_outboard, + lean_obj_arg fec_parity, lean_obj_arg hash, + uint32_t padding, uint32_t chunk_len, + uint8_t format); +extern lean_obj_res l_carbonado_verify_slice(lean_obj_arg body, lean_obj_arg hash, + uint32_t index, uint32_t count, uint8_t format); +extern lean_obj_res l_carbonado_verify_slice_outboard(lean_obj_arg main, lean_obj_arg outboard, + lean_obj_arg hash, uint32_t index, + uint32_t count, uint8_t format); + +static pthread_once_t g_lean_once = PTHREAD_ONCE_INIT; +static int g_lean_init_rc = -1; + +static void lean_init_once(void) { + lean_initialize_runtime_module(); + lean_obj_res res = initialize_Carbonado_Ffi(1); + if (!lean_io_result_is_ok(res)) { + lean_io_result_show_error(res); + lean_dec(res); + g_lean_init_rc = -1; + return; + } + lean_dec_ref(res); + lean_io_mark_end_initialization(); + g_lean_init_rc = 0; +} + +static int ensure_lean(void) { + (void)pthread_once(&g_lean_once, lean_init_once); + return g_lean_init_rc; +} + uint32_t carbonado_abi_version(void) { return CARBONADO_ABI_VERSION; } @@ -20,54 +95,631 @@ void carbonado_free(void *p) { free(p); } -/* Weak stubs: real implementations may be provided by Lean @[export] objects - * when the full static archive is linked. These provide a defined symbol so - * partial links still resolve. */ -__attribute__((weak)) int carbonado_encode( +/* Build a Lean ByteArray; takes a copy of `data` (may be NULL when len==0). */ +static lean_object *mk_byte_array(const uint8_t *data, size_t len) { + lean_object *ba = lean_alloc_sarray(1, len, len); + if (len > 0) { + if (data == NULL) { + lean_dec(ba); + return NULL; + } + memcpy(lean_sarray_cptr(ba), data, len); + } + return ba; +} + +static uint32_t read_u32_le(const uint8_t *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | + ((uint32_t)p[3] << 24); +} + +/* Copy Lean ByteArray payload into malloc'd buffer. */ +static int copy_sarray_payload(lean_object *ba, size_t offset, uint8_t **out, size_t *out_len) { + size_t n = lean_sarray_size(ba); + if (offset > n) { + return CARBONADO_ERR_INTERNAL; + } + size_t len = n - offset; + if (len == 0) { + *out = NULL; + *out_len = 0; + return CARBONADO_OK; + } + uint8_t *buf = (uint8_t *)malloc(len); + if (buf == NULL) { + return CARBONADO_ERR_INTERNAL; + } + memcpy(buf, lean_sarray_cptr(ba) + offset, len); + *out = buf; + *out_len = len; + return CARBONADO_OK; +} + +/* Unpack `[status:4][payload…]` → malloc payload on OK. */ +static int unpack_status_payload(lean_object *packed, uint8_t **out, size_t *out_len) { + size_t n = lean_sarray_size(packed); + if (n < 4) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + const uint8_t *p = lean_sarray_cptr(packed); + uint32_t status = read_u32_le(p); + if (status != CARBONADO_OK) { + lean_dec(packed); + return (int)status; + } + int rc = copy_sarray_payload(packed, 4, out, out_len); + lean_dec(packed); + return rc; +} + +/* Read length-prefixed segment; advances *off. */ +static int read_len_prefixed(const uint8_t *p, size_t n, size_t *off, uint8_t **out, + size_t *out_len) { + if (*off + 4 > n) { + return CARBONADO_ERR_INTERNAL; + } + uint32_t len = read_u32_le(p + *off); + *off += 4; + if (*off + len > n) { + return CARBONADO_ERR_INTERNAL; + } + if (len == 0) { + *out = NULL; + *out_len = 0; + return CARBONADO_OK; + } + uint8_t *buf = (uint8_t *)malloc(len); + if (buf == NULL) { + return CARBONADO_ERR_INTERNAL; + } + memcpy(buf, p + *off, len); + *off += len; + *out = buf; + *out_len = len; + return CARBONADO_OK; +} + +int carbonado_verification_key(uint8_t format, uint8_t key_out[32]) { + if (key_out == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + lean_obj_res ba = l_carbonado_verification_key(format); + if (lean_sarray_size(ba) != 32) { + lean_dec(ba); + return CARBONADO_ERR_INTERNAL; + } + memcpy(key_out, lean_sarray_cptr(ba), 32); + lean_dec(ba); + return CARBONADO_OK; +} + +int carbonado_encode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *plaintext, size_t plaintext_len, + uint8_t format, + const uint8_t *nonce, size_t nonce_len, + const uint8_t *slh_pk, + const uint8_t *metadata, + uint8_t **out, size_t *out_len, + uint32_t *padding_out, + uint32_t *chunk_len_out, + uint32_t *bytes_ecc_out, + uint32_t *verifiable_slice_count_out, + uint32_t *bytes_compressed_out, + uint32_t *bytes_encrypted_out) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (padding_out) *padding_out = 0; + if (chunk_len_out) *chunk_len_out = 0; + if (bytes_ecc_out) *bytes_ecc_out = 0; + if (verifiable_slice_count_out) *verifiable_slice_count_out = 0; + if (bytes_compressed_out) *bytes_compressed_out = 0; + if (bytes_encrypted_out) *bytes_encrypted_out = 0; + if (master == NULL || (plaintext == NULL && plaintext_len != 0)) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (nonce == NULL && nonce_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *m = mk_byte_array(master, master_len); + lean_object *n = mk_byte_array(nonce, nonce_len); + lean_object *pt = mk_byte_array(plaintext, plaintext_len); + /* Empty ByteArray when null → Lean zeros SLH/meta fields. */ + lean_object *slh = mk_byte_array(slh_pk, slh_pk != NULL ? 32 : 0); + lean_object *meta = mk_byte_array(metadata, metadata != NULL ? 8 : 0); + if (m == NULL || n == NULL || pt == NULL || slh == NULL || meta == NULL) { + if (m) lean_dec(m); + if (n) lean_dec(n); + if (pt) lean_dec(pt); + if (slh) lean_dec(slh); + if (meta) lean_dec(meta); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_encode_headered(m, n, pt, slh, meta, format); + size_t nlen = lean_sarray_size(packed); + /* Status-first: errors are packed as [status:4] only (see packEncodeErr). */ + if (nlen < 4) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + const uint8_t *p = lean_sarray_cptr(packed); + uint32_t status = read_u32_le(p); + if (status != CARBONADO_OK) { + lean_dec(packed); + return (int)status; + } + /* Success: status(4)+pad(4)+chunk(4)+ecc(4)+vsc(4)+comp(4)+enc(4)+archive = 28 + archive. */ + if (nlen < 28) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + if (padding_out) *padding_out = read_u32_le(p + 4); + if (chunk_len_out) *chunk_len_out = read_u32_le(p + 8); + if (bytes_ecc_out) *bytes_ecc_out = read_u32_le(p + 12); + if (verifiable_slice_count_out) *verifiable_slice_count_out = read_u32_le(p + 16); + if (bytes_compressed_out) *bytes_compressed_out = read_u32_le(p + 20); + if (bytes_encrypted_out) *bytes_encrypted_out = read_u32_le(p + 24); + int rc = copy_sarray_payload(packed, 28, out, out_len); + lean_dec(packed); + return rc; +} + +int carbonado_decode_headered( + const uint8_t *master, size_t master_len, + const uint8_t *archive, size_t archive_len, + uint8_t **out, size_t *out_len) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (master == NULL || (archive == NULL && archive_len != 0)) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *m = mk_byte_array(master, master_len); + lean_object *a = mk_byte_array(archive, archive_len); + if (m == NULL || a == NULL) { + if (m) lean_dec(m); + if (a) lean_dec(a); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_decode_headered(m, a); + return unpack_status_payload(packed, out, out_len); +} + +int carbonado_encode( const uint8_t *master, size_t master_len, const uint8_t *plaintext, size_t plaintext_len, uint8_t format, const uint8_t *nonce, size_t nonce_len, uint8_t **out, size_t *out_len, - uint8_t hash_out[32]) { - (void)master; (void)master_len; (void)plaintext; (void)plaintext_len; - (void)format; (void)nonce; (void)nonce_len; (void)out; (void)out_len; (void)hash_out; - return CARBONADO_ERR_NOT_IMPLEMENTED; + uint8_t hash_out[32], + uint32_t *padding_out, + uint32_t *chunk_len_out, + uint32_t *bytes_ecc_out, + uint32_t *verifiable_slice_count_out, + uint32_t *bytes_compressed_out, + uint32_t *bytes_encrypted_out) { + if (out == NULL || out_len == NULL || hash_out == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (padding_out) *padding_out = 0; + if (chunk_len_out) *chunk_len_out = 0; + if (bytes_ecc_out) *bytes_ecc_out = 0; + if (verifiable_slice_count_out) *verifiable_slice_count_out = 0; + if (bytes_compressed_out) *bytes_compressed_out = 0; + if (bytes_encrypted_out) *bytes_encrypted_out = 0; + if (master == NULL || (plaintext == NULL && plaintext_len != 0)) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (nonce == NULL && nonce_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *m = mk_byte_array(master, master_len); + lean_object *n = mk_byte_array(nonce, nonce_len); + lean_object *pt = mk_byte_array(plaintext, plaintext_len); + if (m == NULL || n == NULL || pt == NULL) { + if (m) lean_dec(m); + if (n) lean_dec(n); + if (pt) lean_dec(pt); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_encode(m, n, pt, format); + size_t nlen = lean_sarray_size(packed); + /* Status-first: errors are packed as [status:4] only (see packEncodeErr). */ + if (nlen < 4) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + const uint8_t *p = lean_sarray_cptr(packed); + uint32_t status = read_u32_le(p); + if (status != CARBONADO_OK) { + lean_dec(packed); + return (int)status; + } + /* Success: status(4)+pad(4)+chunk(4)+ecc(4)+vsc(4)+comp(4)+enc(4)+hash(32)+body = 60 + body. */ + if (nlen < 60) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + if (padding_out) *padding_out = read_u32_le(p + 4); + if (chunk_len_out) *chunk_len_out = read_u32_le(p + 8); + if (bytes_ecc_out) *bytes_ecc_out = read_u32_le(p + 12); + if (verifiable_slice_count_out) *verifiable_slice_count_out = read_u32_le(p + 16); + if (bytes_compressed_out) *bytes_compressed_out = read_u32_le(p + 20); + if (bytes_encrypted_out) *bytes_encrypted_out = read_u32_le(p + 24); + memcpy(hash_out, p + 28, 32); + int rc = copy_sarray_payload(packed, 60, out, out_len); + lean_dec(packed); + return rc; } -__attribute__((weak)) int carbonado_decode( +int carbonado_decode( const uint8_t *master, size_t master_len, const uint8_t *hash, size_t hash_len, const uint8_t *body, size_t body_len, uint32_t padding, uint8_t format, uint8_t **out, size_t *out_len) { - (void)master; (void)master_len; (void)hash; (void)hash_len; - (void)body; (void)body_len; (void)padding; (void)format; (void)out; (void)out_len; - return CARBONADO_ERR_NOT_IMPLEMENTED; + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (master == NULL || hash == NULL || hash_len != 32) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (body == NULL && body_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *m = mk_byte_array(master, master_len); + lean_object *h = mk_byte_array(hash, hash_len); + lean_object *b = mk_byte_array(body, body_len); + if (m == NULL || h == NULL || b == NULL) { + if (m) lean_dec(m); + if (h) lean_dec(h); + if (b) lean_dec(b); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_decode(m, h, b, padding, format); + return unpack_status_payload(packed, out, out_len); } -__attribute__((weak)) int carbonado_encode_headered( +int carbonado_encode_outboard( const uint8_t *master, size_t master_len, const uint8_t *plaintext, size_t plaintext_len, uint8_t format, const uint8_t *nonce, size_t nonce_len, - uint8_t **out, size_t *out_len) { - (void)master; (void)master_len; (void)plaintext; (void)plaintext_len; - (void)format; (void)nonce; (void)nonce_len; (void)out; (void)out_len; - return CARBONADO_ERR_NOT_IMPLEMENTED; + uint8_t header_path, + uint8_t **main_out, size_t *main_len, + uint8_t **outboard_out, size_t *outboard_len, + uint8_t **parity_out, size_t *parity_len, + uint8_t hash_out[32], + uint32_t *padding_out, + uint32_t *chunk_len_out, + uint32_t *bytes_compressed_out, + uint32_t *bytes_encrypted_out) { + if (main_out == NULL || main_len == NULL || outboard_out == NULL || outboard_len == NULL || + parity_out == NULL || parity_len == NULL || hash_out == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *main_out = NULL; + *main_len = 0; + *outboard_out = NULL; + *outboard_len = 0; + *parity_out = NULL; + *parity_len = 0; + if (padding_out) *padding_out = 0; + if (chunk_len_out) *chunk_len_out = 0; + if (bytes_compressed_out) *bytes_compressed_out = 0; + if (bytes_encrypted_out) *bytes_encrypted_out = 0; + if (master == NULL || (plaintext == NULL && plaintext_len != 0)) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (nonce == NULL && nonce_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *m = mk_byte_array(master, master_len); + lean_object *n = mk_byte_array(nonce, nonce_len); + lean_object *pt = mk_byte_array(plaintext, plaintext_len); + if (m == NULL || n == NULL || pt == NULL) { + if (m) lean_dec(m); + if (n) lean_dec(n); + if (pt) lean_dec(pt); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_encode_outboard(m, n, pt, format, header_path); + size_t nlen = lean_sarray_size(packed); + if (nlen < 4) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + const uint8_t *p = lean_sarray_cptr(packed); + uint32_t status = read_u32_le(p); + if (status != CARBONADO_OK) { + lean_dec(packed); + return (int)status; + } + /* status(4)+pad(4)+chunk(4)+comp(4)+enc(4)+hash(32) = 52 */ + if (nlen < 52) { + lean_dec(packed); + return CARBONADO_ERR_INTERNAL; + } + if (padding_out) *padding_out = read_u32_le(p + 4); + if (chunk_len_out) *chunk_len_out = read_u32_le(p + 8); + if (bytes_compressed_out) *bytes_compressed_out = read_u32_le(p + 12); + if (bytes_encrypted_out) *bytes_encrypted_out = read_u32_le(p + 16); + memcpy(hash_out, p + 20, 32); + size_t off = 52; + int rc = read_len_prefixed(p, nlen, &off, main_out, main_len); + if (rc != CARBONADO_OK) { + lean_dec(packed); + return rc; + } + rc = read_len_prefixed(p, nlen, &off, outboard_out, outboard_len); + if (rc != CARBONADO_OK) { + free(*main_out); + *main_out = NULL; + *main_len = 0; + lean_dec(packed); + return rc; + } + rc = read_len_prefixed(p, nlen, &off, parity_out, parity_len); + if (rc != CARBONADO_OK) { + free(*main_out); + free(*outboard_out); + *main_out = NULL; + *main_len = 0; + *outboard_out = NULL; + *outboard_len = 0; + lean_dec(packed); + return rc; + } + lean_dec(packed); + return CARBONADO_OK; } -__attribute__((weak)) int carbonado_decode_headered( +int carbonado_decode_outboard( const uint8_t *master, size_t master_len, - const uint8_t *archive, size_t archive_len, + const uint8_t *hash, size_t hash_len, + const uint8_t *main, size_t main_len, + const uint8_t *outboard, size_t outboard_len, + const uint8_t *parity, size_t parity_len, + uint32_t padding, + uint8_t format, + uint8_t header_path, + const uint8_t *nonce, size_t nonce_len, uint8_t **out, size_t *out_len) { - (void)master; (void)master_len; (void)archive; (void)archive_len; - (void)out; (void)out_len; - return CARBONADO_ERR_NOT_IMPLEMENTED; + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (master == NULL || hash == NULL || hash_len != 32) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (main == NULL && main_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (outboard == NULL && outboard_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (parity == NULL && parity_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (nonce == NULL && nonce_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *m = mk_byte_array(master, master_len); + lean_object *h = mk_byte_array(hash, hash_len); + lean_object *mn = mk_byte_array(main, main_len); + lean_object *ob = mk_byte_array(outboard, outboard_len); + lean_object *pr = mk_byte_array(parity, parity_len); + lean_object *n = mk_byte_array(nonce, nonce_len); + if (m == NULL || h == NULL || mn == NULL || ob == NULL || pr == NULL || n == NULL) { + if (m) lean_dec(m); + if (h) lean_dec(h); + if (mn) lean_dec(mn); + if (ob) lean_dec(ob); + if (pr) lean_dec(pr); + if (n) lean_dec(n); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = + l_carbonado_decode_outboard(m, h, mn, ob, pr, padding, format, header_path, n); + return unpack_status_payload(packed, out, out_len); } -__attribute__((weak)) int carbonado_verification_key(uint8_t format, uint8_t key_out[32]) { - (void)format; (void)key_out; - return CARBONADO_ERR_NOT_IMPLEMENTED; +int carbonado_scrub( + const uint8_t *body, size_t body_len, + const uint8_t *hash, size_t hash_len, + uint32_t padding, + uint8_t format, + uint8_t **out, size_t *out_len) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (hash == NULL || hash_len != 32) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (body == NULL && body_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *b = mk_byte_array(body, body_len); + lean_object *h = mk_byte_array(hash, hash_len); + if (b == NULL || h == NULL) { + if (b) lean_dec(b); + if (h) lean_dec(h); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_scrub(b, h, padding, format); + return unpack_status_payload(packed, out, out_len); +} + +int carbonado_scrub_outboard( + const uint8_t *main, size_t main_len, + const uint8_t *outboard, size_t outboard_len, + const uint8_t *parity, size_t parity_len, + const uint8_t *hash, size_t hash_len, + uint32_t padding, + uint32_t chunk_len, + uint8_t format, + uint8_t **out, size_t *out_len) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (hash == NULL || hash_len != 32) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (main == NULL && main_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (outboard == NULL && outboard_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (parity == NULL && parity_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *mn = mk_byte_array(main, main_len); + lean_object *ob = mk_byte_array(outboard, outboard_len); + lean_object *pr = mk_byte_array(parity, parity_len); + lean_object *h = mk_byte_array(hash, hash_len); + if (mn == NULL || ob == NULL || pr == NULL || h == NULL) { + if (mn) lean_dec(mn); + if (ob) lean_dec(ob); + if (pr) lean_dec(pr); + if (h) lean_dec(h); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = + l_carbonado_scrub_outboard(mn, ob, pr, h, padding, chunk_len, format); + return unpack_status_payload(packed, out, out_len); +} + +int carbonado_verify_slice( + const uint8_t *body, size_t body_len, + const uint8_t *hash, size_t hash_len, + uint32_t index, + uint32_t count, + uint8_t format, + uint8_t **out, size_t *out_len) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (hash == NULL || hash_len != 32) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (body == NULL && body_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *b = mk_byte_array(body, body_len); + lean_object *h = mk_byte_array(hash, hash_len); + if (b == NULL || h == NULL) { + if (b) lean_dec(b); + if (h) lean_dec(h); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = l_carbonado_verify_slice(b, h, index, count, format); + return unpack_status_payload(packed, out, out_len); +} + +int carbonado_verify_slice_outboard( + const uint8_t *main, size_t main_len, + const uint8_t *outboard, size_t outboard_len, + const uint8_t *hash, size_t hash_len, + uint32_t index, + uint32_t count, + uint8_t format, + uint8_t **out, size_t *out_len) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (hash == NULL || hash_len != 32) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (main == NULL && main_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (outboard == NULL && outboard_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (ensure_lean() != 0) { + return CARBONADO_ERR_INTERNAL; + } + + lean_object *mn = mk_byte_array(main, main_len); + lean_object *ob = mk_byte_array(outboard, outboard_len); + lean_object *h = mk_byte_array(hash, hash_len); + if (mn == NULL || ob == NULL || h == NULL) { + if (mn) lean_dec(mn); + if (ob) lean_dec(ob); + if (h) lean_dec(h); + return CARBONADO_ERR_INVALID_ARGUMENT; + } + + lean_obj_res packed = + l_carbonado_verify_slice_outboard(mn, ob, h, index, count, format); + return unpack_status_payload(packed, out, out_len); } diff --git a/nix/native/carbonado_slh.c b/nix/native/carbonado_slh.c new file mode 100644 index 0000000..b2fbfc0 --- /dev/null +++ b/nix/native/carbonado_slh.c @@ -0,0 +1,199 @@ +/** + * Carbonado SLH-DSA-SHA2-128s FFI (R9 / G10). + * + * Links libbitcoinpqc SLH sources (sphincsplus + slh_dsa wrappers) only — + * no secp256k1 / ML-DSA. Dual-suite product SLH may still use Rust bitcoinpqc; + * this path makes pure Lean AOT / libcarbonado self-contained. + * + * Lean @[extern] wire (status-prefixed ByteArray, like zstd): + * carbonado_slh_keygen_raw : @& ByteArray → ByteArray + * status 0 + pk(32) + sk(64); else status only + * carbonado_slh_sign_raw : @& ByteArray → @& ByteArray → ByteArray + * sk + message → status 0 + sig(7856); else status only + * carbonado_slh_verify_raw : @& ByteArray → @& ByteArray → @& ByteArray → UInt8 + * pk + message + sig → 1 accept / 0 reject + * + * Status codes (Lean decodeSlhStatusPayload): + * 0 OK + * 1 short entropy (keygen) + * 2 other bad argument (wrong sk/pk/sig sizes) + * 3 crypto failure (keygen/sign library error) + * + * Public C ABI (include/carbonado.h): carbonado_slh_keygen / _sign / _verify. + * Keygen library failure → CARBONADO_ERR_INTERNAL (not AUTHENTICATION). + * Verify reject → CARBONADO_ERR_AUTHENTICATION. + */ +#include +#include +#include +#include +#include + +#include "libbitcoinpqc/slh_dsa.h" +#include "carbonado.h" + +enum { + SLH_ST_OK = 0, + SLH_ST_BAD_ENTROPY = 1, + SLH_ST_BAD_ARG = 2, + SLH_ST_CRYPTO = 3 +}; + +/* Non-NULL empty buffer for SLH APIs that reject NULL message pointers. */ +static const uint8_t g_empty_msg[1] = {0}; + +static lean_obj_res mk_status(uint8_t status, const uint8_t *payload, size_t payload_len) { + size_t total = 1 + payload_len; + lean_obj_res out = lean_alloc_sarray(1, total, total); + uint8_t *p = lean_sarray_cptr(out); + p[0] = status; + if (payload_len > 0 && payload != NULL) { + memcpy(p + 1, payload, payload_len); + } + return out; +} + +static lean_obj_res mk_status_only(uint8_t status) { + return mk_status(status, NULL, 0); +} + +static const uint8_t *msg_ptr(const uint8_t *message, size_t message_len) { + if (message_len == 0) { + return g_empty_msg; + } + return message; +} + +/* carbonado_slh_keygen_raw : @& ByteArray → ByteArray */ +LEAN_EXPORT lean_obj_res carbonado_slh_keygen_raw(b_lean_obj_arg entropy) { + size_t ent_len = lean_sarray_size(entropy); + const uint8_t *ent = lean_sarray_cptr(entropy); + if (ent_len < 128 || ent == NULL) { + return mk_status_only(SLH_ST_BAD_ENTROPY); + } + uint8_t pk[SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE]; + uint8_t sk[SLH_DSA_SHA2_128S_SECRET_KEY_SIZE]; + if (slh_dsa_sha2_128s_keygen(pk, sk, ent, ent_len) != 0) { + memset(sk, 0, sizeof sk); + return mk_status_only(SLH_ST_CRYPTO); + } + uint8_t payload[SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE + SLH_DSA_SHA2_128S_SECRET_KEY_SIZE]; + memcpy(payload, pk, SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE); + memcpy(payload + SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE, sk, SLH_DSA_SHA2_128S_SECRET_KEY_SIZE); + lean_obj_res out = mk_status(SLH_ST_OK, payload, sizeof payload); + memset(sk, 0, sizeof sk); + memset(payload + SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE, 0, SLH_DSA_SHA2_128S_SECRET_KEY_SIZE); + return out; +} + +/* carbonado_slh_sign_raw : @& ByteArray → @& ByteArray → ByteArray (sk, message) */ +LEAN_EXPORT lean_obj_res carbonado_slh_sign_raw(b_lean_obj_arg sk, b_lean_obj_arg message) { + size_t sk_len = lean_sarray_size(sk); + size_t m_len = lean_sarray_size(message); + const uint8_t *sk_p = lean_sarray_cptr(sk); + const uint8_t *m_p = lean_sarray_cptr(message); + if (sk_len != SLH_DSA_SHA2_128S_SECRET_KEY_SIZE || sk_p == NULL) { + return mk_status_only(SLH_ST_BAD_ARG); + } + if (m_p == NULL && m_len != 0) { + return mk_status_only(SLH_ST_BAD_ARG); + } + const uint8_t *msg = msg_ptr(m_p, m_len); + uint8_t sig[SLH_DSA_SHA2_128S_SIGNATURE_SIZE]; + size_t siglen = 0; + if (slh_dsa_sha2_128s_sign(sig, &siglen, msg, m_len, sk_p) != 0 || + siglen != SLH_DSA_SHA2_128S_SIGNATURE_SIZE) { + return mk_status_only(SLH_ST_CRYPTO); + } + return mk_status(SLH_ST_OK, sig, SLH_DSA_SHA2_128S_SIGNATURE_SIZE); +} + +/* carbonado_slh_verify_raw : @& ByteArray → @& ByteArray → @& ByteArray → UInt8 */ +LEAN_EXPORT uint8_t carbonado_slh_verify_raw(b_lean_obj_arg pk, b_lean_obj_arg message, + b_lean_obj_arg signature) { + size_t pk_len = lean_sarray_size(pk); + size_t m_len = lean_sarray_size(message); + size_t sig_len = lean_sarray_size(signature); + const uint8_t *pk_p = lean_sarray_cptr(pk); + const uint8_t *m_p = lean_sarray_cptr(message); + const uint8_t *sig_p = lean_sarray_cptr(signature); + if (pk_len != SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE || pk_p == NULL) { + return 0; + } + if (sig_len != SLH_DSA_SHA2_128S_SIGNATURE_SIZE || sig_p == NULL) { + return 0; + } + if (m_p == NULL && m_len != 0) { + return 0; + } + const uint8_t *msg = msg_ptr(m_p, m_len); + return slh_dsa_sha2_128s_verify(sig_p, sig_len, msg, m_len, pk_p) == 0 ? 1 : 0; +} + +/* ── Public C ABI ─────────────────────────────────────────────────────────── */ + +int carbonado_slh_keygen( + const uint8_t *entropy, size_t entropy_len, + uint8_t pk_out[32], + uint8_t sk_out[64]) { + if (entropy == NULL || entropy_len < 128 || pk_out == NULL || sk_out == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (slh_dsa_sha2_128s_keygen(pk_out, sk_out, entropy, entropy_len) != 0) { + memset(sk_out, 0, 64); + /* Keygen failure is not an auth reject — map to INTERNAL. */ + return CARBONADO_ERR_INTERNAL; + } + return CARBONADO_OK; +} + +int carbonado_slh_sign( + const uint8_t *secret_key, size_t secret_key_len, + const uint8_t *message, size_t message_len, + uint8_t **out, size_t *out_len) { + if (out == NULL || out_len == NULL) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + *out = NULL; + *out_len = 0; + if (secret_key == NULL || secret_key_len != SLH_DSA_SHA2_128S_SECRET_KEY_SIZE) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (message == NULL && message_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + const uint8_t *msg = msg_ptr(message, message_len); + uint8_t *sig = (uint8_t *)malloc(SLH_DSA_SHA2_128S_SIGNATURE_SIZE); + if (sig == NULL) { + return CARBONADO_ERR_INTERNAL; + } + size_t siglen = 0; + if (slh_dsa_sha2_128s_sign(sig, &siglen, msg, message_len, secret_key) != 0 || + siglen != SLH_DSA_SHA2_128S_SIGNATURE_SIZE) { + free(sig); + return CARBONADO_ERR_INTERNAL; + } + *out = sig; + *out_len = SLH_DSA_SHA2_128S_SIGNATURE_SIZE; + return CARBONADO_OK; +} + +int carbonado_slh_verify( + const uint8_t *public_key, size_t public_key_len, + const uint8_t *message, size_t message_len, + const uint8_t *signature, size_t signature_len) { + if (public_key == NULL || public_key_len != SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (signature == NULL || signature_len != SLH_DSA_SHA2_128S_SIGNATURE_SIZE) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + if (message == NULL && message_len != 0) { + return CARBONADO_ERR_INVALID_ARGUMENT; + } + const uint8_t *msg = msg_ptr(message, message_len); + if (slh_dsa_sha2_128s_verify(signature, signature_len, msg, message_len, public_key) != 0) { + return CARBONADO_ERR_AUTHENTICATION; + } + return CARBONADO_OK; +} diff --git a/nix/native/default.nix b/nix/native/default.nix index e3c8a7a..65ccd60 100644 --- a/nix/native/default.nix +++ b/nix/native/default.nix @@ -1,13 +1,16 @@ -# Static FFI glue for Carbonado AOT (zstd wrappers + libzstd objects). +# Static FFI glue for Carbonado AOT (zstd + SLH-DSA + C ABI). # Output: $out/libcarbonado_native.a (linked via buildLeanPackage.staticLibDeps). # -# Embeds single-threaded libzstd from the **pinned** `ref/zstd` tree (v1.5.7) -# so product frames track the git submodule SSOT — not a floating nixpkgs.src. +# Embeds: +# * single-threaded libzstd from the **pinned** `ref/zstd` tree (v1.5.7) +# * SLH-DSA-SHA2-128s from pinned libbitcoinpqc (sphincsplus + slh_dsa only) +# so product frames track git submodule SSOTs — not floating nixpkgs.src. # Static archive only (no shared -lzstd / pthread shlib issues under lld). { pkgs, leanAll, # pkgs.lean.lean-all — provides lean/lean.h - zstdSrc, # flake: ./ref/zstd (must be checked-out submodule pin) + zstdSrc, # flake: pinned zstd fetch + bitcoinpqcSrc, # flake: pinned libbitcoinpqc fetch (R9 / G10) carbonadoInclude ? ../.. + "/include", # repo include/carbonado.h (ABI) }: pkgs.stdenv.mkDerivation { @@ -15,6 +18,9 @@ pkgs.stdenv.mkDerivation { version = "0.1.0"; src = ./.; + # Static .a only; strip of archives trips nixpkgs strip.sh under `set -u` on some hosts. + dontStrip = true; + nativeBuildInputs = [pkgs.binutils]; # Fail-closed: every .c compile must succeed (no `|| true` / silent stderr). @@ -23,6 +29,7 @@ pkgs.stdenv.mkDerivation { set -euo pipefail ZSTD_LIB="${zstdSrc}/lib" + PQC="${bitcoinpqcSrc}" ABI_INC="${carbonadoInclude}" if [ ! -f "$ABI_INC/carbonado.h" ]; then echo "carbonado-native: missing $ABI_INC/carbonado.h" >&2 @@ -36,6 +43,10 @@ pkgs.stdenv.mkDerivation { echo "carbonado-native: missing $ZSTD_LIB/zstd.h" >&2 exit 1 fi + if [ ! -f "$PQC/include/libbitcoinpqc/slh_dsa.h" ]; then + echo "carbonado-native: missing libbitcoinpqc at $PQC" >&2 + exit 1 + fi # Portable single-thread objects (no assembly). Explicit loops — fail on first error. compile_dir() { @@ -44,10 +55,10 @@ pkgs.stdenv.mkDerivation { for f in "$dir"/*.c; do [ -f "$f" ] || continue base=$(basename "$f" .c) - echo " CC $base.c" + echo " CC zstd/$base.c" $CC -c -O2 -fPIC -DZSTD_DISABLE_ASM \ -I"$ZSTD_LIB" -I"$ZSTD_LIB/common" \ - "$f" -o "$base.o" + "$f" -o "zstd_$base.o" done } @@ -64,16 +75,53 @@ pkgs.stdenv.mkDerivation { carbonado_zstd.c \ -o carbonado_zstd.o - echo "carbonado-native: compiling carbonado_abi.c (C ABI v0 stubs)" + # SLH-DSA-SHA2-128s only (no secp / ML-DSA). Unique object basenames avoid + # clobbering sphincsplus/ref/utils.o vs src/slh_dsa/utils.o. + PQC_CFLAGS="-O2 -fPIC -DPARAMS=sphincs-sha2-128s -DCUSTOM_RANDOMBYTES=1" + PQC_INCLUDES="-I$PQC/include -I$PQC/src -I$PQC/sphincsplus/ref" + compile_pqc() { + local src="$1" + local base="$2" + echo " CC slh/$base.c" + $CC -c $PQC_CFLAGS $PQC_INCLUDES "$src" -o "slh_$base.o" + } + echo "carbonado-native: compiling libbitcoinpqc SLH-DSA (pinned)" + compile_pqc "$PQC/sphincsplus/ref/address.c" spx_address + compile_pqc "$PQC/sphincsplus/ref/fors.c" spx_fors + compile_pqc "$PQC/sphincsplus/ref/hash_sha2.c" spx_hash_sha2 + compile_pqc "$PQC/sphincsplus/ref/merkle.c" spx_merkle + compile_pqc "$PQC/sphincsplus/ref/sign.c" spx_sign + compile_pqc "$PQC/sphincsplus/ref/thash_sha2_simple.c" spx_thash_sha2_simple + compile_pqc "$PQC/sphincsplus/ref/utils.c" spx_utils + compile_pqc "$PQC/sphincsplus/ref/utilsx1.c" spx_utilsx1 + compile_pqc "$PQC/sphincsplus/ref/wots.c" spx_wots + compile_pqc "$PQC/sphincsplus/ref/wotsx1.c" spx_wotsx1 + compile_pqc "$PQC/sphincsplus/ref/sha2.c" spx_sha2 + compile_pqc "$PQC/src/randombytes_custom.c" pqc_randombytes + compile_pqc "$PQC/src/slh_dsa/utils.c" slh_utils + compile_pqc "$PQC/src/slh_dsa/keygen.c" slh_keygen + compile_pqc "$PQC/src/slh_dsa/sign.c" slh_sign + compile_pqc "$PQC/src/slh_dsa/verify.c" slh_verify + + echo "carbonado-native: compiling carbonado_slh.c (Lean extern + C ABI)" $CC -c -O2 -fPIC \ + -I${leanAll}/include \ + -I"$ABI_INC" \ + -I"$PQC/include" \ + carbonado_slh.c \ + -o carbonado_slh.o + + echo "carbonado-native: compiling carbonado_abi.c (C ABI v1 + Lean glue)" + $CC -c -O2 -fPIC \ + -I${leanAll}/include \ -I"$ABI_INC" \ carbonado_abi.c \ -o carbonado_abi.o # Fail-closed: must have more than just the FFI object. ocount=$(ls -1 ./*.o 2>/dev/null | wc -l) - if [ "$ocount" -lt 10 ]; then - echo "carbonado-native: expected many zstd objects, found $ocount" >&2 + if [ "$ocount" -lt 20 ]; then + echo "carbonado-native: expected many zstd+slh objects, found $ocount" >&2 ls -la ./*.o >&2 || true exit 1 fi @@ -95,6 +143,6 @@ pkgs.stdenv.mkDerivation { ''; meta = { - description = "Carbonado Lean AOT native glue (static zstd + C ABI stubs)"; + description = "Carbonado Lean AOT native glue (static zstd + SLH-DSA + C ABI Lean bridge)"; }; } diff --git a/nix/tooling-purity.nix b/nix/tooling-purity.nix index 802f174..3e2c735 100644 --- a/nix/tooling-purity.nix +++ b/nix/tooling-purity.nix @@ -1,9 +1,11 @@ -# checks.tooling-purity — product Lean/Nix tree must not grow non-ref impurity. -# Transition: existing Rust under src/, tests/, benches/, examples/ is legacy product -# until moved to ref/carbonado-rust. This check: +# checks.tooling-purity — dual-backend product tree purity constraints. +# Dual-backend SSOT (AGENTS / G1 W5a): Rust under src/, tests/, benches/, examples/ +# is permanent first-class product + dual-suite contract — NOT transitional and NOT +# moved to ref/carbonado-rust (permanent no product pin). ref/ is third-party oracles only. +# This check: # * requires product Lean roots to exist and be Lean-only # * bans product shell/python glue outside nix/ and ref/ -# * allowlists known top-level roots (transitional Rust included) +# * allowlists known top-level roots (both engines + docs/tooling expected permanently) { pkgs, src }: pkgs.runCommand "carbonado-tooling-purity" { inherit src; @@ -58,8 +60,9 @@ pkgs.runCommand "carbonado-tooling-purity" { fi # Positive allowlist for top-level names. - # Transitional Rust (src, tests, benches, examples, Cargo.*) until freeze. - # productSrc often excludes those; allowlist still names them for full-tree runs. + # Permanent dual product roots: src/, tests/, benches/, examples/, Cargo.* (Rust) + # plus Carbonado/, CarbonadoTest/ (Lean). productSrc may exclude Rust; allowlist + # still names them for full-tree runs (expected, not temporary). is_allowed() { local base="$1" case "$base" in diff --git a/ref/README.md b/ref/README.md index 66afad5..488cd77 100644 --- a/ref/README.md +++ b/ref/README.md @@ -2,7 +2,14 @@ Any language. Used to **prove and bit-match** Lean AOT analogues (verik1 / beastdb pattern). -Product code lives only under `Carbonado/` (Lean) and is built with Nix flakes. +Trees under `ref/` are **third-party oracles / vendors only** — not product engines. Product engines live outside `ref/`: + +| Engine | Location | +|--------|----------| +| **Rust** (first-class + dual-suite SSOT) | live `src/`, `tests/` (also benches/examples/CLI) | +| **Lean 4** (proofs + AOT `libcarbonado`) | `Carbonado/`, `CarbonadoTest/`; built via Nix flakes | + +**G1/W5a permanent policy:** no `ref/carbonado-rust` product pin. Do not invent a submodule that freezes or demotes live Rust. See [docs/PARITY.md](../docs/PARITY.md) for pin table and [docs/SPEC-MATRIX.md](../docs/SPEC-MATRIX.md) for coverage. @@ -18,7 +25,7 @@ See [docs/PARITY.md](../docs/PARITY.md) for pin table and [docs/SPEC-MATRIX.md]( | `blake3` | Hash / Bao leaves | **pinned** | | `zstd` | Compression C (Nix-linked) | **pinned** | | `bitcoinpqc` | SLH-DSA-SHA2-128s bindings | **pinned** | -| `carbonado-rust` | Frozen historical Rust product | pending freeze | +| ~~`carbonado-rust`~~ | Product pin (not used) | **absent by policy** (G1/W5a permanent no-pin — live `src/` + `tests/` SSOT; see [PARITY.md](../docs/PARITY.md)) | | `parity-harness/` | Compare drivers | skeleton (README) | | `crates/` | crates.io vendors (e.g. `ctr` 0.9.2) | skeleton (README) | diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 62e20c6..b07e09b 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -5,6 +5,88 @@ //! //! Both features must not be enabled together for a single build that links both //! engines into conflicting paths; prefer one engine per `cargo test` invocation. +//! +//! ## Phase 2–3 dispatch surface (`backend-lean`) +//! +//! Lean C ABI is used for: low-level [`crate::encode`]/crate::decode`], +//! [`crate::encode_outboard`]/crate::decode_outboard`], +//! [`crate::scrub`]/crate::scrub_outboard`], [`crate::verify_slice`]/crate::extract_slice`], +//! headered [`crate::file::encode`]/crate::file::decode`], stream buffer helpers +//! (`stream_encode_buffer` / `stream_decode_buffer` / outboard buffer), **R5/W1 E1** +//! stream I/O (`stream_encode_inboard` / `stream_decode` / **encrypted** +//! `stream_*_outboard` via spool-to-buffer; **W1b public** `stream_*_outboard` is rust S4 +//! composition — not Lean spool), and [`lean::verification_key`] for AOT parity checks. +//! +//! ### Phase 3 directory (composition — no new directory C symbols) +//! +//! [`crate::file::encode_directory`] / [`crate::file::decode_directory`] keep: +//! - **Rust:** FS walk, path fail-closed checks, **rkyv** `FilepackManifest` v2 wire +//! (normative Adamantine payload body), Adamantine envelope/payload framing, +//! centralized Bao+FEC bundle assembly, catalog COTS trailer. +//! - **Lean (via existing ABI):** segment bare mains via `encode_outboard` / +//! `decode_outboard` (embedded-nonce layout); catalog container via +//! `encode_headered` / `decode_headered` (`file::encode` / `file::decode`). +//! +//! Dual-suite catalogs are therefore **rkyv** (same bytes as `backend-rust`), not +//! Lean-only CFP2. **W3:** pure Lean Directory/CLI also emit rkyv (wire-compatible); +//! dual-suite encode remains Rust rkyv composition SSOT. CFP2 is dual-decode fallback only. +//! +//! ### Phase 4 PQC / CLI / OTS (composition) +//! +//! - **SLH-DSA (G10-A dual-suite + R9 pure Lean):** dual-suite product path may use +//! Rust `crypto::slh_*` + `bitcoinpqc` under both backends (composition). Lean holds +//! wire parse/build + bind-to-root (`Carbonado/Slh.lean`). **R9/G10:** pure Lean +//! `signRoot` / `verifyRoot` are live via `carbonado_slh_*` + libbitcoinpqc pin in +//! `libcarbonado` — dual-suite need not switch from composition. +//! - **CLI dual path (honest):** +//! - **Dual-engine:** directory CLI → `encode_directory` / `decode_directory` +//! (Lean segment/catalog crypto); buffer single-file APIs (`file::encode`, +//! `encode_outboard`, headered decode) dispatch to Lean under `backend-lean`. +//! - **R5 E1 + W1 stream dual:** `stream_encode_inboard` / `stream_decode` and +//! **encrypted** `stream_*_outboard` spool-to-buffer → Lean C ABI (O(logical) E1). +//! **W1b public** `stream_encode_outboard` / `stream_decode_outboard` use rust S4 +//! geometric composition under lean (**E2 O(chunk/stripe)** when !Compression; +//! Compression under lean is O(logical) bulk zstd; G9 no-compress wire bit-match; +//! not pure Lean stream). `encode_stream` uses Lean via `stream_encode_inboard`. +//! **W1a:** `file::decode_stream` verifies `header_mac` then spools body → Lean +//! `decode_headered` (E1). +//! - Build with `--features "backend-lean,pqc,ots,cli"` + `CARBONADO_LEAN_*` for a +//! lean-**linked** binary. Default `cargo build --bin carbonado` remains rust-engine. +//! - **Directory OTS:** offline CBOTS stubs in Rust (`ots` feature); composition +//! over Lean container crypto — no Lean-native stamping. +//! +//! **G8 full closed at R7** under `backend-lean`: freeze = unfiltered +//! `just test-lean-ci` (`cargo test --no-default-features --features +//! "backend-lean,pqc,ots,cli"`). Former residual suites (`format`, `codec`, +//! `header_tamper`, `format_amplification`, `streaming`/`streaming_limits`, +//! `seekable_slices`, `sharding`, `fec_chaos`, `bin_*`) are freeze-green. +//! +//! **Post-G8 residuals** (purity / feature-policy / composition honesty — not dual-suite red): +//! ~~pure-Lean rkyv encode~~ **W3 closed** (Lean `encodeRkyvManifest` + Directory/CLI; dual-suite +//! catalog encode still Rust rkyv composition SSOT), ~~stream E2 / dual honesty~~ **W1a+W1b closed** (public outboard S4 +//! composition E2; pure Lean chunked C residual), G9 Compression/directory encode bit-match +//! (**W2** permanent W2a/W2b), ~~codecode/decodec~~ **W2d closed**, ~~W4a inboard O(slice) retain~~ +//! **closed**, **W4b** permanent full-buffer C outboard slice, **W4c** permanent buffer-only zstd +//! under lean, **W4d** permanent FEC O(body) + async encoded spool, +//! `streaming_async` / `parallel_determinism` permanently feature-gated off freeze (R10: +//! freeze never requires `async` / `parallel`; Lean RS is serial). +//! +//! **R9 closed (optional pure Lean depth):** G10 SLH-DSA live in `libcarbonado` +//! (`carbonado_slh_*` + Lean `@[extern]`); seekable outboard slice C +//! (`carbonado_verify_slice_outboard` / Lean range verify); rkyv dual-decode residual +//! documented (composition remains SSOT for dual-suite directory wire). +//! +//! **R10 closed (async dual policy):** dual freeze **never** requires `async` +//! (`streaming_async` stays feature-gated → 0 tests under lean freeze). Optional +//! `stream_decode_async` stages the encoded body then calls dual-aware +//! [`crate::stream::stream_decode`] (R5 E1 under `backend-lean`; S4 pipeline under +//! `backend-rust`) — no silent pure-Rust pipeline when lean+async are both enabled. +//! WASM async remains `NotImplemented`. +//! +//! **R1 fail-closed (outboard Option semantics):** `None` means missing sidecar and +//! must error when the format bit requires it (`MissingVerificationOutboard` / +//! `MissingFecParity`). `Some(&[])` is a present empty outboard (valid for single-leaf +//! Bao trees) and is allowed through. Guarded in [`lean::decode_outboard`] before C. #[cfg(all(feature = "backend-lean", feature = "backend-rust"))] compile_error!( @@ -25,6 +107,7 @@ pub mod rust_engine { pub mod lean { //! Lean AOT backend via C ABI (`carbonado-sys` / `libcarbonado`). use crate::error::CarbonadoError; + use crate::structs::{EncodeInfo, Encoded, OutboardEncoded}; use carbonado_sys as sys; pub const NAME: &str = "lean"; @@ -34,47 +117,215 @@ pub mod lean { unsafe { sys::carbonado_abi_version() } } - /// Map C ABI codes to `CarbonadoError` (docs/ABI.md). Refined as mapping matures. + /// Map C ABI codes to `CarbonadoError` (docs/ABI.md). pub fn map_err(code: i32) -> CarbonadoError { match code { - sys::CARBONADO_ERR_INVALID_ARGUMENT => CarbonadoError::InvalidHeaderLength, - sys::CARBONADO_ERR_INVALID_KEY_LENGTH => { - CarbonadoError::HashDecodeError(32, 0) // refine when dedicated key variant exists + // P2 residual: no dedicated InvalidArgument variant (docs/ABI.md error table). + // Includes Lean-only wrong-length SLH/meta on encodeHeaderedBytes if ever + // surfaced via C; typed Rust Option<&[u8; N]> cannot express those lengths. + sys::CARBONADO_ERR_INVALID_ARGUMENT => { + CarbonadoError::InternalStateError("lean-backend invalid argument".into()) } + sys::CARBONADO_ERR_INVALID_KEY_LENGTH => CarbonadoError::InvalidKeyLength, sys::CARBONADO_ERR_AUTHENTICATION => CarbonadoError::AuthenticationFailed, sys::CARBONADO_ERR_INVALID_MAGIC => { CarbonadoError::InvalidMagicNumber("lean-backend".into()) } + // Truncated/malformed header, body bounds, or short inboard Bao prefix + // (`invalidPrefix` / `invalidHeaderLength` via ofPipelineError). sys::CARBONADO_ERR_INVALID_HEADER => CarbonadoError::InvalidHeaderLength, sys::CARBONADO_ERR_FEC => CarbonadoError::UnevenFecChunks, - sys::CARBONADO_ERR_BAO => CarbonadoError::InvalidScrubbedHash, + // Stream truncation / trailing data / root-length / residual slice geometry + // that was not pre-checked in `lean::verify_slice`. Bao *auth* failures use + // CARBONADO_ERR_AUTHENTICATION (R4). + sys::CARBONADO_ERR_BAO => { + CarbonadoError::BaoResponseTruncated("lean-backend bao/verify".into()) + } sys::CARBONADO_ERR_ZSTD => CarbonadoError::ZstdError("lean-backend zstd".into()), sys::CARBONADO_ERR_SCRUB_UNNECESSARY => CarbonadoError::UnnecessaryScrub, sys::CARBONADO_ERR_SCRUB_FAILED => CarbonadoError::InvalidScrubbedHash, - sys::CARBONADO_ERR_NOT_IMPLEMENTED => CarbonadoError::ZstdError( - "lean-backend: C ABI encode/decode not fully wired (Phase 1; stubs return NOT_IMPLEMENTED)" - .into(), - ), + sys::CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION => { + CarbonadoError::ScrubRequiresVerification + } + sys::CARBONADO_ERR_NOT_IMPLEMENTED => CarbonadoError::NotImplemented, sys::CARBONADO_ERR_INTERNAL => { - CarbonadoError::ZstdError("lean-backend internal error".into()) + CarbonadoError::InternalStateError("lean-backend internal error".into()) } - _ => CarbonadoError::ZstdError(format!("lean-backend unknown error {code}")), + _ => CarbonadoError::InternalStateError(format!("lean-backend unknown error {code}")), } } + /// Copy a libcarbonado `malloc` buffer into a Rust `Vec`, then free via C. + /// + /// Avoids `Vec::from_raw_parts` over foreign allocators (jemalloc/mimalloc-safe). + fn take_buf(out: *mut u8, out_len: usize) -> Result, CarbonadoError> { + if out.is_null() { + if out_len == 0 { + return Ok(Vec::new()); + } + return Err(map_err(sys::CARBONADO_ERR_INTERNAL)); + } + let mut v = Vec::with_capacity(out_len); + // SAFETY: `out` is a non-null malloc buffer of length `out_len` from libcarbonado. + unsafe { + v.extend_from_slice(std::slice::from_raw_parts(out, out_len)); + sys::carbonado_free(out as *mut _); + } + Ok(v) + } + + /// Free a raw libcarbonado buffer if non-null (best-effort cleanup on multi-out failures). + fn free_raw(p: *mut u8) { + if !p.is_null() { + unsafe { sys::carbonado_free(p as *mut _) }; + } + } + + /// Format-keyed Bao verification key (32 bytes). + pub fn verification_key(format: u8) -> Result<[u8; 32], CarbonadoError> { + let mut key = [0u8; 32]; + let rc = unsafe { sys::carbonado_verification_key(format, key.as_mut_ptr()) }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + Ok(key) + } + + /// Low-level body encode (≈ `encoding::encode`). Returns body, Bao root, padding + meta. + /// + /// # `EncodeInfo` (R3) + /// + /// Stage counters (`bytes_compressed`, `bytes_encrypted`) and FEC/Bao geometry + /// (`padding_len`, `chunk_len`, `bytes_ecc`, `verifiable_slice_count`) are filled + /// from the Lean pack. Skipped stages report **0** (matches Rust stream path). + pub fn encode( + master: &[u8], + plaintext: &[u8], + format: u8, + nonce: Option<&[u8; 16]>, + ) -> Result { + let (nonce_ptr, nonce_len) = match nonce { + Some(n) => (n.as_ptr(), 16usize), + None => (std::ptr::null(), 0usize), + }; + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let mut hash_out = [0u8; 32]; + let mut padding_len = 0u32; + let mut chunk_len = 0u32; + let mut bytes_ecc = 0u32; + let mut verifiable_slice_count = 0u32; + let mut bytes_compressed = 0u32; + let mut bytes_encrypted = 0u32; + let rc = unsafe { + sys::carbonado_encode( + master.as_ptr(), + master.len(), + plaintext.as_ptr(), + plaintext.len(), + format, + nonce_ptr, + nonce_len, + &mut out, + &mut out_len, + hash_out.as_mut_ptr(), + &mut padding_len, + &mut chunk_len, + &mut bytes_ecc, + &mut verifiable_slice_count, + &mut bytes_compressed, + &mut bytes_encrypted, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + let body = take_buf(out, out_len)?; + let hash = crate::utils::decode_bao_hash(&hash_out)?; + let chunk_slice_count = if verifiable_slice_count > 0 { + verifiable_slice_count / 8 + } else { + 0 + }; + let input_len = plaintext.len() as u32; + let info = EncodeInfo { + input_len, + output_len: body.len() as u32, + bytes_compressed, + compression_factor: bytes_compressed as f32 / input_len.max(1) as f32, + bytes_encrypted, + bytes_ecc, + bytes_verifiable: body.len() as u32, + // Match Rust stream: empty input → 0.0 (not 1.0). + amplification_factor: body.len() as f32 / input_len.max(1) as f32, + padding_len, + chunk_len, + verifiable_slice_count, + chunk_slice_count, + }; + Ok(Encoded(body, hash, info)) + } + + /// Low-level body decode (≈ `decoding::decode` buffer path). + pub fn decode( + master: &[u8], + hash: &[u8], + body: &[u8], + padding: u32, + format: u8, + ) -> Result, CarbonadoError> { + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_decode( + master.as_ptr(), + master.len(), + hash.as_ptr(), + hash.len(), + body.as_ptr(), + body.len(), + padding, + format, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + take_buf(out, out_len) + } + /// Headered encode via Lean AOT (explicit 16-byte nonce when encrypted). + /// + /// `slh_public_key` / `metadata`: when `None`, header fields are zeros (matches Rust). + /// + /// Returns the full archive (`Header || body`) and pipeline [`EncodeInfo`] stage + /// counters (R3: compress/encrypt + FEC/Bao geometry from Lean pack). pub fn encode_headered( master: &[u8], plaintext: &[u8], format: u8, nonce: Option<&[u8; 16]>, - ) -> Result, CarbonadoError> { + slh_public_key: Option<&[u8; 32]>, + metadata: Option<&[u8; 8]>, + ) -> Result<(Vec, EncodeInfo), CarbonadoError> { let (nonce_ptr, nonce_len) = match nonce { Some(n) => (n.as_ptr(), 16usize), None => (std::ptr::null(), 0usize), }; + let slh_ptr = slh_public_key + .map(|s| s.as_ptr()) + .unwrap_or(std::ptr::null()); + let meta_ptr = metadata.map(|m| m.as_ptr()).unwrap_or(std::ptr::null()); let mut out: *mut u8 = std::ptr::null_mut(); let mut out_len: usize = 0; + let mut padding_len = 0u32; + let mut chunk_len = 0u32; + let mut bytes_ecc = 0u32; + let mut verifiable_slice_count = 0u32; + let mut bytes_compressed = 0u32; + let mut bytes_encrypted = 0u32; let rc = unsafe { sys::carbonado_encode_headered( master.as_ptr(), @@ -84,23 +335,52 @@ pub mod lean { format, nonce_ptr, nonce_len, + slh_ptr, + meta_ptr, &mut out, &mut out_len, + &mut padding_len, + &mut chunk_len, + &mut bytes_ecc, + &mut verifiable_slice_count, + &mut bytes_compressed, + &mut bytes_encrypted, ) }; if rc != sys::CARBONADO_OK { return Err(map_err(rc)); } - if out.is_null() { - return Err(map_err(sys::CARBONADO_ERR_INTERNAL)); - } - let v = unsafe { Vec::from_raw_parts(out, out_len, out_len) }; - // from_raw_parts takes ownership; do not free via carbonado_free. - Ok(v) + let archive = take_buf(out, out_len)?; + let body_len = archive.len().saturating_sub(crate::file::Header::LEN) as u32; + let chunk_slice_count = if verifiable_slice_count > 0 { + verifiable_slice_count / 8 + } else { + 0 + }; + let input_len = plaintext.len() as u32; + let info = EncodeInfo { + input_len, + output_len: body_len, + bytes_compressed, + compression_factor: bytes_compressed as f32 / input_len.max(1) as f32, + bytes_encrypted, + bytes_ecc, + bytes_verifiable: body_len, + // Match Rust stream: empty input → 0.0 (not 1.0). + amplification_factor: body_len as f32 / input_len.max(1) as f32, + padding_len, + chunk_len, + verifiable_slice_count, + chunk_slice_count, + }; + Ok((archive, info)) } - /// Headered decode via Lean AOT. - pub fn decode_headered(master: &[u8], archive: &[u8]) -> Result, CarbonadoError> { + /// Headered decode via Lean AOT → (Header, plaintext). + pub fn decode_headered( + master: &[u8], + archive: &[u8], + ) -> Result<(crate::file::Header, Vec), CarbonadoError> { let mut out: *mut u8 = std::ptr::null_mut(); let mut out_len: usize = 0; let rc = unsafe { @@ -116,9 +396,406 @@ pub mod lean { if rc != sys::CARBONADO_OK { return Err(map_err(rc)); } - if out.is_null() { - return Err(map_err(sys::CARBONADO_ERR_INTERNAL)); + let plaintext = take_buf(out, out_len)?; + // Reconstruct Header from archive prefix (already MAC-verified inside Lean). + if archive.len() < crate::file::Header::LEN { + return Err(CarbonadoError::InvalidHeaderLength); + } + let header = crate::file::Header::try_from(&archive[..crate::file::Header::LEN])?; + Ok((header, plaintext)) + } + + /// Outboard encode via Lean AOT. + /// + /// `header_path`: when true (and encrypted), bare main is `[tag|ct]` with nonce + /// out-of-band (matches `file::encode_outboard`). When false, embedded `[nonce|tag|ct]` + /// (matches low-level `encoding::encode_outboard`). + pub fn encode_outboard( + master: &[u8], + plaintext: &[u8], + format: u8, + nonce: Option<&[u8; 16]>, + header_path: bool, + ) -> Result { + let (nonce_ptr, nonce_len) = match nonce { + Some(n) => (n.as_ptr(), 16usize), + None => (std::ptr::null(), 0usize), + }; + let mut main_out: *mut u8 = std::ptr::null_mut(); + let mut main_len: usize = 0; + let mut ob_out: *mut u8 = std::ptr::null_mut(); + let mut ob_len: usize = 0; + let mut par_out: *mut u8 = std::ptr::null_mut(); + let mut par_len: usize = 0; + let mut hash_out = [0u8; 32]; + let mut padding_len = 0u32; + let mut chunk_len = 0u32; + let mut bytes_compressed = 0u32; + let mut bytes_encrypted = 0u32; + let rc = unsafe { + sys::carbonado_encode_outboard( + master.as_ptr(), + master.len(), + plaintext.as_ptr(), + plaintext.len(), + format, + nonce_ptr, + nonce_len, + u8::from(header_path), + &mut main_out, + &mut main_len, + &mut ob_out, + &mut ob_len, + &mut par_out, + &mut par_len, + hash_out.as_mut_ptr(), + &mut padding_len, + &mut chunk_len, + &mut bytes_compressed, + &mut bytes_encrypted, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + // Take all three buffers with cleanup on any failure (no leaked mallocs). + let main = match take_buf(main_out, main_len) { + Ok(v) => v, + Err(e) => { + free_raw(ob_out); + free_raw(par_out); + return Err(e); + } + }; + let ob_bytes = match take_buf(ob_out, ob_len) { + Ok(v) => v, + Err(e) => { + free_raw(par_out); + return Err(e); + } + }; + let par_bytes = take_buf(par_out, par_len)?; + let fmt = crate::constants::Format::from(format); + // Match Rust stream_encode_outboard_buffer: Verification ⇒ Some(ob) even when + // empty (valid single-leaf post-order outboard); Fec ⇒ Some(parity) similarly. + let verification_outboard = if fmt.contains(crate::constants::Format::Verification) { + Some(ob_bytes) + } else { + None + }; + let fec_parity = if fmt.contains(crate::constants::Format::Fec) { + Some(par_bytes) + } else { + None + }; + let hash = crate::utils::decode_bao_hash(&hash_out)?; + // Match Rust: outboard bytes_ecc is the FEC parity sidecar length, not main.len(). + let bytes_ecc = fec_parity.as_ref().map(|p| p.len() as u32).unwrap_or(0); + let verifiable_slice_count = if fmt.contains(crate::constants::Format::Fec) && chunk_len > 0 + { + // 8 shards × chunk_len / SLICE_LEN for inboard-equivalent bookkeeping. + (chunk_len * 8) / crate::constants::SLICE_LEN + } else { + 0 + }; + let input_len = plaintext.len() as u32; + let info = EncodeInfo { + input_len, + output_len: main.len() as u32, + bytes_compressed, + compression_factor: bytes_compressed as f32 / input_len.max(1) as f32, + bytes_encrypted, + bytes_ecc, + bytes_verifiable: main.len() as u32, + // Match Rust stream: empty input → 0.0 (not 1.0). + amplification_factor: main.len() as f32 / input_len.max(1) as f32, + padding_len, + chunk_len, + verifiable_slice_count, + chunk_slice_count: if verifiable_slice_count > 0 { + verifiable_slice_count / 8 + } else { + 0 + }, + }; + Ok(OutboardEncoded { + main, + verification_outboard, + fec_parity, + hash, + info, + }) + } + + /// Outboard decode via Lean AOT. + /// + /// `header_path` / `nonce` must match encode-time layout (see [`encode_outboard`]). + #[allow(clippy::too_many_arguments)] + pub fn decode_outboard( + master: &[u8], + hash: &[u8], + main: &[u8], + verification_outboard: Option<&[u8]>, + fec_parity: Option<&[u8]>, + padding: u32, + format: u8, + nonce: Option<&[u8; 16]>, + header_path: bool, + ) -> Result, CarbonadoError> { + // Mirror Rust stream_decode_outboard: `None` (missing sidecar) is distinct from + // `Some(&[])` (empty outboard for single-leaf trees). Fail closed before C. + let fmt = crate::constants::Format::from(format); + if fmt.contains(crate::constants::Format::Verification) && verification_outboard.is_none() { + return Err(CarbonadoError::MissingVerificationOutboard); + } + if fmt.contains(crate::constants::Format::Fec) && fec_parity.is_none() { + return Err(CarbonadoError::MissingFecParity); + } + let ob = verification_outboard.unwrap_or(&[]); + let par = fec_parity.unwrap_or(&[]); + let (nonce_ptr, nonce_len) = match nonce { + Some(n) => (n.as_ptr(), 16usize), + None => (std::ptr::null(), 0usize), + }; + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_decode_outboard( + master.as_ptr(), + master.len(), + hash.as_ptr(), + hash.len(), + main.as_ptr(), + main.len(), + ob.as_ptr(), + ob.len(), + par.as_ptr(), + par.len(), + padding, + format, + u8::from(header_path), + nonce_ptr, + nonce_len, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + take_buf(out, out_len) + } + + /// Inboard scrub via Lean AOT. + pub fn scrub( + body: &[u8], + hash: &[u8], + padding: u32, + format: u8, + ) -> Result, CarbonadoError> { + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_scrub( + body.as_ptr(), + body.len(), + hash.as_ptr(), + hash.len(), + padding, + format, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + take_buf(out, out_len) + } + + /// Outboard scrub via Lean AOT. + pub fn scrub_outboard( + main: &[u8], + verification_outboard: Option<&[u8]>, + fec_parity: Option<&[u8]>, + hash: &[u8], + padding: u32, + chunk_len: u32, + format: u8, + ) -> Result, CarbonadoError> { + let fmt = crate::constants::Format::from(format); + if !fmt.contains(crate::constants::Format::Verification) { + return Err(CarbonadoError::ScrubRequiresVerification); + } + let Some(ob) = verification_outboard else { + return Err(CarbonadoError::MissingVerificationOutboard); + }; + // Fec + None: do not fail closed here. Pristine path can still return + // UnnecessaryScrub without parity when verify ok; recovery needs parity. + // Call Lean with empty parity (`unwrap_or`); if FEC err on recovery, + // map MissingFecParity when parity was None (post-C remap below). + let par = fec_parity.unwrap_or(&[]); + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_scrub_outboard( + main.as_ptr(), + main.len(), + ob.as_ptr(), + ob.len(), + par.as_ptr(), + par.len(), + hash.as_ptr(), + hash.len(), + padding, + chunk_len, + format, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + // Distinct missing-sidecar modes before collapsing to scrub/FEC. + if rc == sys::CARBONADO_ERR_FEC + && fmt.contains(crate::constants::Format::Fec) + && fec_parity.is_none() + { + return Err(CarbonadoError::MissingFecParity); + } + return Err(map_err(rc)); + } + take_buf(out, out_len) + } + + /// Inboard verify_slice via Lean AOT (W4a: O(slice) retained output in Lean). + /// + /// Geometry pre-checks mirror pure-Rust [`crate::stream::slice::verify_slice_inboard_seekable`] + /// order so dual-suite diagnostics keep `InvalidHeaderLength` / `HashDecodeError` / + /// `InvalidSliceIndex {..}` fields. Bao auth failures map via ABI + /// `CARBONADO_ERR_AUTHENTICATION` (R4 / docs/ABI.md). + /// + /// **Memory honesty:** Lean retains O(slice) after auth walk; this dispatcher still + /// passes the full `body` buffer to C (caller-owned input). `count == 0` returns + /// empty here without calling C (matches pure-Rust short-circuit). + pub fn verify_slice( + body: &[u8], + index: u32, + count: u32, + hash: &[u8], + format: u8, + ) -> Result, CarbonadoError> { + // Match pure-Rust order (stream/slice.rs::verify_slice_inboard_seekable): + // 1. count==0 → Ok([]) + // 2. content_len prefix (InvalidHeaderLength if < 8) + // 3. content_len==0 → InvalidSliceIndex + // 4. decode_bao_hash (HashDecodeError if len != 32) + // 5. OOB slice_byte_range → InvalidSliceIndex + // 6. C verify (auth / truncation / …) + if count == 0 { + return Ok(vec![]); + } + // Short inboard prefix (<8 B) → InvalidHeaderLength (Rust `inboard_bao_content_len_prefix`). + // Also enforced in Lean (`invalidPrefix` → ERR_INVALID_HEADER after R4 ofPipelineError). + if body.len() < 8 { + return Err(CarbonadoError::InvalidHeaderLength); + } + let content_len = u64::from_le_bytes( + body[0..8] + .try_into() + .map_err(|_| CarbonadoError::InvalidHeaderLength)?, + ); + // Empty-content slice index — structured fields the C ABI cannot carry. + if content_len == 0 { + return Err(CarbonadoError::InvalidSliceIndex { index, content_len }); + } + // Hash length before OOB (pure-Rust order: bad-hash+OOB → HashDecodeError first). + let _root = crate::utils::decode_bao_hash(hash)?; + // OOB slice index — need structured fields the C ABI cannot carry. + let slice_byte_start = u64::from(index) * u64::from(crate::constants::SLICE_LEN); + if slice_byte_start >= content_len { + return Err(CarbonadoError::InvalidSliceIndex { index, content_len }); + } + + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_verify_slice( + body.as_ptr(), + body.len(), + hash.as_ptr(), + hash.len(), + index, + count, + format, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); + } + take_buf(out, out_len) + } + + /// Seekable outboard verify_slice via Lean AOT (R9; W4b permanent full-buffer C). + /// + /// Geometry pre-checks mirror pure-Rust + /// [`crate::stream::slice::verify_slice_outboard`] for dual-suite diagnostics. + /// C ABI takes full main + outboard buffers (no ReadAt callback); Lean walks + /// O(slice + height) over the requested range. + pub fn verify_slice_outboard( + data: &[u8], + outboard_bytes: &[u8], + data_len: u64, + index: u32, + count: u32, + hash: &[u8], + format: u8, + ) -> Result, CarbonadoError> { + if count == 0 { + return Ok(vec![]); + } + if data_len == 0 { + return Err(CarbonadoError::InvalidSliceIndex { + index, + content_len: data_len, + }); + } + if data_len as usize != data.len() { + return Err(CarbonadoError::OutboardVerificationFailed(format!( + "data_len {data_len} != data buffer {}", + data.len() + ))); + } + let _root = crate::utils::decode_bao_hash(hash)?; + let slice_byte_start = u64::from(index) * u64::from(crate::constants::SLICE_LEN); + if slice_byte_start >= data_len { + return Err(CarbonadoError::InvalidSliceIndex { + index, + content_len: data_len, + }); + } + + let mut out: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = unsafe { + sys::carbonado_verify_slice_outboard( + data.as_ptr(), + data.len(), + outboard_bytes.as_ptr(), + outboard_bytes.len(), + hash.as_ptr(), + hash.len(), + index, + count, + format, + &mut out, + &mut out_len, + ) + }; + if rc != sys::CARBONADO_OK { + return Err(map_err(rc)); } - Ok(unsafe { Vec::from_raw_parts(out, out_len, out_len) }) + take_buf(out, out_len) } } diff --git a/src/bin/carbonado/key_store.rs b/src/bin/carbonado/key_store.rs index d725795..58861aa 100644 --- a/src/bin/carbonado/key_store.rs +++ b/src/bin/carbonado/key_store.rs @@ -30,17 +30,22 @@ pub struct MnemonicInitNotice { /// Resolve the on-disk path for the persisted mnemonic. /// /// Precedence: `CARBONADO_MNEMONIC_PATH` → XDG-style config dir (`directories` crate). -pub fn mnemonic_path() -> PathBuf { +/// Returns `Err` when no home/config dir is available and the env override is unset +/// (no `.expect` in production paths). +pub fn mnemonic_path() -> Result { if let Ok(p) = std::env::var("CARBONADO_MNEMONIC_PATH") { - return PathBuf::from(p); + return Ok(PathBuf::from(p)); } - let proj = directories::ProjectDirs::from("com", "bitmask-stack", "carbonado") - .expect("home directory required for default mnemonic path"); - proj.config_dir().join(MNEMONIC_FILENAME) + let proj = + directories::ProjectDirs::from("com", "bitmask-stack", "carbonado").ok_or_else(|| { + "home directory required for default mnemonic path; set CARBONADO_MNEMONIC_PATH" + .to_string() + })?; + Ok(proj.config_dir().join(MNEMONIC_FILENAME)) } pub fn mnemonic_exists() -> bool { - mnemonic_path().is_file() + mnemonic_path().map(|p| p.is_file()).unwrap_or(false) } /// Generate a new English BIP39 mnemonic (`word_count` must be 12, 15, 18, 21, or 24). @@ -70,7 +75,7 @@ pub fn ensure_mnemonic() -> Result, String> { /// Write mnemonic to [`mnemonic_path`] (fails if file exists unless `force`). pub fn save_mnemonic(mnemonic: &Mnemonic, force: bool) -> Result { - let path = mnemonic_path(); + let path = mnemonic_path()?; if path.exists() && !force { return Err(format!( "mnemonic already exists at {}; use --force to overwrite or `carbonado key import`", @@ -87,7 +92,7 @@ pub fn save_mnemonic(mnemonic: &Mnemonic, force: bool) -> Result Result { - let path = mnemonic_path(); + let path = mnemonic_path()?; let raw = fs::read_to_string(&path) .map_err(|e| format!("failed to read mnemonic at {}: {e}", path.display()))?; let phrase = raw.trim(); @@ -95,7 +100,7 @@ pub fn load_mnemonic() -> Result { return Err(format!("mnemonic file at {} is empty", path.display())); } Mnemonic::parse_in(Language::English, phrase) - .map_err(|e| format!("invalid mnemonic in {}: {e}", mnemonic_path().display())) + .map_err(|e| format!("invalid mnemonic in {}: {e}", path.display())) } /// Derive the 32-byte Carbonado master key from a BIP39 mnemonic (empty BIP39 passphrase). diff --git a/src/bin/carbonado/main.rs b/src/bin/carbonado/main.rs index fa4c955..32c652e 100644 --- a/src/bin/carbonado/main.rs +++ b/src/bin/carbonado/main.rs @@ -197,7 +197,12 @@ fn run_key_command(command: KeyCommands) -> Result<(), Box { - println!("{}", key_store::mnemonic_path().display()); + println!( + "{}", + key_store::mnemonic_path() + .map_err(cli_string_err)? + .display() + ); } } Ok(()) diff --git a/src/crypto.rs b/src/crypto.rs index 1c8ca37..80af737 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -183,6 +183,9 @@ pub fn derive_subkey(master: &[u8], label: &str) -> Result<[u8; 64], CarbonadoEr /// /// See AGENTS.md §2.1.5 (Keyed Bao KDF) and `tests/bao_keyed_contract.rs`. pub fn carbonado_verification_key(format: u8) -> [u8; 32] { + // Public KDF (not secret-key material). Always pure blake3 so the API is + // infallible on both backends. Lean AOT parity is checked via + // `backend::lean::verification_key` in `tests/lean_backend_smoke.rs`. blake3::derive_key("carbonado-v2/verification", &[format]) } diff --git a/src/decoding.rs b/src/decoding.rs index 2a709c3..48cf3b5 100644 --- a/src/decoding.rs +++ b/src/decoding.rs @@ -1,23 +1,29 @@ use std::io::Cursor; -use log::{debug, info, trace, warn}; +use log::trace; pub use crate::stream::compress::decompress_buffer as decompress; +#[cfg(feature = "backend-rust")] pub use crate::stream::decode::{stream_decode_buffer, stream_decode_outboard_buffer}; use crate::{ - constants::{Format, FEC_K, FEC_M}, - encoding, + constants::{FEC_K, FEC_M}, error::CarbonadoError, - stream::{extract_slice_inboard_for_scrub, verify_slice_inboard_seekable}, structs::EncodeInfo, - utils::decode_bao_hash, }; use reed_solomon_erasure::galois_8::Field; use reed_solomon_erasure::ReedSolomon; -use crate::constants::SLICE_LEN; +#[cfg(feature = "backend-rust")] +use crate::{ + constants::{Format, SLICE_LEN}, + encoding, + stream::{extract_slice_inboard_for_scrub, verify_slice_inboard_seekable}, + utils::decode_bao_hash, +}; +#[cfg(feature = "backend-rust")] +use log::{debug, info, warn}; fn fec_chunks(chunked_bytes: &[(usize, &[u8])], padding: u32) -> Result, CarbonadoError> { let data_shards = FEC_K; @@ -66,6 +72,7 @@ fn fec_chunks(chunked_bytes: &[(usize, &[u8])], padding: u32) -> Result, Ok(decoded) } +#[cfg_attr(feature = "backend-lean", allow(dead_code))] // used by rust scrub_outboard path pub fn verification_with_outboard( bare: &[u8], outboard: &[u8], @@ -200,7 +207,14 @@ pub fn decode( padding: u32, format: u8, ) -> Result, CarbonadoError> { - stream_decode_buffer(master_key, hash, input, padding, format) + #[cfg(feature = "backend-lean")] + { + crate::backend::lean::decode(master_key, hash, input, padding, format) + } + #[cfg(feature = "backend-rust")] + { + stream_decode_buffer(master_key, hash, input, padding, format) + } } pub fn decode_outboard( @@ -212,16 +226,34 @@ pub fn decode_outboard( padding: u32, format: u8, ) -> Result, CarbonadoError> { - stream_decode_outboard_buffer( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - None, - ) + #[cfg(feature = "backend-lean")] + { + // Low-level path: embedded-nonce when encrypted (header_path = false). + crate::backend::lean::decode_outboard( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + None, + false, + ) + } + #[cfg(feature = "backend-rust")] + { + stream_decode_outboard_buffer( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + None, + ) + } } pub fn extract_slice( @@ -241,7 +273,14 @@ pub fn verify_slice( format: u8, ) -> Result, CarbonadoError> { trace!("verify_slice seekable index={index} count={count} format=0x{format:02x}"); - verify_slice_inboard_seekable(input, index, count, hash, format) + #[cfg(feature = "backend-lean")] + { + crate::backend::lean::verify_slice(input, index, count, hash, format) + } + #[cfg(feature = "backend-rust")] + { + verify_slice_inboard_seekable(input, index, count, hash, format) + } } /// Recover a damaged inboard Bao+FEC archive via RS subset search and re-encode oracle. @@ -250,11 +289,31 @@ pub fn verify_slice( /// `InvalidHeaderLength`, `BaoResponseTruncated`, `StdIoError`, etc.) route into combinatorial /// FEC recovery — the API does not distinguish tamper from truncation before attempting recovery. /// Pristine archives return [`CarbonadoError::UnnecessaryScrub`]. +/// +/// Under `backend-lean`, uses C ABI `carbonado_scrub` (geometry peel + RS search). pub fn scrub( input: &[u8], hash: &[u8], encode_info: &EncodeInfo, format: u8, +) -> Result, CarbonadoError> { + #[cfg(feature = "backend-lean")] + { + let _ = encode_info; // padding is the normative scrub input; chunk geometry from body + crate::backend::lean::scrub(input, hash, encode_info.padding_len, format) + } + #[cfg(feature = "backend-rust")] + { + scrub_rust(input, hash, encode_info, format) + } +} + +#[cfg(feature = "backend-rust")] +fn scrub_rust( + input: &[u8], + hash: &[u8], + encode_info: &EncodeInfo, + format: u8, ) -> Result, CarbonadoError> { let fmt = Format::from(format); if !fmt.contains(Format::Verification) { @@ -331,6 +390,40 @@ pub fn scrub_outboard( encode_info: &EncodeInfo, format: u8, hash: &[u8], +) -> Result, CarbonadoError> { + #[cfg(feature = "backend-lean")] + { + crate::backend::lean::scrub_outboard( + bare, + verification_outboard, + fec_parity, + hash, + encode_info.padding_len, + encode_info.chunk_len, + format, + ) + } + #[cfg(feature = "backend-rust")] + { + scrub_outboard_rust( + bare, + verification_outboard, + fec_parity, + encode_info, + format, + hash, + ) + } +} + +#[cfg(feature = "backend-rust")] +fn scrub_outboard_rust( + bare: &[u8], + verification_outboard: Option<&[u8]>, + fec_parity: Option<&[u8]>, + encode_info: &EncodeInfo, + format: u8, + hash: &[u8], ) -> Result, CarbonadoError> { let fmt = Format::from(format); if !fmt.contains(Format::Verification) { diff --git a/src/encoding.rs b/src/encoding.rs index f4c7ea1..e2c13d3 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,27 +1,75 @@ use log::trace; -use crate::{ - error::CarbonadoError, - stream::encode::{stream_encode_buffer, stream_encode_outboard_buffer}, - structs::Encoded, -}; +use crate::{error::CarbonadoError, structs::Encoded}; -/// Encode data into Carbonado format (delegates to streaming pipeline). +use crate::stream::encode::stream_encode_buffer_with_nonce; +#[cfg(feature = "backend-rust")] +use crate::stream::encode::stream_encode_outboard_buffer; + +/// Encode data into Carbonado format (delegates to streaming pipeline, or Lean AOT). +/// +/// Under `backend-lean`, uses C ABI `carbonado_encode`. See [`crate::backend::lean`] +/// for EncodeInfo stage counters (R3: compress/encrypt + FEC/Bao geometry). +/// +/// Encrypted formats use a CSPRNG nonce (embedded layout). For deterministic encrypted +/// bodies (G9 fixtures), use [`encode_with_nonce`]. pub fn encode(master_key: &[u8], input: &[u8], format: u8) -> Result { - let (verifiable, hash, info) = stream_encode_buffer(master_key, input, format)?; + encode_with_nonce(master_key, input, format, None) +} + +/// Low-level body encode with optional fixed nonce for encrypted formats. +/// +/// When `explicit_nonce` is `Some(n)` and Encryption is set, the blob uses `n` in +/// embedded layout `[nonce|tag|ct]` (including all-zero — dual-backend identical). +/// When `None`, encrypted formats draw a CSPRNG nonce (production default). Public +/// formats ignore it. +/// +/// # Safety / intended use +/// +/// Fixed nonces are for **tests and determinism only** (e.g. G9 goldens). Prefer +/// [`encode`] for production. AES-CTR requires the nonce to be unique per +/// `(master_key, encryption operation)` — **reuse is catastrophic** (keystream reuse). +/// See AGENTS.md §2.1.4. +pub fn encode_with_nonce( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, +) -> Result { + let (verifiable, hash, info) = + stream_encode_buffer_with_nonce(master_key, input, format, explicit_nonce)?; Ok(Encoded(verifiable, hash, info)) } /// Outboard variant for public and encrypted formats. +/// +/// Under `backend-lean`, uses C ABI `carbonado_encode_outboard`. pub fn encode_outboard( master_key: &[u8], input: &[u8], format: u8, ) -> Result { trace!("encode_outboard format=0x{format:02x}"); - stream_encode_outboard_buffer(master_key, input, format, None) + #[cfg(feature = "backend-lean")] + { + // Low-level path: embedded-nonce when encrypted (header_path = false). + let nonce = if format & 1 != 0 { + let mut n = [0u8; 16]; + getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; + Some(n) + } else { + None + }; + crate::backend::lean::encode_outboard(master_key, input, format, nonce.as_ref(), false) + } + #[cfg(feature = "backend-rust")] + { + stream_encode_outboard_buffer(master_key, input, format, None) + } } -// Scrub recovery re-exports +// Scrub recovery re-exports (Rust scrub path; lean scrub is in-engine) +#[cfg(feature = "backend-rust")] pub use crate::stream::bao::verification_inboard_buffer; +#[cfg(feature = "backend-rust")] pub use crate::stream::fec::encode_inboard_buffer; diff --git a/src/file.rs b/src/file.rs index 649bc5e..f1b0d8a 100644 --- a/src/file.rs +++ b/src/file.rs @@ -9,6 +9,8 @@ use bao::Hash; // nom imports removed — legacy parse_bytes / old header parsing was deleted as part of the v2 replacement. // (secp256k1 imports removed - clean break, legacy Header parsing deleted) +#[cfg(feature = "backend-rust")] +use crate::stream::decode::stream_decrypt_header_path; use crate::{ adamantine::{ decode_adamantine, encode_adamantine, AdamantineHeader, ADAMANTINE_CARBONADO_FMT_ENCRYPTED, @@ -28,10 +30,7 @@ use crate::{ FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, MAX_SEGMENT_MAIN_LEN, }, paths::parse_bao_root_from_filename, - stream::{ - decode::stream_decrypt_header_path, encode::stream_encode_outboard, - DEFAULT_SEGMENT_PLAINTEXT_BUDGET, - }, + stream::{encode::stream_encode_outboard, DEFAULT_SEGMENT_PLAINTEXT_BUDGET}, structs::{EncodeInfo, OutboardEncoded}, utils::{calc_padding_len, decode_bao_hash, encode_bao_hash}, }; @@ -307,114 +306,183 @@ impl Header { /// Stream decode from headered inboard archive. /// +/// Headered inboard decode over [`Read`] / [`Write`]. +/// +/// Reads the 177-byte [`Header`], verifies integrity, then reverses the body pipeline +/// (Bao/FEC → decrypt → decompress as format bits require). +/// +/// # `backend-rust` +/// /// Bao/FEC reverse pipes into a [`crate::stream::spool::SeekableSpool`] via /// [`crate::stream::decode::stream_decode_inboard_bao_fec_into`], bounded by /// [`Header::encoded_len`]. Encrypted segments use [`crate::stream::stream_decrypt_header_path`] /// (streaming MAC-then-decrypt on `[tag(64) | ct]` with explicit `payload_nonce`). /// -/// **Memory tiers:** +/// **Memory tiers (rust):** /// - **(A) Verification formats:** Bao verify streams incrementally; FEC reverse may still retain /// **O(logical)** at the Bao/FEC step (see `doc/STREAMING_PARALLELISM.md`). /// - **(B) Post-Bao/FEC spool:** O(chunk) RAM during disk-backed staging (no body `Vec`). /// - **(C) Encrypted:** streaming EtM via spool two-pass MAC verify then CTR decrypt. /// - **(D) c4/c8:** bounded by `encoded_len` when known; incremental FEC/decompress otherwise. +/// +/// # `backend-lean` (W1a) +/// +/// Reads the 177-byte header, verifies `header_mac` **before** body I/O (same fail-closed +/// order as rust), then spools `encoded_len` body into an archive buffer and calls Lean +/// [`crate::backend::lean::decode_headered`] for dual body pipeline. Peak RAM +/// **O(header + body + plaintext)** — E1 honesty, not stream E2. (W1b public **non-compress** +/// outboard stream uses S4 O(chunk) composition; this headered path remains Lean E1.) pub fn decode_stream( master_key: &[u8], mut input: R, output: &mut W, ) -> Result<(Header, u64), CarbonadoError> { - let mut header_bytes = [0u8; Header::LEN]; - input - .read_exact(&mut header_bytes) - .map_err(CarbonadoError::StdIoError)?; - let header = Header::try_from(&header_bytes[..])?; - - let auth_data = build_header_auth_data(&header); - let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; - if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { - return Err(CarbonadoError::AuthenticationFailed); - } - - let fmt = header.format; - let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; - let mut body_reader = std::io::Read::by_ref(&mut input).take(header.encoded_len as u64); - crate::stream::decode::stream_decode_inboard_bao_fec_into( - &mut body_reader, - header.hash.as_bytes(), - header.padding_len, - fmt, - Some(header.encoded_len as u64), - &mut post_preprocess, - )?; - post_preprocess.rewind()?; - let out_len = if fmt.contains(Format::Encryption) { - stream_decrypt_header_path( - master_key, - header.payload_nonce, + #[cfg(feature = "backend-lean")] + { + use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; + + let mut header_bytes = [0u8; Header::LEN]; + input + .read_exact(&mut header_bytes) + .map_err(CarbonadoError::StdIoError)?; + let header_probe = Header::try_from(&header_bytes[..])?; + // MAC-before-body (parity with rust path): reject unauthenticated peers before + // allocating/reading up to MAX_SEGMENT_MAIN_LEN body bytes. Lean re-verifies MAC + // inside decode_headered for dual body/pipeline honesty. + let auth_data = build_header_auth_data(&header_probe); + let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; + if !crate::crypto::ct_eq(&expected_mac, &header_probe.header_mac) { + return Err(CarbonadoError::AuthenticationFailed); + } + let body_len = header_probe.encoded_len as u64; + if body_len > MAX_SEGMENT_MAIN_LEN { + return Err(CarbonadoError::InternalStateError(format!( + "header encoded_len {body_len} exceeds MAX_SEGMENT_MAIN_LEN {MAX_SEGMENT_MAIN_LEN}" + ))); + } + let mut body = vec![0u8; body_len as usize]; + if let Err(e) = input.read_exact(&mut body) { + // Match rust pipeline taxonomy: short body after a valid header prefix is + // `InvalidHeaderLength` (not bare UnexpectedEof), e.g. truncated Bao bodies. + return Err(if e.kind() == std::io::ErrorKind::UnexpectedEof { + CarbonadoError::InvalidHeaderLength + } else { + CarbonadoError::StdIoError(e) + }); + } + let mut archive = Vec::with_capacity(Header::LEN + body.len()); + archive.extend_from_slice(&header_bytes); + archive.extend_from_slice(&body); + drop(body); // free body copy before Lean allocates plaintext + let (header, plaintext) = crate::backend::lean::decode_headered(master_key, &archive)?; + drop(archive); + output + .write_all(&plaintext) + .map_err(CarbonadoError::StdIoError)?; + Ok((header, plaintext.len() as u64)) + } + #[cfg(feature = "backend-rust")] + { + let mut header_bytes = [0u8; Header::LEN]; + input + .read_exact(&mut header_bytes) + .map_err(CarbonadoError::StdIoError)?; + let header = Header::try_from(&header_bytes[..])?; + + let auth_data = build_header_auth_data(&header); + let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; + if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { + return Err(CarbonadoError::AuthenticationFailed); + } + + let fmt = header.format; + let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; + let mut body_reader = std::io::Read::by_ref(&mut input).take(header.encoded_len as u64); + crate::stream::decode::stream_decode_inboard_bao_fec_into( + &mut body_reader, + header.hash.as_bytes(), + header.padding_len, + fmt, + Some(header.encoded_len as u64), &mut post_preprocess, - fmt.bits(), - output, - )? - } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(post_preprocess, output)? - } else { - std::io::copy(&mut post_preprocess, output).map_err(CarbonadoError::StdIoError)? - }; + )?; + post_preprocess.rewind()?; + let out_len = if fmt.contains(Format::Encryption) { + stream_decrypt_header_path( + master_key, + header.payload_nonce, + &mut post_preprocess, + fmt.bits(), + output, + )? + } else if fmt.contains(Format::Compression) { + crate::stream::compress::stream_decompress(post_preprocess, output)? + } else { + std::io::copy(&mut post_preprocess, output).map_err(CarbonadoError::StdIoError)? + }; - Ok((header, out_len)) + Ok((header, out_len)) + } } pub fn decode(master_key: &[u8], encoded: &[u8]) -> Result<(Header, Vec), CarbonadoError> { - if encoded.len() < Header::LEN { - return Err(CarbonadoError::InvalidHeaderLength); - } - let (header_bytes, body) = encoded.split_at(Header::LEN); - let header = Header::try_from(header_bytes)?; - - // Verify header_mac - let auth_data = build_header_auth_data(&header); - let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; - - // Constant-time comparison for the header MAC to avoid timing side-channels. - // (See AGENTS.md for the constant-time review of EtM + header auth paths.) - if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { - return Err(CarbonadoError::AuthenticationFailed); - } - - // Same fused spool pipeline as decode_stream (streaming MAC-then-decrypt on header path). - // Body may include trailers (e.g. catalog COTS) after `encoded_len` bytes — limit the reader. - let fmt = header.format; - let body_len = header.encoded_len as usize; - if body.len() < body_len { - return Err(CarbonadoError::InvalidHeaderLength); - } - let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; - crate::stream::decode::stream_decode_inboard_bao_fec_into( - std::io::Cursor::new(&body[..body_len]), - header.hash.as_bytes(), - header.padding_len, - fmt, - Some(header.encoded_len as u64), - &mut post_preprocess, - )?; - post_preprocess.rewind()?; - let mut decompressed = Vec::new(); - if fmt.contains(Format::Encryption) { - crate::stream::stream_decrypt_header_path( - master_key, - header.payload_nonce, + #[cfg(feature = "backend-lean")] + { + crate::backend::lean::decode_headered(master_key, encoded) + } + #[cfg(feature = "backend-rust")] + { + if encoded.len() < Header::LEN { + return Err(CarbonadoError::InvalidHeaderLength); + } + let (header_bytes, body) = encoded.split_at(Header::LEN); + let header = Header::try_from(header_bytes)?; + + // Verify header_mac + let auth_data = build_header_auth_data(&header); + let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; + + // Constant-time comparison for the header MAC to avoid timing side-channels. + // (See AGENTS.md for the constant-time review of EtM + header auth paths.) + if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { + return Err(CarbonadoError::AuthenticationFailed); + } + + // Same fused spool pipeline as decode_stream (streaming MAC-then-decrypt on header path). + // Body may include trailers (e.g. catalog COTS) after `encoded_len` bytes — limit the reader. + let fmt = header.format; + let body_len = header.encoded_len as usize; + if body.len() < body_len { + return Err(CarbonadoError::InvalidHeaderLength); + } + let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; + crate::stream::decode::stream_decode_inboard_bao_fec_into( + std::io::Cursor::new(&body[..body_len]), + header.hash.as_bytes(), + header.padding_len, + fmt, + Some(header.encoded_len as u64), &mut post_preprocess, - fmt.bits(), - &mut decompressed, )?; - } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(post_preprocess, &mut decompressed)?; - } else { - std::io::copy(&mut post_preprocess, &mut decompressed) - .map_err(CarbonadoError::StdIoError)?; - } + post_preprocess.rewind()?; + let mut decompressed = Vec::new(); + if fmt.contains(Format::Encryption) { + crate::stream::stream_decrypt_header_path( + master_key, + header.payload_nonce, + &mut post_preprocess, + fmt.bits(), + &mut decompressed, + )?; + } else if fmt.contains(Format::Compression) { + crate::stream::compress::stream_decompress(post_preprocess, &mut decompressed)?; + } else { + std::io::copy(&mut post_preprocess, &mut decompressed) + .map_err(CarbonadoError::StdIoError)?; + } - Ok((header, decompressed)) + Ok((header, decompressed)) + } } /// High-level encode using the new v2 symmetric model (always inboard with Header prepended). @@ -426,17 +494,92 @@ pub fn decode(master_key: &[u8], encoded: &[u8]) -> Result<(Header, Vec), Ca /// (public and encrypted formats share the same artifact split; optional out-of-band Header). /// Sidecar naming convention: .cXX.out (Bao), .cXX.par (FEC parity). /// See AGENTS §11.2 (completed) and low-level `encoding::encode_outboard`. +/// +/// # `backend-lean` (Phase 2) +/// +/// Dispatches to Lean AOT via C ABI (`carbonado_encode_headered`). +/// +/// - **`metadata` / SLH pk:** plumbed through C ABI (nullable → zero fields). +/// - **`EncodeInfo`:** full stage counters from Lean pack (R3) — compress/encrypt when +/// those bits ran, FEC/Bao geometry, padding, and `output_len`/`bytes_verifiable` +/// from body length (matches header `encoded_len`). +/// - **Live dual under lean:** body/headered encode-decode, outboard (header-path when +/// `file::encode_outboard` supplies `Some(payload_nonce)`), scrub, verify_slice, +/// stream buffer + **R5 E1** stream I/O (inboard/encrypted outboard spool→Lean), +/// **W1a** `decode_stream` → Lean `decode_headered`, **W1b** public outboard stream +/// S4 O(chunk/stripe) composition, seekable outboard slice C (**R9**), optional **R10** +/// `stream_decode_async` under lean+`async` (dual-aware via E1; freeze never requires `async`). +/// - **Post-G8 residuals (honest):** pure Lean chunked stream residual (W1b public outboard +/// E2 is rust geometric composition under lean; encrypted/inboard stream remain E1); +/// dual-suite catalog encode remains Rust rkyv composition SSOT (**W3** pure Lean rkyv +/// also available); ~~W4a~~ O(slice) inboard retain closed; **W4b** full-buffer C outboard +/// slice permanent; **W4c** buffer-only zstd under lean; **W4d** FEC/async spool permanent. +/// Dual-suite SLH may use Rust `bitcoinpqc` composition. pub fn encode( master_key: &[u8], input: &[u8], level: u8, metadata: Option<[u8; 8]>, ) -> Result<(Vec, EncodeInfo), CarbonadoError> { - let mut out = Vec::new(); - let (header, info) = encode_stream(master_key, input, level, metadata, &mut out)?; - let mut body = header.try_to_vec()?; - body.extend_from_slice(&out); - Ok((body, info)) + encode_with_nonce(master_key, input, level, metadata, None) +} + +/// Headered inboard encode with optional fixed `payload_nonce` for encrypted formats. +/// +/// When `explicit_nonce` is `Some(n)` and Encryption is set, **both backends** use `n` +/// literally (including all-zero). When `None`, encrypted formats draw a CSPRNG nonce +/// (production default). Public formats use a zero `payload_nonce` field regardless. +/// +/// # Safety / intended use +/// +/// Fixed nonces are for **tests and determinism only** (e.g. G9 goldens). Prefer +/// [`encode`] for production so a fresh CSPRNG nonce is drawn. AES-CTR requires the +/// nonce to be unique per `(master_key, encryption operation)` — **reuse is catastrophic** +/// (keystream reuse → plaintext recovery). See AGENTS.md §2.1.4. +pub fn encode_with_nonce( + master_key: &[u8], + input: &[u8], + level: u8, + metadata: Option<[u8; 8]>, + explicit_nonce: Option<[u8; 16]>, +) -> Result<(Vec, EncodeInfo), CarbonadoError> { + #[cfg(feature = "backend-lean")] + { + let format = level; + let nonce = if format & 1 != 0 { + match explicit_nonce { + Some(n) => Some(n), + None => { + let mut n = [0u8; 16]; + getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; + Some(n) + } + } + } else { + None + }; + let (archive, info) = crate::backend::lean::encode_headered( + master_key, + input, + format, + nonce.as_ref(), + None, // slh_public_key: dual-suite sets via sidecar path / Header APIs + metadata.as_ref(), + )?; + if archive.len() < Header::LEN { + return Err(CarbonadoError::InvalidHeaderLength); + } + Ok((archive, info)) + } + #[cfg(feature = "backend-rust")] + { + let mut out = Vec::new(); + let (header, info) = + encode_stream_with_nonce(master_key, input, level, metadata, &mut out, explicit_nonce)?; + let mut body = header.try_to_vec()?; + body.extend_from_slice(&out); + Ok((body, info)) + } } /// Headered inboard encode over [`Read`] / [`Write`]. Header is returned for staging; body @@ -446,21 +589,37 @@ pub fn encode( /// multi-segment archives use [`encode_shard_stream`](crate::stream::encode_shard_stream) /// with monotonic `chunk_index` values and [`decode_shards_stream`](crate::stream::decode_shards_stream). pub fn encode_stream( + master_key: &[u8], + input: R, + level: u8, + metadata: Option<[u8; 8]>, + output: &mut W, +) -> Result<(Header, EncodeInfo), CarbonadoError> { + encode_stream_with_nonce(master_key, input, level, metadata, output, None) +} + +/// Like [`encode_stream`], with optional fixed `payload_nonce` for encrypted formats. +/// +/// When `explicit_nonce` is `Some(n)`, both backends use `n` literally (including +/// all-zero). See [`encode_with_nonce`] for safety notes (test/determinism only). +pub fn encode_stream_with_nonce( master_key: &[u8], mut input: R, level: u8, metadata: Option<[u8; 8]>, output: &mut W, + explicit_nonce: Option<[u8; 16]>, ) -> Result<(Header, EncodeInfo), CarbonadoError> { let format = Format::from(level); let mut payload_nonce = [0u8; 16]; - let (hash, info, _stats) = crate::stream::encode::stream_encode_inboard( + let (hash, info, _stats) = crate::stream::encode::stream_encode_inboard_with_nonce( master_key, &mut input, level, output, &mut payload_nonce, true, + explicit_nonce, )?; let header = Header::new( diff --git a/src/lib.rs b/src/lib.rs index 926c916..e0c102b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,6 +115,7 @@ pub mod paths; pub mod stream; pub use encoding::encode; +pub use encoding::encode_with_nonce; pub use encoding::encode_outboard; @@ -139,8 +140,8 @@ pub use stream::stream_decode_async; pub use stream::{ decode_shards_stream, encode_shard_stream, stream_decode, stream_decode_buffer, stream_decode_outboard, stream_decode_outboard_buffer, stream_encode_buffer, - stream_encode_outboard_buffer, verify_slice_inboard_seekable, verify_slice_outboard, - ShardEncodeResult, ShardSource, DEFAULT_SEGMENT_PLAINTEXT_BUDGET, + stream_encode_buffer_with_nonce, stream_encode_outboard_buffer, verify_slice_inboard_seekable, + verify_slice_outboard, ShardEncodeResult, ShardSource, DEFAULT_SEGMENT_PLAINTEXT_BUDGET, }; pub use bao; diff --git a/src/stream/compress.rs b/src/stream/compress.rs index 579d8aa..778dc5b 100644 --- a/src/stream/compress.rs +++ b/src/stream/compress.rs @@ -34,15 +34,50 @@ impl Write for CountWriter { } /// Stream-compress `input` into `output` at level 20. Returns compressed bytes written. +/// +/// Under `backend-lean`, materializes the input and uses the zstd **buffer** API +/// (`ZSTD_compress` / `zstd::bulk`) so frames match Lean AOT (`Carbonado.Compress`). +/// Streaming `copy_encode` frames differ byte-for-byte from the buffer API at the same +/// level — that mismatch broke stream-vs-buffer parity under dual-engine (R5). +/// +/// **W4c permanent residual:** buffer-only under lean (no dual-safe multi-chunk streaming +/// frames). Cross-engine compress re-encode remains non-bit-identical (**W2a**); do not +/// invent streaming-frame bit-match claims. Peak: **O(logical)** RAM for compress under +/// lean — public outboard formats with the Compression bit (c2/c6/c10/c14) are **not** +/// W1b E2 under lean; E2 MVP is **non-compress** public outboard (c0/c4/c8/c12). +/// See docs/LIMITS.md Stream E1/E2 matrix. pub fn stream_compress(mut input: R, output: W) -> Result { - let mut counter = CountWriter { - inner: output, - count: 0, - max: None, - }; - zstd::stream::copy_encode(&mut input, &mut counter, ZSTD_LEVEL) - .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; - Ok(counter.count) + #[cfg(feature = "backend-lean")] + { + let mut plaintext = Vec::new(); + input + .read_to_end(&mut plaintext) + .map_err(CarbonadoError::StdIoError)?; + let compressed = zstd::bulk::Compressor::new(ZSTD_LEVEL) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))? + .compress(&plaintext) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + let mut counter = CountWriter { + inner: output, + count: 0, + max: None, + }; + counter + .write_all(&compressed) + .map_err(CarbonadoError::StdIoError)?; + Ok(counter.count) + } + #[cfg(feature = "backend-rust")] + { + let mut counter = CountWriter { + inner: output, + count: 0, + max: None, + }; + zstd::stream::copy_encode(&mut input, &mut counter, ZSTD_LEVEL) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + Ok(counter.count) + } } /// Stream-decompress `input` into `output`. Returns decompressed bytes written. diff --git a/src/stream/crypto_stream.rs b/src/stream/crypto_stream.rs index 61d4e54..f3f5233 100644 --- a/src/stream/crypto_stream.rs +++ b/src/stream/crypto_stream.rs @@ -95,14 +95,20 @@ pub fn stream_encrypt_with_nonce_seek( stream_encrypt_with_nonce(master_key, nonce, input, output) } -/// Low-level path: random nonce embedded in output `[nonce(16) | tag(64) | ct]`. -pub fn stream_encrypt( +/// Low-level path with caller-supplied nonce: output `[nonce(16) | tag(64) | ct]`. +/// +/// Used by deterministic fixture generators (G9) and by [`stream_encrypt`] after drawing +/// a random nonce. Production callers should prefer [`stream_encrypt`] (CSPRNG nonce). +/// +/// **Safety:** AES-CTR requires the nonce to be unique per `(master_key, operation)`. +/// Reuse under the same master is catastrophic (keystream reuse). Fixed nonces are for +/// tests/determinism only — see AGENTS.md §2.1.4. +pub fn stream_encrypt_embedded_with_nonce( master_key: &[u8], + nonce: [u8; 16], input: R, output: &mut W, -) -> Result<(u64, [u8; 16]), CarbonadoError> { - let mut nonce = [0u8; 16]; - getrandom::getrandom(&mut nonce).map_err(|_| CarbonadoError::RandomnessError)?; +) -> Result { output .write_all(&nonce) .map_err(CarbonadoError::StdIoError)?; @@ -110,7 +116,19 @@ pub fn stream_encrypt( .stream_position() .map_err(CarbonadoError::StdIoError)?; let inner = stream_encrypt_with_nonce_at(master_key, nonce, input, output, tag_offset)?; - Ok((NONCE_LEN as u64 + inner, nonce)) + Ok(NONCE_LEN as u64 + inner) +} + +/// Low-level path: random nonce embedded in output `[nonce(16) | tag(64) | ct]`. +pub fn stream_encrypt( + master_key: &[u8], + input: R, + output: &mut W, +) -> Result<(u64, [u8; 16]), CarbonadoError> { + let mut nonce = [0u8; 16]; + getrandom::getrandom(&mut nonce).map_err(|_| CarbonadoError::RandomnessError)?; + let len = stream_encrypt_embedded_with_nonce(master_key, nonce, input, output)?; + Ok((len, nonce)) } /// Header-path encrypt with tag placeholder at `tag_offset` (not necessarily 0). diff --git a/src/stream/decode.rs b/src/stream/decode.rs index 40a821e..537260b 100644 --- a/src/stream/decode.rs +++ b/src/stream/decode.rs @@ -14,6 +14,8 @@ use crate::{ }; /// Primary inboard decode (buffer). Used by [`crate::decoding::decode`]. +/// +/// Under `backend-lean`, composes over Lean C ABI body decode. pub fn stream_decode_buffer( master_key: &[u8], hash: &[u8], @@ -21,20 +23,30 @@ pub fn stream_decode_buffer( padding: u32, format: u8, ) -> Result, CarbonadoError> { - let mut out = Vec::new(); - stream_decode_inboard_pipeline( - master_key, - hash, - Cursor::new(input), - padding, - format, - None, - &mut out, - )?; - Ok(out) + #[cfg(feature = "backend-lean")] + { + crate::backend::lean::decode(master_key, hash, input, padding, format) + } + #[cfg(feature = "backend-rust")] + { + let mut out = Vec::new(); + stream_decode_inboard_pipeline( + master_key, + hash, + Cursor::new(input), + padding, + format, + None, + &mut out, + )?; + Ok(out) + } } /// Primary outboard decode (buffer). Used by [`crate::decoding::decode_outboard`]. +/// +/// Under `backend-lean`, composes over Lean C ABI outboard decode. +/// `explicit_nonce.is_some()` → header-path decrypt (`[tag|ct]`); else embedded-nonce. #[allow(clippy::too_many_arguments)] pub fn stream_decode_outboard_buffer( master_key: &[u8], @@ -46,19 +58,37 @@ pub fn stream_decode_outboard_buffer( format: u8, explicit_nonce: Option<[u8; 16]>, ) -> Result, CarbonadoError> { - let mut out = Vec::new(); - stream_decode_outboard( - master_key, - hash, - Cursor::new(main), - verification_outboard.map(Cursor::new), - fec_parity.map(Cursor::new), - padding, - format, - explicit_nonce, - &mut out, - )?; - Ok(out) + #[cfg(feature = "backend-lean")] + { + let header_path = explicit_nonce.is_some(); + crate::backend::lean::decode_outboard( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + explicit_nonce.as_ref(), + header_path, + ) + } + #[cfg(feature = "backend-rust")] + { + let mut out = Vec::new(); + stream_decode_outboard( + master_key, + hash, + Cursor::new(main), + verification_outboard.map(Cursor::new), + fec_parity.map(Cursor::new), + padding, + format, + explicit_nonce, + &mut out, + )?; + Ok(out) + } } /// Stream inboard decode from `input` to `output`. @@ -71,6 +101,13 @@ pub fn stream_decode_outboard_buffer( /// /// Pass `encoded_body_len` when the reader may contain trailing bytes after the encoded /// body (FEC c8, compressed c4). When `Some`, excess or truncated input is rejected. +/// +/// Under `backend-lean` (R5 E1 / W1b disk-backed): spool body (O(chunk) ingest) → Lean +/// [`crate::backend::lean::decode`] → write plaintext. Peak RAM **O(encoded + logical)** at +/// the Lean buffer boundary (not stream E2). See docs/LIMITS.md E1/E2 matrix. +/// +/// **R10:** [`super::stream_decode_async`] stages the encoded body then calls this function +/// (dual-aware; freeze never requires `async`). pub fn stream_decode( master_key: &[u8], hash: &[u8], @@ -80,18 +117,76 @@ pub fn stream_decode( encoded_body_len: Option, output: &mut W, ) -> Result { - stream_decode_inboard_pipeline( - master_key, - hash, - input, - padding, - format, - encoded_body_len, - output, - ) + #[cfg(feature = "backend-lean")] + { + let mut input = input; + let body = read_encoded_body(&mut input, encoded_body_len)?; + let plaintext = crate::backend::lean::decode(master_key, hash, &body, padding, format)?; + // Free encoded body before writing plaintext (avoid simultaneous body+pt peak). + drop(body); + output + .write_all(&plaintext) + .map_err(CarbonadoError::StdIoError)?; + Ok(plaintext.len() as u64) + } + #[cfg(feature = "backend-rust")] + { + stream_decode_inboard_pipeline( + master_key, + hash, + input, + padding, + format, + encoded_body_len, + output, + ) + } +} + +/// Read a bounded or unbounded encoded body for Lean E1 spool decode. +/// +/// `encoded_body_len` must be a **trusted** length (typically header-derived +/// `encoded_len`). Declared lengths above [`crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN`] +/// are rejected before allocation (DoS soft cap; same order as segment main limits). +/// +/// **W1b:** unbounded path disk-spools first (O(chunk) ingest) then materializes once for +/// the Lean buffer ABI. Bounded path pre-sizes exactly `declared` (same as prior E1). +#[cfg(feature = "backend-lean")] +fn read_encoded_body( + input: &mut R, + encoded_body_len: Option, +) -> Result, CarbonadoError> { + use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; + + match encoded_body_len { + Some(declared) => { + if declared > MAX_SEGMENT_MAIN_LEN { + return Err(CarbonadoError::InternalStateError(format!( + "encoded_body_len {declared} exceeds MAX_SEGMENT_MAIN_LEN {MAX_SEGMENT_MAIN_LEN}" + ))); + } + let mut body = vec![0u8; declared as usize]; + input + .read_exact(&mut body) + .map_err(CarbonadoError::StdIoError)?; + // Reject trailing bytes beyond declared length (same contract as rust path). + let mut extra = [0u8; 1]; + match input.read(&mut extra) { + Ok(0) => Ok(body), + Ok(_) => Err(CarbonadoError::EncodedBodyExceedsDeclaredLength { declared }), + Err(e) => Err(CarbonadoError::StdIoError(e)), + } + } + None => { + // Unbounded: disk-spool with DoS cap, then materialize for Lean (W1b E1.5). + let body = SeekableSpool::spool_then_materialize(input, Some(MAX_SEGMENT_MAIN_LEN))?; + Ok(body) + } + } } /// Core inboard decode: Bao verify → FEC → decrypt → decompress. +#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_decode / buffer path only pub(crate) fn stream_decode_inboard_pipeline( master_key: &[u8], hash: &[u8], @@ -269,6 +364,7 @@ fn stream_decode_verified_inboard( Ok(()) } +#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_decode_inboard_pipeline only fn stream_decode_post_preprocess_seek( master_key: &[u8], mut input: R, @@ -298,8 +394,135 @@ fn stream_decode_post_preprocess_seek( } /// Stream outboard decode (incremental Bao/FEC/decrypt chain). +/// +/// Stream outboard decode from main + optional sidecars. +/// +/// # Memory / dual-backend matrix (W1b) +/// +/// | Backend | Path | Peak RAM | Engine | +/// |---------|------|----------|--------| +/// | `backend-rust` | all formats | **O(chunk/stripe)** S4 (FEC residual O(segment body)) | rust geometric + streaming EtM | +/// | `backend-lean` | **public non-Compression** (c0/c4/c8/c12) | **O(chunk/stripe) E2** | rust S4 geometric composition (G9 no-compress; c4/c12 evidenced); **not** pure-Lean stream | +/// | `backend-lean` | **public + Compression** (c2/c6/c10/c14) | **O(logical)** if decompress materializes | same composition; not advertised as E2 | +/// | `backend-lean` | **encrypted** | O(logical) E1 | Lean `decode_outboard` (crypto dual) | +/// +/// Buffer APIs remain Lean under `backend-lean` always. See docs/LIMITS.md. #[allow(clippy::too_many_arguments)] pub fn stream_decode_outboard( + master_key: &[u8], + hash: &[u8], + main: M, + verification_outboard: Option, + fec_parity: Option

, + padding: u32, + format: u8, + explicit_nonce: Option<[u8; 16]>, + output: &mut W, +) -> Result { + #[cfg(feature = "backend-lean")] + { + let fmt = Format::from(format); + if !fmt.contains(Format::Encryption) { + // W1b: public S4 composition (E2 only when !Compression; see rustdoc matrix). + stream_decode_outboard_s4( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + explicit_nonce, + output, + ) + } else { + stream_decode_outboard_lean_e1( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + explicit_nonce, + output, + ) + } + } + #[cfg(feature = "backend-rust")] + { + stream_decode_outboard_s4( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + explicit_nonce, + output, + ) + } +} + +/// Lean E1 outboard decode (encrypted dual crypto): disk-spool → buffer ABI → write. +#[cfg(feature = "backend-lean")] +#[allow(clippy::too_many_arguments)] +fn stream_decode_outboard_lean_e1( + master_key: &[u8], + hash: &[u8], + main: M, + verification_outboard: Option, + fec_parity: Option

, + padding: u32, + format: u8, + explicit_nonce: Option<[u8; 16]>, + output: &mut W, +) -> Result { + use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; + + let main_buf = SeekableSpool::spool_then_materialize(main, Some(MAX_SEGMENT_MAIN_LEN))?; + let ob_buf = match verification_outboard { + Some(r) => Some(SeekableSpool::spool_then_materialize( + r, + Some(MAX_SEGMENT_MAIN_LEN), + )?), + None => None, + }; + let par_buf = match fec_parity { + Some(r) => Some(SeekableSpool::spool_then_materialize( + r, + Some(MAX_SEGMENT_MAIN_LEN), + )?), + None => None, + }; + let header_path = explicit_nonce.is_some(); + let plaintext = crate::backend::lean::decode_outboard( + master_key, + hash, + &main_buf, + ob_buf.as_deref(), + par_buf.as_deref(), + padding, + format, + explicit_nonce.as_ref(), + header_path, + )?; + drop(main_buf); + drop(ob_buf); + drop(par_buf); + output + .write_all(&plaintext) + .map_err(CarbonadoError::StdIoError)?; + Ok(plaintext.len() as u64) +} + +/// S4 outboard decode: O(chunk/stripe) peak (public geometric + encrypted EtM spool). +/// +/// Under `backend-lean` this is the **W1b public** composition path (caller gates Encryption). +/// Peak is E2 O(chunk/stripe) only when !Compression; see public matrix on `stream_decode_outboard`. +#[allow(clippy::too_many_arguments)] +fn stream_decode_outboard_s4( master_key: &[u8], hash: &[u8], mut main: M, diff --git a/src/stream/decode_async.rs b/src/stream/decode_async.rs index efd40a1..21991f9 100644 --- a/src/stream/decode_async.rs +++ b/src/stream/decode_async.rs @@ -1,35 +1,62 @@ -//! Async adapter for inboard decode — delegates to the canonical sync pipeline (Phase 2). +//! Async adapter for inboard decode — stages encoded body, then dual-aware sync decode (R10). use crate::{ constants::Format, error::CarbonadoError, stream::{ - decode::stream_decode_inboard_pipeline, io::{ async_copy_all, async_copy_bounded, async_reject_trailing, AsyncPipelineSink, AsyncPipelineSource, BoundedCopyTruncation, }, spool::SeekableSpool, + stream_decode, }, }; /// Async inboard decode from [`AsyncPipelineSource`] to [`AsyncPipelineSink`]. /// -/// Same semantics as [`super::stream_decode`]: Bao verify → FEC reverse → decrypt → decompress. +/// Same high-level semantics as [`super::stream_decode`]: Bao verify → FEC reverse → decrypt → +/// decompress (embedded-nonce layout). +/// +/// ## Dual-backend policy (R10 closed) +/// +/// | Concern | Policy | +/// |---------|--------| +/// | Dual freeze / `just test-lean-ci` | **Never requires `async`** — permanent. Features stay `"backend-lean,pqc,ots,cli"`. | +/// | `tests/streaming_async.rs` | `#![cfg(feature = "async")]` → **0 tests** under freeze (feature-gated; not dual-suite red). | +/// | Engine after spool | Calls dual-aware [`super::stream_decode`] (R5 E1), **not** pure-Rust-only `stream_decode_inboard_pipeline`. | +/// | `backend-rust` + `async` | Same S4 inboard pipeline as sync `stream_decode`. | +/// | `backend-lean` + `async` | Spool → E1 `stream_decode` → Lean `decode` (see costs; not stream E2). | +/// | WASM + `async` | [`CarbonadoError::NotImplemented`] (host temp spool). | +/// +/// Sync stream dual E1 remains the dual-suite contract for streaming. Async is an optional +/// concurrency adapter (disk spool bridge), not part of the freeze bar. +/// +/// Optional dual smoke (not freeze): +/// `cargo test --no-default-features --features "backend-lean,pqc,ots,async,async-tokio" --test streaming_async` +/// with `CARBONADO_LEAN_LIB` / `LD_LIBRARY_PATH` set. /// /// ## Phase 2 materialization tradeoff /// -/// Unlike sync [`super::stream_decode`], which streams incrementally from [`std::io::Read`] -/// into Bao/FEC (S4), this adapter **fully stages the encoded body** to a disk-backed -/// [`SeekableSpool`] before invoking the sync pipeline. Every async decode therefore pays +/// Unlike sync [`super::stream_decode`] under `backend-rust`, which streams incrementally from +/// [`std::io::Read`] into Bao/FEC (S4), this adapter **fully stages the encoded body** to a +/// disk-backed [`SeekableSpool`] before invoking the sync path. Every async decode therefore pays /// **O(encoded_body)** disk write + read for the input boundary, plus a plaintext spool before -/// [`async_copy_all`]. Peak RAM stays O(chunk) via spool files, but disk I/O is higher than sync -/// incremental decode. Phase 3 will stream Bao via `bao_tree::io::fsm` (or equivalent) to -/// eliminate the encoded-body spool where possible. +/// [`async_copy_all`]. +/// +/// **Peak costs (honest):** +/// - **Disk (all engines):** O(encoded) staging + O(logical) plaintext spool traffic. +/// - **`backend-rust` peak RAM:** spool/chunk-oriented (FEC verification still O(FEC body) +/// shard buffers on the sync S4 path where applicable). +/// - **`backend-lean` peak RAM:** O(**encoded** + **logical**) — E1 `read_encoded_body` +/// materializes a full body `Vec` before Lean decode, then O(logical) plaintext. Do **not** +/// treat lean+async as O(logical) RAM only. +/// +/// Not stream E2 / true chunked async Bao. /// /// ## Executor blocking /// -/// The sync pipeline (`stream_decode_inboard_pipeline`) runs as a **blocking** section inside +/// The dual-aware sync path ([`super::stream_decode`]) runs as a **blocking** section inside /// this `async fn`. On Tokio/async-std this can starve the executor for large payloads. /// Integrators should either: /// - enable the `async-tokio` feature (uses `tokio::task::spawn_blocking` for the sync section), or @@ -42,10 +69,14 @@ use crate::{ /// /// Non-verification formats (c4, c8) surface staging truncation as /// `StdIoError(UnexpectedEof, "truncated encoded body")` or `"truncated FEC body"` — aligned with -/// sync `take(limit)` paths. **Verification formats (c6/c12/c14/c15)** diverge: sync -/// [`super::stream_decode`] fails during incremental Bao read (`BaoResponseTruncated`), while this -/// adapter fails earlier at [`async_copy_bounded`] staging with the encoded-body message. Callers -/// must not assume identical error variants across sync/async for verification truncated bodies. +/// sync `take(limit)` paths. **Verification formats (c6/c12/c14/c15):** +/// - **`backend-rust`:** sync fails during incremental Bao (`BaoResponseTruncated`); this +/// adapter fails earlier at [`async_copy_bounded`] with the encoded-body staging message. +/// - **`backend-lean`:** both fail closed **before Bao**, but **not** at the same site/message — +/// async fails at adapter staging (`"truncated encoded body"`); sync E1 fails later in +/// `read_encoded_body` / `read_exact` as generic `UnexpectedEof` (`"failed to fill whole buffer"`). +/// +/// Callers must not assume identical error variants or messages across sync/async or engines. #[cfg(all(feature = "async", not(target_arch = "wasm32")))] pub async fn stream_decode_async( master_key: &[u8], @@ -74,8 +105,10 @@ where } encoded_spool.rewind()?; + // Body length already enforced by staging; pass None so dual-aware stream_decode + // (R5 E1 under backend-lean, S4 pipeline under backend-rust) reads the whole spool. let (nbytes, mut plaintext_spool) = - run_sync_inboard_pipeline(master_key, hash, encoded_spool, padding, format).await?; + run_sync_stream_decode(master_key, hash, encoded_spool, padding, format).await?; async_copy_all(&mut plaintext_spool, output).await?; Ok(nbytes) } @@ -98,8 +131,11 @@ where Err(CarbonadoError::NotImplemented) } +/// Blocking dual-aware inboard decode after async staging. +/// +/// Uses [`stream_decode`] so `backend-lean` hits Lean E1 (no silent pure-Rust pipeline). #[cfg(all(feature = "async", not(target_arch = "wasm32")))] -async fn run_sync_inboard_pipeline( +async fn run_sync_stream_decode( master_key: &[u8], hash: &[u8], encoded_spool: SeekableSpool, @@ -117,7 +153,7 @@ async fn run_sync_inboard_pipeline( .map_err(|_| CarbonadoError::HashDecodeError(32, hash_len))?; tokio::task::spawn_blocking(move || { let mut plaintext_spool = SeekableSpool::new()?; - let nbytes = stream_decode_inboard_pipeline( + let nbytes = stream_decode( &master_key, &hash, encoded_spool, @@ -138,7 +174,7 @@ async fn run_sync_inboard_pipeline( #[cfg(not(feature = "async-tokio"))] { let mut plaintext_spool = SeekableSpool::new()?; - let nbytes = stream_decode_inboard_pipeline( + let nbytes = stream_decode( master_key, hash, encoded_spool, diff --git a/src/stream/encode.rs b/src/stream/encode.rs index 5ef8974..140430a 100644 --- a/src/stream/encode.rs +++ b/src/stream/encode.rs @@ -8,22 +8,27 @@ use crate::{ constants::{Format, FEC_M, SLICE_LEN}, error::CarbonadoError, stream::{ - bao::{ - stream_verification_outboard, verification_inboard_buffer, verification_outboard_buffer, - }, - compress::{compress_buffer, stream_compress}, + compress::stream_compress, crypto_stream::{ - stream_encrypt, stream_encrypt_with_nonce, stream_encrypt_with_nonce_seek, - }, - fec::{ - encode_inboard_buffer, encode_outboard_parity_buffer, feed_inboard_fec_stripe, - write_inboard_stripe, FecStripeReadAt, + stream_encrypt, stream_encrypt_embedded_with_nonce, stream_encrypt_with_nonce_seek, }, + fec::{feed_inboard_fec_stripe, write_inboard_stripe, FecStripeReadAt}, spool::SeekableSpool, }, structs::{EncodeInfo, OutboardEncoded}, }; +// Outboard S4 geometric pipeline (backend-rust always; backend-lean W1b public E2). +use crate::stream::bao::stream_verification_outboard; +// Buffer-path helpers (rust engine encode_buffer only). +#[cfg(feature = "backend-rust")] +use crate::stream::{ + bao::{verification_inboard_buffer, verification_outboard_buffer}, + compress::compress_buffer, + crypto_stream::stream_encrypt_with_nonce, + fec::{encode_inboard_buffer, encode_outboard_parity_buffer}, +}; + struct CountingReader { inner: R, count: u64, @@ -48,8 +53,13 @@ pub struct PreprocessStats { /// Run compress → encrypt into `body_sink`. /// /// When `header_path_encrypt` is true (file layer), encrypted output is `[tag|ct]` with -/// random nonce written to `payload_nonce`. When false (CLI/low-level), nonce is embedded +/// nonce written to `payload_nonce`. When false (CLI/low-level), nonce is embedded /// in the sink as `[nonce|tag|ct]`. +/// +/// **`fixed_nonce`:** when `Some(n)` and Encryption is set, use `n` literally (including +/// all-zero — dual-backend identical). When `None`, draw a CSPRNG nonce (production). +/// Prefer CSPRNG for live archives; fixed nonces are for tests/determinism only — see +/// AGENTS §2.1.4 (nonce uniqueness; reuse under the same master is catastrophic). pub fn stream_preprocess( master_key: &[u8], format: Format, @@ -57,6 +67,7 @@ pub fn stream_preprocess( body_sink: &mut W, payload_nonce: &mut [u8; 16], header_path_encrypt: bool, + fixed_nonce: Option<[u8; 16]>, ) -> Result { body_sink .seek(std::io::SeekFrom::Start(0)) @@ -96,16 +107,35 @@ pub fn stream_preprocess( .seek(SeekFrom::Start(0)) .map_err(CarbonadoError::StdIoError)?; if header_path_encrypt { - getrandom::getrandom(payload_nonce).map_err(|_| CarbonadoError::RandomnessError)?; + match fixed_nonce { + Some(n) => *payload_nonce = n, + None => { + getrandom::getrandom(payload_nonce) + .map_err(|_| CarbonadoError::RandomnessError)?; + } + } encrypt_preprocess_sink(master_key, *payload_nonce, body_sink, comp_len)?; } else { let mut encrypted_spool = SeekableSpool::new()?; - let (_len, nonce) = stream_encrypt( - master_key, - std::io::Read::by_ref(body_sink).take(comp_len), - &mut encrypted_spool, - )?; - *payload_nonce = nonce; + match fixed_nonce { + Some(n) => { + stream_encrypt_embedded_with_nonce( + master_key, + n, + std::io::Read::by_ref(body_sink).take(comp_len), + &mut encrypted_spool, + )?; + *payload_nonce = n; + } + None => { + let (_len, nonce) = stream_encrypt( + master_key, + std::io::Read::by_ref(body_sink).take(comp_len), + &mut encrypted_spool, + )?; + *payload_nonce = nonce; + } + } replace_preprocess_encrypted(body_sink, &mut encrypted_spool)?; } reader_len(body_sink)? @@ -122,6 +152,7 @@ pub fn stream_preprocess( /// [`stream_preprocess`] for [`SeekableSpool`] sinks — encrypt replace uses /// [`SeekableSpool::overwrite_from`] so file size matches ciphertext (no stale tail bytes). +#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_encode_inboard only pub(crate) fn stream_preprocess_spool( master_key: &[u8], format: Format, @@ -129,6 +160,7 @@ pub(crate) fn stream_preprocess_spool( body_sink: &mut SeekableSpool, payload_nonce: &mut [u8; 16], header_path_encrypt: bool, + fixed_nonce: Option<[u8; 16]>, ) -> Result { body_sink.rewind()?; let mut input_len = 0u64; @@ -164,16 +196,35 @@ pub(crate) fn stream_preprocess_spool( let comp_len = body_sink.content_len()?; body_sink.rewind()?; if header_path_encrypt { - getrandom::getrandom(payload_nonce).map_err(|_| CarbonadoError::RandomnessError)?; + match fixed_nonce { + Some(n) => *payload_nonce = n, + None => { + getrandom::getrandom(payload_nonce) + .map_err(|_| CarbonadoError::RandomnessError)?; + } + } encrypt_preprocess_spool(master_key, *payload_nonce, body_sink, comp_len)?; } else { let mut encrypted_spool = SeekableSpool::new()?; - let (_len, nonce) = stream_encrypt( - master_key, - std::io::Read::by_ref(body_sink).take(comp_len), - &mut encrypted_spool, - )?; - *payload_nonce = nonce; + match fixed_nonce { + Some(n) => { + stream_encrypt_embedded_with_nonce( + master_key, + n, + std::io::Read::by_ref(body_sink).take(comp_len), + &mut encrypted_spool, + )?; + *payload_nonce = n; + } + None => { + let (_len, nonce) = stream_encrypt( + master_key, + std::io::Read::by_ref(body_sink).take(comp_len), + &mut encrypted_spool, + )?; + *payload_nonce = nonce; + } + } body_sink.overwrite_from(&mut encrypted_spool)?; } body_sink.content_len()? @@ -208,6 +259,7 @@ fn encrypt_preprocess_sink( } /// Header-path encrypt for [`SeekableSpool`] preprocess sinks (uses [`SeekableSpool::overwrite_from`]). +#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_preprocess_spool only pub(crate) fn encrypt_preprocess_spool( master_key: &[u8], nonce: [u8; 16], @@ -245,10 +297,68 @@ fn reader_len(r: &mut R) -> Result { } /// Primary inboard encode (buffer). Used by [`crate::encoding::encode`]. +/// +/// Under `backend-lean`, composes over Lean C ABI body encode (same engine as +/// [`crate::encode`]) so streaming buffer tests do not silently use pure Rust. +/// +/// Encrypted formats draw a random nonce (embedded layout). For a fixed nonce +/// (G9 fixtures), use [`stream_encode_buffer_with_nonce`]. pub fn stream_encode_buffer( master_key: &[u8], input: &[u8], format: u8, +) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { + stream_encode_buffer_with_nonce(master_key, input, format, None) +} + +/// Inboard body encode with optional fixed nonce for encrypted formats. +/// +/// When `explicit_nonce` is `Some(n)` and the Encryption bit is set, the low-level +/// embedded layout is `[nonce(16) | tag(64) | ct]` with `n` (including all-zero). +/// When `None`, encrypted formats use a CSPRNG nonce. Public formats ignore the nonce. +/// +/// Production defaults are unchanged: [`stream_encode_buffer`] / [`crate::encode`] pass `None`. +/// +/// # Safety / intended use +/// +/// Fixed nonces are for **tests and determinism only**. Prefer [`stream_encode_buffer`] +/// for production. Nonce reuse under the same master is catastrophic (AGENTS §2.1.4). +pub fn stream_encode_buffer_with_nonce( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, +) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { + #[cfg(feature = "backend-lean")] + { + let nonce = if format & 1 != 0 { + match explicit_nonce { + Some(n) => Some(n), + None => { + let mut n = [0u8; 16]; + getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; + Some(n) + } + } + } else { + None + }; + let crate::structs::Encoded(body, hash, info) = + crate::backend::lean::encode(master_key, input, format, nonce.as_ref())?; + Ok((body, hash, info)) + } + #[cfg(feature = "backend-rust")] + { + stream_encode_buffer_rust(master_key, input, format, explicit_nonce) + } +} + +#[cfg(feature = "backend-rust")] +fn stream_encode_buffer_rust( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, ) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { let fmt = Format::from(format); let input_len = input.len() as u32; @@ -263,7 +373,20 @@ pub fn stream_encode_buffer( if fmt.contains(Format::Encryption) { body = { let mut out = SeekableSpool::new()?; - let (_len, _nonce) = stream_encrypt(master_key, std::io::Cursor::new(&body), &mut out)?; + match explicit_nonce { + Some(nonce) => { + stream_encrypt_embedded_with_nonce( + master_key, + nonce, + std::io::Cursor::new(&body), + &mut out, + )?; + } + None => { + let (_len, _nonce) = + stream_encrypt(master_key, std::io::Cursor::new(&body), &mut out)?; + } + } let mut buf = Vec::new(); out.rewind()?; std::io::copy(&mut out, &mut buf).map_err(CarbonadoError::StdIoError)?; @@ -322,11 +445,49 @@ pub fn stream_encode_buffer( /// /// When `explicit_nonce` is `Some`, encrypted output is `[tag(64) | ct]` (header path). /// When `None`, encrypted output embeds the nonce (low-level path). +/// +/// Under `backend-lean`, composes over Lean C ABI with matching layout: +/// `explicit_nonce.is_some()` → `header_path=true` (`[tag|ct]`); else embedded-nonce. pub fn stream_encode_outboard_buffer( master_key: &[u8], input: &[u8], format: u8, explicit_nonce: Option<[u8; 16]>, +) -> Result { + #[cfg(feature = "backend-lean")] + { + let header_path = explicit_nonce.is_some(); + let nonce = if format & 1 != 0 { + if let Some(n) = explicit_nonce { + Some(n) + } else { + let mut n = [0u8; 16]; + getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; + Some(n) + } + } else { + None + }; + crate::backend::lean::encode_outboard( + master_key, + input, + format, + nonce.as_ref(), + header_path, + ) + } + #[cfg(feature = "backend-rust")] + { + stream_encode_outboard_buffer_rust(master_key, input, format, explicit_nonce) + } +} + +#[cfg(feature = "backend-rust")] +fn stream_encode_outboard_buffer_rust( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, ) -> Result { let fmt = Format::from(format); let input_len = input.len() as u32; @@ -419,8 +580,165 @@ pub fn stream_encode_outboard_buffer( } /// Stream outboard encode to writers (public + encrypted). +/// +/// # Memory / dual-backend matrix (W1b) +/// +/// | Backend | Path | Peak RAM | Engine | +/// |---------|------|----------|--------| +/// | `backend-rust` | all formats | **O(chunk/stripe)** S4 (compress streams) | rust geometric + streaming EtM | +/// | `backend-lean` | **public non-Compression** (c0/c4/c8/c12) | **O(chunk/stripe) E2** | rust S4 geometric composition (G9 **no-compress** wire bit-match; c4/c12 evidenced); **not** pure-Lean stream | +/// | `backend-lean` | **public + Compression** (c2/c6/c10/c14) | **O(logical)** at bulk zstd | same S4 composition; compress uses Lean-parity buffer zstd (not E2) | +/// | `backend-lean` | **encrypted** | O(logical) E1 | Lean `encode_outboard` (crypto dual) | +/// +/// Buffer APIs ([`stream_encode_outboard_buffer`]) remain Lean under `backend-lean` always. +/// Pure Lean chunked stream residual remains (no streaming C ABI). See docs/LIMITS.md. +/// +/// **Encrypted nonces:** both backends always draw a CSPRNG nonce for this stream API +/// (dual-identical). For a fixed nonce (tests/G9), use [`stream_encode_outboard_buffer`] +/// with `Some(nonce)` (header-path layout when `Some`). #[allow(clippy::too_many_arguments)] pub fn stream_encode_outboard( + master_key: &[u8], + input: impl Read, + format: u8, + main_out: &mut M, + bao_out: Option<&mut O>, + parity_out: Option<&mut P>, + payload_nonce: &mut [u8; 16], + header_path_encrypt: bool, +) -> Result<(Hash, EncodeInfo), CarbonadoError> { + #[cfg(feature = "backend-lean")] + { + let fmt = Format::from(format); + // W1b: public → S4 composition (E2 only when !Compression; see rustdoc matrix). + // Encrypted stays Lean E1 dual crypto. + if !fmt.contains(Format::Encryption) { + stream_encode_outboard_s4( + master_key, + input, + format, + main_out, + bao_out, + parity_out, + payload_nonce, + header_path_encrypt, + ) + } else { + stream_encode_outboard_lean( + master_key, + input, + format, + main_out, + bao_out, + parity_out, + payload_nonce, + header_path_encrypt, + ) + } + } + #[cfg(feature = "backend-rust")] + { + stream_encode_outboard_s4( + master_key, + input, + format, + main_out, + bao_out, + parity_out, + payload_nonce, + header_path_encrypt, + ) + } +} + +/// Lean E1: disk-spool plaintext (O(chunk) ingest) → `lean::encode_outboard` → write-all. +/// +/// Peak RAM remains O(logical) at the Lean buffer boundary (encrypted dual crypto). +#[cfg(feature = "backend-lean")] +#[allow(clippy::too_many_arguments)] +fn stream_encode_outboard_lean( + master_key: &[u8], + input: impl Read, + format: u8, + main_out: &mut M, + mut bao_out: Option<&mut O>, + mut parity_out: Option<&mut P>, + payload_nonce: &mut [u8; 16], + header_path_encrypt: bool, +) -> Result<(Hash, EncodeInfo), CarbonadoError> { + use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; + + let fmt = Format::from(format); + // Fail-closed before Lean work: required sidecar writers must be present when + // format bits demand them (decode-side lean::decode_outboard already enforces + // the symmetric Missing* contract). Rust stream path hard-requires Bao writer + // for Verification; FEC writer is fail-closed here for dual-backend symmetry + // with decode (silent drop of Lean-produced parity would diverge API contracts). + if fmt.contains(Format::Verification) && bao_out.is_none() { + return Err(CarbonadoError::MissingVerificationOutboard); + } + if fmt.contains(Format::Fec) && parity_out.is_none() { + return Err(CarbonadoError::MissingFecParity); + } + + // Disk-backed ingest (O(chunk) during copy); materialize once for Lean buffer ABI. + let plaintext = SeekableSpool::spool_then_materialize(input, Some(MAX_SEGMENT_MAIN_LEN))?; + + // Always CSPRNG when encrypted (matches rust `stream_preprocess(..., fixed_nonce=None)`). + // Fixed-nonce outboard: `stream_encode_outboard_buffer(..., Some(nonce))`. + let encrypted = fmt.contains(Format::Encryption); + if encrypted { + getrandom::getrandom(payload_nonce).map_err(|_| CarbonadoError::RandomnessError)?; + } else { + *payload_nonce = [0u8; 16]; + } + let nonce = if encrypted { + Some(*payload_nonce) + } else { + None + }; + + let oenc = crate::backend::lean::encode_outboard( + master_key, + &plaintext, + format, + nonce.as_ref(), + header_path_encrypt, + )?; + // Free plaintext before writing outputs (avoid simultaneous pt + main peak). + drop(plaintext); + + main_out + .seek(SeekFrom::Start(0)) + .map_err(CarbonadoError::StdIoError)?; + main_out + .write_all(&oenc.main) + .map_err(CarbonadoError::StdIoError)?; + + if let Some(ob_writer) = bao_out.as_mut() { + if let Some(ref ob) = oenc.verification_outboard { + ob_writer + .write_all(ob) + .map_err(CarbonadoError::StdIoError)?; + } + } + if let Some(par_writer) = parity_out.as_mut() { + if let Some(ref par) = oenc.fec_parity { + par_writer + .write_all(par) + .map_err(CarbonadoError::StdIoError)?; + } + } + + Ok((oenc.hash, oenc.info)) +} + +/// S4 outboard encode: O(chunk/stripe) peak RAM (public geometric + encrypted EtM spool). +/// +/// Under `backend-lean` this is the **W1b public** composition path (caller gates Encryption). +/// Peak is E2 O(chunk/stripe) only when !Compression; Compression under lean is O(logical) bulk zstd. +#[allow(clippy::too_many_arguments)] +fn stream_encode_outboard_s4( master_key: &[u8], input: impl Read, format: u8, @@ -431,6 +749,16 @@ pub fn stream_encode_outboard( header_path_encrypt: bool, ) -> Result<(Hash, EncodeInfo), CarbonadoError> { let fmt = Format::from(format); + // Fail-closed: required sidecar writers when format bits demand them (matches + // lean E1 stream_encode_outboard_lean + decode Missing* contract). + if fmt.contains(Format::Verification) && bao_out.is_none() { + return Err(CarbonadoError::MissingVerificationOutboard); + } + if fmt.contains(Format::Fec) && parity_out.is_none() { + return Err(CarbonadoError::MissingFecParity); + } + // Stream outboard uses CSPRNG when encrypted (`fixed_nonce = None`). Deterministic + // encrypted outboard goldens use [`stream_encode_outboard_buffer`] with `Some(nonce)`. let stats = stream_preprocess( master_key, fmt, @@ -438,6 +766,7 @@ pub fn stream_encode_outboard( main_out, payload_nonce, header_path_encrypt, + None, )?; let bare_len = stats.bare_len; main_out.rewind().map_err(CarbonadoError::StdIoError)?; @@ -447,10 +776,11 @@ pub fn stream_encode_outboard( (0, 0, 0, 0) } else { let (stripe, pl, cl) = feed_inboard_fec_stripe(bare_len as usize, &mut *main_out)?; - let mut par_len = 0u64; - if let Some(par) = parity_out.as_mut() { - par_len = crate::stream::fec::write_outboard_parity(&stripe, par)?; - } + // parity_out is Some after fail-closed check above. + let par = parity_out + .as_mut() + .ok_or(CarbonadoError::MissingFecParity)?; + let par_len = crate::stream::fec::write_outboard_parity(&stripe, par)?; main_out.rewind().map_err(CarbonadoError::StdIoError)?; (pl, cl, par_len as u32, par_len as u32) } @@ -657,6 +987,15 @@ fn stream_copy( } /// Fused inboard encode: preprocess into a disk spool, then FEC/Bao directly to `output`. +/// +/// Under `backend-lean` (R5 E1 / W1b residual): disk-spool plaintext → Lean body encode +/// (embedded-nonce via [`crate::backend::lean::encode`]) or headered encode when +/// `header_path_encrypt` (strip header, write body only). Peak RAM O(logical) at Lean +/// buffer boundary — not stream E2. Public **outboard** stream is W1b E2 (see +/// [`stream_encode_outboard`]). See docs/LIMITS.md. +/// +/// Encrypted formats draw a CSPRNG nonce. For a fixed nonce (including all-zero), use +/// [`stream_encode_inboard_with_nonce`]. pub fn stream_encode_inboard( master_key: &[u8], input: R, @@ -665,16 +1004,129 @@ pub fn stream_encode_inboard( payload_nonce: &mut [u8; 16], header_path_encrypt: bool, ) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { - let fmt = Format::from(format); - let mut spool = SeekableSpool::new()?; - let stats = stream_preprocess_spool( + stream_encode_inboard_with_nonce( master_key, - fmt, input, - &mut spool, + format, + output, payload_nonce, header_path_encrypt, - )?; - let (hash, info) = stream_encode_inboard_body(&mut spool, stats, format, output)?; + None, + ) +} + +/// Like [`stream_encode_inboard`], with optional fixed AES-CTR nonce when encrypted. +/// +/// When `fixed_nonce` is `Some(n)`, both backends use `n` literally (including all-zero). +/// When `None`, a CSPRNG nonce is drawn. **Test/determinism only** for fixed nonces — +/// nonce reuse under the same master is catastrophic (AGENTS §2.1.4). Prefer +/// [`stream_encode_inboard`] for production. +pub fn stream_encode_inboard_with_nonce( + master_key: &[u8], + input: R, + format: u8, + output: &mut W, + payload_nonce: &mut [u8; 16], + header_path_encrypt: bool, + fixed_nonce: Option<[u8; 16]>, +) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { + #[cfg(feature = "backend-lean")] + { + stream_encode_inboard_lean( + master_key, + input, + format, + output, + payload_nonce, + header_path_encrypt, + fixed_nonce, + ) + } + #[cfg(feature = "backend-rust")] + { + let fmt = Format::from(format); + let mut spool = SeekableSpool::new()?; + let stats = stream_preprocess_spool( + master_key, + fmt, + input, + &mut spool, + payload_nonce, + header_path_encrypt, + fixed_nonce, + )?; + let (hash, info) = stream_encode_inboard_body(&mut spool, stats, format, output)?; + Ok((hash, info, stats)) + } +} + +/// Lean E1 fused inboard: disk-spool plaintext → lean encode / encode_headered → write body. +/// +/// Peak RAM O(logical) at Lean buffer boundary (W1b residual for inboard stream). +#[cfg(feature = "backend-lean")] +fn stream_encode_inboard_lean( + master_key: &[u8], + input: R, + format: u8, + output: &mut W, + payload_nonce: &mut [u8; 16], + header_path_encrypt: bool, + fixed_nonce: Option<[u8; 16]>, +) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { + use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; + + let plaintext = SeekableSpool::spool_then_materialize(input, Some(MAX_SEGMENT_MAIN_LEN))?; + + let encrypted = format & 1 != 0; + if encrypted { + match fixed_nonce { + Some(n) => *payload_nonce = n, + None => { + getrandom::getrandom(payload_nonce).map_err(|_| CarbonadoError::RandomnessError)?; + } + } + } else { + *payload_nonce = [0u8; 16]; + } + + let nonce_ref: Option<&[u8; 16]> = if encrypted { Some(payload_nonce) } else { None }; + + let (body, hash, info) = if header_path_encrypt { + // Header-path: Lean builds Header||body; return body only + nonce from header. + let (archive, info) = crate::backend::lean::encode_headered( + master_key, &plaintext, format, nonce_ref, None, None, + )?; + drop(plaintext); + if archive.len() < crate::file::Header::LEN { + return Err(CarbonadoError::InvalidHeaderLength); + } + let header = crate::file::Header::try_from(&archive[..crate::file::Header::LEN])?; + *payload_nonce = header.payload_nonce; + let body = archive[crate::file::Header::LEN..].to_vec(); + drop(archive); + (body, header.hash, info) + } else { + let crate::structs::Encoded(body, hash, info) = + crate::backend::lean::encode(master_key, &plaintext, format, nonce_ref)?; + drop(plaintext); + (body, hash, info) + }; + + output + .write_all(&body) + .map_err(CarbonadoError::StdIoError)?; + + let bare_len = if encrypted { + info.bytes_encrypted as u64 + } else if info.bytes_compressed > 0 { + info.bytes_compressed as u64 + } else { + info.input_len as u64 + }; + let stats = PreprocessStats { + bare_len, + input_len: info.input_len as u64, + bytes_compressed: info.bytes_compressed, + }; Ok((hash, info, stats)) } diff --git a/src/stream/mod.rs b/src/stream/mod.rs index d191bef..2a40d9c 100644 --- a/src/stream/mod.rs +++ b/src/stream/mod.rs @@ -22,6 +22,7 @@ pub mod shard; pub mod slice; pub(crate) mod spool; +#[cfg(feature = "backend-rust")] pub(crate) use slice::extract_slice_inboard_for_scrub; pub use slice::{slice_to_chunk_ranges, verify_slice_inboard_seekable, verify_slice_outboard}; @@ -32,8 +33,9 @@ pub use decode::{ #[cfg(feature = "async")] pub use decode_async::stream_decode_async; pub use encode::{ - stream_encode_buffer, stream_encode_inboard, stream_encode_inboard_body, - stream_encode_outboard, stream_encode_outboard_buffer, stream_preprocess, + stream_encode_buffer, stream_encode_buffer_with_nonce, stream_encode_inboard, + stream_encode_inboard_body, stream_encode_inboard_with_nonce, stream_encode_outboard, + stream_encode_outboard_buffer, stream_preprocess, }; pub use shard::{ decode_shards_stream, encode_shard_stream, ShardEncodeResult, ShardSource, diff --git a/src/stream/slice.rs b/src/stream/slice.rs index a4876e1..fe3f502 100644 --- a/src/stream/slice.rs +++ b/src/stream/slice.rs @@ -1,9 +1,11 @@ use std::io::{Cursor, Read}; +#[cfg(feature = "backend-rust")] +use bao_tree::io::{outboard::PostOrderMemOutboard, sync::keyed_valid_ranges}; use bao_tree::{ io::{ - outboard::{EmptyOutboard, PostOrderMemOutboard}, - sync::{keyed_decode_ranges, keyed_valid_ranges, ReadAt, WriteAt}, + outboard::EmptyOutboard, + sync::{keyed_decode_ranges, ReadAt, WriteAt}, DecodeError, }, iter::BaoChunk, @@ -51,6 +53,7 @@ fn map_valid_ranges_read_error(err: std::io::Error) -> CarbonadoError { )) } +#[cfg(feature = "backend-rust")] fn chunk_count(ranges: &ChunkRanges) -> u64 { ranges .boundaries() @@ -80,7 +83,8 @@ fn slice_byte_range( /// In-memory [`WriteAt`] target that retains only the requested byte sub-range. /// /// Used with a full-layout (`ChunkRanges::all()`) keyed decode over inboard responses; -/// discards writes outside the slice window so memory stays O(slice). +/// discards writes outside the slice window so **retained output** stays O(slice). +/// Peak RSS still includes the caller-owned full inboard body when that blob is resident. struct SliceRegionWriter { region_start: u64, region_end: u64, @@ -126,11 +130,17 @@ impl WriteAt for SliceRegionWriter { /// Verified read of `count` contiguous 4 KiB slices at `index` from an inboard bao /// response (`[u64le content_len | response_bytes]`). /// -/// **Memory:** O(slice) via [`SliceRegionWriter`]. +/// **Retained output:** O(slice) via [`SliceRegionWriter`]. +/// +/// **Input:** full inboard body (`input: &[u8]`) — not streaming `ReadAt`. Peak RSS is +/// O(body) whenever the caller already holds the blob (same honesty class as W4a C input). /// /// **Time / I/O:** O(N) over the embedded bao response bytes. Inboard artifacts store a /// full `ChunkRanges::all()` response; partial keyed decode desyncs the sequential reader, /// so verification walks the entire encoded stream even when only one slice is requested. +/// +/// **`count == 0`:** empty success immediately (no auth) — pure-Rust and dual +/// `lean::verify_slice` short-circuit. Pure Lean C `carbonado_verify_slice` is auth-first. pub fn verify_slice_inboard_seekable( input: &[u8], index: u32, @@ -173,6 +183,7 @@ pub fn verify_slice_inboard_seekable( /// Does not perform keyed hash checks; RS + re-bao oracle in scrub filters bad candidates. /// Returns [`CarbonadoError::BaoResponseTruncated`] if the response ends before the slice /// window is fully populated. +#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust scrub path only pub(crate) fn extract_slice_inboard_for_scrub( input: &[u8], index: u32, @@ -250,8 +261,13 @@ pub(crate) fn extract_slice_inboard_for_scrub( /// Verified read of `count` contiguous 4 KiB slices at `index` from bare data plus a /// post-order outboard sidecar. /// -/// **Memory and time:** O(slice) — validates only the requested chunk ranges via -/// `keyed_valid_ranges`, then reads the corresponding bare bytes. +/// **Memory and time (backend-rust):** O(slice) — validates only the requested chunk +/// ranges via `keyed_valid_ranges`, then reads the corresponding bare bytes. +/// +/// **backend-lean (R9 / W4b permanent):** dispatches to `carbonado_verify_slice_outboard` +/// (Lean range verify, O(slice+height) hash). The C ABI takes a full main buffer — when +/// `data` is not already contiguous in process memory this path materializes `data_len` +/// bytes once (honest LIMITS vs pure-Rust streaming `ReadAt`; no callback C ABI). pub fn verify_slice_outboard( data: D, outboard_bytes: &[u8], @@ -270,30 +286,55 @@ pub fn verify_slice_outboard( content_len: data_len, }); } - let root = decode_bao_hash(hash)?; - let tree = BaoTree::new(data_len, BAO_BLOCK_SIZE); - let ob = PostOrderMemOutboard { - root, - tree, - data: outboard_bytes, - }; - let key = carbonado_verification_key(format); - let ranges = slice_to_chunk_ranges(index, count); - let expected_chunks = u64::from(count) * CHUNKS_PER_SLICE; - - let mut validated = ChunkRanges::empty(); - for item in keyed_valid_ranges(&ob, &data, &ranges, &key) { - let range = item.map_err(map_valid_ranges_read_error)?; - validated |= ChunkRanges::from(range); - } - if chunk_count(&validated) < expected_chunks { - return Err(CarbonadoError::AuthenticationFailed); + #[cfg(feature = "backend-lean")] + { + let len = usize::try_from(data_len).map_err(|_| { + CarbonadoError::OutboardVerificationFailed("data_len exceeds usize".into()) + })?; + let mut buf = vec![0u8; len]; + data.read_exact_at(0, &mut buf) + .map_err(map_valid_ranges_read_error)?; + crate::backend::lean::verify_slice_outboard( + &buf, + outboard_bytes, + data_len, + index, + count, + hash, + format, + ) } + #[cfg(feature = "backend-rust")] + { + let root = decode_bao_hash(hash)?; + let tree = BaoTree::new(data_len, BAO_BLOCK_SIZE); + let ob = PostOrderMemOutboard { + root, + tree, + data: outboard_bytes, + }; + let key = carbonado_verification_key(format); + let ranges = slice_to_chunk_ranges(index, count); + // Cap expected chunks at content length (partial last leaf / short files). + let content_chunks = data_len.div_ceil(1024); + let expected_chunks = (u64::from(count) * CHUNKS_PER_SLICE) + .min(content_chunks.saturating_sub(u64::from(index) * CHUNKS_PER_SLICE)); - let (slice_byte_start, _slice_byte_end, actual_len) = slice_byte_range(index, count, data_len)?; + let mut validated = ChunkRanges::empty(); + for item in keyed_valid_ranges(&ob, &data, &ranges, &key) { + let range = item.map_err(map_valid_ranges_read_error)?; + validated |= ChunkRanges::from(range); + } + if chunk_count(&validated) < expected_chunks { + return Err(CarbonadoError::AuthenticationFailed); + } - let mut out = vec![0u8; actual_len as usize]; - data.read_exact_at(slice_byte_start, &mut out) - .map_err(map_valid_ranges_read_error)?; - Ok(out) + let (slice_byte_start, _slice_byte_end, actual_len) = + slice_byte_range(index, count, data_len)?; + + let mut out = vec![0u8; actual_len as usize]; + data.read_exact_at(slice_byte_start, &mut out) + .map_err(map_valid_ranges_read_error)?; + Ok(out) + } } diff --git a/src/stream/spool.rs b/src/stream/spool.rs index 9844fd1..c27be8e 100644 --- a/src/stream/spool.rs +++ b/src/stream/spool.rs @@ -60,6 +60,7 @@ impl SeekableSpool { } /// Truncate and replace contents from `src` (used after encrypt preprocess). + #[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_preprocess_spool only pub fn overwrite_from(&mut self, src: &mut Self) -> Result<(), CarbonadoError> { src.rewind()?; self.file.set_len(0).map_err(CarbonadoError::StdIoError)?; @@ -69,6 +70,39 @@ impl SeekableSpool { src.rewind()?; Ok(()) } + + /// Spool a reader to this temp file with O(chunk) RAM, then materialize for a buffer ABI. + /// + /// **W1b disk-backed E1.5:** ingest peak RAM is O(copy buffer), not O(N). The returned + /// `Vec` is still O(N) — required for Lean buffer C ABI. Prefer this over `read_to_end` + /// when the source is unbounded / adversarial so intermediate growth stays on disk until + /// the final materialize (with optional DoS cap via [`Read::take`]). + #[cfg(feature = "backend-lean")] + pub fn spool_then_materialize( + mut input: R, + max_len: Option, + ) -> Result, CarbonadoError> { + let mut spool = Self::new()?; + match max_len { + Some(cap) => { + let mut limited = input.by_ref().take(cap.saturating_add(1)); + io::copy(&mut limited, &mut spool).map_err(CarbonadoError::StdIoError)?; + let len = spool.content_len()?; + if len > cap { + return Err(CarbonadoError::InternalStateError(format!( + "spool materialize exceeds max_len {cap}" + ))); + } + } + None => { + io::copy(&mut input, &mut spool).map_err(CarbonadoError::StdIoError)?; + } + } + spool.rewind()?; + let mut out = Vec::new(); + io::copy(&mut spool, &mut out).map_err(CarbonadoError::StdIoError)?; + Ok(out) + } } impl Read for SeekableSpool { diff --git a/tests/common/inboard_parity.rs b/tests/common/inboard_parity.rs index e4ddd69..502fca6 100644 --- a/tests/common/inboard_parity.rs +++ b/tests/common/inboard_parity.rs @@ -123,6 +123,7 @@ pub fn preprocess_and_body( &mut staging, &mut nonce, true, + None, // CSPRNG when encrypted (production path) ) .expect("preprocess"); (stats, staging.into_inner(), nonce) diff --git a/tests/determinism_roundtrip.rs b/tests/determinism_roundtrip.rs new file mode 100644 index 0000000..fb397a4 --- /dev/null +++ b/tests/determinism_roundtrip.rs @@ -0,0 +1,831 @@ +//! Wave 2 / **W2d**: codecode (EDE) + decodec (DED) determinism contracts. +//! +//! ## Contracts (normative) +//! +//! | Contract | Steps | Assert | +//! |----------|-------|--------| +//! | **codecode** (EDE) | encode → decode → encode | `pt' == pt` and `A' == A` under fixed pins | +//! | **decodec** (DED) | decode archive A → encode → decode | `pt' == pt` and `B == A` when encode is deterministic | +//! +//! Pins: same MASTER / NONCE / plaintext as G9 (`g9_matrix_v1`). Encrypted formats +//! use fixed `NONCE` so wire identity is in scope. +//! +//! ## Matrix (no-compress — full wire equality) +//! +//! | Layout | Formats | +//! |--------|---------| +//! | body | c0, c1, c4, c5, c8, c9, c12, c13 | +//! | headered | c4, c5, c12, c13 | +//! | outboard | c4, c5, c12, c13 | +//! +//! ## Compression (**W2a** — same-engine determinism only) +//! +//! Body + headered + outboard compress formats (public + fixed-nonce encrypted): +//! same-engine codecode/decodec require `A' == A`. **Cross-backend** encode +//! bit-match is a **permanent residual** (LIMITS / GAPS): Lean AOT zstd frames +//! differ from Rust `zstd` (measured on G9 `outboard_c14` mains; hard-asserted). +//! +//! ## Directory (**W2b**) +//! +//! Same-engine catalog+segment codecode/decodec under pinned options. Cross-engine +//! encode residual is **live rust root vs live lean root** (hard-asserted pins), +//! not the committed decode seed. `tests/fixtures/phase3_g9_directory` is +//! **decode-only SSOT** (catalog root may lag live re-encode). +//! +//! Runs under default `backend-rust` and lean freeze features (auto-include). + +use std::fs; +use std::path::{Path, PathBuf}; + +use carbonado::{ + constants::Format, decode, decode_outboard, encode_with_nonce, file, + stream_encode_outboard_buffer, structs::Encoded, OutboardEncoded, +}; + +/// Same master as G9 / Phase 2. +const MASTER: [u8; 32] = [ + 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, +]; + +/// Fixed 16-byte nonce for encrypted pins (G9 / Phase 2). +const NONCE: [u8; 16] = [ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, +]; + +/// Same plaintext as G9 matrix (`plaintext_id = g9_matrix_v1`). +const PLAINTEXT: &[u8] = b"g9 cross-backend matrix v1"; + +/// No-compress body formats (G9-green). +const BODY_NO_COMPRESS: &[u8] = &[0, 1, 4, 5, 8, 9, 12, 13]; +/// Headered no-compress subset. +const HEADERED_NO_COMPRESS: &[u8] = &[4, 5, 12, 13]; +/// Outboard no-compress subset. +const OUTBOARD_NO_COMPRESS: &[u8] = &[4, 5, 12, 13]; + +/// Compression body formats for same-engine W2a determinism (public + encrypted fixed-nonce). +const BODY_COMPRESS: &[u8] = &[2, 3, 6, 7, 10, 11, 14, 15]; +/// Headered compress subset (Verification + Compression; public + encrypted). +const HEADERED_COMPRESS: &[u8] = &[6, 7, 14, 15]; +/// Outboard compress subset (public + encrypted header-path). +const OUTBOARD_COMPRESS: &[u8] = &[6, 7, 14, 15]; + +/// Live `backend-rust` catalog Bao root for [`dir_files`] + zero master + default options. +/// +/// **Not** the committed `phase3_g9_directory` catalog (that seed is decode-only SSOT; +/// catalog packaging can lag while segment mains stay stable). +const LIVE_RUST_DIR_CATALOG_ROOT: &str = + "0b119f121a003dd4136f340cdcb8de9dc91d8e15d6f402df485e9ffd123cea4e"; + +/// Live `backend-lean` catalog Bao root for the same tree/options as [`LIVE_RUST_DIR_CATALOG_ROOT`]. +const LIVE_LEAN_DIR_CATALOG_ROOT: &str = + "f67b6f49b9d2f3d8ac8b3906e9771dfaa6e2101fe8501b48d24700c3eb64d189"; + +/// Committed phase3 G9 directory catalog root — **decode seed only**, not live re-encode golden. +const PHASE3_SEED_DIR_CATALOG_ROOT: &str = + "16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f"; + +#[cfg(feature = "backend-lean")] +fn require_lean_lib() { + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + panic!( + "CARBONADO_LEAN_LIB unset. Build and export first:\n \ + nix build .#libcarbonado -o result-libcarbonado\n \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ + export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ + export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ + # or: just test-lean-ci" + ); + } +} + +fn is_encrypted(format: u8) -> bool { + Format::from(format).contains(Format::Encryption) +} + +fn nonce_for(format: u8) -> Option<[u8; 16]> { + if is_encrypted(format) { + Some(NONCE) + } else { + None + } +} + +fn active_engine() -> &'static str { + if cfg!(feature = "backend-lean") { + "lean" + } else { + "rust" + } +} + +// --------------------------------------------------------------------------- +// Body helpers +// --------------------------------------------------------------------------- + +fn encode_body(format: u8, pt: &[u8]) -> (Vec, [u8; 32], u32) { + let Encoded(body, hash, info) = encode_with_nonce(&MASTER, pt, format, nonce_for(format)) + .unwrap_or_else(|e| panic!("[{}] encode body c{format}: {e}", active_engine())); + (body, *hash.as_bytes(), info.padding_len) +} + +fn decode_body(format: u8, body: &[u8], hash: &[u8; 32], pad: u32) -> Vec { + decode(&MASTER, hash, body, pad, format) + .unwrap_or_else(|e| panic!("[{}] decode body c{format}: {e}", active_engine())) +} + +/// codecode: E → D → E; assert pt' and A' == A. +fn codecode_body(format: u8, pt: &[u8]) { + let (a, hash, pad) = encode_body(format, pt); + let pt1 = decode_body(format, &a, &hash, pad); + assert_eq!( + pt1, + pt, + "[{}] codecode body c{format}: pt' != pt", + active_engine() + ); + let (a2, hash2, pad2) = encode_body(format, &pt1); + assert_eq!( + a2, + a, + "[{}] codecode body c{format}: A' != A (wire)", + active_engine() + ); + assert_eq!( + hash2, + hash, + "[{}] codecode body c{format}: hash", + active_engine() + ); + assert_eq!( + pad2, + pad, + "[{}] codecode body c{format}: pad", + active_engine() + ); +} + +/// decodec: start from A, D → E → D; assert pt' and B == A. +fn decodec_body(format: u8, pt: &[u8]) { + let (a, hash, pad) = encode_body(format, pt); + let pt1 = decode_body(format, &a, &hash, pad); + assert_eq!(pt1, pt, "[{}] decodec body c{format}: pt", active_engine()); + let (b, hash_b, pad_b) = encode_body(format, &pt1); + assert_eq!( + b, + a, + "[{}] decodec body c{format}: B != A (wire)", + active_engine() + ); + let pt2 = decode_body(format, &b, &hash_b, pad_b); + assert_eq!( + pt2, + pt, + "[{}] decodec body c{format}: pt' after re-encode", + active_engine() + ); +} + +// --------------------------------------------------------------------------- +// Headered helpers +// --------------------------------------------------------------------------- + +fn encode_headered(format: u8, pt: &[u8]) -> Vec { + let (archive, _) = file::encode_with_nonce(&MASTER, pt, format, None, nonce_for(format)) + .unwrap_or_else(|e| panic!("[{}] encode headered c{format}: {e}", active_engine())); + archive +} + +fn decode_headered(archive: &[u8]) -> (file::Header, Vec) { + file::decode(&MASTER, archive) + .unwrap_or_else(|e| panic!("[{}] decode headered: {e}", active_engine())) +} + +fn codecode_headered(format: u8, pt: &[u8]) { + let a = encode_headered(format, pt); + let (hdr, pt1) = decode_headered(&a); + assert_eq!( + pt1, + pt, + "[{}] codecode headered c{format}: pt'", + active_engine() + ); + assert_eq!(hdr.format.bits(), format); + let a2 = encode_headered(format, &pt1); + assert_eq!( + a2, + a, + "[{}] codecode headered c{format}: A' != A", + active_engine() + ); +} + +fn decodec_headered(format: u8, pt: &[u8]) { + let a = encode_headered(format, pt); + let (_, pt1) = decode_headered(&a); + assert_eq!(pt1, pt); + let b = encode_headered(format, &pt1); + assert_eq!( + b, + a, + "[{}] decodec headered c{format}: B != A", + active_engine() + ); + let (_, pt2) = decode_headered(&b); + assert_eq!(pt2, pt); +} + +// --------------------------------------------------------------------------- +// Outboard helpers +// --------------------------------------------------------------------------- + +struct OutboardWire { + oenc: OutboardEncoded, + header: Option>, +} + +fn encode_outboard_wire(format: u8, pt: &[u8]) -> OutboardWire { + if is_encrypted(format) { + let oenc = stream_encode_outboard_buffer(&MASTER, pt, format, Some(NONCE)) + .unwrap_or_else(|e| panic!("[{}] outboard enc c{format}: {e}", active_engine())); + let hdr = file::Header::new( + &MASTER, + NONCE, + oenc.hash.as_bytes(), + [0u8; 32], + Format::from(format), + 0, + oenc.info.bytes_verifiable, + oenc.info.padding_len, + None, + ) + .expect("header for outboard"); + let hdr_bytes = hdr.try_to_vec().expect("hdr vec"); + OutboardWire { + oenc, + header: Some(hdr_bytes), + } + } else { + let oenc = carbonado::encode_outboard(&MASTER, pt, format) + .unwrap_or_else(|e| panic!("[{}] outboard pub c{format}: {e}", active_engine())); + OutboardWire { oenc, header: None } + } +} + +fn decode_outboard_wire(format: u8, w: &OutboardWire) -> Vec { + if is_encrypted(format) { + file::decode_outboard( + &MASTER, + w.oenc.hash.as_bytes(), + w.header.as_deref(), + &w.oenc.main, + w.oenc.verification_outboard.as_deref(), + w.oenc.fec_parity.as_deref(), + w.oenc.info.padding_len, + format, + ) + .unwrap_or_else(|e| panic!("[{}] decode outboard enc c{format}: {e}", active_engine())) + } else { + decode_outboard( + &MASTER, + w.oenc.hash.as_bytes(), + &w.oenc.main, + w.oenc.verification_outboard.as_deref(), + w.oenc.fec_parity.as_deref(), + w.oenc.info.padding_len, + format, + ) + .unwrap_or_else(|e| panic!("[{}] decode outboard pub c{format}: {e}", active_engine())) + } +} + +fn assert_outboard_wire_eq(label: &str, format: u8, a: &OutboardWire, b: &OutboardWire) { + assert_eq!( + a.oenc.main, + b.oenc.main, + "[{}] {label} outboard c{format}: main", + active_engine() + ); + assert_eq!( + a.oenc.verification_outboard.as_deref(), + b.oenc.verification_outboard.as_deref(), + "[{}] {label} outboard c{format}: verification_outboard", + active_engine() + ); + assert_eq!( + a.oenc.fec_parity.as_deref(), + b.oenc.fec_parity.as_deref(), + "[{}] {label} outboard c{format}: fec_parity", + active_engine() + ); + assert_eq!( + a.oenc.hash.as_bytes(), + b.oenc.hash.as_bytes(), + "[{}] {label} outboard c{format}: hash", + active_engine() + ); + assert_eq!( + a.oenc.info.padding_len, + b.oenc.info.padding_len, + "[{}] {label} outboard c{format}: pad", + active_engine() + ); + assert_eq!( + a.header.as_deref(), + b.header.as_deref(), + "[{}] {label} outboard c{format}: header", + active_engine() + ); +} + +fn codecode_outboard(format: u8, pt: &[u8]) { + let a = encode_outboard_wire(format, pt); + let pt1 = decode_outboard_wire(format, &a); + assert_eq!( + pt1, + pt, + "[{}] codecode outboard c{format}: pt'", + active_engine() + ); + let a2 = encode_outboard_wire(format, &pt1); + assert_outboard_wire_eq("codecode", format, &a, &a2); +} + +fn decodec_outboard(format: u8, pt: &[u8]) { + let a = encode_outboard_wire(format, pt); + let pt1 = decode_outboard_wire(format, &a); + assert_eq!(pt1, pt); + let b = encode_outboard_wire(format, &pt1); + assert_outboard_wire_eq("decodec", format, &a, &b); + let pt2 = decode_outboard_wire(format, &b); + assert_eq!(pt2, pt); +} + +// --------------------------------------------------------------------------- +// W2d — no-compress matrix +// --------------------------------------------------------------------------- + +#[test] +fn codecode_body_no_compress_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in BODY_NO_COMPRESS { + codecode_body(format, PLAINTEXT); + } +} + +#[test] +fn decodec_body_no_compress_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in BODY_NO_COMPRESS { + decodec_body(format, PLAINTEXT); + } +} + +#[test] +fn codecode_headered_no_compress_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in HEADERED_NO_COMPRESS { + codecode_headered(format, PLAINTEXT); + } +} + +#[test] +fn decodec_headered_no_compress_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in HEADERED_NO_COMPRESS { + decodec_headered(format, PLAINTEXT); + } +} + +#[test] +fn codecode_outboard_no_compress_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in OUTBOARD_NO_COMPRESS { + codecode_outboard(format, PLAINTEXT); + } +} + +#[test] +fn decodec_outboard_no_compress_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in OUTBOARD_NO_COMPRESS { + decodec_outboard(format, PLAINTEXT); + } +} + +// --------------------------------------------------------------------------- +// W2a — compression same-engine codecode/decodec (wire equality on one engine) +// --------------------------------------------------------------------------- + +#[test] +fn codecode_body_compress_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in BODY_COMPRESS { + codecode_body(format, PLAINTEXT); + } +} + +#[test] +fn decodec_body_compress_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in BODY_COMPRESS { + decodec_body(format, PLAINTEXT); + } +} + +#[test] +fn codecode_headered_compress_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in HEADERED_COMPRESS { + codecode_headered(format, PLAINTEXT); + } +} + +#[test] +fn decodec_headered_compress_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in HEADERED_COMPRESS { + decodec_headered(format, PLAINTEXT); + } +} + +#[test] +fn codecode_outboard_compress_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in OUTBOARD_COMPRESS { + codecode_outboard(format, PLAINTEXT); + } +} + +#[test] +fn decodec_outboard_compress_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + for &format in OUTBOARD_COMPRESS { + decodec_outboard(format, PLAINTEXT); + } +} + +/// Documented residual: rust↔lean Compression encode is **not** bit-identical. +/// +/// Evidence from committed G9 fixtures (`tests/fixtures/g9/{rust,lean}/outboard_c14/`): +/// - both mains 35 bytes; frame descriptor differs (`28b5 2ffd 00…` vs `28b5 2ffd 20…`) +/// - Bao roots differ (`0abe5781…` vs `129b4518…`) +/// - FEC parity shards differ (same length 16384) +/// +/// Decode interop remains green (`g9_cross_backend` outboard_c14 both directions). +/// Re-encode bit-match across engines is permanently out of scope (LIMITS W2a). +/// Fail-closed if fixtures are stripped (permanent residual must stay assertable). +#[test] +fn compress_cross_engine_encode_not_bit_identical_documented() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/g9"); + let rust_main = root.join("rust/outboard_c14/main.bin"); + let lean_main = root.join("lean/outboard_c14/main.bin"); + assert!( + rust_main.is_file(), + "missing committed W2a residual fixture {}", + rust_main.display() + ); + assert!( + lean_main.is_file(), + "missing committed W2a residual fixture {}", + lean_main.display() + ); + let r = fs::read(&rust_main).expect("rust main"); + let l = fs::read(&lean_main).expect("lean main"); + assert_eq!( + r.len(), + l.len(), + "c14 mains same length (frame residual, not size)" + ); + assert_ne!( + r, l, + "W2a residual evidence: rust vs lean c14 main must still differ \ + (if this fails, re-check zstd alignment — residual may have closed)" + ); + // Frame magic is zstd in both; descriptor byte (offset 4) is the known diverge point. + assert_eq!(&r[..4], b"\x28\xb5\x2f\xfd"); + assert_eq!(&l[..4], b"\x28\xb5\x2f\xfd"); + assert_ne!( + r[4], l[4], + "expected zstd frame descriptor byte to differ (measured residual)" + ); +} + +// --------------------------------------------------------------------------- +// W2d optional strength: DED starting from committed G9 goldens (no-compress body) +// --------------------------------------------------------------------------- + +/// decodec from **committed** G9 body goldens for the active engine (not live-encode A). +/// +/// Complements live-encode DED matrices: proves re-encode of decoded fixture plaintext +/// bit-matches the golden wire under the same pins. +#[test] +fn decodec_body_from_g9_fixture_no_compress() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + let engine = active_engine(); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/g9") + .join(engine); + + for &format in BODY_NO_COMPRESS { + let name = format!("body_c{format}"); + let body_path = root.join(format!("{name}.bin")); + let meta_path = root.join(format!("{name}.meta.json")); + assert!( + body_path.is_file(), + "missing G9 golden {}", + body_path.display() + ); + assert!( + meta_path.is_file(), + "missing G9 meta {}", + meta_path.display() + ); + let a = fs::read(&body_path).expect("read golden body"); + let meta: serde_json::Value = + serde_json::from_slice(&fs::read(&meta_path).expect("read meta")).expect("meta json"); + let hash_hex = meta["hash_hex"].as_str().expect("hash_hex"); + let pad = meta["padding_len"].as_u64().expect("padding_len") as u32; + let mut hash = [0u8; 32]; + let hb = from_hex(hash_hex); + assert_eq!(hb.len(), 32); + hash.copy_from_slice(&hb); + + let pt = decode_body(format, &a, &hash, pad); + assert_eq!(pt, PLAINTEXT, "[{engine}] golden body c{format} plaintext"); + let (b, hash_b, pad_b) = encode_body(format, &pt); + assert_eq!( + b, a, + "[{engine}] DED from G9 golden: re-encode must bit-match body_c{format}" + ); + assert_eq!(hash_b, hash); + assert_eq!(pad_b, pad); + let pt2 = decode_body(format, &b, &hash_b, pad_b); + assert_eq!(pt2, PLAINTEXT); + } +} + +fn from_hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("hex")) + .collect() +} + +// --------------------------------------------------------------------------- +// W2b — directory same-engine codecode/decodec +// --------------------------------------------------------------------------- + +fn tempdir(name: &str) -> PathBuf { + let p = std::env::temp_dir().join(format!( + "carbonado-w2-{}-{}-{}", + name, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).expect("tempdir"); + p +} + +fn write_tree(src: &Path, files: &[(&str, &[u8])]) { + for (rel, data) in files { + let path = src.join(rel); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(&path, data).expect("write"); + } +} + +fn read_tree_file(dec: &Path, rel: &str) -> Vec { + fs::read(dec.join(rel)).unwrap_or_else(|e| panic!("read {rel}: {e}")) +} + +fn hex32(bytes: &[u8; 32]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn list_archive_artifacts(dir: &Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + for entry in fs::read_dir(dir).expect("read_dir") { + let entry = entry.expect("entry"); + let path = entry.path(); + if path.is_file() { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + let bytes = fs::read(&path).expect("read artifact"); + out.push((name, bytes)); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + out +} + +/// Public directory uses conventional zero master (matches phase3 G9 seed). +const ZERO_MASTER: [u8; 32] = [0u8; 32]; + +/// Public directory encode with default policy (matches phase3 G9 seed shape). +fn dir_files() -> [(&'static str, &'static [u8]); 2] { + [("a.txt", b"phase3 g9 hello"), ("sub/b.bin", b"nested data")] +} + +#[test] +fn codecode_directory_public_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + let src = tempdir("dir_src"); + write_tree(&src, &dir_files()); + + let enc1 = tempdir("dir_enc1"); + let arch1 = file::encode_directory(&ZERO_MASTER, &src, &enc1) + .unwrap_or_else(|e| panic!("[{}] dir encode1: {e}", active_engine())); + let artifacts1 = list_archive_artifacts(&enc1); + + let dec = tempdir("dir_dec"); + let catalog1 = enc1.join(format!( + "{}.adam.c{}", + hex32(&arch1.catalog_bao_root), + file::DIRECTORY_ARCHIVE_FORMAT + )); + file::decode_directory(&ZERO_MASTER, &catalog1, &dec) + .unwrap_or_else(|e| panic!("[{}] dir decode1: {e}", active_engine())); + assert_eq!(read_tree_file(&dec, "a.txt"), b"phase3 g9 hello"); + assert_eq!(read_tree_file(&dec, "sub/b.bin"), b"nested data"); + + // codecode: re-encode from extracted tree → same roots + wire bytes + let enc2 = tempdir("dir_enc2"); + let arch2 = file::encode_directory(&ZERO_MASTER, &dec, &enc2) + .unwrap_or_else(|e| panic!("[{}] dir encode2: {e}", active_engine())); + assert_eq!( + arch2.catalog_bao_root, + arch1.catalog_bao_root, + "[{}] directory codecode: catalog root must match", + active_engine() + ); + assert_eq!(arch2.entry_count, arch1.entry_count); + let artifacts2 = list_archive_artifacts(&enc2); + assert_eq!( + artifacts2, + artifacts1, + "[{}] directory codecode: full archive tree must bit-match", + active_engine() + ); + + let _ = fs::remove_dir_all(&src); + let _ = fs::remove_dir_all(&enc1); + let _ = fs::remove_dir_all(&enc2); + let _ = fs::remove_dir_all(&dec); +} + +#[test] +fn decodec_directory_public_same_engine() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + let src = tempdir("dir_src_ded"); + write_tree(&src, &dir_files()); + + let enc_a = tempdir("dir_enc_a"); + let arch_a = file::encode_directory(&ZERO_MASTER, &src, &enc_a) + .unwrap_or_else(|e| panic!("[{}] dir encode A: {e}", active_engine())); + let artifacts_a = list_archive_artifacts(&enc_a); + + let dec = tempdir("dir_dec_ded"); + let catalog = enc_a.join(format!( + "{}.adam.c{}", + hex32(&arch_a.catalog_bao_root), + file::DIRECTORY_ARCHIVE_FORMAT + )); + file::decode_directory(&ZERO_MASTER, &catalog, &dec) + .unwrap_or_else(|e| panic!("[{}] dir decode: {e}", active_engine())); + + // decodec: D → E → D; B == A wire + let enc_b = tempdir("dir_enc_b"); + let arch_b = file::encode_directory(&ZERO_MASTER, &dec, &enc_b) + .unwrap_or_else(|e| panic!("[{}] dir encode B: {e}", active_engine())); + assert_eq!(arch_b.catalog_bao_root, arch_a.catalog_bao_root); + let artifacts_b = list_archive_artifacts(&enc_b); + assert_eq!( + artifacts_b, + artifacts_a, + "[{}] directory decodec: B != A wire", + active_engine() + ); + + let dec2 = tempdir("dir_dec2"); + let catalog_b = enc_b.join(format!( + "{}.adam.c{}", + hex32(&arch_b.catalog_bao_root), + file::DIRECTORY_ARCHIVE_FORMAT + )); + file::decode_directory(&ZERO_MASTER, &catalog_b, &dec2) + .unwrap_or_else(|e| panic!("[{}] dir decode2: {e}", active_engine())); + assert_eq!(read_tree_file(&dec2, "a.txt"), b"phase3 g9 hello"); + assert_eq!(read_tree_file(&dec2, "sub/b.bin"), b"nested data"); + + let _ = fs::remove_dir_all(&src); + let _ = fs::remove_dir_all(&enc_a); + let _ = fs::remove_dir_all(&enc_b); + let _ = fs::remove_dir_all(&dec); + let _ = fs::remove_dir_all(&dec2); +} + +/// W2b residual canary: **live-vs-live** catalog roots under identical pins. +/// +/// Compares pinned live rust encode root vs pinned live lean encode root for +/// [`dir_files`] + zero master + default options. Does **not** use +/// `phase3_g9_directory` catalog as a re-encode golden (that seed is decode-only; +/// its catalog root lags live rust while segment mains may still match). +/// +/// Hard asserts: +/// - active engine live root matches its pin (`LIVE_RUST_*` / `LIVE_LEAN_*`) +/// - `LIVE_RUST_DIR_CATALOG_ROOT != LIVE_LEAN_DIR_CATALOG_ROOT` (cross-engine residual) +/// - live rust root ≠ phase3 seed catalog (documents seed packaging drift) +/// - seed catalog file still present (decode SSOT) +#[test] +fn directory_cross_engine_live_roots_residual() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + // Pin table integrity: residual is live-vs-live, not seed-vs-live. + assert_ne!( + LIVE_RUST_DIR_CATALOG_ROOT, LIVE_LEAN_DIR_CATALOG_ROOT, + "W2b residual pin table: live rust and live lean catalog roots must differ" + ); + assert_ne!( + LIVE_RUST_DIR_CATALOG_ROOT, PHASE3_SEED_DIR_CATALOG_ROOT, + "phase3_g9_directory catalog is decode-only SSOT — not equal to live rust re-encode" + ); + + let fixture = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/phase3_g9_directory"); + let seed_catalog = fixture.join(format!("{PHASE3_SEED_DIR_CATALOG_ROOT}.adam.c14")); + assert!( + seed_catalog.is_file(), + "missing phase3_g9_directory decode seed catalog {}", + seed_catalog.display() + ); + + let src = tempdir("dir_xeng_src"); + write_tree(&src, &dir_files()); + let enc = tempdir("dir_xeng_enc"); + let arch = file::encode_directory(&ZERO_MASTER, &src, &enc) + .unwrap_or_else(|e| panic!("[{}] dir encode for residual: {e}", active_engine())); + let live = hex32(&arch.catalog_bao_root); + + #[cfg(feature = "backend-rust")] + { + assert_eq!( + live, LIVE_RUST_DIR_CATALOG_ROOT, + "live rust directory catalog root drifted from W2b pin — update \ + LIVE_RUST_DIR_CATALOG_ROOT (and re-check lean residual) if intentional" + ); + assert_ne!( + live, PHASE3_SEED_DIR_CATALOG_ROOT, + "live rust catalog must not silently equal stale seed (decode-only SSOT)" + ); + assert_ne!( + live, LIVE_LEAN_DIR_CATALOG_ROOT, + "W2b residual: live rust catalog must still differ from live lean pin" + ); + } + + #[cfg(feature = "backend-lean")] + { + assert_eq!( + live, LIVE_LEAN_DIR_CATALOG_ROOT, + "live lean directory catalog root drifted from W2b pin — update \ + LIVE_LEAN_DIR_CATALOG_ROOT (and re-check rust residual) if intentional" + ); + assert_ne!( + live, LIVE_RUST_DIR_CATALOG_ROOT, + "W2b residual evidence: live lean catalog must still differ from live rust pin \ + (if equal, residual may have closed — investigate zstd/catalog packaging)" + ); + assert_ne!( + live, PHASE3_SEED_DIR_CATALOG_ROOT, + "live lean catalog must not equal decode-only seed root" + ); + } + + let _ = fs::remove_dir_all(&src); + let _ = fs::remove_dir_all(&enc); +} diff --git a/tests/directory_archive.rs b/tests/directory_archive.rs index a1d2f54..d014fa0 100644 --- a/tests/directory_archive.rs +++ b/tests/directory_archive.rs @@ -673,12 +673,11 @@ fn decode_rejects_tampered_catalog_body_returns_verification_failed() { fs::write(&tampered, &bytes).expect("write tampered"); let err = decode_directory(&ZERO_KEY, &tampered, &tempdir("tamper_body_dec")).unwrap_err(); + // Measured both backends (R4): keyed Bao body auth failure → AuthenticationFailed. + // Must not misreport CatalogBaoRootMismatch or collapse to BaoResponseTruncated. assert!( - matches!( - err, - CarbonadoError::OutboardVerificationFailed(_) | CarbonadoError::AuthenticationFailed - ), - "tampered catalog body must not map to CatalogBaoRootMismatch; got {err:?}" + matches!(err, CarbonadoError::AuthenticationFailed), + "tampered catalog body must yield AuthenticationFailed, got {err:?}" ); } diff --git a/tests/filepack_interop.rs b/tests/filepack_interop.rs index b830560..5af072d 100644 --- a/tests/filepack_interop.rs +++ b/tests/filepack_interop.rs @@ -15,6 +15,7 @@ use carbonado::{ }, }; use ciborium::value::Value as CborValue; +#[cfg(feature = "backend-rust")] use serde_json::Value as JsonValue; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; @@ -326,6 +327,7 @@ fn parse_rejects_oversized_rel_path_at_flatten() { ); } +#[cfg(feature = "backend-rust")] fn golden_fixture_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/directory_interop_golden.json") } @@ -395,6 +397,11 @@ fn adamantine_decimal_segment_naming_contract() { } } +/// Pins rust-engine directory roots / rkyv SHA-256 for `tests/samples`. +/// Under `backend-lean`, segment crypto differs until full G9 encode bit-match; +/// dual-suite directory green is exercised by `lean_backend_phase3` + functional +/// `directory_archive` tests (not rust-root checksums). +#[cfg(feature = "backend-rust")] #[test] fn golden_directory_interop_checksums_and_manifest_wire() { let fixture_text = fs::read_to_string(golden_fixture_path()).expect("read golden fixture"); diff --git a/tests/fixtures/g9/README.md b/tests/fixtures/g9/README.md new file mode 100644 index 0000000..546f7e3 --- /dev/null +++ b/tests/fixtures/g9/README.md @@ -0,0 +1,90 @@ +# G9 cross-backend fixtures (Milestone R8) + +Committed wire goldens for **Rust ↔ Lean** encode/decode parity on **no-compress** formats. + +## Pins + +| Pin | Value | +|-----|--------| +| `MASTER` | `0ca1b0da00112233445566778899aabbccddeeff102030405060708090a0b0c0` (same as `lean_backend_phase2`) | +| `NONCE` | `0102030405060708090a0b0c0d0e0f10` (Phase 2 fixed-nonce pattern) | +| `PLAINTEXT` / `plaintext_id` | `g9 cross-backend matrix v1` / `g9_matrix_v1` | + +> **Test-only.** Do **not** reuse `NONCE` (or this `MASTER`) for production encryption. +> AES-CTR nonce reuse under the same master key is **catastrophic** (keystream reuse → +> plaintext recovery). Prefer CSPRNG nonces via `encode` / `file::encode` for live archives. +> See AGENTS.md §2.1.4. + +## Layout + +```text +g9/ + rust/ # encoded under default backend-rust + lean/ # encoded under backend-lean + fixed NONCE when encrypted +``` + +### Body (`body_c{fmt}.bin` + `.meta.json`) + +| Format | Bits | Notes | +|--------|------|--------| +| c0, c4, c8, c12 | public | Deterministic (no RNG) | +| c1, c5, c9, c13 | encrypted | Fixed `NONCE`, **embedded** layout `[nonce\|tag\|ct]` | + +### Headered (`headered_c{fmt}.bin` + `.meta.json`) + +| Format | Notes | +|--------|--------| +| c4, c12 | public headered | +| c5, c13 | encrypted, fixed `payload_nonce` = `NONCE` | + +### Outboard (`outboard_c{fmt}/`) + +| Format | Notes | +|--------|--------| +| c4, c12 | public bare main + optional `out.bin` / `par.bin` (no compress; wire-identical engines) | +| c14 | public **with Compression** — decode interop only; mains may differ (Zstd residual) | +| c5, c13 | **header-path** encrypted: main is `[tag\|ct]`, `header.bin` carries `payload_nonce` | + +`meta.json` records hash, padding, layout flags, and nonce hex when encrypted. + +**Empty `out.bin`:** valid for single-leaf / small mains when Verification is set +(`has_verification_outboard: true` with zero-length outboard). Not a regen bug — bao-tree +geometry yields an empty post-order outboard for some tiny payloads. + +## Matrix scope (R8 DoD) + +- **In scope:** body/headered/outboard public + encrypted fixed-nonce, **both directions**. +- **Continuous re-encode bit-match (CI under lean):** body c0/c1/c4/c5/c8/c9/c12/c13; + headered c4/c5/c12/c13; outboard c4/c5/c12/c13 (no compress). Live lean re-encode vs + rust golden. +- **Committed fixture identity:** rust/ and lean/ trees are regenerated together under the + same pins; c14 outboard mains may differ (Compression residual). +- **Residuals (W2 settled):** cross-engine Compression encode bit-match is **permanent** + (W2a — zstd frames differ; decode interop only). Cross-engine directory encode bit-match + is **permanent** (W2b; `phase3_g9_directory` decode seed remains SSOT). Same-engine + codecode/decodec shipped in `tests/determinism_roundtrip.rs` (W2d). + +## Regeneration + +```bash +# Both engines (recommended) +just g9-gen-fixtures + +# Or manually: +G9_WRITE_FIXTURES=1 cargo test --test g9_cross_backend write_fixtures -- --ignored --nocapture + +eval "$(just _lean-env)" +G9_WRITE_FIXTURES=1 cargo test --no-default-features --features "backend-lean,pqc,ots" \ + --test g9_cross_backend write_fixtures -- --ignored --nocapture +``` + +Do **not** hand-edit binaries; regenerate and commit both `rust/` and `lean/` trees together. + +## Tests + +| Command | Direction | +|---------|-----------| +| `cargo test --test g9_cross_backend` | lean→rust + self RT + zero-nonce contract (no libcarbonado) | +| lean features + `CARBONADO_LEAN_LIB` | rust→lean + continuous re-encode bit-match + self RT | +| `just test-g9` | both directions | +| `just test-lean-ci` | full dual suite (includes this file after R7 freeze) | diff --git a/tests/fixtures/g9/lean/body_c0.bin b/tests/fixtures/g9/lean/body_c0.bin new file mode 100644 index 0000000..87a6d12 --- /dev/null +++ b/tests/fixtures/g9/lean/body_c0.bin @@ -0,0 +1 @@ +g9 cross-backend matrix v1 \ No newline at end of file diff --git a/tests/fixtures/g9/lean/body_c0.meta.json b/tests/fixtures/g9/lean/body_c0.meta.json new file mode 100644 index 0000000..bc4427f --- /dev/null +++ b/tests/fixtures/g9/lean/body_c0.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "lean", + "format": 0, + "hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "padding_len": 0, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/body_c1.bin b/tests/fixtures/g9/lean/body_c1.bin new file mode 100644 index 0000000..c0537d6 --- /dev/null +++ b/tests/fixtures/g9/lean/body_c1.bin @@ -0,0 +1,3 @@ + + OvZRk|I|): +7Jsvˁݾ𡶚qT`GDЯj8SLd|8D;f#]fL \ No newline at end of file diff --git a/tests/fixtures/g9/lean/body_c1.meta.json b/tests/fixtures/g9/lean/body_c1.meta.json new file mode 100644 index 0000000..7a014a0 --- /dev/null +++ b/tests/fixtures/g9/lean/body_c1.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "lean", + "format": 1, + "hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "padding_len": 0, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/lean/body_c12.bin b/tests/fixtures/g9/lean/body_c12.bin new file mode 100644 index 0000000000000000000000000000000000000000..1e43ba0e1c178515725c04fe2e4b372a8e4ea370 GIT binary patch literal 33224 zcmeIu?K2Yq90zb0YRr_J9?-UKImyYWj+rf`Za4A}Wu7Kurj@0eaas>{tx4(X1yQLk z9OACa!*$%qbZM-T8Q0NidRXY`q^Dy}byt7D-nP5%@5S%U@5S%?{n#jM{!UbsQeAh9 z@@6n9nBX2h-(3?#Oq1w_s`lN=A0ixZq51zf?m!d?EGBaABsXY8yq&&&c|m{RKqBpNHiMce&CSd6%MfH9&N?EbN(3^gxR6>9 z@UI^>1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0{_1NVVLYG zxe)oxroL^~FsIm48eb$dC!9AuJh>Rcn-qlTYDc|P+`x#;?3GJ$s!yyK^4L}->N#Ng z9o$*+Sk27%{~ID91Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_< z0+s~qTL_<1#2r&>Q*MNd7Zp9LCZhS;p?vANm%}W28!LhDC_TK2;XP-!MHX68RP~wM zNjXGYMoOcynN0^7d7R6w9lzH2G%tRrSlO%5j=d;y)Am)2@_HtQUEkb2qiVFBQeRC; zjZOclT(?s*W^ca7J<8x8+G%7Lj>lZfNGF!~a7=NrT<-L5gDJE-vW_D%5aJxWIP6(V z$#I7O1Rwwb2tWV=5P$##AOHafK)_lAZWTA^UBXBt->1`h`jyl(4TaN5E(BeJKx8cq zkP`wBfB*y_009U<00Izz00bZa0ZRhxM^3MqU+3@dNL`-aE|fSLUdXooMoj{gyuC4gm;200Izz00bZa0SG_<0uX?JwFqe9)id>!*4DIzzL}UxqEa2y mXROcOtp54US{fiH1Rwwb2tWV=5P$##AOHafKmY=k1pWZps}sKf literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/body_c12.meta.json b/tests/fixtures/g9/lean/body_c12.meta.json new file mode 100644 index 0000000..d43fb32 --- /dev/null +++ b/tests/fixtures/g9/lean/body_c12.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "lean", + "format": 12, + "hash_hex": "974c9d40f5cabcd4bbbe87522e6a281ded02dd6330cabf17db3bf1cf700fe796", + "padding_len": 16358, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/body_c13.bin b/tests/fixtures/g9/lean/body_c13.bin new file mode 100644 index 0000000000000000000000000000000000000000..1cf556d0f66d35f9f3cdab7bddda08806bffae3a GIT binary patch literal 33224 zcmeIu=}!{|7zS`#l~#~rfVxpCMljaF6p;JcK(vTOu12(oQWO`ZcuZ1})=C8|2uK|u zx3LH!#{i`=E7cW2JSJL)j*8s0=PM~saEZM?;VEFXBAKvHN`@H!rN?i2c_(f*v ze|Omi<^&zpRq21JZ*PQ>Gd)dGEdyBITROKZ({I4$u=Q>-e8N$ys?8ry? z?|f$RqjSvIu3brCjVToen&muZCOx3OxYVg4mY5Q_?$m*}qKp*9+vKQ9(IDl+_+~-t z=nD5^RS!pXov67!43V4tK4OW0E7zJzSRI64v4OfGt4WWDE9t+uhLK7>GAykw~kiM&i#Z@IpKp;33B zj7wz+Nt1++g&{dH{O$Y=&)jsK{ZE#26+a8R?~^r>#=-|DRx%^iGjy4+vhIj9J;lW& zpD;L56v{X_$5-gcU3{B6+-DE{Wm#*IWf5_uPoi_^dJE%!FKh@v00Izz00bZa0SG_< z0uX=z1Rwwb2tWV=5P$##AOHafKmY;|fB*zOy};u$29Awa+U8anhB^iNc;;+7ZT#hA zvqeYX-^*^cgthnIw9EZt`hDcReZi+^=xVQJ)D>=4c9#?b!VyZVvM){c=|93n2tWV= z5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1QuFAp-7s$u~>AT`-L)1 z(peE?!-;J8zMok-_1UWpRpg4GJCxOVuL%y!rf=1xvepPmP0%a-m7`<6QH(at%x>0o zPQvk#fWY{H9^MO~HnIN1`UK-nLRjxmcwriQ=(n-_kp0Uws<@Z+dsjV(o&G(dUA}e~ zO>8t|SYA9xt+dvpQk=fxoML&aTGSKLx^o3TWUiul5!=nY?@_FLRwz7_0m@|eCF<(P z?RH*S31*pxRnZYQePm}p20rl@zo7N6?OiV#=h`R6QOfl87`TTlPaf-4--2=sDAS8FI(mtkr03Y1Rwwb z2tWV=5P$##AOHafEVMu_SMBUETrB0Kcy8$#s(E;)LBmVleAIOGdD_#=hP*1?;epx% zwvKFCMcTo%Mv3|fOK-~wbCH%O5C2nN=3Z}X<>bfW2*l^4*Huqv)t*{2+uo07h6TSG zyYSy9(qgzVlW)ePuHU4*?j=+VYj=vbmGtn}NrHtJMST}+Cn1Pz1_x}r;g6q zwB3jyaE^4!O?2N~>1Q%d>QOotJMc<4{R7T63oU~_2tWV=5P$##AOHafKmY;|fB*#M z1x#rkLB8>vtJ2hB&7kU81hHJGmC4&Uv!=Er^WH!~x4>Y>vDEZ;w`{ac>)VL$ukPgs zkg^Nc9qf3cSSoZKDhrR^d6~}MWEy?7 fqJsHGBm^J;0SG_<0uX=z1Rwwb2tWV=3oY<3J%HiH literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/body_c13.meta.json b/tests/fixtures/g9/lean/body_c13.meta.json new file mode 100644 index 0000000..a528f04 --- /dev/null +++ b/tests/fixtures/g9/lean/body_c13.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "lean", + "format": 13, + "hash_hex": "eb11b7f819b5dc0ad237291a5b15fdddc187ffb5633dc32110498d011674b7fc", + "padding_len": 16278, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/lean/body_c4.bin b/tests/fixtures/g9/lean/body_c4.bin new file mode 100644 index 0000000000000000000000000000000000000000..d1e5c88aff12dcb83a87b3deea352f99d358d315 GIT binary patch literal 34 kcmb1QfPi#Mh2*0A;$q#T#N_PMycC7p#FC=S3WYL50D(mbb^rhX literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/body_c4.meta.json b/tests/fixtures/g9/lean/body_c4.meta.json new file mode 100644 index 0000000..4cef76a --- /dev/null +++ b/tests/fixtures/g9/lean/body_c4.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "lean", + "format": 4, + "hash_hex": "98ffd187311ba242f45a808916defe07f2dbb6ddf4238b66425b99dd00c34eaa", + "padding_len": 0, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/body_c5.bin b/tests/fixtures/g9/lean/body_c5.bin new file mode 100644 index 0000000000000000000000000000000000000000..dc40eb83168b4ecdca825fbfbfc17a840578cb97 GIT binary patch literal 114 zcmc~|fB;4&W)@a9b`DN1ZXRAfeu2CGJIiFF8rgp^exDUICA-G6M)Qmnm$_H*(y{}m z8}IJp_^@!>tV4xjAqnq(yYF R3HQs#&U>(sEp6Fh9{^X>E-?TA literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/body_c5.meta.json b/tests/fixtures/g9/lean/body_c5.meta.json new file mode 100644 index 0000000..d974b9f --- /dev/null +++ b/tests/fixtures/g9/lean/body_c5.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "lean", + "format": 5, + "hash_hex": "35bc2f7043e605a1295df8f32199c866b58185482a91d8df0f8f41d40d3c72c7", + "padding_len": 0, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/lean/body_c8.bin b/tests/fixtures/g9/lean/body_c8.bin new file mode 100644 index 0000000000000000000000000000000000000000..21ee9721b5f7c2a17093ddb19bd76ad1fbd23931 GIT binary patch literal 32768 zcmeIuK`X-n0LEd*g&p<@tXxRMNlJ21YdO&5pd>T4B-AFYU737?IqcwKD<|!=mN>DL z!&=T<9CjNimmT;N`=6fo3;f=m%v>~;JX0d1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oN9;kig}5>@%?%k9YpspYspPH?ed(lo-5^Eqn)bK!yMT0t5&UAV7cs z0RjXF5a_$W^+mN6Sqz6KUOTPntIhLjzOxYty;PIwzNZBU5FkK+009C72oNAZfB=Dj z1iqdM_0o2+cz0W_kG*A|3rpo*X?V2ou^!L?83F_d5FkK+009C72oNAZpzi{YYmIJY tq*mMNHM=XNp|i$p^SzRpYW)86JuN_h009C72oNAZfB*pk1PBBq@DGSfK!pGR literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/body_c8.meta.json b/tests/fixtures/g9/lean/body_c8.meta.json new file mode 100644 index 0000000..8e29ca1 --- /dev/null +++ b/tests/fixtures/g9/lean/body_c8.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "lean", + "format": 8, + "hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "padding_len": 16358, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/body_c9.bin b/tests/fixtures/g9/lean/body_c9.bin new file mode 100644 index 0000000000000000000000000000000000000000..1ed2d65d6cb4a38e748dddc7e7009afb71f5ac63 GIT binary patch literal 32768 zcmeIu?K2Yq90%|=50iO1Q{6Ff;jFtQB<3L(=Q0do*Tdwp4tH#&j$1BknTNQLbyLU0 zc^E0O>c-tzT1~FRHRsCIEhCrng45->UHS|5(!1~P<@e3+`~47z7M51lB%A%VWIKBY zM<+wFO5n~BleP$(vebH>Fiv=MJj^yEzMxZ}`6M@=ZDe{!n z6%0xPVPUB$jg_@4)L5$-F>j~BO*y~4`Y3fC87;;(>zpnvYab2*5P$##AOHafKmY;| zfB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U< z00Izz00bZa0SG_<0uX=z1RwwbGXieG$M`onQF`slx>8Zv_`LQIEs{9l8ui-EJNkeo zLYJs354T+D$&3q#th?n}o2$>xcp0s_vzh!cUhut8DR=+@ z2tWV=5P$##AOHafKmY;|fPfhRXYXJphcA+pD!cS!rj=70uKu%|w--fUof*RA5*kmy z+nV{K3_V@(WtQ+{aU)4`rum1^r!7T%ztVWmUmd{>+euSQlP@@**uJ()AMNH5_<7T6 zitSv>JeyTOcAK&YDsJY_{|up*>*P5n&( literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/body_c9.meta.json b/tests/fixtures/g9/lean/body_c9.meta.json new file mode 100644 index 0000000..51954dd --- /dev/null +++ b/tests/fixtures/g9/lean/body_c9.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "lean", + "format": 9, + "hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "padding_len": 16278, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/lean/headered_c12.bin b/tests/fixtures/g9/lean/headered_c12.bin new file mode 100644 index 0000000000000000000000000000000000000000..26fe3ceaaab31764c5e63a994abc2792c713e9ec GIT binary patch literal 33401 zcmeIuTTByX6bEo3*438Fkf0kPmsJOhC>6J0DMc=trre6fQEsi2)?$%PEtg=2Qip{K zULNG4t0-!qYGNjk#ilS?vswz9qZC45LvXGFV-oGA21-z~?8Ug3K1%Zcp3e7uCprJ~ zONisdvs2@CvLmP@mv>J3tYMB*aairz(ldMc6P5OQcysWsTH_v7hbEcOd3i)hE(+fk zRG9IpJS%Abv&Z~n#%f0J>FL&0&6ZiJ;I5)6xjFVlfA?@#PYq{FVaSHL6%T~ee*L=N z==Om_p3{x*jQii!?cW#kaTk}V9dG?9{;N|`s&uVK0v-*dY$y6AEc9HBB=e{*ieqap!=6+f+v!O%# z>Z8!xOS6q@jC#}9ua!Qg-o{a`VRFd((RU}5H(aN67jkmb@?KVLh}Vx19N)7J(RhXN z6EWrE>~Hh)$f{7Ljg!V=&AhhSqI;5BnPMxEL+WP6*qw@xCI~+T-*^NfXhKKF#Cd?w3_*rySQmqB<$ga8B}009U<00Izz00bZa z0SG|ADS?=KZokw2Uicv{*R%AFNJ_FkoUdM&Z7DKjezS@@#$EkKtk>0Sf<9Wq4G%k* znqA`?C4A@*`AzIv@OBLw@gqLFC)YORDV4WxUetyR%S`f}7Z;n{DDufKof?lO2tWV= z5P$##AOHafKmY;|fB*!Rfq*_;_p~jbql3TL`;^^8uF^&JPP7$l(Y^d<8H7hf2tWV= W5P$##AOHafKmY;|fB*!X5_kiOH%XBI literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/headered_c12.meta.json b/tests/fixtures/g9/lean/headered_c12.meta.json new file mode 100644 index 0000000..08b9b6e --- /dev/null +++ b/tests/fixtures/g9/lean/headered_c12.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "headered", + "engine": "lean", + "format": 12, + "hash_hex": "974c9d40f5cabcd4bbbe87522e6a281ded02dd6330cabf17db3bf1cf700fe796", + "padding_len": 16358, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/headered_c13.bin b/tests/fixtures/g9/lean/headered_c13.bin new file mode 100644 index 0000000000000000000000000000000000000000..a1fd38f5e0e58d9be65e0eb5331a325775114119 GIT binary patch literal 33401 zcmeIuYcv#S90qVR3XKd)R2T~7l4Fg_lyNDOl8h#oLn(!&&1Kn+avj4Q31M;2G943z zlGbHIJ4Q8=$W{)mqjFofHfwEEWW;Rt?Ab4UTVK|v=l%3Q=Y7umy#L=`M-K+;kfRgJ z!kk1P%C1=}C$F%Mq_|#*tgLcdm|~+t-ZIr>?{Qk&&|ZqLF$tlE+Y`DeO#;_FZC>;mSgrslep%z=ll@Ok=lJ4CyzF|9&RI>Bt%6_mAw_w zT^jOai`o70Zr%dFzL^Ih;-|+eC(Z{}rM=R6KWE8#IHPVGd9!VXM57ckm}zv26T}iu zbV%>H=%DNMCfRvra_lLQ{k+i(&#H^>!ua+P_kI%HSd_Hyd*uf^?kKcclqisy5g!7@ z3YQ^K@zGv6ev^B7_YqA9*y`2~W6=TQ*Gc3~SorIU6ZjH-whJ>d@BQ+Qj}#Cvc? z!^?kq$ywmkT^rhvnYv4@i12FivL`cWnJtoQ+2thev6Y^Br(dI%qwB{XDU&-@*~&t3XF0*}93} zzPwS{ljhE_FW4WaYO)~K?P2}3uD;c$bF7|P`1|4;|GOvLtXjuhTwnw93F=N1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG`~ zRRp|@)Ot_pOEzcS`K@AEdUi=#BQDWZ)hB{S7dJ}3wkFJNRu?x4Qm6LB?sw|y>@2ohY6#WYlj$vWWouXa&koRR zNJER)qvpapb4*_g>vSlaNIa$VlIm1L%G2KLjYbZ{PEE(VS_TYtkzKc2_j%hj)7Abi zREL~+&Fqku!G84&L6U^^^TFIMqcOcxh?XZf7a zhsi}&S!Nx6zWLf$>8rwr5(q#50uX=z1Rwwb2tWV=5P$##Rv=J5u_eIWyV)yJd*E7= zqPd_}a4h)hOS4R+ZvsA0o6FCRz7sB_ zg(~mC!S0|U4?Uv6m3d9`VM*cy=eg=G_M2p*u;{RbzS#zwJ*mCJgzFRYhjRCq{L0=| z=Fa^tZRDbPi(LSH(X;l2;t^HLj}gnJ?Ki>*;h`^D)JUWJF)wD6qE?TrRYFPl*!WJ< zRaqY;5P$##AOHafKmY;|fB*y_009WBK!8}mofovUhf6n@%WDs_I71w=Yh-%n=#QrJ zLc@0Ao6_TKH~D+)m4hb}n>e7V8BQB9#ZC6^ec(b} qfKFbw4}l$9t&&!}f^tM4009U<00Izz00bZa0SG_<0uWdgfxiK`E407`Kb9C`HGT>rB1A_cjY9Z39J~QesUE9>iGcTZWwf9uZr{>XBYkzFtx^APj z7B}mizn>qt9zT9fRrt&+3+5*_RgJGEeVt%`HE407`Kb9C`HGT>rlVrF4wW9Q)H;^yJy;}=*z+v-bbc`UEv|B626WqrkR&)f3F zH}WhuD)_no+`EJ?^(maM1K#i5%X&1HmZ^R2jlly zK~u78JZm)1SaF$q6)!D2aJupCK8_Cyx6L|KC>E0N?zj6+mkaB&wsf~z2+v@A@^W^t jPs;zA18jRNTn=8g{*(4md7W^-eC)gj3)#|^9rghLY>s42 literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/headered_c5.meta.json b/tests/fixtures/g9/lean/headered_c5.meta.json new file mode 100644 index 0000000..cd74ae3 --- /dev/null +++ b/tests/fixtures/g9/lean/headered_c5.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "headered", + "engine": "lean", + "format": 5, + "hash_hex": "ab4cf6a9bbc86edb05f4adf1178bd21828c6d79bb8864fde3b4edd15b50f95ea", + "padding_len": 0, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/lean/outboard_c12/main.bin b/tests/fixtures/g9/lean/outboard_c12/main.bin new file mode 100644 index 0000000..87a6d12 --- /dev/null +++ b/tests/fixtures/g9/lean/outboard_c12/main.bin @@ -0,0 +1 @@ +g9 cross-backend matrix v1 \ No newline at end of file diff --git a/tests/fixtures/g9/lean/outboard_c12/meta.json b/tests/fixtures/g9/lean/outboard_c12/meta.json new file mode 100644 index 0000000..20efd7e --- /dev/null +++ b/tests/fixtures/g9/lean/outboard_c12/meta.json @@ -0,0 +1,13 @@ +{ + "layout": "outboard", + "engine": "lean", + "format": 12, + "hash_hex": "fb2705239ae261fdcf9555569b2fef5a90c5b2596dabb26e10971dd87118cdbb", + "padding_len": 16358, + "header_path": false, + "has_verification_outboard": true, + "has_fec_parity": true, + "has_header": false, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/outboard_c12/out.bin b/tests/fixtures/g9/lean/outboard_c12/out.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/g9/lean/outboard_c12/par.bin b/tests/fixtures/g9/lean/outboard_c12/par.bin new file mode 100644 index 0000000000000000000000000000000000000000..58e2d55e77e9b298ef74f2292b962d9541b06cb6 GIT binary patch literal 16384 zcmeIuu?hhJ0DxhmaDyi>SV+lckqqj(40JLmiG|&yTRC}yGu&WtS0oO)Es;5Wb|zI2O$|`0|5jOKmY**5I_I{1Q0*~0R$qiqhe`2 zRo$$0D{~*6l&T(RiL~-;fr9`72q1s}0tg_000IagfB*tv5jZq$-$-q@-N5ndT6|&` gorf{Z+uu)E4*5U;0R#|0009ILKmY**5I_KdKli9NDgXcg literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/outboard_c13/header.bin b/tests/fixtures/g9/lean/outboard_c13/header.bin new file mode 100644 index 0000000000000000000000000000000000000000..7158bb120f2f8cd9a06409380c39e4630bd099fd GIT binary patch literal 177 zcmZ>E407`Kb9C`HGT>rlVrF4wW9Q)H;^yJy;};NMd*>(9$YKBFm-Klf@gz;2<6Ln% z?!TBU?`1M;Rr~#oX7$w4wDxSsQ8d5VdOfn5!~9N#Q}S1~tT`@k?gqcxbLM`?x~6EE rPmgqs-9I+)#yV-=e6Zq`SER>B)5#}Vzr1B20C++2Q9!)R9-$Hdv}Q0* literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/outboard_c13/main.bin b/tests/fixtures/g9/lean/outboard_c13/main.bin new file mode 100644 index 0000000..3964138 --- /dev/null +++ b/tests/fixtures/g9/lean/outboard_c13/main.bin @@ -0,0 +1,2 @@ +OvZRk|I|): +7Jsvˁݾ𡶚qT`GDЯj8SLd|8D;f#]fL \ No newline at end of file diff --git a/tests/fixtures/g9/lean/outboard_c13/meta.json b/tests/fixtures/g9/lean/outboard_c13/meta.json new file mode 100644 index 0000000..18eaf7f --- /dev/null +++ b/tests/fixtures/g9/lean/outboard_c13/meta.json @@ -0,0 +1,14 @@ +{ + "layout": "outboard", + "engine": "lean", + "format": 13, + "hash_hex": "ccdf54ae825b1cf2e22d3347f1800d5d422bd9e0a8ea4a5948f13593c885f4ed", + "padding_len": 16294, + "header_path": true, + "has_verification_outboard": true, + "has_fec_parity": true, + "has_header": true, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/lean/outboard_c13/out.bin b/tests/fixtures/g9/lean/outboard_c13/out.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/g9/lean/outboard_c13/par.bin b/tests/fixtures/g9/lean/outboard_c13/par.bin new file mode 100644 index 0000000000000000000000000000000000000000..bd8e33b679a233d2e7db724a1bff2e8a7c84731a GIT binary patch literal 16384 zcmeIuTThY!0LF2l8chlv>e>L6c^F(U1F{w9oO+RX9>&ZxS8t08h4LcjT+k~_ikMSezM z(sHi1+}n|J{EzEKu`JJekJsI>p{;*eNVV-%ekieh5(m=!BFk=z+|vafQK69-;v z?R9yeH!gL}PBy+Z^=Efwy|1se3&xZ&7z7{y0SG_<0uX=z1Rwwb2tWV=Q6k_B3JG=f zyrxO;ZTvM`O1Y^z?bt?E=jq!-NH*_m-m)h5j<25Y`@g$i$`YoF=Oi;XKRP0Vg0$yv zrdK+0sF146k^cMCQYvk4e-`03Fnv|YD!O=WB}xt=0|F3$00bZa0SG_<0uX=z1Rwx` zm@rT3-@6|qzi=-acT8eki|*g%-&czK2cP|n$zZs`c4_E`F>_RH?y+n=meRR|a{sb> zon6B{7rApN@l5F$`o?uSA$EH>peeh{=KGFi8;10OpBE&+g#ZK~009U<00Izz00bZa z0SG`KS_GIA<}gKj^aqJjmcU<0ekVDlO^-UaXC>hUk%zf6a?M!IvO6+w`8JnNm=(XA zb=r!jMa0=e^<2Wqfp#0Y(E6@O;r?ovaj71@AX1)quP}_Qu6WBpv^+!y1Rwwb2tWV= P5P$##AOHafKmYLLR}x}`#LQGRi;Zc<`$c4}UVLT+M7QD%ignIQn*oD6UP literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/lean/outboard_c14/meta.json b/tests/fixtures/g9/lean/outboard_c14/meta.json new file mode 100644 index 0000000..ca55a81 --- /dev/null +++ b/tests/fixtures/g9/lean/outboard_c14/meta.json @@ -0,0 +1,13 @@ +{ + "layout": "outboard", + "engine": "lean", + "format": 14, + "hash_hex": "129b45184c50d861dc6013dcb32406e2b8a9e5087230ceed6ea01850db1f616e", + "padding_len": 16349, + "header_path": false, + "has_verification_outboard": true, + "has_fec_parity": true, + "has_header": false, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/lean/outboard_c14/out.bin b/tests/fixtures/g9/lean/outboard_c14/out.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/g9/lean/outboard_c14/par.bin b/tests/fixtures/g9/lean/outboard_c14/par.bin new file mode 100644 index 0000000000000000000000000000000000000000..e1183ac8c5eda490249f07085d710fccd6c1d152 GIT binary patch literal 16384 zcmeIuF-rn*9LDhYFOfgdu!$1(dV4V#l^cnss}a-c6-8KZD`c^U0lk6C9`G z1N&T$_Ce|qKmY**5I_I{1Q0*~0R#|0Kz@R!N8=$cGVL0!c~+}Aua=wpy6PGG*3wL7 zjqwpm2q1s}0tg_000IagfB*srAP{kZ%;mM~egBGds$SSyZ?{jE{qVp{+E407`Kb9C`HGT>rlVrF4wW9Q)H;^yJy;}?*&SClYkivPkgOD7`luH6?-o#IKE z+}(wLuPJ{tx4(X1yQLk z9OACa!*$%qbZM-T8Q0NidRXY`q^Dy}byt7D-nP5%@5S%U@5S%?{n#jM{!UbsQeAh9 z@@6n9nBX2h-(3?#Oq1w_s`lN=A0ixZq51zf?m!d?EGBaABsXY8yq&&&c|m{RKqBpNHiMce&CSd6%MfH9&N?EbN(3^gxR6>9 z@UI^>1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0{_1NVVLYG zxe)oxroL^~FsIm48eb$dC!9AuJh>Rcn-qlTYDc|P+`x#;?3GJ$s!yyK^4L}->N#Ng z9o$*+Sk27%{~ID91Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_< z0+s~qTL_<1#2r&>Q*MNd7Zp9LCZhS;p?vANm%}W28!LhDC_TK2;XP-!MHX68RP~wM zNjXGYMoOcynN0^7d7R6w9lzH2G%tRrSlO%5j=d;y)Am)2@_HtQUEkb2qiVFBQeRC; zjZOclT(?s*W^ca7J<8x8+G%7Lj>lZfNGF!~a7=NrT<-L5gDJE-vW_D%5aJxWIP6(V z$#I7O1Rwwb2tWV=5P$##AOHafK)_lAZWTA^UBXBt->1`h`jyl(4TaN5E(BeJKx8cq zkP`wBfB*y_009U<00Izz00bZa0ZRhxM^3MqU+3@dNL`-aE|fSLUdXooMoj{gyuC4gm;200Izz00bZa0SG_<0uX?JwFqe9)id>!*4DIzzL}UxqEa2y mXROcOtp54US{fiH1Rwwb2tWV=5P$##AOHafKmY=k1pWZps}sKf literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/body_c12.meta.json b/tests/fixtures/g9/rust/body_c12.meta.json new file mode 100644 index 0000000..b02ef84 --- /dev/null +++ b/tests/fixtures/g9/rust/body_c12.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "rust", + "format": 12, + "hash_hex": "974c9d40f5cabcd4bbbe87522e6a281ded02dd6330cabf17db3bf1cf700fe796", + "padding_len": 16358, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/rust/body_c13.bin b/tests/fixtures/g9/rust/body_c13.bin new file mode 100644 index 0000000000000000000000000000000000000000..1cf556d0f66d35f9f3cdab7bddda08806bffae3a GIT binary patch literal 33224 zcmeIu=}!{|7zS`#l~#~rfVxpCMljaF6p;JcK(vTOu12(oQWO`ZcuZ1})=C8|2uK|u zx3LH!#{i`=E7cW2JSJL)j*8s0=PM~saEZM?;VEFXBAKvHN`@H!rN?i2c_(f*v ze|Omi<^&zpRq21JZ*PQ>Gd)dGEdyBITROKZ({I4$u=Q>-e8N$ys?8ry? z?|f$RqjSvIu3brCjVToen&muZCOx3OxYVg4mY5Q_?$m*}qKp*9+vKQ9(IDl+_+~-t z=nD5^RS!pXov67!43V4tK4OW0E7zJzSRI64v4OfGt4WWDE9t+uhLK7>GAykw~kiM&i#Z@IpKp;33B zj7wz+Nt1++g&{dH{O$Y=&)jsK{ZE#26+a8R?~^r>#=-|DRx%^iGjy4+vhIj9J;lW& zpD;L56v{X_$5-gcU3{B6+-DE{Wm#*IWf5_uPoi_^dJE%!FKh@v00Izz00bZa0SG_< z0uX=z1Rwwb2tWV=5P$##AOHafKmY;|fB*zOy};u$29Awa+U8anhB^iNc;;+7ZT#hA zvqeYX-^*^cgthnIw9EZt`hDcReZi+^=xVQJ)D>=4c9#?b!VyZVvM){c=|93n2tWV= z5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1QuFAp-7s$u~>AT`-L)1 z(peE?!-;J8zMok-_1UWpRpg4GJCxOVuL%y!rf=1xvepPmP0%a-m7`<6QH(at%x>0o zPQvk#fWY{H9^MO~HnIN1`UK-nLRjxmcwriQ=(n-_kp0Uws<@Z+dsjV(o&G(dUA}e~ zO>8t|SYA9xt+dvpQk=fxoML&aTGSKLx^o3TWUiul5!=nY?@_FLRwz7_0m@|eCF<(P z?RH*S31*pxRnZYQePm}p20rl@zo7N6?OiV#=h`R6QOfl87`TTlPaf-4--2=sDAS8FI(mtkr03Y1Rwwb z2tWV=5P$##AOHafEVMu_SMBUETrB0Kcy8$#s(E;)LBmVleAIOGdD_#=hP*1?;epx% zwvKFCMcTo%Mv3|fOK-~wbCH%O5C2nN=3Z}X<>bfW2*l^4*Huqv)t*{2+uo07h6TSG zyYSy9(qgzVlW)ePuHU4*?j=+VYj=vbmGtn}NrHtJMST}+Cn1Pz1_x}r;g6q zwB3jyaE^4!O?2N~>1Q%d>QOotJMc<4{R7T63oU~_2tWV=5P$##AOHafKmY;|fB*#M z1x#rkLB8>vtJ2hB&7kU81hHJGmC4&Uv!=Er^WH!~x4>Y>vDEZ;w`{ac>)VL$ukPgs zkg^Nc9qf3cSSoZKDhrR^d6~}MWEy?7 fqJsHGBm^J;0SG_<0uX=z1Rwwb2tWV=3oY<3J%HiH literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/body_c13.meta.json b/tests/fixtures/g9/rust/body_c13.meta.json new file mode 100644 index 0000000..88cca5f --- /dev/null +++ b/tests/fixtures/g9/rust/body_c13.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "rust", + "format": 13, + "hash_hex": "eb11b7f819b5dc0ad237291a5b15fdddc187ffb5633dc32110498d011674b7fc", + "padding_len": 16278, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/rust/body_c4.bin b/tests/fixtures/g9/rust/body_c4.bin new file mode 100644 index 0000000000000000000000000000000000000000..d1e5c88aff12dcb83a87b3deea352f99d358d315 GIT binary patch literal 34 kcmb1QfPi#Mh2*0A;$q#T#N_PMycC7p#FC=S3WYL50D(mbb^rhX literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/body_c4.meta.json b/tests/fixtures/g9/rust/body_c4.meta.json new file mode 100644 index 0000000..b215abc --- /dev/null +++ b/tests/fixtures/g9/rust/body_c4.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "rust", + "format": 4, + "hash_hex": "98ffd187311ba242f45a808916defe07f2dbb6ddf4238b66425b99dd00c34eaa", + "padding_len": 0, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/rust/body_c5.bin b/tests/fixtures/g9/rust/body_c5.bin new file mode 100644 index 0000000000000000000000000000000000000000..dc40eb83168b4ecdca825fbfbfc17a840578cb97 GIT binary patch literal 114 zcmc~|fB;4&W)@a9b`DN1ZXRAfeu2CGJIiFF8rgp^exDUICA-G6M)Qmnm$_H*(y{}m z8}IJp_^@!>tV4xjAqnq(yYF R3HQs#&U>(sEp6Fh9{^X>E-?TA literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/body_c5.meta.json b/tests/fixtures/g9/rust/body_c5.meta.json new file mode 100644 index 0000000..bd5939d --- /dev/null +++ b/tests/fixtures/g9/rust/body_c5.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "rust", + "format": 5, + "hash_hex": "35bc2f7043e605a1295df8f32199c866b58185482a91d8df0f8f41d40d3c72c7", + "padding_len": 0, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/rust/body_c8.bin b/tests/fixtures/g9/rust/body_c8.bin new file mode 100644 index 0000000000000000000000000000000000000000..21ee9721b5f7c2a17093ddb19bd76ad1fbd23931 GIT binary patch literal 32768 zcmeIuK`X-n0LEd*g&p<@tXxRMNlJ21YdO&5pd>T4B-AFYU737?IqcwKD<|!=mN>DL z!&=T<9CjNimmT;N`=6fo3;f=m%v>~;JX0d1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oN9;kig}5>@%?%k9YpspYspPH?ed(lo-5^Eqn)bK!yMT0t5&UAV7cs z0RjXF5a_$W^+mN6Sqz6KUOTPntIhLjzOxYty;PIwzNZBU5FkK+009C72oNAZfB=Dj z1iqdM_0o2+cz0W_kG*A|3rpo*X?V2ou^!L?83F_d5FkK+009C72oNAZpzi{YYmIJY tq*mMNHM=XNp|i$p^SzRpYW)86JuN_h009C72oNAZfB*pk1PBBq@DGSfK!pGR literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/body_c8.meta.json b/tests/fixtures/g9/rust/body_c8.meta.json new file mode 100644 index 0000000..bc5b731 --- /dev/null +++ b/tests/fixtures/g9/rust/body_c8.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "body", + "engine": "rust", + "format": 8, + "hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "padding_len": 16358, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/rust/body_c9.bin b/tests/fixtures/g9/rust/body_c9.bin new file mode 100644 index 0000000000000000000000000000000000000000..1ed2d65d6cb4a38e748dddc7e7009afb71f5ac63 GIT binary patch literal 32768 zcmeIu?K2Yq90%|=50iO1Q{6Ff;jFtQB<3L(=Q0do*Tdwp4tH#&j$1BknTNQLbyLU0 zc^E0O>c-tzT1~FRHRsCIEhCrng45->UHS|5(!1~P<@e3+`~47z7M51lB%A%VWIKBY zM<+wFO5n~BleP$(vebH>Fiv=MJj^yEzMxZ}`6M@=ZDe{!n z6%0xPVPUB$jg_@4)L5$-F>j~BO*y~4`Y3fC87;;(>zpnvYab2*5P$##AOHafKmY;| zfB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U< z00Izz00bZa0SG_<0uX=z1RwwbGXieG$M`onQF`slx>8Zv_`LQIEs{9l8ui-EJNkeo zLYJs354T+D$&3q#th?n}o2$>xcp0s_vzh!cUhut8DR=+@ z2tWV=5P$##AOHafKmY;|fPfhRXYXJphcA+pD!cS!rj=70uKu%|w--fUof*RA5*kmy z+nV{K3_V@(WtQ+{aU)4`rum1^r!7T%ztVWmUmd{>+euSQlP@@**uJ()AMNH5_<7T6 zitSv>JeyTOcAK&YDsJY_{|up*>*P5n&( literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/body_c9.meta.json b/tests/fixtures/g9/rust/body_c9.meta.json new file mode 100644 index 0000000..b7ad5e9 --- /dev/null +++ b/tests/fixtures/g9/rust/body_c9.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "body", + "engine": "rust", + "format": 9, + "hash_hex": "0000000000000000000000000000000000000000000000000000000000000000", + "padding_len": 16278, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/rust/headered_c12.bin b/tests/fixtures/g9/rust/headered_c12.bin new file mode 100644 index 0000000000000000000000000000000000000000..26fe3ceaaab31764c5e63a994abc2792c713e9ec GIT binary patch literal 33401 zcmeIuTTByX6bEo3*438Fkf0kPmsJOhC>6J0DMc=trre6fQEsi2)?$%PEtg=2Qip{K zULNG4t0-!qYGNjk#ilS?vswz9qZC45LvXGFV-oGA21-z~?8Ug3K1%Zcp3e7uCprJ~ zONisdvs2@CvLmP@mv>J3tYMB*aairz(ldMc6P5OQcysWsTH_v7hbEcOd3i)hE(+fk zRG9IpJS%Abv&Z~n#%f0J>FL&0&6ZiJ;I5)6xjFVlfA?@#PYq{FVaSHL6%T~ee*L=N z==Om_p3{x*jQii!?cW#kaTk}V9dG?9{;N|`s&uVK0v-*dY$y6AEc9HBB=e{*ieqap!=6+f+v!O%# z>Z8!xOS6q@jC#}9ua!Qg-o{a`VRFd((RU}5H(aN67jkmb@?KVLh}Vx19N)7J(RhXN z6EWrE>~Hh)$f{7Ljg!V=&AhhSqI;5BnPMxEL+WP6*qw@xCI~+T-*^NfXhKKF#Cd?w3_*rySQmqB<$ga8B}009U<00Izz00bZa z0SG|ADS?=KZokw2Uicv{*R%AFNJ_FkoUdM&Z7DKjezS@@#$EkKtk>0Sf<9Wq4G%k* znqA`?C4A@*`AzIv@OBLw@gqLFC)YORDV4WxUetyR%S`f}7Z;n{DDufKof?lO2tWV= z5P$##AOHafKmY;|fB*!Rfq*_;_p~jbql3TL`;^^8uF^&JPP7$l(Y^d<8H7hf2tWV= W5P$##AOHafKmY;|fB*!X5_kiOH%XBI literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/headered_c12.meta.json b/tests/fixtures/g9/rust/headered_c12.meta.json new file mode 100644 index 0000000..92d37e7 --- /dev/null +++ b/tests/fixtures/g9/rust/headered_c12.meta.json @@ -0,0 +1,9 @@ +{ + "layout": "headered", + "engine": "rust", + "format": 12, + "hash_hex": "974c9d40f5cabcd4bbbe87522e6a281ded02dd6330cabf17db3bf1cf700fe796", + "padding_len": 16358, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/rust/headered_c13.bin b/tests/fixtures/g9/rust/headered_c13.bin new file mode 100644 index 0000000000000000000000000000000000000000..a1fd38f5e0e58d9be65e0eb5331a325775114119 GIT binary patch literal 33401 zcmeIuYcv#S90qVR3XKd)R2T~7l4Fg_lyNDOl8h#oLn(!&&1Kn+avj4Q31M;2G943z zlGbHIJ4Q8=$W{)mqjFofHfwEEWW;Rt?Ab4UTVK|v=l%3Q=Y7umy#L=`M-K+;kfRgJ z!kk1P%C1=}C$F%Mq_|#*tgLcdm|~+t-ZIr>?{Qk&&|ZqLF$tlE+Y`DeO#;_FZC>;mSgrslep%z=ll@Ok=lJ4CyzF|9&RI>Bt%6_mAw_w zT^jOai`o70Zr%dFzL^Ih;-|+eC(Z{}rM=R6KWE8#IHPVGd9!VXM57ckm}zv26T}iu zbV%>H=%DNMCfRvra_lLQ{k+i(&#H^>!ua+P_kI%HSd_Hyd*uf^?kKcclqisy5g!7@ z3YQ^K@zGv6ev^B7_YqA9*y`2~W6=TQ*Gc3~SorIU6ZjH-whJ>d@BQ+Qj}#Cvc? z!^?kq$ywmkT^rhvnYv4@i12FivL`cWnJtoQ+2thev6Y^Br(dI%qwB{XDU&-@*~&t3XF0*}93} zzPwS{ljhE_FW4WaYO)~K?P2}3uD;c$bF7|P`1|4;|GOvLtXjuhTwnw93F=N1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG`~ zRRp|@)Ot_pOEzcS`K@AEdUi=#BQDWZ)hB{S7dJ}3wkFJNRu?x4Qm6LB?sw|y>@2ohY6#WYlj$vWWouXa&koRR zNJER)qvpapb4*_g>vSlaNIa$VlIm1L%G2KLjYbZ{PEE(VS_TYtkzKc2_j%hj)7Abi zREL~+&Fqku!G84&L6U^^^TFIMqcOcxh?XZf7a zhsi}&S!Nx6zWLf$>8rwr5(q#50uX=z1Rwwb2tWV=5P$##Rv=J5u_eIWyV)yJd*E7= zqPd_}a4h)hOS4R+ZvsA0o6FCRz7sB_ zg(~mC!S0|U4?Uv6m3d9`VM*cy=eg=G_M2p*u;{RbzS#zwJ*mCJgzFRYhjRCq{L0=| z=Fa^tZRDbPi(LSH(X;l2;t^HLj}gnJ?Ki>*;h`^D)JUWJF)wD6qE?TrRYFPl*!WJ< zRaqY;5P$##AOHafKmY;|fB*y_009WBK!8}mofovUhf6n@%WDs_I71w=Yh-%n=#QrJ zLc@0Ao6_TKH~D+)m4hb}n>e7V8BQB9#ZC6^ec(b} qfKFbw4}l$9t&&!}f^tM4009U<00Izz00bZa0SG_<0uWdgfxiK`E407`Kb9C`HGT>rB1A_cjY9Z39J~QesUE9>iGcTZWwf9uZr{>XBYkzFtx^APj z7B}mizn>qt9zT9fRrt&+3+5*_RgJGEeVt%`HE407`Kb9C`HGT>rlVrF4wW9Q)H;^yJy;}=*z+v-bbc`UEv|B626WqrkR&)f3F zH}WhuD)_no+`EJ?^(maM1K#i5%X&1HmZ^R2jlly zK~u78JZm)1SaF$q6)!D2aJupCK8_Cyx6L|KC>E0N?zj6+mkaB&wsf~z2+v@A@^W^t jPs;zA18jRNTn=8g{*(4md7W^-eC)gj3)#|^9rghLY>s42 literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/headered_c5.meta.json b/tests/fixtures/g9/rust/headered_c5.meta.json new file mode 100644 index 0000000..b97db6d --- /dev/null +++ b/tests/fixtures/g9/rust/headered_c5.meta.json @@ -0,0 +1,10 @@ +{ + "layout": "headered", + "engine": "rust", + "format": 5, + "hash_hex": "ab4cf6a9bbc86edb05f4adf1178bd21828c6d79bb8864fde3b4edd15b50f95ea", + "padding_len": 0, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/rust/outboard_c12/main.bin b/tests/fixtures/g9/rust/outboard_c12/main.bin new file mode 100644 index 0000000..87a6d12 --- /dev/null +++ b/tests/fixtures/g9/rust/outboard_c12/main.bin @@ -0,0 +1 @@ +g9 cross-backend matrix v1 \ No newline at end of file diff --git a/tests/fixtures/g9/rust/outboard_c12/meta.json b/tests/fixtures/g9/rust/outboard_c12/meta.json new file mode 100644 index 0000000..0996a9e --- /dev/null +++ b/tests/fixtures/g9/rust/outboard_c12/meta.json @@ -0,0 +1,13 @@ +{ + "layout": "outboard", + "engine": "rust", + "format": 12, + "hash_hex": "fb2705239ae261fdcf9555569b2fef5a90c5b2596dabb26e10971dd87118cdbb", + "padding_len": 16358, + "header_path": false, + "has_verification_outboard": true, + "has_fec_parity": true, + "has_header": false, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/rust/outboard_c12/out.bin b/tests/fixtures/g9/rust/outboard_c12/out.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/g9/rust/outboard_c12/par.bin b/tests/fixtures/g9/rust/outboard_c12/par.bin new file mode 100644 index 0000000000000000000000000000000000000000..58e2d55e77e9b298ef74f2292b962d9541b06cb6 GIT binary patch literal 16384 zcmeIuu?hhJ0DxhmaDyi>SV+lckqqj(40JLmiG|&yTRC}yGu&WtS0oO)Es;5Wb|zI2O$|`0|5jOKmY**5I_I{1Q0*~0R$qiqhe`2 zRo$$0D{~*6l&T(RiL~-;fr9`72q1s}0tg_000IagfB*tv5jZq$-$-q@-N5ndT6|&` gorf{Z+uu)E4*5U;0R#|0009ILKmY**5I_KdKli9NDgXcg literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/outboard_c13/header.bin b/tests/fixtures/g9/rust/outboard_c13/header.bin new file mode 100644 index 0000000000000000000000000000000000000000..7158bb120f2f8cd9a06409380c39e4630bd099fd GIT binary patch literal 177 zcmZ>E407`Kb9C`HGT>rlVrF4wW9Q)H;^yJy;};NMd*>(9$YKBFm-Klf@gz;2<6Ln% z?!TBU?`1M;Rr~#oX7$w4wDxSsQ8d5VdOfn5!~9N#Q}S1~tT`@k?gqcxbLM`?x~6EE rPmgqs-9I+)#yV-=e6Zq`SER>B)5#}Vzr1B20C++2Q9!)R9-$Hdv}Q0* literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/outboard_c13/main.bin b/tests/fixtures/g9/rust/outboard_c13/main.bin new file mode 100644 index 0000000..3964138 --- /dev/null +++ b/tests/fixtures/g9/rust/outboard_c13/main.bin @@ -0,0 +1,2 @@ +OvZRk|I|): +7Jsvˁݾ𡶚qT`GDЯj8SLd|8D;f#]fL \ No newline at end of file diff --git a/tests/fixtures/g9/rust/outboard_c13/meta.json b/tests/fixtures/g9/rust/outboard_c13/meta.json new file mode 100644 index 0000000..e1f567b --- /dev/null +++ b/tests/fixtures/g9/rust/outboard_c13/meta.json @@ -0,0 +1,14 @@ +{ + "layout": "outboard", + "engine": "rust", + "format": 13, + "hash_hex": "ccdf54ae825b1cf2e22d3347f1800d5d422bd9e0a8ea4a5948f13593c885f4ed", + "padding_len": 16294, + "header_path": true, + "has_verification_outboard": true, + "has_fec_parity": true, + "has_header": true, + "nonce_hex": "0102030405060708090a0b0c0d0e0f10", + "plaintext_id": "g9_matrix_v1", + "encrypted": true +} diff --git a/tests/fixtures/g9/rust/outboard_c13/out.bin b/tests/fixtures/g9/rust/outboard_c13/out.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/g9/rust/outboard_c13/par.bin b/tests/fixtures/g9/rust/outboard_c13/par.bin new file mode 100644 index 0000000000000000000000000000000000000000..bd8e33b679a233d2e7db724a1bff2e8a7c84731a GIT binary patch literal 16384 zcmeIuTThY!0LF2l8chlv>e>L6c^F(U1F{w9oO+RX9>&ZxS8t08h4LcjT+k~_ikMSezM z(sHi1+}n|J{EzEKu`JJekJsI>p{;*eNVV-%ekieh5(m=!BFk=z+|vafQK69-;v z?R9yeH!gL}PBy+Z^=Efwy|1se3&xZ&7z7{y0SG_<0uX=z1Rwwb2tWV=Q6k_B3JG=f zyrxO;ZTvM`O1Y^z?bt?E=jq!-NH*_m-m)h5j<25Y`@g$i$`YoF=Oi;XKRP0Vg0$yv zrdK+0sF146k^cMCQYvk4e-`03Fnv|YD!O=WB}xt=0|F3$00bZa0SG_<0uX=z1Rwx` zm@rT3-@6|qzi=-acT8eki|*g%-&czK2cP|n$zZs`c4_E`F>_RH?y+n=meRR|a{sb> zon6B{7rApN@l5F$`o?uSA$EH>peeh{=KGFi8;10OpBE&+g#ZK~009U<00Izz00bZa z0SG`KS_GIA<}gKj^aqJjmcU<0ekVDlO^-UaXC>hUk%zf6a?M!IvO6+w`8JnNm=(XA zb=r!jMa0=e^<2Wqfp#0Y(E6@O;r?ovaj71@AX1)quP}_Qu6WBpv^+!y1Rwwb2tWV= P5P$##AOHafKmYx}`#LQGRi;Zc<`$c4}UVLT+M7QD%ignIQn?zznVc literal 0 HcmV?d00001 diff --git a/tests/fixtures/g9/rust/outboard_c14/meta.json b/tests/fixtures/g9/rust/outboard_c14/meta.json new file mode 100644 index 0000000..7b405cb --- /dev/null +++ b/tests/fixtures/g9/rust/outboard_c14/meta.json @@ -0,0 +1,13 @@ +{ + "layout": "outboard", + "engine": "rust", + "format": 14, + "hash_hex": "0abe578176945b6a8e823cd2af95f5c7ae58fe055d3b6a20d6aae512b555bdcf", + "padding_len": 16349, + "header_path": false, + "has_verification_outboard": true, + "has_fec_parity": true, + "has_header": false, + "plaintext_id": "g9_matrix_v1", + "encrypted": false +} diff --git a/tests/fixtures/g9/rust/outboard_c14/out.bin b/tests/fixtures/g9/rust/outboard_c14/out.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/g9/rust/outboard_c14/par.bin b/tests/fixtures/g9/rust/outboard_c14/par.bin new file mode 100644 index 0000000000000000000000000000000000000000..50a3a06c46013b865849aa788ebd1f837f3e12e7 GIT binary patch literal 16384 zcmeIuF-rmg9ER~fxJY?}xV1Fgk_d-`AVNe#a7aU(oX8DVj-XRcZ=2O=<-dkFQUbR zp?E407`Kb9C`HGT>rlVrF4wW9Q)H;^yJy;}?*&SClYkivPkgOD7`luH6?-o#IKE z+}(wLuPJjtj1uCIOqCa#VCiA9^yxbT8fT+5Ew~~KR#&_qGr6Any4tz# zxk1#9bhtz_bw8$lW@!-Tqo!p8%Gt&`P6V=TvNxZGucn5JWET%%&d&BP@uy}oMB{&4 z?teb=F~4aH1JGChuirwyzn5=~j20V}|C}yOducmBdH0||z;8L}OuTiX%R+{Qo}8o? z(Q8g+sPnYtjarH0wZQ{ykq+BV-2d&Klhf>Tat2om%UjY@bUofP~HShuPn%EcCO@UhyRp| zR<&TpMr+FgjeGIjLruPBm&SFEe47pnk&>9obLAh>2nggo7j7S?7}qrIyxKZ1Z?zlg zTb>A?NSxd~(pT^c;)RRLv~;wLw8&3Y#bi!HzBS+p&>4t+TSI;_NHwiPk6{qJN@9J z>y%Q$`78)|*W_@f5Xn99ydHuR8M7DzqtZiM0Z69iKgs(T(pc z5kdj7Gkcri!8HKKrEPnv8uVh;>n*Jc3fc#Hw}n`ssS{wm@Gx(>?X{ffYr^S{j+!IG z7vn%i5#a91%&3l20UzrER(4oe*omL=y$_pr0wj1fP|~`~t4+(j(qv?$`T-28_Hb6^ z>*cp`ws`^^O6RpokGvK+Y&C2E;Mozi&`}>ypSh69$c8(O(wi1h9?{U-z(^66!O-bS zqL^Ius7Q_GyiEzPH!p?^`+%KT4!uU|qSWW~Py98Lih~XF%VE(0Te(16hHwKKmim$0Te(16hHwKKmim$0Te(16hHwKKmim$ z0Te)izXguZPfL0Pf7BMT>lRg@oEZy|Dk`78@3g`DfPp_%dy1%@nh;|?Hu+w;px2s3 zG&-itXfF$opw)@tek$Le@0b4Pe)ivE;VTND01BW03ZMWApa2S>01BW03ZMWApa2S> z01BW03ZMWApa2S>01BW03ZTIMr@%}si%W*us&6~DrtwEo1(VY^m<{uo$J0t*OJDA4 zpN-E-87s5bKX>&H_1K3eX3JY1{XZ|_1CY#& zc-U6Q`GAv7r$@vs9##=ij&%0gy>YhD@6d`*xml&#zp^%2yp3UR=-wXgF9DAj0pgYi zrw_QP*Q!m>Z3cID>#}RTc2&|FT-8r7yzsbdq~vpWn=+$YxTZ2N4qezLmSYcxIRNs@ zgpFKVpS_HHR_5B$A_LDxLpOn;IY8t?ffDarj^C^~9(w2JC#)5yxd+E40qRL&zT}Wt zr_Zq@vcG@R?hth)sAC0)*wflGPG$4tW7{)+?AX>ufIwFxnycb7MPZt-2|${6Nx%0A zjuQg?0)YS~WZxumXKbffk{BbKnem>Sad|fR)|^)6+~=TrjgGQUwx`;w#Ds#lJ~(Jk z|0*B^O+LT9NZeYpwbI-4QeyCAUTeWg7-k3aSN9#aYc^=#ROZxDuc9`tlV^c!<~Z0; z00mG01yBG5Pyhu`00mG01yJBu1-_r>mmde*DaJC!eZ~vlN)|Mn189XnD+f@KWGr)? zRHL2f!3|XYwecg5FBW*6F&72?b2~&=c?b+ne0#vWk{Xx-@ajGhdcyyN^H6TjUy=uI z1T2}`E`EKmCu^j5)2XGiJid};f<7ZfGQikZ04N^jw`#IdRaNC9J~2!#Q?d4IWuq#Y z*i}qiSo>E})|IQ%cf)d~Y(=vQ>m}hJBcOb_zJrK|{q@vf)fLa#L}#bWw7qv-d+zU9 zuwc7w}(Y9E`}DJ4hO1btz)yB}GKI{HKb#+BRE{qmosGrv2p&h?UpjZ>g==lJ@pPqu#qs#uxXr;nM)htGi?a~y0a zfC4Ch0w{n2D1ZVefC4Ch0x0mS0wzH_p6S`-By_GG-9U=el2W*PLet2RyZP*gMO*dPL!@&fm{EIY7GiuS^0D#6kCBkQT;7xa8VIL!!nciGMKf_|S@k5=K8 z`{$D5S%Z&|R|nj0F_?)E33u0Pwx(ywGP=>v%fbx6&&mPdXZr*{t1CE{+qW8`rvR2j zg`}2cupe4vk18`XZ2z%F;Kv<zBqsCOrz{Va9>A8;LgWH z7-5cs4FymD1yBG5Pyhu`00mG01yBG5epTT6^ZkwuLf@L89YOIycN}}}Y?^u(LxY%l z06AfRM1v$#gVKYtBMxJ6qL+-?hGSVXgSg`?b5 zn?~UdyLbG@_xj&nVe>_lS4pLS<&~PSf&K=;-jNol>=-u3#l$6Bve4%cYq#WP& zjh!QNgsfYS(#IMTdZ$GZY5KZGEVJ=tq{=H+%ca!K5=2oh>X~ZqLS3 z$|nwow($#=7QP!?q%|-f3P`JxgEoMlmHord>H&VX&hN~aO#ugY04@r3#hZZ2dCp48 zo9O5ZKmIG=#~OhjSNx1@GT*FFfAgX8167^MockKk9e5%4Reb)+@(S1C=Xv0#l%$>8 z4DDZCZdP3AyFe#-KdI24+D{#R54K%!b-7l;m#S`#M7dtK*|Y$H^@ro%YcQB&VM75F YKmim$0Te(16hHwKKmim$fnO2$H>B^m+W-In literal 0 HcmV?d00001 diff --git a/tests/fixtures/phase3_g9_directory/90ccff39cc098af6242e5ead3c4cb680f9d7a795002e17a295b904d2b2265466.c14 b/tests/fixtures/phase3_g9_directory/90ccff39cc098af6242e5ead3c4cb680f9d7a795002e17a295b904d2b2265466.c14 new file mode 100644 index 0000000000000000000000000000000000000000..6ca5e802eddf10d5135e43e22e03c5600fbc732c GIT binary patch literal 24 fcmdPcs{favqLP83AS1Cj)mS0jQXwNXCnp~OXaoo| literal 0 HcmV?d00001 diff --git a/tests/fixtures/phase3_g9_directory/README.txt b/tests/fixtures/phase3_g9_directory/README.txt new file mode 100644 index 0000000..ac70ac0 --- /dev/null +++ b/tests/fixtures/phase3_g9_directory/README.txt @@ -0,0 +1,21 @@ +G9 directory seed: rust-encoded Adamantine 1.0 archive (backend-rust / CLI). +Used by tests/lean_backend_phase3.rs for rust-encode → lean-decode extract. + +**Decode-only SSOT** for dual-suite interop. Catalog root is **not** a live +re-encode golden: current backend-rust re-encode of the same tree yields a +different catalog root (see LIVE_RUST_DIR_CATALOG_ROOT in +tests/determinism_roundtrip.rs) while segment mains may still match. Cross-engine +encode residual is live-rust vs live-lean (W2b), not lean-vs-this-seed. + +Source tree (public c14 catalog, zero master): + a.txt = "phase3 g9 hello" + sub/b.bin = "nested data" + +Segment formats follow auto policy (both small texts → c14). +Catalog seed: 16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14 + +Regenerate (only when intentionally refreshing the decode seed; update +lean_backend_phase3 hard-coded path + this README): + printf 'phase3 g9 hello' > /tmp/src/a.txt + mkdir -p /tmp/src/sub && printf 'nested data' > /tmp/src/sub/b.bin + cargo run --features cli --bin carbonado -- encode /tmp/src -o tests/fixtures/phase3_g9_directory diff --git a/tests/fixtures/phase3_g9_directory/c710a801faf330c99d2ce6aeb210823c14caa4723dd3e301528f1e3c534400e6.c14 b/tests/fixtures/phase3_g9_directory/c710a801faf330c99d2ce6aeb210823c14caa4723dd3e301528f1e3c534400e6.c14 new file mode 100644 index 0000000000000000000000000000000000000000..d8e7a713f36869e31ba6a9ce3908139a4b4046d4 GIT binary patch literal 20 bcmdPcs{favB9eh2FSWQNHANvMu_O@yO0owS literal 0 HcmV?d00001 diff --git a/tests/fixtures/rkyv/README.md b/tests/fixtures/rkyv/README.md new file mode 100644 index 0000000..8efcfd6 --- /dev/null +++ b/tests/fixtures/rkyv/README.md @@ -0,0 +1,30 @@ +# R9/W3 rkyv FilepackManifestWire goldens + +Pinned Rust `rkyv` 0.8.16 + `unaligned` wire for pure Lean encode **and** decode +(`Carbonado/RkyvFilepack.lean`). + +| File | Description | +|------|-------------| +| `empty_manifest.bin` | version=2, format_level=c14, 0 entries (13 B) | +| `single_entry.bin` | one entry `a.txt`, one SegmentRef, no OTS (131 B) | +| `multi_entry_ots.bin` | two entries (`a.txt` + ool long path) + OTS Some on second (275 B) | +| `path_inline_8.bin` | exactly 8-byte path (inline boundary) | +| `path_ool_9.bin` | exactly 9-byte path (out-of-line boundary) | +| `two_segments.bin` | one entry, two SegmentRefs | +| `ots_first_only.bin` | OTS Some on first entry only; second None | +| `rkyv_cfp2_prefix.bin` | rkyv body starting with ASCII `CFP2` (dual-decode sniffer regression) | + +Regenerate: + +```bash +cargo run --example dump_rkyv_r9 --features backend-rust +# Also update Carbonado/RkyvFilepack.lean golden*Hex constants (must stay bit-identical). +# CI lock: tests/rkyv_golden_lock.rs asserts fixture bins == Lean hex constants. +``` + +**W3 acceptance:** Lean `encodeRkyvManifest` must bit-match these fixtures (encode twice → same bytes). +AOT demo greps: `rkyv FilepackManifestWire encode/decode goldens ok`. + +**Dual-suite honesty:** directory encode/decode under `backend-lean` still uses **Rust rkyv +composition** as product SSOT (segment/catalog *crypto* via Lean C ABI). Pure Lean path is +wire-compatible when claimed; dual-suite does **not** require pure Lean encode. diff --git a/tests/fixtures/rkyv/empty_manifest.bin b/tests/fixtures/rkyv/empty_manifest.bin new file mode 100644 index 0000000000000000000000000000000000000000..b547c4be2bb4ea004ad5344e300694cc53d0f569 GIT binary patch literal 13 TcmZQ#U|`_;{r~@eAk6>(8fydz literal 0 HcmV?d00001 diff --git a/tests/fixtures/rkyv/multi_entry_ots.bin b/tests/fixtures/rkyv/multi_entry_ots.bin new file mode 100644 index 0000000000000000000000000000000000000000..41eb5473495a12a55d4cd83a316ccaa21d0a41a1 GIT binary patch literal 275 zcmWd>#19yNAO%D~fdh~R;RYa1($C4yOHVD*El4cM(9KKCP1P%@C~+YGfEpQ2fCwmX z0Ma1b0K}`$zGqAX8S?-CeyfG-xRlW_u&!2;C91jKyp IKn{oj0A=Sn^Z)<= literal 0 HcmV?d00001 diff --git a/tests/fixtures/rkyv/ots_first_only.bin b/tests/fixtures/rkyv/ots_first_only.bin new file mode 100644 index 0000000000000000000000000000000000000000..19e7f5e1c5bd430fd3c596ebe4cee6ddc4ed598a GIT binary patch literal 251 zcmWd>#19yNAO%D~fdh~R;RYbSx7LLK04icQ0V1Hl0Z4;z0}v#19yNAO%D~fdh~R;RYZ!G%_|ZH8ZzRA^`Yi|NsA=5vTzUn1C$4gHTZbA> literal 0 HcmV?d00001 diff --git a/tests/fixtures/rkyv/path_ool_9.bin b/tests/fixtures/rkyv/path_ool_9.bin new file mode 100644 index 0000000000000000000000000000000000000000..905272afcb2367106e83abae381187c6b669a622 GIT binary patch literal 140 zcmXpsGBzE3osJI0~mlH1w=rB1CR#c1|Uw#19yNAO%D~fdh~R;RYa1)GMhd`TzgF5&^(B8>om8r~wX`fGoa)P*DK-h!iIP literal 0 HcmV?d00001 diff --git a/tests/fixtures/rkyv/two_segments.bin b/tests/fixtures/rkyv/two_segments.bin new file mode 100644 index 0000000000000000000000000000000000000000..b03210083287440ee8afca2cb89c0e168ae7368c GIT binary patch literal 191 zcmWd>#19yNAO%D~fdh~R;RYZUA^;eHYK(vw1P%bP0}wL+1si}=qFzZw$^ZZVl?VX7 Uc%ULCkVY5)arh1bMM1^`09le3-~a#s literal 0 HcmV?d00001 diff --git a/tests/g9_cross_backend.rs b/tests/g9_cross_backend.rs new file mode 100644 index 0000000..bca0c08 --- /dev/null +++ b/tests/g9_cross_backend.rs @@ -0,0 +1,807 @@ +//! Milestone R8 / G9: full cross-backend encode/decode matrix (no-compress formats). +//! +//! Both directions: +//! - **rust→lean** (`#[cfg(feature = "backend-lean")]`): decode committed `tests/fixtures/g9/rust/*` +//! - **lean→rust** (`#[cfg(feature = "backend-rust")]`): decode committed `tests/fixtures/g9/lean/*` +//! +//! Fixtures are no-compress formats only (c0/c1/c4/c5/c8/c9/c12/c13 + selected headered/outboard) +//! so re-encode bit-match is meaningful. Compression (Zstd) is an intentional residual. +//! +//! Regenerate (writes under `tests/fixtures/g9/{rust,lean}/` for the active backend): +//! ```bash +//! # Rust goldens +//! G9_WRITE_FIXTURES=1 cargo test --test g9_cross_backend write_fixtures -- --ignored --nocapture +//! # Lean goldens (needs libcarbonado) +//! eval "$(just _lean-env)" +//! G9_WRITE_FIXTURES=1 cargo test --no-default-features --features "backend-lean,pqc,ots" \ +//! --test g9_cross_backend write_fixtures -- --ignored --nocapture +//! # or: just g9-gen-fixtures +//! ``` +//! +//! Pins: [`MASTER`], [`NONCE`], [`PLAINTEXT`] — same MASTER/NONCE as `lean_backend_phase2`. + +use std::fs; +use std::path::{Path, PathBuf}; + +use carbonado::{ + constants::Format, decode, decode_outboard, encode_with_nonce, file, + stream_encode_outboard_buffer, structs::Encoded, OutboardEncoded, +}; +use serde::{Deserialize, Serialize}; + +/// Same master as Phase 2 G9 seeds (continuity). +const MASTER: [u8; 32] = [ + 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, +]; + +/// Fixed 16-byte nonce for encrypted goldens (Phase 2 pattern `01..10`). +const NONCE: [u8; 16] = [ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, +]; + +/// Single plaintext for all matrix entries (plaintext_id = `g9_matrix_v1`). +const PLAINTEXT: &[u8] = b"g9 cross-backend matrix v1"; +const PLAINTEXT_ID: &str = "g9_matrix_v1"; + +/// No-compress body formats: public + encrypted fixed-nonce. +const BODY_FORMATS: &[u8] = &[0, 1, 4, 5, 8, 9, 12, 13]; +/// Headered subset. +const HEADERED_FORMATS: &[u8] = &[4, 5, 12, 13]; +/// Outboard subset (public + encrypted header-path). +const OUTBOARD_FORMATS: &[u8] = &[4, 5, 12, 13, 14]; + +fn fixtures_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/g9") +} + +fn engine_dir(engine: &str) -> PathBuf { + fixtures_root().join(engine) +} + +fn to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn from_hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("hex")) + .collect() +} + +fn require_write_env() { + match std::env::var("G9_WRITE_FIXTURES") { + Ok(v) if v == "1" || v.eq_ignore_ascii_case("true") || v.eq_ignore_ascii_case("yes") => {} + _ => panic!("set G9_WRITE_FIXTURES=1 to regenerate fixtures"), + } +} + +#[cfg(feature = "backend-lean")] +fn require_lean_lib() { + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + panic!( + "CARBONADO_LEAN_LIB unset. Build and export first:\n \ + nix build .#libcarbonado -o result-libcarbonado\n \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ + export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ + export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ + # or: just test-lean-ci / just g9-gen-fixtures" + ); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct BodyMeta { + layout: String, + engine: String, + format: u8, + hash_hex: String, + padding_len: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + nonce_hex: Option, + plaintext_id: String, + encrypted: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct HeaderedMeta { + layout: String, + engine: String, + format: u8, + hash_hex: String, + padding_len: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + nonce_hex: Option, + plaintext_id: String, + encrypted: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct OutboardMeta { + layout: String, + engine: String, + format: u8, + hash_hex: String, + padding_len: u32, + /// Encrypted outboard fixtures use header-path layout (`[tag|ct]` + out-of-band header). + header_path: bool, + has_verification_outboard: bool, + has_fec_parity: bool, + has_header: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + nonce_hex: Option, + plaintext_id: String, + encrypted: bool, +} + +fn write_bytes(path: &Path, data: &[u8]) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(path, data).unwrap_or_else(|e| panic!("write {}: {e}", path.display())); +} + +fn write_json(path: &Path, value: &T) { + let s = serde_json::to_string_pretty(value).expect("json"); + write_bytes(path, format!("{s}\n").as_bytes()); +} + +fn active_engine() -> &'static str { + if cfg!(feature = "backend-lean") { + "lean" + } else { + "rust" + } +} + +fn is_encrypted(format: u8) -> bool { + Format::from(format).contains(Format::Encryption) +} + +/// Encode body under the active backend with fixed nonce when encrypted. +fn encode_body(format: u8) -> (Vec, [u8; 32], u32) { + let nonce = if is_encrypted(format) { + Some(NONCE) + } else { + None + }; + let carbonado::structs::Encoded(body, hash, info) = + encode_with_nonce(&MASTER, PLAINTEXT, format, nonce) + .unwrap_or_else(|e| panic!("encode body c{format}: {e}")); + (body, *hash.as_bytes(), info.padding_len) +} + +/// Encode headered under the active backend with fixed nonce when encrypted. +fn encode_headered(format: u8) -> (Vec, [u8; 32], u32, Option<[u8; 16]>) { + let nonce = if is_encrypted(format) { + Some(NONCE) + } else { + None + }; + let (archive, info) = file::encode_with_nonce(&MASTER, PLAINTEXT, format, None, nonce) + .unwrap_or_else(|e| panic!("encode headered c{format}: {e}")); + let (hdr, _) = file::decode(&MASTER, &archive).expect("self-decode headered for hash"); + let hash = *hdr.hash.as_bytes(); + let nonce_out = if is_encrypted(format) { + Some(hdr.payload_nonce) + } else { + None + }; + (archive, hash, info.padding_len, nonce_out) +} + +/// Encode outboard: public via low-level; encrypted via header-path + Header (fixed nonce). +fn encode_outboard_fixture(format: u8) -> (OutboardEncoded, Option>, bool) { + let encrypted = is_encrypted(format); + if encrypted { + // Header-path: stream_encode_outboard_buffer Some(nonce) + Header for file::decode_outboard. + let oenc = stream_encode_outboard_buffer(&MASTER, PLAINTEXT, format, Some(NONCE)) + .unwrap_or_else(|e| panic!("outboard header_path c{format}: {e}")); + let hdr = file::Header::new( + &MASTER, + NONCE, + oenc.hash.as_bytes(), + [0u8; 32], + Format::from(format), + 0, + oenc.info.bytes_verifiable, + oenc.info.padding_len, + None, + ) + .expect("header for outboard"); + let hdr_bytes = hdr.try_to_vec().expect("hdr vec"); + (oenc, Some(hdr_bytes), true) + } else { + let oenc = carbonado::encode_outboard(&MASTER, PLAINTEXT, format) + .unwrap_or_else(|e| panic!("outboard public c{format}: {e}")); + (oenc, None, false) + } +} + +// --------------------------------------------------------------------------- +// Fixture writer (ignored unless G9_WRITE_FIXTURES=1) +// --------------------------------------------------------------------------- + +#[test] +#[ignore = "set G9_WRITE_FIXTURES=1 to regenerate tests/fixtures/g9/{engine}/"] +fn write_fixtures() { + require_write_env(); + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + let engine = active_engine(); + let root = engine_dir(engine); + fs::create_dir_all(&root).expect("mkdir engine"); + + for &format in BODY_FORMATS { + let (body, hash, padding) = encode_body(format); + let name = format!("body_c{format}"); + write_bytes(&root.join(format!("{name}.bin")), &body); + let meta = BodyMeta { + layout: "body".into(), + engine: engine.into(), + format, + hash_hex: to_hex(&hash), + padding_len: padding, + nonce_hex: if is_encrypted(format) { + Some(to_hex(&NONCE)) + } else { + None + }, + plaintext_id: PLAINTEXT_ID.into(), + encrypted: is_encrypted(format), + }; + write_json(&root.join(format!("{name}.meta.json")), &meta); + eprintln!("wrote {engine}/{name} ({} bytes)", body.len()); + } + + for &format in HEADERED_FORMATS { + let (archive, hash, padding, nonce_out) = encode_headered(format); + let name = format!("headered_c{format}"); + write_bytes(&root.join(format!("{name}.bin")), &archive); + let meta = HeaderedMeta { + layout: "headered".into(), + engine: engine.into(), + format, + hash_hex: to_hex(&hash), + padding_len: padding, + nonce_hex: nonce_out.map(|n| to_hex(&n)), + plaintext_id: PLAINTEXT_ID.into(), + encrypted: is_encrypted(format), + }; + write_json(&root.join(format!("{name}.meta.json")), &meta); + eprintln!("wrote {engine}/{name} ({} bytes)", archive.len()); + } + + for &format in OUTBOARD_FORMATS { + let (oenc, header, header_path) = encode_outboard_fixture(format); + let dir = root.join(format!("outboard_c{format}")); + fs::create_dir_all(&dir).expect("mkdir outboard"); + write_bytes(&dir.join("main.bin"), &oenc.main); + let has_ob = oenc.verification_outboard.is_some(); + let has_par = oenc.fec_parity.is_some(); + if let Some(ref ob) = oenc.verification_outboard { + write_bytes(&dir.join("out.bin"), ob); + } + if let Some(ref par) = oenc.fec_parity { + write_bytes(&dir.join("par.bin"), par); + } + let has_header = if let Some(ref h) = header { + write_bytes(&dir.join("header.bin"), h); + true + } else { + false + }; + let meta = OutboardMeta { + layout: "outboard".into(), + engine: engine.into(), + format, + hash_hex: to_hex(oenc.hash.as_bytes()), + padding_len: oenc.info.padding_len, + header_path, + has_verification_outboard: has_ob, + has_fec_parity: has_par, + has_header, + nonce_hex: if is_encrypted(format) { + Some(to_hex(&NONCE)) + } else { + None + }, + plaintext_id: PLAINTEXT_ID.into(), + encrypted: is_encrypted(format), + }; + write_json(&dir.join("meta.json"), &meta); + eprintln!( + "wrote {engine}/outboard_c{format} (main {} bytes)", + oenc.main.len() + ); + } + + eprintln!("G9 fixtures written under {}", root.display()); +} + +// --------------------------------------------------------------------------- +// Load helpers +// --------------------------------------------------------------------------- + +/// Assert encrypted fixture meta carries the pin NONCE (and optional wire check). +fn assert_encrypted_nonce_meta(nonce_hex: &Option, label: &str) { + let hex = nonce_hex + .as_deref() + .unwrap_or_else(|| panic!("{label}: encrypted fixture missing nonce_hex")); + assert_eq!( + hex, + to_hex(&NONCE), + "{label}: nonce_hex must match pin NONCE" + ); +} + +fn load_body(engine: &str, format: u8) -> (Vec, BodyMeta) { + let root = engine_dir(engine); + let name = format!("body_c{format}"); + let body = fs::read(root.join(format!("{name}.bin"))) + .unwrap_or_else(|e| panic!("missing {engine}/{name}.bin: {e}")); + let meta: BodyMeta = serde_json::from_slice( + &fs::read(root.join(format!("{name}.meta.json"))) + .unwrap_or_else(|e| panic!("missing {engine}/{name}.meta.json: {e}")), + ) + .expect("body meta json"); + assert_eq!(meta.format, format); + assert_eq!(meta.plaintext_id, PLAINTEXT_ID); + assert_eq!( + meta.encrypted, + is_encrypted(format), + "{engine}/{name} encrypted flag" + ); + if meta.encrypted { + assert_encrypted_nonce_meta(&meta.nonce_hex, &format!("{engine}/{name}")); + // Pure Encryption (c1): body is embedded `[nonce|tag|ct]` with no Bao/FEC wrap. + // c5/c9/c13 wrap ciphertext in Bao and/or FEC — nonce is not at offset 0. + if format == 1 { + assert!( + body.len() >= 16, + "{engine}/{name}: body too short for embedded nonce" + ); + assert_eq!( + &body[..16], + &NONCE[..], + "{engine}/{name}: c1 embedded body nonce must match pin" + ); + } + } else { + assert!( + meta.nonce_hex.is_none(), + "{engine}/{name}: public body has no nonce_hex" + ); + } + (body, meta) +} + +fn load_headered(engine: &str, format: u8) -> (Vec, HeaderedMeta) { + let root = engine_dir(engine); + let name = format!("headered_c{format}"); + let archive = fs::read(root.join(format!("{name}.bin"))) + .unwrap_or_else(|e| panic!("missing {engine}/{name}.bin: {e}")); + let meta: HeaderedMeta = serde_json::from_slice( + &fs::read(root.join(format!("{name}.meta.json"))) + .unwrap_or_else(|e| panic!("missing {engine}/{name}.meta.json: {e}")), + ) + .expect("headered meta json"); + assert_eq!(meta.format, format); + assert_eq!(meta.plaintext_id, PLAINTEXT_ID); + assert_eq!( + meta.encrypted, + is_encrypted(format), + "{engine}/{name} encrypted flag" + ); + assert!( + archive.len() >= file::Header::LEN, + "{engine}/{name}: archive shorter than header" + ); + // Header wire: payload_nonce at bytes [12..28] (after 12-byte MAGIC). + let wire_nonce = &archive[12..28]; + if meta.encrypted { + assert_encrypted_nonce_meta(&meta.nonce_hex, &format!("{engine}/{name}")); + assert_eq!( + wire_nonce, + &NONCE[..], + "{engine}/{name}: header payload_nonce must match pin" + ); + } else { + assert!( + meta.nonce_hex.is_none(), + "{engine}/{name}: public headered has no nonce_hex" + ); + assert_eq!( + wire_nonce, + &[0u8; 16][..], + "{engine}/{name}: public payload_nonce is zero" + ); + } + (archive, meta) +} + +struct LoadedOutboard { + main: Vec, + verification_outboard: Option>, + fec_parity: Option>, + header: Option>, + meta: OutboardMeta, +} + +fn load_outboard(engine: &str, format: u8) -> LoadedOutboard { + let dir = engine_dir(engine).join(format!("outboard_c{format}")); + let meta: OutboardMeta = serde_json::from_slice( + &fs::read(dir.join("meta.json")) + .unwrap_or_else(|e| panic!("missing {}/meta.json: {e}", dir.display())), + ) + .expect("outboard meta"); + assert_eq!(meta.format, format); + assert_eq!(meta.plaintext_id, PLAINTEXT_ID); + assert_eq!( + meta.encrypted, + is_encrypted(format), + "{engine}/outboard_c{format} encrypted" + ); + let main = fs::read(dir.join("main.bin")).expect("main.bin"); + let verification_outboard = if meta.has_verification_outboard { + Some(fs::read(dir.join("out.bin")).expect("out.bin")) + } else { + None + }; + let fec_parity = if meta.has_fec_parity { + Some(fs::read(dir.join("par.bin")).expect("par.bin")) + } else { + None + }; + let header = if meta.has_header { + Some(fs::read(dir.join("header.bin")).expect("header.bin")) + } else { + None + }; + if meta.encrypted { + assert_encrypted_nonce_meta(&meta.nonce_hex, &format!("{engine}/outboard_c{format}")); + assert!( + meta.header_path, + "{engine}/outboard_c{format}: encrypted is header_path" + ); + assert!( + meta.has_header, + "{engine}/outboard_c{format}: encrypted needs header.bin" + ); + let hdr = header.as_ref().expect("header"); + assert!( + hdr.len() >= file::Header::LEN, + "{engine}/outboard_c{format}: header.bin short" + ); + assert_eq!( + &hdr[12..28], + &NONCE[..], + "{engine}/outboard_c{format}: header payload_nonce must match pin" + ); + } else { + assert!( + meta.nonce_hex.is_none(), + "{engine}/outboard_c{format}: public has no nonce_hex" + ); + } + LoadedOutboard { + main, + verification_outboard, + fec_parity, + header, + meta, + } +} + +fn hash_from_meta(hex: &str) -> [u8; 32] { + let v = from_hex(hex); + assert_eq!(v.len(), 32, "hash must be 32 bytes"); + let mut h = [0u8; 32]; + h.copy_from_slice(&v); + h +} + +fn decode_body_fixture(engine: &str, format: u8) { + let (body, meta) = load_body(engine, format); + let hash = hash_from_meta(&meta.hash_hex); + let decoded = decode(&MASTER, &hash, &body, meta.padding_len, format) + .unwrap_or_else(|e| panic!("{engine}→active body c{format}: {e}")); + assert_eq!(decoded, PLAINTEXT, "{engine} body c{format} plaintext"); +} + +fn decode_headered_fixture(engine: &str, format: u8) { + let (archive, meta) = load_headered(engine, format); + let (hdr, decoded) = file::decode(&MASTER, &archive) + .unwrap_or_else(|e| panic!("{engine}→active headered c{format}: {e}")); + assert_eq!(hdr.format.bits(), format); + assert_eq!(decoded, PLAINTEXT, "{engine} headered c{format}"); + assert_eq!( + to_hex(hdr.hash.as_bytes()), + meta.hash_hex, + "{engine} headered c{format} hash" + ); +} + +fn decode_outboard_fixture(engine: &str, format: u8) { + let loaded = load_outboard(engine, format); + let hash = hash_from_meta(&loaded.meta.hash_hex); + let decoded = if loaded.meta.encrypted { + // Header-path encrypted outboard → high-level file::decode_outboard + file::decode_outboard( + &MASTER, + &hash, + loaded.header.as_deref(), + &loaded.main, + loaded.verification_outboard.as_deref(), + loaded.fec_parity.as_deref(), + loaded.meta.padding_len, + format, + ) + .unwrap_or_else(|e| panic!("{engine}→active outboard c{format}: {e}")) + } else { + decode_outboard( + &MASTER, + &hash, + &loaded.main, + loaded.verification_outboard.as_deref(), + loaded.fec_parity.as_deref(), + loaded.meta.padding_len, + format, + ) + .unwrap_or_else(|e| panic!("{engine}→active outboard c{format}: {e}")) + }; + assert_eq!(decoded, PLAINTEXT, "{engine} outboard c{format}"); +} + +// --------------------------------------------------------------------------- +// lean→rust: decode lean fixtures under default backend-rust +// --------------------------------------------------------------------------- + +#[cfg(feature = "backend-rust")] +mod lean_to_rust { + use super::*; + + #[test] + fn body_matrix() { + for &format in BODY_FORMATS { + decode_body_fixture("lean", format); + } + } + + #[test] + fn headered_matrix() { + for &format in HEADERED_FORMATS { + decode_headered_fixture("lean", format); + } + } + + #[test] + fn outboard_matrix() { + for &format in OUTBOARD_FORMATS { + decode_outboard_fixture("lean", format); + } + } +} + +// --------------------------------------------------------------------------- +// rust→lean: decode rust fixtures under backend-lean (+ optional re-encode bit-match) +// --------------------------------------------------------------------------- + +#[cfg(feature = "backend-lean")] +mod rust_to_lean { + use super::*; + use carbonado::structs::Encoded; + + #[test] + fn body_matrix() { + require_lean_lib(); + for &format in BODY_FORMATS { + decode_body_fixture("rust", format); + } + } + + #[test] + fn headered_matrix() { + require_lean_lib(); + for &format in HEADERED_FORMATS { + decode_headered_fixture("rust", format); + } + } + + #[test] + fn outboard_matrix() { + require_lean_lib(); + for &format in OUTBOARD_FORMATS { + decode_outboard_fixture("rust", format); + } + } + + /// Public no-compress body re-encode under lean must bit-match rust golden. + #[test] + fn public_body_reencode_bit_match() { + require_lean_lib(); + for &format in &[0u8, 4, 8, 12] { + let (rust_body, meta) = load_body("rust", format); + let Encoded(lean_body, lean_hash, _) = + encode_with_nonce(&MASTER, PLAINTEXT, format, None) + .unwrap_or_else(|e| panic!("lean re-encode c{format}: {e}")); + assert_eq!( + lean_body, rust_body, + "lean re-encode must bit-match rust body c{format}" + ); + assert_eq!( + to_hex(lean_hash.as_bytes()), + meta.hash_hex, + "lean re-encode hash c{format}" + ); + } + } + + /// Encrypted fixed-nonce body re-encode under lean must bit-match rust golden. + #[test] + fn encrypted_body_reencode_bit_match() { + require_lean_lib(); + for &format in &[1u8, 5, 9, 13] { + let (rust_body, meta) = load_body("rust", format); + let Encoded(lean_body, lean_hash, _) = + encode_with_nonce(&MASTER, PLAINTEXT, format, Some(NONCE)) + .unwrap_or_else(|e| panic!("lean re-encode enc c{format}: {e}")); + assert_eq!( + lean_body, rust_body, + "lean re-encode must bit-match rust encrypted body c{format}" + ); + assert_eq!( + to_hex(lean_hash.as_bytes()), + meta.hash_hex, + "lean re-encode enc hash c{format}" + ); + } + } + + /// Headered re-encode under lean must bit-match rust golden (no-compress formats). + #[test] + fn headered_reencode_bit_match() { + require_lean_lib(); + for &format in HEADERED_FORMATS { + let (rust_arch, meta) = load_headered("rust", format); + let nonce = if is_encrypted(format) { + Some(NONCE) + } else { + None + }; + let (lean_arch, _) = file::encode_with_nonce(&MASTER, PLAINTEXT, format, None, nonce) + .unwrap_or_else(|e| panic!("lean re-encode headered c{format}: {e}")); + assert_eq!( + lean_arch, rust_arch, + "lean re-encode must bit-match rust headered c{format}" + ); + let (hdr, _) = file::decode(&MASTER, &lean_arch).expect("decode lean headered"); + assert_eq!(to_hex(hdr.hash.as_bytes()), meta.hash_hex); + } + } + + /// Outboard re-encode under lean must bit-match rust golden (skip compressed c14). + #[test] + fn outboard_reencode_bit_match_no_compress() { + require_lean_lib(); + for &format in &[4u8, 5, 12, 13] { + let loaded = load_outboard("rust", format); + let (oenc, header, _) = encode_outboard_fixture(format); + assert_eq!( + oenc.main, loaded.main, + "lean re-encode main must bit-match rust outboard c{format}" + ); + assert_eq!( + oenc.verification_outboard.as_deref(), + loaded.verification_outboard.as_deref(), + "outboard c{format} verification outboard" + ); + assert_eq!( + oenc.fec_parity.as_deref(), + loaded.fec_parity.as_deref(), + "outboard c{format} fec parity" + ); + if is_encrypted(format) { + assert_eq!( + header.as_deref(), + loaded.header.as_deref(), + "outboard c{format} header.bin" + ); + } + assert_eq!( + to_hex(oenc.hash.as_bytes()), + loaded.meta.hash_hex, + "outboard c{format} hash" + ); + } + } +} + +/// `Some([0u8; 16])` must be honored literally on the active backend (no CSPRNG override). +/// +/// Uses c1 (encryption-only body, embedded layout) and c5 headered so the zero nonce is +/// visible on the wire. Dual-backend identity is the same contract on both engines. +#[test] +fn explicit_zero_nonce_is_honored_headered_and_body() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + let zero = [0u8; 16]; + let pt = b"g9 zero nonce dual contract"; + + // Body c1 (embedded only): two encodes with Some(zero) must match (deterministic). + let Encoded(b1, h1, i1) = + encode_with_nonce(&MASTER, pt, 1, Some(zero)).expect("body zero nonce 1"); + let Encoded(b2, h2, i2) = + encode_with_nonce(&MASTER, pt, 1, Some(zero)).expect("body zero nonce 2"); + assert_eq!(b1, b2, "Some([0;16]) body must be deterministic"); + assert_eq!(h1, h2); + assert_eq!(i1.padding_len, i2.padding_len); + assert_eq!( + &b1[..16], + &zero[..], + "c1 embedded layout starts with zero nonce" + ); + let d = decode(&MASTER, h1.as_bytes(), &b1, i1.padding_len, 1).expect("decode body zero"); + assert_eq!(d, pt); + + // Headered c5: payload_nonce in header must be all-zero, and encode is deterministic. + let (a1, _) = file::encode_with_nonce(&MASTER, pt, 5, None, Some(zero)).expect("hdr 1"); + let (a2, _) = file::encode_with_nonce(&MASTER, pt, 5, None, Some(zero)).expect("hdr 2"); + assert_eq!(a1, a2, "Some([0;16]) headered must be deterministic"); + assert_eq!(&a1[12..28], &zero[..], "header payload_nonce is zero"); + let (hdr, d2) = file::decode(&MASTER, &a1).expect("decode headered zero"); + assert_eq!(hdr.payload_nonce, zero); + assert_eq!(d2, pt); +} + +// --------------------------------------------------------------------------- +// Smoke: active backend self roundtrip for matrix formats (always runs) +// --------------------------------------------------------------------------- + +#[test] +fn active_backend_self_roundtrip_matrix() { + #[cfg(feature = "backend-lean")] + require_lean_lib(); + + for &format in BODY_FORMATS { + let (body, hash, pad) = encode_body(format); + let d = decode(&MASTER, &hash, &body, pad, format).expect("body decode"); + assert_eq!(d, PLAINTEXT, "self body c{format}"); + } + for &format in HEADERED_FORMATS { + let (arch, _, _, _) = encode_headered(format); + let (_, d) = file::decode(&MASTER, &arch).expect("headered decode"); + assert_eq!(d, PLAINTEXT, "self headered c{format}"); + } + for &format in OUTBOARD_FORMATS { + let (oenc, header, _) = encode_outboard_fixture(format); + let d = if is_encrypted(format) { + file::decode_outboard( + &MASTER, + oenc.hash.as_bytes(), + header.as_deref(), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + format, + ) + .expect("outboard enc decode") + } else { + decode_outboard( + &MASTER, + oenc.hash.as_bytes(), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + format, + ) + .expect("outboard pub decode") + }; + assert_eq!(d, PLAINTEXT, "self outboard c{format}"); + } +} diff --git a/tests/lean_backend_phase2.rs b/tests/lean_backend_phase2.rs new file mode 100644 index 0000000..f7a6a22 --- /dev/null +++ b/tests/lean_backend_phase2.rs @@ -0,0 +1,512 @@ +//! Phase 2 allowlist for `backend-lean` (docs/TEST_CONTRACT.md). +//! +//! Covers outboard roundtrip, scrub happy/error paths, verify_slice, stream buffer +//! composition, and G9 cross-backend body/headered buffers (Rust encode → Lean decode +//! requires both engines available — under pure `backend-lean` we check Lean self +//! parity and document G9 via lean-encode / lean-decode identity against fixed vectors). +//! +//! ```bash +//! nix build .#libcarbonado -o result-libcarbonado +//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB +//! cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_phase2 +//! # or: just test-lean-phase2 +//! ``` +//! Only compiled under `backend-lean` (avoids breaking default/`backend-rust` clippy of all targets). + +#![cfg(feature = "backend-lean")] + +use carbonado::{ + decode, decode_outboard, encode, encode_outboard, error::CarbonadoError, extract_slice, file, + scrub, scrub_outboard, stream_decode_buffer, stream_encode_buffer, structs::Encoded, + verify_slice, OutboardEncoded, +}; + +const MASTER: [u8; 32] = [ + 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, +]; + +const NONCE: [u8; 16] = [ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, +]; + +fn require_lean_lib() { + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + panic!( + "CARBONADO_LEAN_LIB unset. Build and export first:\n \ + nix build .#libcarbonado -o result-libcarbonado\n \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ + export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ + export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ + # or: just test-lean-phase2" + ); + } +} + +#[test] +fn abi_version_is_one() { + require_lean_lib(); + assert_eq!(carbonado::backend::lean::abi_version(), 1); +} + +#[test] +fn outboard_roundtrip_public_c4_c12_c14() { + require_lean_lib(); + let plaintext = b"phase2 outboard public smoke"; + for format in [4u8, 12, 14] { + let oenc = encode_outboard(&MASTER, plaintext, format) + .unwrap_or_else(|e| panic!("encode_outboard c{format}: {e}")); + assert_eq!(oenc.info.input_len, plaintext.len() as u32); + if format & 0x4 != 0 { + assert!( + oenc.verification_outboard.is_some(), + "c{format}: Verification formats always yield Some(outboard) (may be empty single-leaf)" + ); + } + if format & 0x8 != 0 { + assert!( + oenc.fec_parity.is_some(), + "c{format}: Fec formats always yield Some(parity)" + ); + assert_eq!( + oenc.info.bytes_ecc, + oenc.fec_parity + .as_ref() + .map(|p| p.len() as u32) + .unwrap_or(0), + "c{format}: bytes_ecc must equal parity sidecar length" + ); + // plaintext is non-empty smoke payload → FEC parity sidecar must be non-empty. + assert!( + oenc.fec_parity.as_ref().is_some_and(|p| !p.is_empty()), + "c{format}: expected non-empty FEC parity for non-empty plaintext" + ); + } + let decoded = decode_outboard( + &MASTER, + oenc.hash.as_bytes(), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + format, + ) + .unwrap_or_else(|e| panic!("decode_outboard c{format}: {e}")); + assert_eq!(decoded, plaintext, "c{format} outboard plaintext mismatch"); + } +} + +#[test] +fn outboard_encrypted_fixed_nonce_roundtrip_c5() { + require_lean_lib(); + let plaintext = b"encrypted outboard fixed nonce"; + // c5 = Encryption | Verification — low-level embedded-nonce layout + let oenc = + carbonado::backend::lean::encode_outboard(&MASTER, plaintext, 5, Some(&NONCE), false) + .expect("lean encode_outboard c5 embedded"); + // Embedded layout embeds 16-byte nonce in main. + assert!( + oenc.main.len() >= 16 + 64, + "embedded main must hold nonce+tag" + ); + let decoded = carbonado::backend::lean::decode_outboard( + &MASTER, + oenc.hash.as_bytes(), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + 5, + None, + false, + ) + .expect("lean decode_outboard c5 embedded"); + assert_eq!(decoded, plaintext); +} + +#[test] +fn outboard_header_path_encrypted_matches_file_layout() { + require_lean_lib(); + let plaintext = b"header-path outboard encrypted"; + // c5 with header_path=true → bare main is [tag|ct] (matches file::encode_outboard). + let oenc = carbonado::backend::lean::encode_outboard(&MASTER, plaintext, 5, Some(&NONCE), true) + .expect("lean encode_outboard c5 header_path"); + // Header-path main starts with tag (64 B), not a random-looking nonce prefix alone. + assert!(oenc.main.len() >= 64, "header-path main has at least tag"); + // Embedded would be 16 longer for same pt (nonce prefix); header_path is shorter by 16. + let oenc_emb = + carbonado::backend::lean::encode_outboard(&MASTER, plaintext, 5, Some(&NONCE), false) + .expect("embedded"); + assert_eq!( + oenc_emb.main.len(), + oenc.main.len() + 16, + "header-path main omits 16-byte embedded nonce" + ); + let decoded = carbonado::backend::lean::decode_outboard( + &MASTER, + oenc.hash.as_bytes(), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + 5, + Some(&NONCE), + true, + ) + .expect("lean decode_outboard c5 header_path"); + assert_eq!(decoded, plaintext); + + // High-level file::encode_outboard under lean uses header_path (Some(payload_nonce)). + let (hdr, fo) = + file::encode_outboard(&MASTER, plaintext, 5, None).expect("file encode_outboard"); + let hdr = hdr.expect("encrypted outboard returns Header"); + let hdr_bytes = hdr.try_to_vec().expect("hdr vec"); + let d2 = file::decode_outboard( + &MASTER, + fo.hash.as_bytes(), + Some(&hdr_bytes), + &fo.main, + fo.verification_outboard.as_deref(), + fo.fec_parity.as_deref(), + fo.info.padding_len, + 5, + ) + .expect("file decode_outboard"); + assert_eq!(d2, plaintext); +} + +#[test] +fn inboard_scrub_pristine_unnecessary() { + require_lean_lib(); + let plaintext = b"scrub pristine"; + // c12 = Verification | Fec (public) + let Encoded(body, hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); + let err = scrub(&body, hash.as_bytes(), &info, 12).expect_err("pristine scrub"); + assert!( + matches!(err, CarbonadoError::UnnecessaryScrub), + "expected UnnecessaryScrub, got {err:?}" + ); +} + +#[test] +fn inboard_scrub_requires_verification() { + require_lean_lib(); + let plaintext = b"no verification bit"; + let Encoded(body, hash, info) = encode(&MASTER, plaintext, 0).expect("encode c0"); + let err = scrub(&body, hash.as_bytes(), &info, 0).expect_err("scrub c0"); + assert!( + matches!(err, CarbonadoError::ScrubRequiresVerification), + "expected ScrubRequiresVerification, got {err:?}" + ); +} + +#[test] +fn inboard_scrub_recovers_bitflip_in_fec_body() { + require_lean_lib(); + let plaintext = b"scrub recover bitflip phase2"; + let Encoded(mut body, hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); + // Flip a byte deep in the inboard (past 8-byte length prefix) to taint ≤1 shard. + if body.len() > 64 { + let i = body.len() / 2; + body[i] ^= 0xff; + } else if !body.is_empty() { + let i = body.len() - 1; + body[i] ^= 0x01; + } + let recovered = + scrub(&body, hash.as_bytes(), &info, 12).unwrap_or_else(|e| panic!("scrub recover: {e}")); + assert_eq!( + recovered.len(), + encode(&MASTER, plaintext, 12).unwrap().0.len() + ); + // Decode recovered body + let decoded = decode(&MASTER, hash.as_bytes(), &recovered, info.padding_len, 12) + .expect("decode recovered"); + assert_eq!(decoded, plaintext); +} + +#[test] +fn outboard_scrub_pristine_unnecessary() { + require_lean_lib(); + let plaintext = b"outboard scrub pristine"; + let oenc = encode_outboard(&MASTER, plaintext, 12).expect("encode_outboard c12"); + let err = scrub_outboard( + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + &oenc.info, + 12, + oenc.hash.as_bytes(), + ) + .expect_err("pristine outboard scrub"); + assert!( + matches!(err, CarbonadoError::UnnecessaryScrub), + "expected UnnecessaryScrub, got {err:?}" + ); +} + +#[test] +fn outboard_scrub_recovers_main_damage() { + require_lean_lib(); + let plaintext = b"outboard scrub damage recover"; + let oenc = encode_outboard(&MASTER, plaintext, 12).expect("encode_outboard c12"); + let mut damaged = oenc.main.clone(); + if damaged.len() > 8 { + damaged[0] ^= 0xff; + damaged[1] ^= 0xaa; + } + let recovered = scrub_outboard( + &damaged, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + &oenc.info, + 12, + oenc.hash.as_bytes(), + ) + .unwrap_or_else(|e| panic!("scrub_outboard recover: {e}")); + let decoded = decode_outboard( + &MASTER, + oenc.hash.as_bytes(), + &recovered, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + 12, + ) + .expect("decode recovered bare"); + assert_eq!(decoded, plaintext); +} + +#[test] +fn verify_slice_c4_content() { + require_lean_lib(); + let plaintext = b"slice verify content match!!"; + let Encoded(body, hash, info) = encode(&MASTER, plaintext, 4).expect("encode c4"); + // For non-FEC, verifiable_slice_count is 0; use count=1 for first leaf. + let _ = info.verifiable_slice_count; + let got = verify_slice(&body, 0, 1, hash.as_bytes(), 4).expect("verify_slice"); + let n = got.len().min(plaintext.len()); + assert_eq!(&got[..n], &plaintext[..n]); + let extracted = extract_slice(&body, 0, hash.as_bytes(), 4).expect("extract_slice"); + assert_eq!(extracted, got); +} + +#[test] +fn stream_buffer_composes_over_lean_body() { + require_lean_lib(); + // Under backend-lean, stream_encode_buffer / stream_decode_buffer compose over Lean C ABI. + let plaintext = b"stream buffer compose"; + let (body, hash, info) = + stream_encode_buffer(&MASTER, plaintext, 4).expect("stream_encode_buffer"); + let decoded = stream_decode_buffer(&MASTER, hash.as_bytes(), &body, info.padding_len, 4) + .expect("stream_decode_buffer"); + assert_eq!(decoded, plaintext); + // Same engine as crate::encode + let Encoded(body2, hash2, _) = encode(&MASTER, plaintext, 4).expect("encode"); + assert_eq!(body, body2); + assert_eq!(hash, hash2); +} + +#[test] +fn g9_headered_public_roundtrip_matrix() { + require_lean_lib(); + // G9 start: lean encode → lean decode for headered public formats (same ABI). + // Full Rust↔Lean cross process needs both engines; buffer identity under lean is + // the Phase 2 G9 seed. Cross-process G9 continues as residual until CI freezes both. + let plaintext = b"g9 headered public"; + for level in [0u8, 4, 12, 14] { + let (archive, info) = file::encode(&MASTER, plaintext, level, None) + .unwrap_or_else(|e| panic!("headered encode c{level}: {e}")); + assert!(archive.len() >= file::Header::LEN); + assert_eq!(info.input_len, plaintext.len() as u32); + let (header, decoded) = file::decode(&MASTER, &archive) + .unwrap_or_else(|e| panic!("headered decode c{level}: {e}")); + assert_eq!(header.format.bits(), level); + assert_eq!(decoded, plaintext, "c{level}"); + } +} + +#[test] +fn g9_body_public_formats_deterministic() { + require_lean_lib(); + let plaintext = b"g9 body deterministic"; + for format in [0u8, 4, 12, 14] { + let Encoded(b1, h1, i1) = encode(&MASTER, plaintext, format).expect("e1"); + let Encoded(b2, h2, i2) = encode(&MASTER, plaintext, format).expect("e2"); + assert_eq!(b1, b2, "c{format} body deterministic"); + assert_eq!(h1, h2); + assert_eq!(i1.padding_len, i2.padding_len); + assert_eq!(i1.chunk_len, i2.chunk_len); + let d = decode(&MASTER, h1.as_bytes(), &b1, i1.padding_len, format).expect("decode"); + assert_eq!(d, plaintext); + } +} + +#[test] +fn g9_fixed_nonce_encrypted_body() { + require_lean_lib(); + // Encrypted body with fixed nonce via lean helper (public encode uses random). + let plaintext = b"g9 enc fixed nonce"; + let Encoded(body, hash, info) = + carbonado::backend::lean::encode(&MASTER, plaintext, 5, Some(&NONCE)).expect("enc c5"); + let decoded = + carbonado::backend::lean::decode(&MASTER, hash.as_bytes(), &body, info.padding_len, 5) + .expect("dec c5"); + assert_eq!(decoded, plaintext); + // Second encode with same nonce matches (deterministic). + let Encoded(body2, hash2, _) = + carbonado::backend::lean::encode(&MASTER, plaintext, 5, Some(&NONCE)).expect("enc2"); + assert_eq!(body, body2); + assert_eq!(hash, hash2); +} + +#[test] +fn encode_info_fec_fields_populated() { + require_lean_lib(); + let plaintext = b"encode info meta"; + let Encoded(body, _hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); + assert_eq!(info.bytes_verifiable as usize, body.len()); + assert!(info.chunk_len > 0, "chunk_len must be set for FEC"); + assert!(info.bytes_ecc > 0, "bytes_ecc must be set for FEC"); + assert!( + info.verifiable_slice_count > 0, + "verifiable_slice_count must be set for FEC+V" + ); +} + +#[test] +fn outboard_missing_verification_maps() { + require_lean_lib(); + let plaintext = b"missing ob"; + let OutboardEncoded { + main, + verification_outboard: _, + fec_parity, + hash, + info, + } = encode_outboard(&MASTER, plaintext, 12).expect("encode"); + let err = scrub_outboard( + &main, + None, + fec_parity.as_deref(), + &info, + 12, + hash.as_bytes(), + ) + .expect_err("missing outboard"); + assert!( + matches!(err, CarbonadoError::MissingVerificationOutboard), + "expected MissingVerificationOutboard, got {err:?}" + ); +} + +#[test] +fn inboard_scrub_invalid_scrubbed_hash_excess_damage() { + require_lean_lib(); + let plaintext = b"scrub fail excess damage"; + let Encoded(mut body, hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); + // Zero most of the body past the length prefix so >4 shards are wiped. + if body.len() > 8 { + for b in &mut body[8..] { + *b = 0; + } + } + let err = scrub(&body, hash.as_bytes(), &info, 12).expect_err("irrecoverable"); + assert!( + matches!(err, CarbonadoError::InvalidScrubbedHash), + "expected InvalidScrubbedHash, got {err:?}" + ); +} + +#[test] +fn outboard_scrub_requires_verification() { + require_lean_lib(); + // c0 has no Verification bit + let plaintext = b"outboard scrub no V"; + let oenc = encode_outboard(&MASTER, plaintext, 0).expect("encode c0"); + let err = scrub_outboard( + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + &oenc.info, + 0, + oenc.hash.as_bytes(), + ) + .expect_err("scrub without V"); + assert!( + matches!(err, CarbonadoError::ScrubRequiresVerification), + "expected ScrubRequiresVerification, got {err:?}" + ); +} + +#[test] +fn outboard_scrub_missing_fec_parity() { + require_lean_lib(); + let plaintext = b"outboard scrub missing parity"; + let oenc = encode_outboard(&MASTER, plaintext, 12).expect("encode c12"); + let mut damaged = oenc.main.clone(); + if !damaged.is_empty() { + damaged[0] ^= 0xff; + } + let err = scrub_outboard( + &damaged, + oenc.verification_outboard.as_deref(), + None, // missing parity after verify fail + &oenc.info, + 12, + oenc.hash.as_bytes(), + ) + .expect_err("missing parity"); + assert!( + matches!(err, CarbonadoError::MissingFecParity), + "expected MissingFecParity, got {err:?}" + ); +} + +fn from_hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("hex")) + .collect() +} + +/// G9: Rust-engine goldens (generated under default `backend-rust`) decoded by Lean AOT. +/// +/// Vectors from `encode` / `file::encode` with MASTER + public formats (deterministic). +#[test] +fn g9_rust_encode_lean_decode_body_c0_c4() { + require_lean_lib(); + // c0: body == plaintext (no verification) + let c0_body = from_hex("67392063726f73732d6261636b656e6420626f6479206330"); + let c0_hash = [0u8; 32]; + let pt0 = b"g9 cross-backend body c0"; + assert_eq!(c0_body, pt0); + let d0 = decode(&MASTER, &c0_hash, &c0_body, 0, 0).expect("lean decode rust c0"); + assert_eq!(d0, pt0); + + // c4: bao inboard over plaintext + let c4_body = from_hex("180000000000000067392063726f73732d6261636b656e6420626f6479206334"); + let c4_hash = from_hex("4174c6c3b5a0cf2d734a243b9cf3766afaf2c0e0e913fb09944bbcc9a8556c48"); + let pt4 = b"g9 cross-backend body c4"; + let d4 = decode(&MASTER, &c4_hash, &c4_body, 0, 4).expect("lean decode rust c4"); + assert_eq!(d4, pt4); + + // Lean re-encode of same input must match the Rust body (wire identity). + let Encoded(lean_body, lean_hash, _) = encode(&MASTER, pt4, 4).expect("lean encode c4"); + assert_eq!(lean_body, c4_body, "lean encode must bit-match rust body"); + assert_eq!(lean_hash.as_bytes(), c4_hash.as_slice()); +} + +#[test] +fn g9_rust_encode_lean_decode_headered_c4() { + require_lean_lib(); + let arch = from_hex( + "434152424f4e41444f32300a00000000000000000000000000000000fa7760aa360e9c232b9d7b544f8172f5dc64d99404fd77f94a7a934906034bdcf82b5662847ab76c4f45594cee2faeb45d97bc2ebed05e2e9df3936b4ff9c5f94174c6c3b5a0cf2d734a243b9cf3766afaf2c0e0e913fb09944bbcc9a8556c480000000000000000000000000000000000000000000000000000000000000000040000000020000000000000000000000000000000180000000000000067392063726f73732d6261636b656e6420626f6479206334", + ); + let pt = b"g9 cross-backend body c4"; + let (header, decoded) = file::decode(&MASTER, &arch).expect("lean decode rust headered c4"); + assert_eq!(header.format.bits(), 4); + assert_eq!(decoded, pt); +} diff --git a/tests/lean_backend_phase3.rs b/tests/lean_backend_phase3.rs new file mode 100644 index 0000000..07ba991 --- /dev/null +++ b/tests/lean_backend_phase3.rs @@ -0,0 +1,447 @@ +//! Phase 3 allowlist for `backend-lean` (docs/TEST_CONTRACT.md). +//! +//! Directory dual-backend via **composition**: +//! - Rust: FS + **rkyv** FilepackManifest v2 + Adamantine framing + path policy +//! - Lean C ABI: segment outboard encode/decode + catalog headered encode/decode +//! +//! OTS entry/catalog proof cases remain Phase 4. Full rust-root checksum goldens +//! (`filepack_interop::golden_directory_interop_*`) stay `backend-rust`-only. +//! +//! ```bash +//! nix build .#libcarbonado -o result-libcarbonado +//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB +//! cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_phase3 +//! # or: just test-lean-phase3 +//! ``` +//! Only compiled under `backend-lean` (avoids breaking default/`backend-rust` clippy of all targets). + +#![cfg(feature = "backend-lean")] + +use std::fs; +use std::path::{Path, PathBuf}; + +use carbonado::directory::format_policy::{ + is_likely_incompressible, SegmentFormatPolicy, SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, + SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, +}; +use carbonado::error::CarbonadoError; +use carbonado::file::{ + decode_directory, encode_directory, encode_directory_with_options, DirectoryEncodeOptions, + DIRECTORY_ARCHIVE_FORMAT, +}; +use carbonado::filepack_manifest::{ + FilepackEntry, FilepackManifest, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + FILEPACK_MANIFEST_VERSION, MAX_REL_PATH_LEN, +}; +use carbonado::{ + build_adamantine_payload, decode_adamantine, encode_adamantine, split_adamantine_payload, + ADAMANTINE_CARBONADO_FMT_PUBLIC, +}; + +const ZERO_KEY: [u8; 32] = [0u8; 32]; + +const TEST_MASTER: [u8; 32] = [ + 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, +]; + +fn require_lean_lib() { + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + panic!( + "CARBONADO_LEAN_LIB unset. Build and export first:\n \ + nix build .#libcarbonado -o result-libcarbonado\n \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ + export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ + export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ + # or: just test-lean-phase3" + ); + } +} + +fn tempdir(name: &str) -> PathBuf { + let p = std::env::temp_dir().join(format!( + "carbonado-lean-p3-{}-{}-{}", + name, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).expect("tempdir"); + p +} + +fn hex32(bytes: &[u8; 32]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn adam_catalog_path(enc_dir: &Path, root: &[u8; 32], format: u8) -> PathBuf { + enc_dir.join(format!("{}.adam.c{format}", hex32(root))) +} + +fn write_tree(src: &Path, files: &[(&str, &[u8])]) { + for (rel, data) in files { + let path = src.join(rel); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(&path, data).expect("write"); + } +} + +fn read_tree_file(dec: &Path, rel: &str) -> Vec { + fs::read(dec.join(rel)).unwrap_or_else(|e| panic!("read {rel}: {e}")) +} + +#[test] +fn abi_version_is_one() { + require_lean_lib(); + assert_eq!(carbonado::backend::lean::abi_version(), 1); +} + +#[test] +fn directory_public_roundtrip_under_lean() { + require_lean_lib(); + let src = tempdir("pub_src"); + write_tree( + &src, + &[ + ("hello.txt", b"phase3 public lean directory"), + ("nested/x.bin", b"\x00\x01\x02nested"), + ], + ); + let enc = tempdir("pub_enc"); + let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode_directory public"); + assert_eq!(archive.entry_count, 2); + + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + assert!(catalog.is_file(), "missing catalog {}", catalog.display()); + // Catalog filename uses decimal c14 (not hex .c0e). + let name = catalog.file_name().unwrap().to_string_lossy(); + assert!( + name.ends_with(".adam.c14"), + "expected decimal .adam.c14, got {name}" + ); + + let dec = tempdir("pub_dec"); + decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode_directory public"); + assert_eq!( + read_tree_file(&dec, "hello.txt"), + b"phase3 public lean directory" + ); + assert_eq!(read_tree_file(&dec, "nested/x.bin"), b"\x00\x01\x02nested"); +} + +#[test] +fn directory_encrypted_roundtrip_under_lean() { + require_lean_lib(); + let src = tempdir("enc_src"); + write_tree(&src, &[("secret.txt", b"encrypted catalog+segments")]); + let enc = tempdir("enc_enc"); + let options = DirectoryEncodeOptions { + encrypted: true, + ..Default::default() + }; + let archive = encode_directory_with_options(&TEST_MASTER, &src, &enc, options) + .expect("encode_directory encrypted"); + assert_eq!(archive.entry_count, 1); + + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, 0x0F); + let name = catalog.file_name().unwrap().to_string_lossy(); + assert!( + name.ends_with(".adam.c15"), + "expected .adam.c15, got {name}" + ); + + let dec = tempdir("enc_dec"); + decode_directory(&TEST_MASTER, &catalog, &dec).expect("decode encrypted"); + assert_eq!( + read_tree_file(&dec, "secret.txt"), + b"encrypted catalog+segments" + ); +} + +#[test] +fn empty_directory_roundtrip_under_lean() { + require_lean_lib(); + let src = tempdir("empty_src"); + let enc = tempdir("empty_enc"); + let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode empty"); + assert_eq!(archive.entry_count, 0); + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + let dec = tempdir("empty_dec"); + decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode empty"); +} + +#[test] +fn encode_rejects_zero_master_on_encrypted_strict() { + require_lean_lib(); + let src = tempdir("zmk_src"); + write_tree(&src, &[("a.txt", b"x")]); + let enc = tempdir("zmk_enc"); + let options = DirectoryEncodeOptions { + encrypted: true, + ..Default::default() + }; + let err = encode_directory_with_options(&ZERO_KEY, &src, &enc, options).unwrap_err(); + assert!( + matches!(err, CarbonadoError::ZeroMasterKeyNotAllowed), + "expected ZeroMasterKeyNotAllowed, got {err:?}" + ); +} + +#[test] +fn decode_rejects_nonzero_master_on_public_strict() { + require_lean_lib(); + let src = tempdir("nz_src"); + write_tree(&src, &[("a.txt", b"public")]); + let enc = tempdir("nz_enc"); + let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode"); + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + let err = decode_directory(&TEST_MASTER, &catalog, &tempdir("nz_dec")).unwrap_err(); + assert!( + matches!(err, CarbonadoError::EncryptedDirectoryNotRequested), + "expected EncryptedDirectoryNotRequested, got {err:?}" + ); +} + +#[test] +fn decode_rejects_path_traversal_writes_no_files() { + require_lean_lib(); + // Unit SSOT: validate_rel_path / FilepackManifest::validate reject `..`. + let pe = FilepackManifest::validate_rel_path("../escape.txt").unwrap_err(); + assert!( + matches!(pe, CarbonadoError::InvalidFilepackManifest(_)), + "expected InvalidFilepackManifest from validate_rel_path, got {pe:?}" + ); + let _ = MAX_REL_PATH_LEN; + + // Full fail-closed: encode good tree → rewrite manifest rel_path → re-encode catalog → + // decode_directory must fail before any extract write. + let src = tempdir("mal_src"); + write_tree(&src, &[("one.txt", b"hello")]); + let enc = tempdir("mal_enc"); + let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode"); + let good_catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + + let main_raw = fs::read(&good_catalog).expect("read catalog"); + let (_, body) = carbonado::file::decode(&ZERO_KEY, &main_raw).expect("headered decode"); + let (adam_payload, hdr) = decode_adamantine(&body).expect("adamantine"); + let (rkyv, bundle) = split_adamantine_payload(&adam_payload).expect("split"); + let good_index = + FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); + let entry = good_index.entries.first().expect("one entry"); + + let malicious = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "../escape.txt".into(), + content_blake3: entry.content_blake3, + segment_format: entry.segment_format, + segments: entry.segments.clone(), + ots_proof: None, + }], + }; + // Structural validate also fails on the hand-built index (secondary assert). + let unit_err = malicious.validate().unwrap_err(); + assert!( + matches!(unit_err, CarbonadoError::InvalidFilepackManifest(_)), + "expected InvalidFilepackManifest from validate, got {unit_err:?}" + ); + + let mal_rkyv = malicious.to_bytes().expect("malicious rkyv"); + let mal_payload = build_adamantine_payload(&mal_rkyv, &bundle).expect("build payload"); + let mal_adam = encode_adamantine(&mal_payload, ADAMANTINE_CARBONADO_FMT_PUBLIC, hdr.flags); + let (mal_encoded, _) = + carbonado::file::encode(&ZERO_KEY, &mal_adam, DIRECTORY_ARCHIVE_FORMAT, None) + .expect("encode malicious catalog"); + let mal_header = + carbonado::file::Header::try_from(&mal_encoded[..carbonado::file::Header::LEN]) + .expect("header"); + let mal_root = *mal_header.hash.as_bytes(); + let mal_catalog = adam_catalog_path(&enc, &mal_root, DIRECTORY_ARCHIVE_FORMAT); + fs::write(&mal_catalog, &mal_encoded).expect("write malicious catalog"); + + let dec = tempdir("mal_dec"); + let err = decode_directory(&ZERO_KEY, &mal_catalog, &dec).unwrap_err(); + assert!( + matches!( + err, + CarbonadoError::InvalidFilepackManifest(ref msg) if msg.contains("..") + ), + "expected InvalidFilepackManifest containing '..', got {err:?}" + ); + assert!( + fs::read_dir(&dec) + .map(|mut d| d.next()) + .expect("read_dir") + .is_none(), + "decode_directory must not write files on path traversal" + ); +} + +#[test] +fn format_policy_pure_logic_under_lean() { + require_lean_lib(); + // Pure Rust policy (no crypto) — must stay available under backend-lean builds. + assert!(!is_likely_incompressible( + b"hello world text that compresses well" + )); + assert!(is_likely_incompressible(&[ + 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10 + ])); + + let text = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let fmt = SegmentFormatPolicy::Auto + .resolve_segment_format(false, text) + .expect("auto public text"); + assert_eq!( + fmt, SEGMENT_FORMAT_PUBLIC_COMPRESSED, + "compressible public → c14" + ); + + let jpeg = [0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]; + let fmt = SegmentFormatPolicy::Auto + .resolve_segment_format(false, &jpeg) + .expect("auto public jpeg"); + assert_eq!( + fmt, SEGMENT_FORMAT_PUBLIC_RAW, + "incompressible public → c12" + ); + + let fmt = SegmentFormatPolicy::Auto + .resolve_segment_format(true, text) + .expect("auto encrypted text"); + assert_eq!( + fmt, SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, + "compressible encrypted → c15" + ); + + let err = SegmentFormatPolicy::ForceC12 + .resolve_segment_format(true, text) + .unwrap_err(); + assert!( + matches!(err, CarbonadoError::SegmentFormatMismatch(_)), + "ForceC12 on encrypted catalog must fail, got {err:?}" + ); +} + +#[test] +fn catalog_rkyv_wire_roundtrip_under_lean_encode() { + require_lean_lib(); + // Dual-suite claim: under backend-lean, directory catalogs still carry **rkyv** + // FilepackManifest v2 (not CFP2) inside Adamantine payload. + let src = tempdir("rkyv_src"); + write_tree(&src, &[("only.txt", b"rkyv normative wire")]); + let enc = tempdir("rkyv_enc"); + let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode"); + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + + // Peel catalog: headered decode → adamantine → rkyv body. + let main_raw = fs::read(&catalog).expect("read catalog"); + let (header, body) = carbonado::file::decode(&ZERO_KEY, &main_raw).expect("headered decode"); + assert_eq!(header.format.bits(), DIRECTORY_ARCHIVE_FORMAT); + assert_eq!(header.hash.as_bytes(), &archive.catalog_bao_root); + + let (adam_payload, adam_hdr) = carbonado::decode_adamantine(&body).expect("adamantine"); + assert_eq!(adam_hdr.carbonado_fmt, 0x0E); + let (rkyv_payload, _bundle) = + carbonado::split_adamantine_payload(&adam_payload).expect("split"); + // CFP2 magic would be b"CFP2"; rkyv has no that prefix at byte 0 typically, but + // definitive check is successful FilepackManifest deserialize + version 2. + assert!( + !rkyv_payload.starts_with(b"CFP2"), + "dual-suite catalog must not be Lean-only CFP2 wire" + ); + let manifest = FilepackManifest::from_bytes_with_root(&rkyv_payload, archive.catalog_bao_root) + .expect("rkyv FilepackManifest v2"); + assert_eq!(manifest.version, FILEPACK_MANIFEST_VERSION); + assert_eq!(manifest.format_level, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC); + assert_eq!(manifest.entries.len(), 1); + assert_eq!(manifest.entries[0].rel_path, "only.txt"); +} + +/// G9 directory seed: rust-encoded fixture (tests/fixtures/phase3_g9_directory) → lean decode. +#[test] +fn g9_rust_encode_lean_decode_directory_fixture() { + require_lean_lib(); + let fixture = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/phase3_g9_directory"); + assert!( + fixture.is_dir(), + "missing G9 fixture dir {}", + fixture.display() + ); + + let catalog = + fixture.join("16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14"); + assert!( + catalog.is_file(), + "missing G9 catalog {}", + catalog.display() + ); + + // Fixture segments must sit next to the catalog (decode looks in parent dir). + let dec = tempdir("g9_dec"); + // Copy entire fixture archive next to a writable extract root so decode can + // resolve segment mains relative to the catalog path without mutating fixtures. + let work = tempdir("g9_work"); + for entry in fs::read_dir(&fixture).expect("read fixture") { + let entry = entry.expect("entry"); + let path = entry.path(); + if path.is_file() + && path + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n != "README.txt") + { + fs::copy(&path, work.join(path.file_name().unwrap())).expect("copy artifact"); + } + } + let work_catalog = work.join(catalog.file_name().unwrap()); + decode_directory(&ZERO_KEY, &work_catalog, &dec).expect("lean decode of rust G9 fixture"); + assert_eq!(read_tree_file(&dec, "a.txt"), b"phase3 g9 hello"); + assert_eq!(read_tree_file(&dec, "sub/b.bin"), b"nested data"); +} + +#[test] +fn multi_segment_sharding_under_lean() { + require_lean_lib(); + let src = tempdir("shard_src"); + // 5 bytes with budget 2 → 3 segments. + write_tree(&src, &[("shard.bin", b"abcde")]); + let enc = tempdir("shard_enc"); + let options = DirectoryEncodeOptions { + segment_plaintext_budget: 2, + ..Default::default() + }; + let archive = encode_directory_with_options(&ZERO_KEY, &src, &enc, options).expect("encode"); + assert_eq!(archive.entry_count, 1); + + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + let mains: Vec<_> = fs::read_dir(&enc) + .expect("read_dir") + .filter_map(|e| e.ok()) + .filter(|e| { + let n = e.file_name().to_string_lossy().into_owned(); + n.contains(".c") && !n.contains(".adam.") + }) + .collect(); + assert_eq!( + mains.len(), + 3, + "expected exactly 3 segment mains for budget=2 on 5-byte file, got {}", + mains.len() + ); + + let dec = tempdir("shard_dec"); + decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode sharded"); + assert_eq!(read_tree_file(&dec, "shard.bin"), b"abcde"); +} diff --git a/tests/lean_backend_phase4.rs b/tests/lean_backend_phase4.rs new file mode 100644 index 0000000..98f4dc0 --- /dev/null +++ b/tests/lean_backend_phase4.rs @@ -0,0 +1,676 @@ +//! Phase 4 allowlist for `backend-lean` (docs/TEST_CONTRACT.md, docs/GAPS.md). +//! +//! Dual-backend composition (G10 strategy A): +//! - **Container crypto** (outboard / headered / directory segments+catalog): Lean C ABI +//! - **SLH-DSA** (`crypto::slh_*`, SLH1 sidecar, header `slh_public_key`): Rust `bitcoinpqc` +//! under both backends until pure Lean/libbitcoinpqc FFI lands (G10 residual) +//! - **OTS** offline CBOTS stubs: pure Rust (`ots` feature) — same path under lean +//! - **CLI dual path (honest):** +//! - Directory library + subprocess → Lean composition (dual-engine) +//! - Buffer APIs (`file::encode` / `encode_outboard`) → Lean +//! - Single-file CLI streaming (`encode_stream` / `stream_*_outboard`) remains pure Rust +//! under lean builds; lean-linked binary subprocess for single-file is link/smoke only +//! +//! ```bash +//! nix build .#libcarbonado -o result-libcarbonado +//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB +//! cargo test --no-default-features --features "backend-lean,pqc,ots,cli" --test lean_backend_phase4 +//! # or: just test-lean-phase4 +//! ``` +//! Only compiled under `backend-lean` + `pqc` (avoids breaking default/`backend-rust` clippy of all targets). + +#![cfg(all(feature = "backend-lean", feature = "pqc"))] + +use std::fs; +use std::path::{Path, PathBuf}; + +use carbonado::constants::Format; +use carbonado::crypto::{ + read_slh_sidecar, slh_dsa_generate_keypair, slh_dsa_sign, slh_dsa_verify, write_slh_sidecar, + Algorithm, PublicKey, Signature, SLH1_MAGIC, SLH1_SIDECAR_LEN, SLH1_SIGNATURE_LEN, +}; +use carbonado::error::CarbonadoError; +use carbonado::file::{ + self, decode_directory, encode_directory, encode_directory_with_options, encode_stream, + DirectoryEncodeOptions, Header, DIRECTORY_ARCHIVE_FORMAT, +}; +use carbonado::{ + build_adamantine_payload, decode_adamantine, encode_adamantine, split_adamantine_payload, + ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, +}; +use getrandom::getrandom; +use rand::RngCore; + +#[cfg(feature = "ots")] +use carbonado::filepack_manifest::FilepackManifest; +#[cfg(feature = "ots")] +use carbonado::ots::{verify_stamp, OtsPolicy}; + +const ZERO_KEY: [u8; 32] = [0u8; 32]; + +const TEST_MASTER: [u8; 32] = [ + 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, +]; + +/// Header wire offset of `slh_public_key` (AGENTS § Header layout; 12+16+64+32 = 124). +mod offsets { + pub const SLH_PK: usize = 124; +} + +fn require_lean_lib() { + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + panic!( + "CARBONADO_LEAN_LIB unset. Build and export first:\n \ + nix build .#libcarbonado -o result-libcarbonado\n \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ + export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ + export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ + # or: just test-lean-phase4" + ); + } +} + +fn tempdir(name: &str) -> PathBuf { + let p = std::env::temp_dir().join(format!( + "carbonado-lean-p4-{}-{}-{}", + name, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).expect("tempdir"); + p +} + +fn hex32(bytes: &[u8; 32]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn adam_catalog_path(enc_dir: &Path, root: &[u8; 32], format: u8) -> PathBuf { + enc_dir.join(format!("{}.adam.c{format}", hex32(root))) +} + +fn random_master() -> [u8; 32] { + let mut k = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut k); + k +} + +fn slh_entropy() -> [u8; 128] { + let mut e = [0u8; 128]; + getrandom(&mut e).expect("entropy"); + e +} + +fn slh_public_key_bytes(pk: &PublicKey) -> [u8; 32] { + let mut out = [0u8; 32]; + out.copy_from_slice(&pk.bytes[..32]); + out +} + +#[test] +fn abi_version_is_one() { + require_lean_lib(); + assert_eq!(carbonado::backend::lean::abi_version(), 1); +} + +#[test] +fn backend_name_is_lean() { + require_lean_lib(); + assert_eq!(carbonado::backend::lean::NAME, "lean"); +} + +/// G10 dual-suite: Lean outboard encode/decode + Rust bitcoinpqc SLH sidecar over Bao root. +#[test] +fn slh_outboard_sidecar_binds_header_public_key_under_lean() { + require_lean_lib(); + let key = random_master(); + let input = b"Phase4 SLH under lean: Lean container + Rust SLH-DSA-SHA2-128s"; + + let (hdr_opt, oenc) = file::encode_outboard(&key, input, 14, None).expect("encode_outboard"); + let base_hdr = hdr_opt.expect("header for outboard high-level path"); + let bao_root = base_hdr.hash.as_bytes(); + + let keypair = slh_dsa_generate_keypair(&slh_entropy()).expect("slh keygen"); + let slh_pk = slh_public_key_bytes(&keypair.public_key); + let signature = slh_dsa_sign(&keypair.secret_key, bao_root).expect("slh sign"); + + let signed_hdr = Header::new( + &key, + base_hdr.payload_nonce, + bao_root, + slh_pk, + Format::from(14), + base_hdr.chunk_index, + base_hdr.encoded_len, + base_hdr.padding_len, + base_hdr.metadata, + ) + .expect("Header::new with slh_pk"); + assert_eq!(signed_hdr.slh_public_key, slh_pk); + + let hdr_bytes = signed_hdr.try_to_vec().expect("header wire"); + let rec = file::decode_outboard( + &key, + signed_hdr.hash.as_bytes(), + Some(&hdr_bytes), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + 14, + ) + .expect("decode_outboard lean"); + assert_eq!(rec, input); + + let sidecar_path = tempdir("slh_sc").join(format!("{}.slh", signed_hdr.file_name())); + write_slh_sidecar(&sidecar_path, &signature.bytes).expect("write slh"); + let sig_bytes = read_slh_sidecar(&sidecar_path).expect("read slh"); + assert_eq!(sig_bytes.len(), SLH1_SIGNATURE_LEN); + let on_disk = fs::read(&sidecar_path).expect("raw sidecar"); + assert_eq!(on_disk.len(), SLH1_SIDECAR_LEN); + assert_eq!(&on_disk[..4], SLH1_MAGIC); + + let sig = Signature { + algorithm: Algorithm::SLH_DSA_SHA2_128S, + bytes: sig_bytes, + }; + assert!( + slh_dsa_verify(&keypair.public_key, bao_root, &sig).expect("verify"), + "signature must verify over Bao root" + ); + + let hdr_pk = PublicKey { + algorithm: Algorithm::SLH_DSA_SHA2_128S, + bytes: signed_hdr.slh_public_key.to_vec(), + }; + assert!( + slh_dsa_verify(&hdr_pk, bao_root, &sig).expect("verify via header pk"), + "header slh_public_key must verify sidecar" + ); + + // Fail-closed: wrong root + let mut bad_root = *bao_root; + bad_root[0] ^= 0x01; + assert!( + !slh_dsa_verify(&hdr_pk, &bad_root, &sig).expect("verify bad root"), + "wrong Bao root must not verify" + ); + + // Fail-closed: wrong public key + let other = slh_dsa_generate_keypair(&slh_entropy()).expect("other keygen"); + let wrong_pk = PublicKey { + algorithm: Algorithm::SLH_DSA_SHA2_128S, + bytes: slh_public_key_bytes(&other.public_key).to_vec(), + }; + assert!( + !slh_dsa_verify(&wrong_pk, bao_root, &sig).expect("verify wrong pk"), + "wrong header pk must not verify" + ); + + // Fail-closed: tampered slh_public_key fails header_mac (lean headered path) + let mut bad_hdr_bytes = hdr_bytes.clone(); + bad_hdr_bytes[offsets::SLH_PK] ^= 0x01; + let err_pk = file::decode_outboard( + &key, + signed_hdr.hash.as_bytes(), + Some(&bad_hdr_bytes), + &oenc.main, + oenc.verification_outboard.as_deref(), + oenc.fec_parity.as_deref(), + oenc.info.padding_len, + 14, + ) + .unwrap_err(); + assert!( + matches!(err_pk, CarbonadoError::AuthenticationFailed), + "tampered slh_public_key must fail header_mac, got {err_pk:?}" + ); +} + +#[test] +fn slh_sidecar_bad_magic_and_length_fail_closed() { + require_lean_lib(); + let dir = tempdir("slh_wire"); + + // Bad magic, correct length + let bad_magic = dir.join("bad_magic.slh"); + let mut wire = vec![0u8; SLH1_SIDECAR_LEN]; + wire[..4].copy_from_slice(b"XXXX"); + fs::write(&bad_magic, &wire).expect("write"); + let err = read_slh_sidecar(&bad_magic).unwrap_err(); + assert!( + matches!(err, CarbonadoError::InvalidMagicNumber(_)), + "bad SLH1 magic must be InvalidMagicNumber, got {err:?}" + ); + + // Truncated (good magic prefix) + let short = dir.join("short.slh"); + fs::write(&short, b"SLH1").expect("write"); + let err2 = read_slh_sidecar(&short).unwrap_err(); + assert!( + matches!(err2, CarbonadoError::OutboardVerificationFailed(_)), + "short sidecar must be OutboardVerificationFailed, got {err2:?}" + ); + + // Wrong signature length on write. + // Taxonomy freeze (P4): short sig / short sidecar map to `OutboardVerificationFailed` + // (pre-existing; not a dedicated SlhWire error). Update these asserts if refined later. + let err3 = write_slh_sidecar(dir.join("short_sig.slh"), b"short").unwrap_err(); + assert!( + matches!(err3, CarbonadoError::OutboardVerificationFailed(_)), + "short signature write must fail, got {err3:?}" + ); +} + +#[cfg(feature = "ots")] +#[test] +fn directory_ots_entry_and_catalog_under_lean() { + require_lean_lib(); + let src = tempdir("ots_src"); + fs::write(src.join("one.txt"), b"phase4 ots lean payload").expect("write"); + + let enc_dir = tempdir("ots_enc"); + let dec_dir = tempdir("ots_dec"); + let options = DirectoryEncodeOptions { + ots_policy: Some(OtsPolicy { + stamp_entries: true, + stamp_catalog: true, + }), + ..DirectoryEncodeOptions::default() + }; + let archive = + encode_directory_with_options(&ZERO_KEY, &src, &enc_dir, options).expect("encode"); + let catalog_path = adam_catalog_path( + &enc_dir, + &archive.catalog_bao_root, + DIRECTORY_ARCHIVE_FORMAT, + ); + + let catalog_bytes = fs::read(&catalog_path).expect("read catalog"); + assert!( + catalog_bytes.windows(4).any(|w| w == b"COTS"), + "catalog must contain COTS trailer when stamp_catalog is set" + ); + + let (_, body) = carbonado::file::decode(&ZERO_KEY, &catalog_bytes).expect("headered decode"); + let (adam_payload, hdr) = decode_adamantine(&body).expect("adam"); + assert_ne!( + hdr.flags & ADAMANTINE_FLAG_REQUIRE_OTS, + 0, + "REQUIRE_OTS must be set when stamp_entries" + ); + let (rkyv, _) = split_adamantine_payload(&adam_payload).expect("split"); + let index = + FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); + let proof = index.entries[0].ots_proof.as_ref().expect("entry ots"); + let primary_root = index.entries[0].segments[0].segment_bao_root; + assert!( + verify_stamp(proof, &primary_root) + .expect("verify entry") + .valid, + "entry OTS must verify primary segment Bao root" + ); + let catalog_ots = catalog_ots_proof_from_cots_trailer(&catalog_bytes).expect("catalog ots"); + assert!( + verify_stamp(&catalog_ots, &archive.catalog_bao_root) + .expect("verify catalog") + .valid, + "catalog OTS must verify catalog Bao root" + ); + + decode_directory(&ZERO_KEY, &catalog_path, &dec_dir).expect("decode_directory"); + assert_eq!( + fs::read(dec_dir.join("one.txt")).expect("read"), + b"phase4 ots lean payload" + ); +} + +#[cfg(feature = "ots")] +#[test] +fn directory_ots_tampered_entry_fails_under_lean() { + require_lean_lib(); + let src = tempdir("ots_tamper_src"); + fs::write(src.join("one.txt"), b"tamper me lean").expect("write"); + let enc_dir = tempdir("ots_tamper_enc"); + let dec_dir = tempdir("ots_tamper_dec"); + let options = DirectoryEncodeOptions { + ots_policy: Some(OtsPolicy { + stamp_entries: true, + stamp_catalog: false, + }), + ..DirectoryEncodeOptions::default() + }; + let archive = + encode_directory_with_options(&ZERO_KEY, &src, &enc_dir, options).expect("encode"); + let catalog_path = adam_catalog_path( + &enc_dir, + &archive.catalog_bao_root, + DIRECTORY_ARCHIVE_FORMAT, + ); + + let (_, body) = carbonado::file::decode(&ZERO_KEY, &fs::read(&catalog_path).expect("read")) + .expect("decode"); + let (adam_payload, hdr) = decode_adamantine(&body).expect("adam"); + let (rkyv, bundle) = split_adamantine_payload(&adam_payload).expect("split"); + let mut index = + FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); + let proof = index.entries[0].ots_proof.as_mut().expect("proof"); + if let Some(byte) = proof.first_mut() { + *byte ^= 0xFF; + } + let tampered_rkyv = index.to_bytes().expect("to_bytes"); + let tampered_payload = build_adamantine_payload(&tampered_rkyv, &bundle).expect("payload"); + let tampered_adam = encode_adamantine( + &tampered_payload, + ADAMANTINE_CARBONADO_FMT_PUBLIC, + hdr.flags, + ); + let (tampered_encoded, _) = + carbonado::file::encode(&ZERO_KEY, &tampered_adam, DIRECTORY_ARCHIVE_FORMAT, None) + .expect("re-encode"); + let tampered_header = Header::try_from(&tampered_encoded[..Header::LEN]).expect("header"); + let tampered_root = *tampered_header.hash.as_bytes(); + let tampered_catalog = adam_catalog_path(&enc_dir, &tampered_root, DIRECTORY_ARCHIVE_FORMAT); + fs::write(&tampered_catalog, &tampered_encoded).expect("write tampered"); + + let err = decode_directory(&ZERO_KEY, &tampered_catalog, &dec_dir).unwrap_err(); + assert!( + matches!(err, CarbonadoError::OtsVerificationFailed), + "tampered entry OTS must be OtsVerificationFailed, got {err:?}" + ); +} + +#[cfg(feature = "ots")] +#[test] +fn directory_ots_missing_when_required_fails_under_lean() { + require_lean_lib(); + let src = tempdir("ots_req_src"); + fs::write(src.join("one.txt"), b"x").expect("write"); + let enc_dir = tempdir("ots_req_enc"); + let archive = encode_directory(&ZERO_KEY, &src, &enc_dir).expect("encode"); + let catalog_path = adam_catalog_path( + &enc_dir, + &archive.catalog_bao_root, + DIRECTORY_ARCHIVE_FORMAT, + ); + let (_, body) = carbonado::file::decode(&ZERO_KEY, &fs::read(&catalog_path).expect("read")) + .expect("decode"); + let (adam_payload, _) = decode_adamantine(&body).expect("adam"); + let (rkyv, bundle) = split_adamantine_payload(&adam_payload).expect("split"); + let mut index = + FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); + index.entries[0].ots_proof = None; + let payload = build_adamantine_payload(&index.to_bytes().expect("bytes"), &bundle).expect("p"); + let adam = encode_adamantine( + &payload, + ADAMANTINE_CARBONADO_FMT_PUBLIC, + ADAMANTINE_FLAG_REQUIRE_OTS, + ); + let (encoded, _) = + carbonado::file::encode(&ZERO_KEY, &adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); + let header = Header::try_from(&encoded[..Header::LEN]).expect("hdr"); + let root = *header.hash.as_bytes(); + let bad_catalog = adam_catalog_path(&enc_dir, &root, DIRECTORY_ARCHIVE_FORMAT); + fs::write(&bad_catalog, &encoded).expect("write"); + let err = decode_directory(&ZERO_KEY, &bad_catalog, &tempdir("ots_req_dec")).unwrap_err(); + assert!( + matches!( + err, + CarbonadoError::OtsProofRequired(ref rel) if rel == "one.txt" + ), + "expected OtsProofRequired(one.txt), got {err:?}" + ); +} + +/// CLI-shaped library paths under lean — engines called out per half. +#[test] +fn cli_library_encode_decode_paths_under_lean() { + require_lean_lib(); + + // --- Cross-engine (mirrors CLI single-file inboard wire assembly) --- + // Encode: pure-Rust `encode_stream` (same as bin/carbonado; no stream→Lean dispatch). + // Decode: Lean headered `file::decode` (G9-style rust-encode → lean-decode). + let input = b"phase4 cli library single-file lean"; + let mut body_bytes = Vec::new(); + let (enc_hdr, _info) = encode_stream(&ZERO_KEY, &mut &input[..], 14, None, &mut body_bytes) + .expect("encode_stream"); + assert_eq!(enc_hdr.hash.as_bytes().len(), 32); + assert!(!body_bytes.is_empty()); + let mut archive = enc_hdr.try_to_vec().expect("header wire"); + archive.extend_from_slice(&body_bytes); + + let (hdr, body) = carbonado::file::decode(&ZERO_KEY, &archive).expect("lean headered decode"); + assert_eq!(hdr.hash, enc_hdr.hash); + assert_eq!(body, input); + + // --- Same-engine dual path (buffer API the CLI does *not* use for streaming) --- + // `file::encode` under backend-lean → Lean `carbonado_encode_headered`. + let (lean_archive, _) = + carbonado::file::encode(&ZERO_KEY, input, 14, None).expect("lean file::encode"); + let (lean_hdr, lean_body) = + carbonado::file::decode(&ZERO_KEY, &lean_archive).expect("lean file::decode"); + assert_eq!(lean_hdr.format.bits(), 14); + assert_eq!(lean_body, input); + + // --- Directory path (CLI `encode

` / `decode .adam.c14`) — dual-engine --- + let src = tempdir("cli_lib_src"); + fs::write(src.join("hi.txt"), b"cli dir lean").expect("write"); + let enc = tempdir("cli_lib_enc"); + let dir_archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode_directory"); + let catalog = adam_catalog_path( + &enc, + &dir_archive.catalog_bao_root, + DIRECTORY_ARCHIVE_FORMAT, + ); + let dec = tempdir("cli_lib_dec"); + decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode_directory"); + assert_eq!(fs::read(dec.join("hi.txt")).expect("read"), b"cli dir lean"); +} + +/// Encrypted directory under lean (CLI `--encrypted --master`). +#[test] +fn cli_library_encrypted_directory_under_lean() { + require_lean_lib(); + let src = tempdir("cli_enc_src"); + fs::write(src.join("secret.txt"), b"encrypted cli lean").expect("write"); + let enc = tempdir("cli_enc_enc"); + let options = DirectoryEncodeOptions { + encrypted: true, + ..Default::default() + }; + let archive = encode_directory_with_options(&TEST_MASTER, &src, &enc, options).expect("enc"); + let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, 0x0F); + let dec = tempdir("cli_enc_dec"); + decode_directory(&TEST_MASTER, &catalog, &dec).expect("dec"); + assert_eq!( + fs::read(dec.join("secret.txt")).expect("read"), + b"encrypted cli lean" + ); +} + +/// Lean-linked binary: single-file `--outboard` is **link/smoke only** (pure Rust streaming). +/// Does **not** exercise Lean C ABI — see `cli_subprocess_directory_roundtrip_under_lean`. +#[cfg(feature = "cli")] +#[test] +fn cli_subprocess_single_file_link_smoke_under_lean() { + require_lean_lib(); + let bin = PathBuf::from(env!("CARGO_BIN_EXE_carbonado")); + assert!( + bin.is_file(), + "carbonado binary missing at {} — build with features backend-lean,cli", + bin.display() + ); + + let work = tempdir("cli_sub_sf"); + let input = work.join("input.txt"); + let outdir = work.join("enc"); + let recovered = work.join("recovered.bin"); + fs::create_dir_all(&outdir).expect("outdir"); + fs::write(&input, b"phase4 subprocess single-file link smoke").expect("write"); + + let enc = std::process::Command::new(&bin) + .args([ + "encode", + input.to_str().unwrap(), + "--format", + "14", + "--outboard", + "--output", + outdir.to_str().unwrap(), + ]) + .env( + "LD_LIBRARY_PATH", + std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), + ) + .output() + .expect("spawn encode"); + assert!( + enc.status.success(), + "encode failed: status={:?} stderr={}", + enc.status, + String::from_utf8_lossy(&enc.stderr) + ); + + // One bare main archive (not .out/.par) + let archive = fs::read_dir(&outdir) + .expect("read") + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| { + p.is_file() + && p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| !n.ends_with(".out") && !n.ends_with(".par")) + }) + .expect("archive main missing"); + + let dec = std::process::Command::new(&bin) + .args([ + "decode", + archive.to_str().unwrap(), + "--output", + recovered.to_str().unwrap(), + ]) + .env( + "LD_LIBRARY_PATH", + std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), + ) + .output() + .expect("spawn decode"); + assert!( + dec.status.success(), + "decode failed: status={:?} stderr={}", + dec.status, + String::from_utf8_lossy(&dec.stderr) + ); + assert_eq!( + fs::read(&recovered).expect("read recovered"), + b"phase4 subprocess single-file link smoke" + ); +} + +/// Dual-engine CLI subprocess: directory encode/decode hits Lean segment/catalog crypto. +#[cfg(feature = "cli")] +#[test] +fn cli_subprocess_directory_roundtrip_under_lean() { + require_lean_lib(); + let bin = PathBuf::from(env!("CARGO_BIN_EXE_carbonado")); + assert!( + bin.is_file(), + "carbonado binary missing at {} — build with features backend-lean,cli", + bin.display() + ); + + let work = tempdir("cli_sub_dir"); + let src = work.join("src"); + let outdir = work.join("enc"); + let recovered = work.join("recovered"); + fs::create_dir_all(&src).expect("src"); + fs::create_dir_all(&outdir).expect("outdir"); + fs::write(src.join("hi.txt"), b"phase4 subprocess directory lean").expect("write"); + + let enc = std::process::Command::new(&bin) + .args([ + "encode", + src.to_str().unwrap(), + "--output", + outdir.to_str().unwrap(), + ]) + .env( + "LD_LIBRARY_PATH", + std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), + ) + .output() + .expect("spawn directory encode"); + assert!( + enc.status.success(), + "directory encode failed: status={:?} stderr={}", + enc.status, + String::from_utf8_lossy(&enc.stderr) + ); + + let catalog = fs::read_dir(&outdir) + .expect("read enc") + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| { + p.is_file() + && p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.ends_with(".adam.c14")) + }) + .expect("catalog .adam.c14 missing"); + + let dec = std::process::Command::new(&bin) + .args([ + "decode", + catalog.to_str().unwrap(), + "--output", + recovered.to_str().unwrap(), + ]) + .env( + "LD_LIBRARY_PATH", + std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), + ) + .output() + .expect("spawn directory decode"); + assert!( + dec.status.success(), + "directory decode failed: status={:?} stderr={}", + dec.status, + String::from_utf8_lossy(&dec.stderr) + ); + assert_eq!( + fs::read(recovered.join("hi.txt")).expect("read recovered"), + b"phase4 subprocess directory lean" + ); +} + +#[cfg(feature = "ots")] +fn catalog_ots_proof_from_cots_trailer(bytes: &[u8]) -> Option> { + if bytes.len() < Header::LEN + 8 { + return None; + } + let max_scan = carbonado::filepack_manifest::MAX_OTS_PROOF_LEN + 8; + let scan_start = bytes.len().saturating_sub(max_scan).max(Header::LEN); + for i in (scan_start..=bytes.len().saturating_sub(8)).rev() { + if bytes.get(i..i + 4)? != b"COTS" { + continue; + } + let ots_len = u32::from_le_bytes(bytes[i + 4..i + 8].try_into().ok()?) as usize; + if ots_len > carbonado::filepack_manifest::MAX_OTS_PROOF_LEN { + return None; + } + if i + 8 + ots_len == bytes.len() { + return Some(bytes[i + 8..].to_vec()); + } + } + None +} diff --git a/tests/lean_backend_smoke.rs b/tests/lean_backend_smoke.rs new file mode 100644 index 0000000..0d30dc7 --- /dev/null +++ b/tests/lean_backend_smoke.rs @@ -0,0 +1,277 @@ +//! Phase 1 allowlist smoke for `backend-lean` (docs/TEST_CONTRACT.md). +//! +//! ```bash +//! nix build .#libcarbonado -o result-libcarbonado +//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib +//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include +//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB +//! cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_smoke +//! # or: just test-lean-smoke +//! ``` +//! +//! Primary allowlist: public (even) formats c0, c4, c12. Fixed master keys. +//! R2 also smokes encrypted headered + non-zero SLH with a fixed nonce (no RNG). +//! Only compiled under `backend-lean` (avoids breaking default/`backend-rust` clippy of all targets). + +#![cfg(feature = "backend-lean")] + +mod common; + +use carbonado::{ + carbonado_verification_key, constants::Format, decode, encode, error::CarbonadoError, file, + file::Header, structs::Encoded, +}; +use common::header_layout::offsets; + +/// Fixed 32-byte master (public formats may use zeros; we use a non-zero pattern). +const MASTER: [u8; 32] = [ + 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, + 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, +]; + +fn require_lean_lib() { + if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { + panic!( + "CARBONADO_LEAN_LIB unset. Build and export first:\n \ + nix build .#libcarbonado -o result-libcarbonado\n \ + export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ + export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ + export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ + # or: just test-lean-smoke" + ); + } +} + +#[test] +fn abi_version_is_one() { + require_lean_lib(); + let v = carbonado::backend::lean::abi_version(); + assert_eq!(v, 1, "CARBONADO_ABI_VERSION"); +} + +#[test] +fn verification_key_lean_abi_matches_blake3_formula() { + require_lean_lib(); + // Public API is pure blake3; parity is via the Lean C ABI helper. + for format in [0u8, 4, 6, 12, 14, 15] { + let lean = carbonado::backend::lean::verification_key(format) + .unwrap_or_else(|e| panic!("lean verification_key c{format}: {e}")); + let rust_formula = carbonado_verification_key(format); + assert_eq!( + lean, rust_formula, + "Lean AOT verification key mismatch for format {format}" + ); + assert_eq!( + rust_formula, + blake3::derive_key("carbonado-v2/verification", &[format]) + ); + } +} + +#[test] +fn headered_roundtrip_public_formats() { + require_lean_lib(); + let plaintext = b"carbonado phase1 lean headered smoke"; + // c0 raw, c4 bao, c12 bao+fec (padding lives in Header) + for level in [0u8, 4, 12] { + let (archive, info) = file::encode(&MASTER, plaintext, level, None) + .unwrap_or_else(|e| panic!("encode c{level}: {e}")); + assert!( + archive.len() >= file::Header::LEN, + "c{level}: archive shorter than header" + ); + assert_eq!(info.input_len, plaintext.len() as u32); + let (header, decoded) = + file::decode(&MASTER, &archive).unwrap_or_else(|e| panic!("decode c{level}: {e}")); + assert_eq!(header.format.bits(), level); + assert_eq!(decoded, plaintext, "c{level} plaintext mismatch"); + } +} + +#[test] +fn low_level_roundtrip_c0_c4() { + require_lean_lib(); + let plaintext = b"low-level body smoke c0/c4"; + for format in [0u8, 4] { + let Encoded(body, hash, info) = encode(&MASTER, plaintext, format) + .unwrap_or_else(|e| panic!("encode body c{format}: {e}")); + assert_eq!(info.bytes_verifiable as usize, body.len()); + let decoded = decode(&MASTER, hash.as_bytes(), &body, info.padding_len, format) + .unwrap_or_else(|e| panic!("decode body c{format}: {e}")); + assert_eq!(decoded, plaintext, "c{format} body plaintext mismatch"); + } +} + +#[test] +fn low_level_encode_short_master_invalid_key_length() { + require_lean_lib(); + let short = [0u8; 16]; + let err = match encode(&short, b"x", 0) { + Ok(_) => panic!("short master low-level encode must fail"), + Err(e) => e, + }; + assert!( + matches!(err, CarbonadoError::InvalidKeyLength), + "expected InvalidKeyLength (not InternalStateError from packEncodeErr bug), got {err:?}" + ); +} + +#[test] +fn low_level_decode_wrong_hash_fails() { + require_lean_lib(); + let plaintext = b"low-level wrong hash"; + let Encoded(body, hash, info) = encode(&MASTER, plaintext, 4).expect("encode"); + let mut bad_hash = *hash.as_bytes(); + bad_hash[0] ^= 0xff; + let err = decode(&MASTER, &bad_hash, &body, info.padding_len, 4).expect_err("bad hash"); + // R4: Bao root/auth mismatch maps to AuthenticationFailed (same as pure Rust + // map_decode_error Parent/LeafHashMismatch). Truncation alone stays BaoResponseTruncated. + assert!( + matches!(err, CarbonadoError::AuthenticationFailed), + "expected AuthenticationFailed, got {err:?}" + ); +} + +#[test] +fn headered_bad_magic_fails() { + require_lean_lib(); + let plaintext = b"tamper magic"; + let (mut archive, _) = file::encode(&MASTER, plaintext, 4, None).expect("encode"); + // Corrupt MAGICNO first byte (CARBONADO20\n) + archive[0] ^= 0xff; + let err = file::decode(&MASTER, &archive).expect_err("bad magic must fail"); + assert!( + matches!(err, CarbonadoError::InvalidMagicNumber(_)), + "expected InvalidMagicNumber, got {err:?}" + ); +} + +#[test] +fn headered_auth_fail_on_header_mac_tamper() { + require_lean_lib(); + let plaintext = b"tamper header mac"; + let (mut archive, _) = file::encode(&MASTER, plaintext, 4, None).expect("encode"); + // header_mac sits at offset 28 (12 magic + 16 nonce); flip one byte + let mac_off = 28; + archive[mac_off] ^= 0x01; + let err = file::decode(&MASTER, &archive).expect_err("tampered MAC must fail"); + assert!( + matches!(err, CarbonadoError::AuthenticationFailed), + "expected AuthenticationFailed, got {err:?}" + ); +} + +#[test] +fn invalid_key_length_rejected_headered() { + require_lean_lib(); + let short = [0u8; 16]; + let err = file::encode(&short, b"x", 0, None).expect_err("short master"); + assert!( + matches!(err, CarbonadoError::InvalidKeyLength), + "expected InvalidKeyLength, got {err:?}" + ); +} + +#[test] +fn headered_metadata_roundtrip_and_mac_binding() { + require_lean_lib(); + let meta = *b"metameta"; + let (arch, _) = file::encode(&MASTER, b"meta payload", 0, Some(meta)).expect("encode meta"); + let hdr = Header::try_from(&arch[..Header::LEN]).expect("header"); + assert_eq!(hdr.metadata, Some(meta)); + let (_hdr2, pt) = file::decode(&MASTER, &arch).expect("decode"); + assert_eq!(pt, b"meta payload"); + + // Tamper metadata byte → header_mac fail. + let mut tampered = arch.clone(); + tampered[offsets::METADATA] ^= 0xff; + let err = file::decode(&MASTER, &tampered).expect_err("tampered meta"); + assert!( + matches!(err, CarbonadoError::AuthenticationFailed), + "expected AuthenticationFailed, got {err:?}" + ); +} + +/// R2: non-zero SLH pk via `lean::encode_headered` (file::encode leaves SLH zeroed by design). +#[test] +fn headered_slh_pk_roundtrip_and_mac_binding() { + require_lean_lib(); + let slh = [0xABu8; 32]; + let meta = *b"slh-meta"; + let (arch, info) = carbonado::backend::lean::encode_headered( + &MASTER, + b"slh payload", + 0, + None, + Some(&slh), + Some(&meta), + ) + .expect("encode with slh+meta"); + assert_eq!(info.bytes_compressed, 0); + assert_eq!(info.bytes_encrypted, 0); + let hdr = Header::try_from(&arch[..Header::LEN]).expect("header"); + assert_eq!(hdr.slh_public_key, slh); + assert_eq!(hdr.metadata, Some(meta)); + let (hdr2, pt) = file::decode(&MASTER, &arch).expect("decode"); + assert_eq!(pt, b"slh payload"); + assert_eq!(hdr2.slh_public_key, slh); + assert_eq!(hdr2.metadata, Some(meta)); + + // Tamper SLH pk byte → header_mac fail. + let mut tampered = arch.clone(); + tampered[offsets::SLH_PUBLIC_KEY] ^= 0xff; + let err = file::decode(&MASTER, &tampered).expect_err("tampered slh"); + assert!( + matches!(err, CarbonadoError::AuthenticationFailed), + "expected AuthenticationFailed, got {err:?}" + ); +} + +/// R2: non-zero SLH + metadata on encrypted headered path (fixed nonce; no RNG). +#[test] +fn headered_encrypted_slh_pk_and_metadata_roundtrip() { + require_lean_lib(); + let slh = [0xCDu8; 32]; + let meta = *b"enc-meta"; + let nonce = [0x11u8; 16]; + let format = Format::Encryption.bits(); // c1 + let (arch, info) = carbonado::backend::lean::encode_headered( + &MASTER, + b"enc slh payload", + format, + Some(&nonce), + Some(&slh), + Some(&meta), + ) + .expect("encrypted encode with slh+meta"); + assert_eq!(info.bytes_compressed, 0); + assert!(info.bytes_encrypted > 0); + let hdr = Header::try_from(&arch[..Header::LEN]).expect("header"); + assert_eq!(hdr.slh_public_key, slh); + assert_eq!(hdr.metadata, Some(meta)); + assert_eq!(hdr.payload_nonce, nonce); + let (hdr2, pt) = file::decode(&MASTER, &arch).expect("decode"); + assert_eq!(pt, b"enc slh payload"); + assert_eq!(hdr2.slh_public_key, slh); + assert_eq!(hdr2.metadata, Some(meta)); + + let mut tampered = arch.clone(); + tampered[offsets::SLH_PUBLIC_KEY] ^= 0xff; + let err = file::decode(&MASTER, &tampered).expect_err("tampered slh on encrypted"); + assert!( + matches!(err, CarbonadoError::AuthenticationFailed), + "expected AuthenticationFailed, got {err:?}" + ); +} + +#[test] +fn format_bits_even_are_public() { + // Sanity: Phase 1 allowlist uses even formats only. + for f in [0u8, 4, 12] { + let fmt = Format::from(f); + assert!( + !fmt.contains(Format::Encryption), + "format {f} should be public (even)" + ); + } +} diff --git a/tests/parallel_determinism.rs b/tests/parallel_determinism.rs index 8994b73..5d84002 100644 --- a/tests/parallel_determinism.rs +++ b/tests/parallel_determinism.rs @@ -2,7 +2,9 @@ //! //! Run with: `cargo test --test parallel_determinism` (default features include `parallel`). //! -//! Serial-path coverage without `parallel`: `cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path`. +//! Serial-path coverage without `parallel`: +//! `cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path` +//! (must name `backend-rust` under `--no-default-features`; never `--all-features`). #![cfg(feature = "parallel")] diff --git a/tests/rkyv_golden_lock.rs b/tests/rkyv_golden_lock.rs new file mode 100644 index 0000000..d3b30e1 --- /dev/null +++ b/tests/rkyv_golden_lock.rs @@ -0,0 +1,233 @@ +//! W3 golden lock: fixture `.bin` files must match Rust `rkyv::to_bytes` **and** +//! the Lean-embedded `golden*Hex` constants in `Carbonado/RkyvFilepack.lean`. +//! +//! Regen (updates bins + prints hex to sync into Lean): +//! ```bash +//! cargo run --example dump_rkyv_r9 --features backend-rust +//! ``` + +use std::fs; +use std::path::PathBuf; + +use carbonado::filepack_manifest::*; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rkyv") + .join(name) +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn assert_bin_hex(name: &str, lean_hex: &str) { + let bytes = fs::read(fixture(name)).unwrap_or_else(|e| panic!("read {name}: {e}")); + let got = hex(&bytes); + assert_eq!( + got, lean_hex, + "fixture {name} drifted from Lean golden*Hex — run dump_rkyv_r9 and update both sides" + ); +} + +fn seg(root_fill: u8, main_len: u64, chunk: u32, vo: u32) -> SegmentRef { + SegmentRef { + segment_bao_root: [root_fill; 32], + chunk_index: chunk, + main_len, + verification_outboard_offset: vo, + verification_outboard_len: 64, + fec_parity_offset: vo + 64, + fec_parity_len: 128, + } +} + +/// Must stay bit-identical to `Carbonado/RkyvFilepack.lean` golden*Hex (Issue 3 lock). +mod lean_hex { + pub const EMPTY: &str = "020000000efbffffff00000000"; + pub const SINGLE: &str = concat!( + "1111111111111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "612e747874ffffff", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e9bffffff01000000000000000000000000", + "020000000ec1ffffff01000000", + ); + pub const MULTI_OTS: &str = concat!( + "1111111111111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "622f6c6f6e6765722d706174682d6e616d652e747874", + "4444444444444444444444444444444444444444444444444444444444444444", + "00000000c80000000000000000000000400000004000000080000000", + "abcdef01", + "612e747874ffffff", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e45ffffff01000000000000000000000000", + "9600000070ffffff", + "3333333333333333333333333333333333333333333333333333333333333333", + "0e5dffffff010000000190ffffff04000000", + "020000000e87ffffff02000000", + ); + pub const PATH_INLINE_8: &str = concat!( + "1111111111111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "3132333435363738", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e9bffffff01000000000000000000000000", + "020000000ec1ffffff01000000", + ); + pub const PATH_OOL_9: &str = concat!( + "313233343536373839", + "1111111111111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "89000000bbffffff", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e9bffffff01000000000000000000000000", + "020000000ec1ffffff01000000", + ); + pub const TWO_SEGMENTS: &str = concat!( + "1111111111111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "1212121212121212121212121212121212121212121212121212121212121212", + "010000003200000000000000c0000000400000000001000080000000", + "612e747874ffffff", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e5fffffff02000000000000000000000000", + "020000000ec1ffffff01000000", + ); + pub const OTS_FIRST_ONLY: &str = concat!( + "1111111111111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "dead", + "4444444444444444444444444444444444444444444444444444444444444444", + "00000000c80000000000000000000000400000004000000080000000", + "612e747874ffffff", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e5dffffff010000000190ffffff02000000", + "622e747874ffffff", + "3333333333333333333333333333333333333333333333333333333333333333", + "0e61ffffff01000000000000000000000000", + "020000000e87ffffff02000000", + ); + pub const RKYV_CFP2_PREFIX: &str = concat!( + "4346503211111111111111111111111111111111111111111111111111111111", + "00000000640000000000000000000000400000004000000080000000", + "612e747874ffffff", + "2222222222222222222222222222222222222222222222222222222222222222", + "0e9bffffff01000000000000000000000000", + "020000000ec1ffffff01000000", + ); +} + +#[test] +fn fixture_bins_match_lean_hex_constants() { + assert_bin_hex("empty_manifest.bin", lean_hex::EMPTY); + assert_bin_hex("single_entry.bin", lean_hex::SINGLE); + assert_bin_hex("multi_entry_ots.bin", lean_hex::MULTI_OTS); + assert_bin_hex("path_inline_8.bin", lean_hex::PATH_INLINE_8); + assert_bin_hex("path_ool_9.bin", lean_hex::PATH_OOL_9); + assert_bin_hex("two_segments.bin", lean_hex::TWO_SEGMENTS); + assert_bin_hex("ots_first_only.bin", lean_hex::OTS_FIRST_ONLY); + assert_bin_hex("rkyv_cfp2_prefix.bin", lean_hex::RKYV_CFP2_PREFIX); +} + +#[test] +fn rust_rkyv_to_bytes_matches_fixtures() { + let empty = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![], + }; + assert_eq!( + empty.to_bytes().unwrap(), + fs::read(fixture("empty_manifest.bin")).unwrap() + ); + + let e1 = FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: None, + }; + let single = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0x33; 32], + catalog_ots_proof: None, + entries: vec![e1.clone()], + }; + assert_eq!( + single.to_bytes().unwrap(), + fs::read(fixture("single_entry.bin")).unwrap() + ); + + let e2 = FilepackEntry { + rel_path: "b/longer-path-name.txt".into(), + content_blake3: [0x33; 32], + segment_format: 0x0E, + segments: vec![seg(0x44, 200, 0, 0)], + ots_proof: Some(vec![0xAB, 0xCD, 0xEF, 0x01]), + }; + let multi = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0x55; 32], + catalog_ots_proof: None, + entries: vec![e1, e2], + }; + assert_eq!( + multi.to_bytes().unwrap(), + fs::read(fixture("multi_entry_ots.bin")).unwrap() + ); + + // CFP2-prefix rkyv regression fixture + let mut cfp2_root = [0x11u8; 32]; + cfp2_root[0..4].copy_from_slice(b"CFP2"); + let e_cfp2 = FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![SegmentRef { + segment_bao_root: cfp2_root, + chunk_index: 0, + main_len: 100, + verification_outboard_offset: 0, + verification_outboard_len: 64, + fec_parity_offset: 64, + fec_parity_len: 128, + }], + ots_proof: None, + }; + let m_cfp2 = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0; 32], + catalog_ots_proof: None, + entries: vec![e_cfp2], + }; + let b = m_cfp2.to_bytes().unwrap(); + assert_eq!(&b[0..4], b"CFP2"); + assert_eq!(b, fs::read(fixture("rkyv_cfp2_prefix.bin")).unwrap()); +} + +#[test] +fn rel_path_max_is_utf8_bytes() { + // Rust SSOT: MAX_REL_PATH_LEN is byte length. + let mut s = String::new(); + while s.len() < MAX_REL_PATH_LEN { + s.push('你'); // 3 UTF-8 bytes + } + // May overshoot slightly; trim to exact max bytes without splitting a char. + while s.len() > MAX_REL_PATH_LEN { + s.pop(); + } + assert!(s.len() <= MAX_REL_PATH_LEN); + assert!(FilepackManifest::validate_rel_path(&s).is_ok()); + + s.push('你'); + assert!(s.len() > MAX_REL_PATH_LEN); + assert!(FilepackManifest::validate_rel_path(&s).is_err()); +} diff --git a/tests/seekable_slices.rs b/tests/seekable_slices.rs index 309ac33..8406037 100644 --- a/tests/seekable_slices.rs +++ b/tests/seekable_slices.rs @@ -54,18 +54,75 @@ fn large_payload_seekable_slices_without_full_decode() -> Result<()> { Ok(()) } +/// Short single-leaf bare main + empty post-order outboard must verify successfully +/// (regression for the pre-R9 `expected_chunks == count * 4` false-fail on short files). +#[test] +fn single_leaf_empty_outboard_verify_slice_succeeds() -> Result<()> { + // c12 (no zstd) keeps main ≈ plaintext; short payload → single 4 KiB leaf → empty .out. + const C12: u8 = 0x0C; + let input = b"single-leaf short outboard ok"; + let master_key = [0u8; 32]; + let oenc = encode_outboard(&master_key, input, C12)?; + let bao_ob = oenc + .verification_outboard + .as_ref() + .expect("bao sidecar present"); + assert_eq!( + bao_ob.len(), + 0, + "short c12 main should be single-leaf (empty outboard), got {} bytes", + bao_ob.len() + ); + assert!( + oenc.main.len() < SLICE_LEN as usize, + "expected main under one slice, got {}", + oenc.main.len() + ); + + let got = verify_slice_outboard( + oenc.main.as_slice(), + bao_ob, + oenc.main.len() as u64, + 0, + 1, + oenc.hash.as_bytes(), + C12, + )?; + assert_eq!(got, oenc.main, "verified slice must equal bare main"); + + // count==0: empty success without geometry (parity with Rust extract semantics). + let empty = verify_slice_outboard( + oenc.main.as_slice(), + bao_ob, + oenc.main.len() as u64, + 99, // OOB index ignored when count==0 + 0, + oenc.hash.as_bytes(), + C12, + )?; + assert!(empty.is_empty()); + Ok(()) +} + #[test] fn tamper_outboard_parent_hash_fails_verification() -> Result<()> { - let input = b"tamper outboard parent hash"; + // Multi-leaf bare main so post-order outboard contains parent hash pairs. + // Use c12 (no zstd) + patterned payload so main stays multi-leaf; highly + // compressible c14 can shrink to a single leaf (empty .out). + const C12: u8 = 0x0C; + let input = patterned_payload((SLICE_LEN as usize) * 3 + 100); let master_key = [0u8; 32]; - let oenc = encode_outboard(&master_key, input, C14)?; + let oenc = encode_outboard(&master_key, &input, C12)?; let bao_ob = oenc.verification_outboard.as_ref().expect("bao sidecar"); let hash = oenc.hash; + assert!( + bao_ob.len() >= 64, + "fixture must produce parent pairs in outboard (got {} bytes)", + bao_ob.len() + ); let mut bad_ob = bao_ob.clone(); - if bad_ob.len() >= 32 { - bad_ob[0] ^= 0xFF; - } + bad_ob[0] ^= 0xFF; let err = verify_slice_outboard( oenc.main.as_slice(), @@ -74,7 +131,7 @@ fn tamper_outboard_parent_hash_fails_verification() -> Result<()> { 0, 1, hash.as_bytes(), - C14, + C12, ) .unwrap_err(); diff --git a/tests/serial_fec_path.rs b/tests/serial_fec_path.rs index 26d3c19..c3656a5 100644 --- a/tests/serial_fec_path.rs +++ b/tests/serial_fec_path.rs @@ -1,6 +1,8 @@ //! Serial FEC encode path (`fec.rs` `rs.encode` branch) when `parallel` is disabled. //! -//! CI runs this via `cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path`. +//! CI / `just test-serial`: +//! `cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path` +//! (must name `backend-rust` under `--no-default-features`; never `--all-features`). #![cfg(not(feature = "parallel"))] diff --git a/tests/slh_outboard.rs b/tests/slh_outboard.rs index cb796fd..3bba0ee 100644 --- a/tests/slh_outboard.rs +++ b/tests/slh_outboard.rs @@ -1,7 +1,9 @@ //! Phase 1B: SLH-DSA sidecar E2E (requires `pqc` feature). //! -//! CI must run `cargo test --all-features` (or `--features pqc`) for this crate; -//! `--no-default-features` skips all tests here (`bitcoinpqc` / `pqc` is optional). +//! Default features already enable `pqc`. Also in the lean freeze allowlist: +//! `just test-lean-ci` / `cargo test --no-default-features --features "backend-lean,pqc,ots,cli" --test slh_outboard`. +//! Never `cargo test --all-features` (enables both backends → `compile_error!`). +//! Builds without `pqc` skip this crate (`#![cfg(feature = "pqc")]`). #![cfg(feature = "pqc")] diff --git a/tests/streaming.rs b/tests/streaming.rs index 1deaa57..16144fb 100644 --- a/tests/streaming.rs +++ b/tests/streaming.rs @@ -9,7 +9,7 @@ use carbonado::file::{decode_stream, encode_stream}; use carbonado::stream::{ decode::stream_decode_outboard, encode::{stream_encode_buffer, stream_encode_outboard, stream_encode_outboard_buffer}, - stream_decode_buffer, + stream_decode_buffer, stream_decode_outboard_buffer, }; use proptest::prelude::*; use rand::RngCore; @@ -222,6 +222,44 @@ fn multi_mib_file_stream_smoke() { assert_eq!(recovered, buffer_recovered); } +/// W1a smoke: codecode (encode→decode→encode) + decodec (decode→encode→decode) for public c14. +/// Full matrix lives in post-R10 W2d; this pins stream dual determinism for one format. +#[test] +fn decode_stream_codecode_decodec_public_c14() { + const FORMAT: u8 = 14; + let pt: Vec = (0..4096).map(|i| (i % 251) as u8).collect(); + + let mut body = Vec::new(); + let (h1, _) = + encode_stream(&MASTER, std::io::Cursor::new(&pt), FORMAT, None, &mut body).expect("enc1"); + let mut a = h1.try_to_vec().expect("hdr"); + a.extend_from_slice(&body); + + let mut out = Vec::new(); + let (_h, n) = decode_stream(&MASTER, std::io::Cursor::new(&a), &mut out).expect("dec1"); + assert_eq!(n as usize, pt.len()); + assert_eq!(out, pt, "decode_stream plaintext"); + + // codecode: re-encode must match wire when public (deterministic) + let mut body2 = Vec::new(); + let (h2, _) = encode_stream( + &MASTER, + std::io::Cursor::new(&out), + FORMAT, + None, + &mut body2, + ) + .expect("enc2"); + let mut a2 = h2.try_to_vec().expect("hdr2"); + a2.extend_from_slice(&body2); + assert_eq!(a2, a, "codecode: second encode must match first archive"); + + // decodec: decode → encode → decode recovers plaintext; wire stable + let mut out2 = Vec::new(); + decode_stream(&MASTER, std::io::Cursor::new(&a2), &mut out2).expect("dec2"); + assert_eq!(out2, pt, "decodec: plaintext roundtrip"); +} + /// `encode_stream` / `decode_stream` format sweep (~64 KiB) vs buffer path. #[test] fn file_stream_format_sweep() { @@ -294,3 +332,116 @@ fn file_stream_format_sweep() { ); } } + +/// W1b: public **non-Compression** outboard stream (c4 Bao, c12 Bao+FEC) multi-MiB +/// codecode/decodec. +/// +/// Under `backend-lean` this is the **S4 O(chunk/stripe) composition E2** path (not pure Lean +/// buffer; not Compression — bulk zstd under lean is O(logical)). Under `backend-rust` it is +/// the same S4 pipeline. Wire must match buffer path; public re-encode is deterministic. +/// +/// **Peak RAM:** architectural O(chunk/stripe) claim (SeekableSpool / stripe FEC / leaf Bao); +/// not RSS-instrumented here (optional W4 measurement residual). +#[test] +fn stream_outboard_public_e2_codecode_decodec_c4_c12() { + const PAYLOAD_LEN: usize = 2 * 1024 * 1024; // multi-MiB — exercises O(chunk) spool path + let pt: Vec = (0..PAYLOAD_LEN).map(|i| (i % 251) as u8).collect(); + + for &format in &[4u8, 12u8] { + let has_bao = format & 4 != 0; + let has_fec = format & 8 != 0; + + let mut main1 = Cursor::new(Vec::new()); + let mut bao1 = Vec::new(); + let mut par1 = Vec::new(); + let mut nonce = [0u8; 16]; + let (hash1, info1) = stream_encode_outboard( + &MASTER, + Cursor::new(&pt), + format, + &mut main1, + has_bao.then_some(&mut bao1), + has_fec.then_some(&mut par1), + &mut nonce, + false, + ) + .expect("encode1"); + let main1_bytes = main1.into_inner(); + + // Decode recovers plaintext + let mut out = Vec::new(); + stream_decode_outboard( + &MASTER, + hash1.as_bytes(), + Cursor::new(&main1_bytes), + has_bao.then_some(Cursor::new(&bao1)), + has_fec.then_some(Cursor::new(&par1)), + info1.padding_len, + format, + None, + &mut out, + ) + .expect("decode1"); + assert_eq!(out, pt, "c{format} decode plaintext"); + + // codecode: re-encode public must bit-match (deterministic) + let mut main2 = Cursor::new(Vec::new()); + let mut bao2 = Vec::new(); + let mut par2 = Vec::new(); + let mut nonce2 = [0u8; 16]; + let (hash2, info2) = stream_encode_outboard( + &MASTER, + Cursor::new(&out), + format, + &mut main2, + has_bao.then_some(&mut bao2), + has_fec.then_some(&mut par2), + &mut nonce2, + false, + ) + .expect("encode2"); + let main2_bytes = main2.into_inner(); + assert_eq!(hash2, hash1, "c{format} codecode hash"); + assert_eq!(main2_bytes, main1_bytes, "c{format} codecode main"); + if has_bao { + assert_eq!(bao2, bao1, "c{format} codecode bao outboard"); + } + if has_fec { + assert_eq!(par2, par1, "c{format} codecode fec parity"); + assert_eq!(info2.padding_len, info1.padding_len); + } + + // decodec: decode second wire → plaintext + let mut out2 = Vec::new(); + stream_decode_outboard( + &MASTER, + hash2.as_bytes(), + Cursor::new(&main2_bytes), + has_bao.then_some(Cursor::new(&bao2)), + has_fec.then_some(Cursor::new(&par2)), + info2.padding_len, + format, + None, + &mut out2, + ) + .expect("decode2"); + assert_eq!(out2, pt, "c{format} decodec plaintext"); + + // Match buffer path (Lean dual under backend-lean for buffer APIs) + let buf = stream_encode_outboard_buffer(&MASTER, &pt, format, None).expect("buf encode"); + assert_eq!(buf.hash, hash1, "c{format} stream vs buffer hash"); + assert_eq!(buf.main, main1_bytes, "c{format} stream vs buffer main"); + let buf_dec = stream_decode_outboard_buffer( + &MASTER, + buf.hash.as_bytes(), + &buf.main, + buf.verification_outboard.as_deref(), + buf.fec_parity.as_deref(), + buf.info.padding_len, + format, + None, + ) + .expect("buf decode"); + assert_eq!(buf_dec, pt, "c{format} buffer decode"); + } +} diff --git a/tests/streaming_async.rs b/tests/streaming_async.rs index aca8401..924193e 100644 --- a/tests/streaming_async.rs +++ b/tests/streaming_async.rs @@ -1,4 +1,8 @@ //! Async streaming decode parity tests (Phase 2). Requires `--features async`. +//! +//! **R10:** Dual freeze (`just test-lean-ci`) never enables `async` → this file is 0 tests under +//! freeze (permanent). Optional dual smoke: lean features + `async`/`async-tokio` + +//! `CARBONADO_LEAN_LIB` — `stream_decode_async` is dual-aware via R5 E1 `stream_decode`. #![cfg(feature = "async")] @@ -207,7 +211,10 @@ async fn stream_decode_async_truncated_bounded_body_staging_errors_c4_c8() { } } -/// Verification c12: spool staging fails before Bao; sync fails at Bao (`BaoResponseTruncated`). +/// Verification c12 truncated body: async always fails at spool staging +/// (`truncated encoded body`). Sync taxonomy is engine-dependent (R10): +/// - `backend-rust`: incremental Bao → `BaoResponseTruncated` (divergence from async). +/// - `backend-lean`: R5 E1 spool `read_exact` → `UnexpectedEof` (both paths fail before Bao). #[tokio::test] async fn stream_decode_async_truncated_bounded_verification_diverges_from_sync_c12() { let input: Vec = (0..8192).map(|i| (i % 251) as u8).collect(); @@ -226,9 +233,18 @@ async fn stream_decode_async_truncated_bounded_verification_diverges_from_sync_c &mut sync_out, ) .expect_err("sync truncated bounded c12"); + #[cfg(feature = "backend-rust")] assert!( matches!(err_sync, CarbonadoError::BaoResponseTruncated(_)), - "sync must yield BaoResponseTruncated, got {err_sync:?}" + "sync rust must yield BaoResponseTruncated, got {err_sync:?}" + ); + #[cfg(feature = "backend-lean")] + assert!( + matches!( + err_sync, + CarbonadoError::StdIoError(ref e) if e.kind() == ErrorKind::UnexpectedEof + ), + "sync lean E1 must yield UnexpectedEof on short body, got {err_sync:?}" ); assert!(sync_out.is_empty()); @@ -403,7 +419,8 @@ async fn stream_decode_async_short_bao_body_invalid_header_length() { assert!(out.is_empty()); } -/// `async-tokio` compiles the `spawn_blocking` offload path (exercised under `--all-features` CI). +/// `async-tokio` compiles the `spawn_blocking` offload path (desktop optional matrix: +/// `cargo test --features "async,async-tokio,man-gen"`; never `--all-features`). #[cfg(feature = "async-tokio")] #[test] fn async_tokio_spawn_blocking_path_enabled() { diff --git a/tests/streaming_limits.rs b/tests/streaming_limits.rs index 0b0d24b..ca42a67 100644 --- a/tests/streaming_limits.rs +++ b/tests/streaming_limits.rs @@ -443,6 +443,60 @@ fn stream_decode_encrypted_bounded_read_matches_buffer_path_c15() { assert_stream_decode_parity(&enc_master, 15, &input, Some(512)); } +/// W1a / M1: invalid `header_mac` fails before body I/O under both backends. +/// +/// Uses a huge unauthenticated `encoded_len` so a MAC-after-body path would attempt +/// a multi-MiB read. A `Read` that panics on body bytes proves fail-closed order. +#[test] +fn decode_stream_rejects_bad_header_mac_before_body_read() { + use std::io::{self, Read}; + + /// First `Header::LEN` bytes are the forged header; any further read panics. + struct HeaderOnlyThenPanic { + header: Cursor>, + body_reads: u64, + } + + impl Read for HeaderOnlyThenPanic { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.header.read(buf)?; + if n > 0 { + return Ok(n); + } + self.body_reads += 1; + panic!( + "body read attempted after invalid header_mac (body_reads={})", + self.body_reads + ); + } + } + + let mut archive = encode(&[0u8; 32], b"mac-before-body", 14, None) + .expect("encode public c14") + .0; + assert!(archive.len() > Header::LEN); + // Flip a header_mac byte (bytes 28..92 of the 177-byte header). + archive[40] ^= 0xFF; + // Claim a large body so MAC-after-body would be expensive / force body read. + // encoded_len is at offset 12+16+64+32+32+1+4 = 161 (u32 LE). + let huge = (16 * 1024 * 1024u32).to_le_bytes(); + archive[161..165].copy_from_slice(&huge); + + let mut reader = HeaderOnlyThenPanic { + header: Cursor::new(archive[..Header::LEN].to_vec()), + body_reads: 0, + }; + let mut out = Vec::new(); + let err = + decode_stream(&[0u8; 32], &mut reader, &mut out).expect_err("bad header_mac must fail"); + assert!( + matches!(err, CarbonadoError::AuthenticationFailed), + "expected AuthenticationFailed before body read, got {err:?}" + ); + assert!(out.is_empty()); + assert_eq!(reader.body_reads, 0, "must not touch body after bad MAC"); +} + /// Header-path encode_stream / decode_stream roundtrip without intermediate body staging. #[test] fn encode_stream_decode_stream_roundtrip_c14_c15() { From f77980f4703b341c4acb27a0c5aaa21ec0e13dd5 Mon Sep 17 00:00:00 2001 From: Hunter Beast Date: Thu, 13 Aug 2026 15:43:35 -0600 Subject: [PATCH 3/6] fix ci --- .agents/reports/pr32-ci-checkout-fix.md | 67 ++++++++++++++++ .agents/reports/pr32-ci.md | 54 +++++++++++++ .github/actions/ci-patch-bao-tree/action.yml | 35 +++++++++ .github/workflows/rust.yaml | 80 ++++++++++++++------ .gitignore | 4 + 5 files changed, 215 insertions(+), 25 deletions(-) create mode 100644 .agents/reports/pr32-ci-checkout-fix.md create mode 100644 .agents/reports/pr32-ci.md create mode 100644 .github/actions/ci-patch-bao-tree/action.yml diff --git a/.agents/reports/pr32-ci-checkout-fix.md b/.agents/reports/pr32-ci-checkout-fix.md new file mode 100644 index 0000000..7bd86e9 --- /dev/null +++ b/.agents/reports/pr32-ci-checkout-fix.md @@ -0,0 +1,67 @@ +# PR 32 CI: restore bao-tree checkout on a legal path + +**Date:** 2026-08-13 +**Workspace:** `/home/hunter/Projects/surmount/carbonado` +**HEAD (committed YAML still had the illegal sibling checkout):** `b8c070ae64da98a9a2743dfdb48e8f75322b4f93` + +## What changed + +The previous working-tree change deleted all six `actions/checkout` steps that fetch bao-tree. That is reversed. + +Every job that had **Checkout bao-tree keyed fork (sibling for path dep)** now has the step again. The path is inside `GITHUB_WORKSPACE`. Cargo is pointed at that tree before any cargo/just step. + +`git diff HEAD -- .github/workflows/rust.yaml` is a restore-and-rewire (repo/ref/path + a patch step), not a net deletion of the checkout feature. + +## In-workspace path + +| Field | Old (illegal) | New | +|-------|---------------|-----| +| `repository` | `SurmountSystems/bao-tree` | `n0-computer/bao-tree` | +| `ref` | `76-keyed-bao` | `keyed-bao` | +| `path` | `../bao-tree` | `bao-tree` | + +`path: bao-tree` is `${GITHUB_WORKSPACE}/bao-tree`, i.e. `/home/runner/work/carbonado/carbonado/bao-tree`. `actions/checkout@v4` accepts that. + +Product `Cargo.toml` already uses `git = "https://github.com/n0-computer/bao-tree.git"`, `branch = "keyed-bao"`. The checkout matches that source, not the SurmountSystems `76-keyed-bao` sibling used only by local `just setup-bao-tree`. + +Local optional path is unchanged: `just setup-bao-tree` / `.cargo/config.toml.example` still talk about `../bao-tree`. + +`/bao-tree` is gitignored so an in-workspace clone is not committed. + +## How cargo is patched (CI-only) + +Committed `.cargo/config.toml` still only has the bitcoinpqc `[patch.crates-io]` block. That file is not overwritten. + +Each job, after the bao-tree checkout, runs the composite +`.github/actions/ci-patch-bao-tree`. That step: + +1. Fail-closes unless `${GITHUB_WORKSPACE}/bao-tree/Cargo.toml` exists. +2. **Appends** (does not replace) this table to `.cargo/config.toml`: + +```toml +[patch."https://github.com/n0-computer/bao-tree.git"] +bao-tree = { path = "${GITHUB_WORKSPACE}/bao-tree" } +``` + +The path is absolute so it does not depend on whether cargo resolves patch paths relative to `.cargo/` or the workspace root. The git URL matches `Cargo.toml` exactly. The bitcoinpqc patch stays in place. + +This append is job-local. It is not committed. Developers without a sibling or in-workspace tree keep using the public git dep. + +## Jobs covered + +Same six jobs as `git show HEAD:.github/workflows/rust.yaml`: + +| Job | Checkout restored | Patch before cargo | +|-----|-------------------|--------------------| +| `lint` | yes | yes (before `just fmt` / `just lint`) | +| `lint-wasm` | yes | yes (before `just lint-wasm`) | +| `desktop` | yes | yes (before `cargo test` / just) | +| `test-matrix` | yes | yes (before `cargo check`) | +| `web-check` | yes | yes (before `cargo check`) | +| `dual-backend-lean` | yes | yes (before nix + `just test-lean-ci`) | + +No `--all-features`. Tests were not weakened. No commit or push. + +## Process note + +This L2 session could not launch a workflow (host: workflows only from a top-level session). Inventory used `git show HEAD:.github/workflows/rust.yaml`, `Cargo.toml`, `.cargo/config.toml`, `.cargo/config.toml.example`, and `justfile`. diff --git a/.agents/reports/pr32-ci.md b/.agents/reports/pr32-ci.md new file mode 100644 index 0000000..0b0f0b7 --- /dev/null +++ b/.agents/reports/pr32-ci.md @@ -0,0 +1,54 @@ +# PR 32 CI diagnosis + +**PR:** https://github.com/bitmask-stack/carbonado/pull/32 +**Branch:** `lean` → `main` (draft, title “proven”) +**HEAD:** `b8c070ae64da98a9a2743dfdb48e8f75322b4f93` +**Date observed:** 2026-08-13 + +## What ran + +Workflows are not skipped, pending forever, or missing. One workflow exists (`Rust` / `.github/workflows/rust.yaml`). Both the `push` run ([101](https://github.com/bitmask-stack/carbonado/actions/runs/31745640857)) and the `pull_request` run ([102](https://github.com/bitmask-stack/carbonado/actions/runs/31745690786)) completed in about ten seconds as **failure**. Combined commit status is `pending` with **zero** commit statuses. That is normal: this repo uses Actions check runs, not the old status API. + +| Check name | Conclusion | Why | +|------------|------------|-----| +| `lint` | **failure** | Illegal sibling checkout (below) | +| `lint-wasm` | **failure** | Same | +| `desktop` | skipped | `needs: lint` | +| `test-matrix` | skipped | `needs: lint` | +| `dual-backend-lean` | skipped | `needs: lint` | +| `web-check` | skipped | `needs: lint-wasm` | + +Job names match `docs/TEST_CONTRACT.md` (`desktop`, `dual-backend-lean`, plus lint/matrix). This is not a branch-protection name mismatch. There are no PR comments about CI. Rustc, clippy, and tests never started. + +## Root cause + +Every job checked out `SurmountSystems/bao-tree` at `76-keyed-bao` with `path: ../bao-tree`. `actions/checkout@v4` refuses any path outside `GITHUB_WORKSPACE`. + +Quoted from `lint` / `lint-wasm` on run 102: + +``` +Repository path '/home/runner/work/carbonado/bao-tree' is not under '/home/runner/work/carbonado/carbonado' +``` + +Failed step: **Checkout bao-tree keyed fork (sibling for path dep)**. + +That step is leftover from an optional local path patch (`just dev-local-bao` / `.cargo/config.toml.example`). CI does not copy that example. Product `Cargo.toml` already uses a public git dep: + +`git+https://github.com/n0-computer/bao-tree.git?branch=keyed-bao` (lock pin `e82e744…`; branch is public). + +Committed `.cargo/config.toml` only patches `bitcoinpqc` to a public git rev. It does **not** force `../bao-tree`. So CI does not need a sibling checkout. + +Local `just check` looks fine because it never runs `actions/checkout`. Same illegal step already fails `main` the same way (run 99, 2026-07-09). + +Not the cause: YAML syntax, path filters, permissions, n0-computer fetch (never reached), dual-backend `--all-features` (PR already avoids that). + +## Fix applied (not committed, not pushed) + +Removed all six sibling `actions/checkout` steps from [`.github/workflows/rust.yaml`](../../.github/workflows/rust.yaml) and left a short comment. Cargo on CI will fetch `n0-computer/bao-tree` `keyed-bao` as `Cargo.toml` already says. + +After this file is on `lean`, re-run the PR workflow. Later jobs may still fail on real compile/test; this change only unblocks the first step. + +## Residual (not this outage) + +- `keyed-bao` is a moving branch; `/Cargo.lock` is gitignored, so CI is not pinned to a lockfile. +- `just setup-bao-tree` still clones `SurmountSystems` `76-keyed-bao`. Local optional path patch vs product git source can drift. Separate from this CI break. diff --git a/.github/actions/ci-patch-bao-tree/action.yml b/.github/actions/ci-patch-bao-tree/action.yml new file mode 100644 index 0000000..fa673b9 --- /dev/null +++ b/.github/actions/ci-patch-bao-tree/action.yml @@ -0,0 +1,35 @@ +name: Patch cargo to workspace bao-tree +description: > + Append a CI-only [patch] so cargo uses the in-workspace bao-tree checkout. + Does not overwrite the committed bitcoinpqc crates-io patch. + +runs: + using: composite + steps: + - name: Append bao-tree path patch + shell: bash + run: | + set -euo pipefail + cfg=".cargo/config.toml" + dest="${GITHUB_WORKSPACE}/bao-tree" + mkdir -p .cargo + if [[ ! -f "${dest}/Cargo.toml" ]]; then + echo "bao-tree checkout missing at ${dest} (expected path: bao-tree)" + exit 1 + fi + if [[ ! -f "${cfg}" ]]; then + echo "expected committed ${cfg} (bitcoinpqc patch); creating empty" + : > "${cfg}" + fi + if grep -Fq '[patch."https://github.com/n0-computer/bao-tree.git"]' "${cfg}"; then + echo "bao-tree cargo patch already present in ${cfg}" + else + { + printf '\n' + printf '[patch."https://github.com/n0-computer/bao-tree.git"]\n' + printf 'bao-tree = { path = "%s" }\n' "${dest}" + } >> "${cfg}" + echo "appended bao-tree path patch -> ${dest}" + fi + echo "---- ${cfg} ----" + cat "${cfg}" diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 3117469..1f5db73 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -7,6 +7,12 @@ on: env: CARGO_TERM_COLOR: always +# bao-tree is a git dep (Cargo.toml: n0-computer/bao-tree branch keyed-bao). +# Checkout it *inside* GITHUB_WORKSPACE (`path: bao-tree`). actions/checkout +# rejects anything outside the workspace, so `path: ../bao-tree` fails before rustc. +# After checkout, append a CI-only [patch] so cargo uses that tree (do not commit +# a machine-local path). Local optional sibling remains `just setup-bao-tree`. + jobs: lint: runs-on: ubuntu-latest @@ -14,12 +20,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (sibling for path dep) + - name: Checkout bao-tree keyed fork (in-workspace for path dep) uses: actions/checkout@v4 with: - repository: SurmountSystems/bao-tree - ref: 76-keyed-bao - path: ../bao-tree + repository: n0-computer/bao-tree + ref: keyed-bao + path: bao-tree + persist-credentials: false + + - name: Patch cargo to use workspace bao-tree + uses: ./.github/actions/ci-patch-bao-tree - uses: extractions/setup-just@v2 @@ -39,12 +49,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (sibling for path dep) + - name: Checkout bao-tree keyed fork (in-workspace for path dep) uses: actions/checkout@v4 with: - repository: SurmountSystems/bao-tree - ref: 76-keyed-bao - path: ../bao-tree + repository: n0-computer/bao-tree + ref: keyed-bao + path: bao-tree + persist-credentials: false + + - name: Patch cargo to use workspace bao-tree + uses: ./.github/actions/ci-patch-bao-tree - uses: extractions/setup-just@v2 @@ -64,12 +78,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (sibling for path dep) + - name: Checkout bao-tree keyed fork (in-workspace for path dep) uses: actions/checkout@v4 with: - repository: SurmountSystems/bao-tree - ref: 76-keyed-bao - path: ../bao-tree + repository: n0-computer/bao-tree + ref: keyed-bao + path: bao-tree + persist-credentials: false + + - name: Patch cargo to use workspace bao-tree + uses: ./.github/actions/ci-patch-bao-tree - uses: extractions/setup-just@v2 @@ -121,12 +139,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (sibling for path dep) + - name: Checkout bao-tree keyed fork (in-workspace for path dep) uses: actions/checkout@v4 with: - repository: SurmountSystems/bao-tree - ref: 76-keyed-bao - path: ../bao-tree + repository: n0-computer/bao-tree + ref: keyed-bao + path: bao-tree + persist-credentials: false + + - name: Patch cargo to use workspace bao-tree + uses: ./.github/actions/ci-patch-bao-tree - uses: dtolnay/rust-toolchain@stable with: @@ -151,12 +173,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (sibling for path dep) + - name: Checkout bao-tree keyed fork (in-workspace for path dep) uses: actions/checkout@v4 with: - repository: SurmountSystems/bao-tree - ref: 76-keyed-bao - path: ../bao-tree + repository: n0-computer/bao-tree + ref: keyed-bao + path: bao-tree + persist-credentials: false + + - name: Patch cargo to use workspace bao-tree + uses: ./.github/actions/ci-patch-bao-tree - uses: dtolnay/rust-toolchain@stable with: @@ -179,12 +205,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (sibling for path dep) + - name: Checkout bao-tree keyed fork (in-workspace for path dep) uses: actions/checkout@v4 with: - repository: SurmountSystems/bao-tree - ref: 76-keyed-bao - path: ../bao-tree + repository: n0-computer/bao-tree + ref: keyed-bao + path: bao-tree + persist-credentials: false + + - name: Patch cargo to use workspace bao-tree + uses: ./.github/actions/ci-patch-bao-tree - uses: extractions/setup-just@v2 @@ -221,4 +251,4 @@ jobs: RUST_BACKTRACE: 1 CARBONADO_LEAN_LIB: ${{ github.workspace }}/result-libcarbonado/lib CARBONADO_LEAN_INCLUDE: ${{ github.workspace }}/result-libcarbonado/include - LD_LIBRARY_PATH: ${{ github.workspace }}/result-libcarbonado/lib \ No newline at end of file + LD_LIBRARY_PATH: ${{ github.workspace }}/result-libcarbonado/lib diff --git a/.gitignore b/.gitignore index 93f1fa5..330ccf2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ /Cargo.lock *.7z +# CI (and optional local) in-workspace bao-tree checkout. Local `just setup-bao-tree` +# still uses the sibling ../bao-tree; do not commit a machine-local tree here. +/bao-tree + # Carbonado encode artifacts when CLI `-o` is omitted (defaults to `.`) recovered.bin recovered_dir/ From 1717286d08e4e8dedcc98390e8b8f137283d7e18 Mon Sep 17 00:00:00 2001 From: Hunter Beast Date: Mon, 24 Aug 2026 19:00:49 -0600 Subject: [PATCH 4/6] Update bao-tree and bump version to 0.7 --- .cargo/config.toml.example | 4 +- .github/workflows/rust.yaml | 106 +-- AGENTS.md | 47 +- CHANGELOG.md | 17 +- Carbonado.lean | 3 +- Carbonado/Compress.lean | 340 +++++++- Carbonado/Ffi.lean | 514 ----------- Carbonado/Filepack.lean | 7 +- Carbonado/Main.lean | 17 + Carbonado/Pipeline.lean | 2 +- Carbonado/RkyvFilepack.lean | 5 +- Carbonado/Slh.lean | 11 +- CarbonadoTest/Compress.lean | 52 +- CarbonadoTest/Pipeline.lean | 29 +- Cargo.toml | 22 +- README.md | 18 +- benches/crypto_bench.rs | 2 +- benches/parallel_bench.rs | 4 +- build.rs | 22 - carbonado-sys/Cargo.toml | 16 - carbonado-sys/build.rs | 79 -- carbonado-sys/src/lib.rs | 220 ----- doc/TEST_STRATEGY.md | 8 +- docs/ABI.md | 334 -------- docs/GAPS.md | 42 +- docs/LIMITS.md | 31 +- docs/PARITY.md | 18 +- docs/PROOFS.md | 6 +- docs/SPEC-MATRIX.md | 6 +- docs/TEST_CONTRACT.md | 333 +------- docs/VISION.md | 18 +- examples/slh_dsa_sidecar.rs | 6 +- flake.lock | 37 + flake.nix | 181 ++-- include/carbonado.h | 240 ------ justfile | 491 ++++++++--- nix/cargo-quality.nix | 217 +++++ nix/native/carbonado_abi.c | 725 ---------------- nix/native/carbonado_slh.c | 76 +- nix/native/carbonado_zstd.c | 4 +- nix/native/default.nix | 27 +- nix/tooling-purity.nix | 10 +- ref/README.md | 6 +- .../drivers/bao-vectors/src/main.rs | 2 +- rust-toolchain.toml | 8 + src/backend/mod.rs | 801 ------------------ src/bin/carbonado/main.rs | 8 +- src/constants.rs | 20 +- src/crypto.rs | 11 +- src/decoding.rs | 138 +-- src/directory/format_policy.rs | 8 +- src/directory/mod.rs | 5 +- src/encoding.rs | 31 +- src/error.rs | 16 +- src/file.rs | 476 +++++------ src/filepack.rs | 10 +- src/filepack_manifest.rs | 50 +- src/lib.rs | 63 +- src/paths.rs | 20 +- src/stream/bao.rs | 4 +- src/stream/compress.rs | 76 +- src/stream/crypto_stream.rs | 26 +- src/stream/decode.rs | 283 +------ src/stream/decode_async.rs | 69 +- src/stream/encode.rs | 355 +------- src/stream/fec.rs | 2 +- src/stream/mod.rs | 4 +- src/stream/parallel.rs | 2 +- src/stream/shard.rs | 2 +- src/stream/slice.rs | 91 +- src/stream/spool.rs | 34 - src/utils.rs | 2 +- tests/bao_keyed_contract.rs | 3 +- tests/codec.rs | 4 +- tests/common/inboard_parity.rs | 2 +- tests/common/mod.rs | 1 + tests/common/zstd_frame.rs | 159 ++++ tests/deprecation_aliases.rs | 5 +- tests/determinism_roundtrip.rs | 138 +-- tests/directory_archive.rs | 84 +- tests/fec_chaos.rs | 4 +- tests/fec_scrub_matrix.rs | 2 +- tests/filepack_interop.rs | 12 +- tests/fixtures/g9/README.md | 35 +- tests/fixtures/rkyv/README.md | 5 +- tests/format.rs | 36 +- tests/format_policy.rs | 6 +- tests/g9_cross_backend.rs | 195 +---- tests/lean_backend_phase2.rs | 512 ----------- tests/lean_backend_phase3.rs | 447 ---------- tests/lean_backend_phase4.rs | 676 --------------- tests/lean_backend_smoke.rs | 277 ------ tests/parallel_determinism.rs | 10 +- tests/serial_fec_path.rs | 2 +- tests/shard_fec_scrub.rs | 2 +- tests/sharding.rs | 4 +- tests/slh_outboard.rs | 10 +- tests/streaming.rs | 6 +- tests/streaming_async.rs | 15 +- tests/streaming_limits.rs | 8 +- tests/udp_fec_sim.rs | 6 +- tests/zstd_frame_params.rs | 204 +++++ 102 files changed, 2473 insertions(+), 7367 deletions(-) delete mode 100644 Carbonado/Ffi.lean delete mode 100644 build.rs delete mode 100644 carbonado-sys/Cargo.toml delete mode 100644 carbonado-sys/build.rs delete mode 100644 carbonado-sys/src/lib.rs delete mode 100644 docs/ABI.md delete mode 100644 include/carbonado.h create mode 100644 nix/cargo-quality.nix delete mode 100644 nix/native/carbonado_abi.c create mode 100644 rust-toolchain.toml delete mode 100644 src/backend/mod.rs create mode 100644 tests/common/zstd_frame.rs delete mode 100644 tests/lean_backend_phase2.rs delete mode 100644 tests/lean_backend_phase3.rs delete mode 100644 tests/lean_backend_phase4.rs delete mode 100644 tests/lean_backend_smoke.rs create mode 100644 tests/zstd_frame_params.rs diff --git a/.cargo/config.toml.example b/.cargo/config.toml.example index 8e4efb3..fdd4b89 100644 --- a/.cargo/config.toml.example +++ b/.cargo/config.toml.example @@ -1,7 +1,7 @@ # Optional local development overrides (copy to `.cargo/config.toml`). # -# Faster iteration: use a sibling checkout of the keyed bao-tree fork instead of -# fetching from git on every clean build. +# Faster iteration: use a sibling checkout of n0-computer/bao-tree (PR 78 merge +# SHA, see just setup-bao-tree) instead of fetching from git on every clean build. # # just setup-bao-tree # cp .cargo/config.toml.example .cargo/config.toml diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 1f5db73..458978b 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -4,14 +4,14 @@ on: - push - pull_request -env: - CARGO_TERM_COLOR: always - -# bao-tree is a git dep (Cargo.toml: n0-computer/bao-tree branch keyed-bao). +# bao-tree is a git dep (Cargo.toml: n0-computer/bao-tree rev = PR 78 merge SHA). # Checkout it *inside* GITHUB_WORKSPACE (`path: bao-tree`). actions/checkout # rejects anything outside the workspace, so `path: ../bao-tree` fails before rustc. # After checkout, append a CI-only [patch] so cargo uses that tree (do not commit # a machine-local path). Local optional sibling remains `just setup-bao-tree`. +env: + CARGO_TERM_COLOR: always + BAO_TREE_REV: dbc952e32cbda8ffd14c106b770e72987b01618e jobs: lint: @@ -20,11 +20,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (in-workspace for path dep) + - name: Checkout bao-tree (PR 78 merge, in-workspace for path dep) uses: actions/checkout@v4 with: repository: n0-computer/bao-tree - ref: keyed-bao + ref: ${{ env.BAO_TREE_REV }} path: bao-tree persist-credentials: false @@ -33,14 +33,17 @@ jobs: - uses: extractions/setup-just@v2 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "1.98.0" + components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 - name: Format check run: just fmt - - name: Lint (native; backend-rust + optional features, never --all-features) + - name: Lint (native; backend-rust + optional features) run: just lint lint-wasm: @@ -49,11 +52,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (in-workspace for path dep) + - name: Checkout bao-tree (PR 78 merge, in-workspace for path dep) uses: actions/checkout@v4 with: repository: n0-computer/bao-tree - ref: keyed-bao + ref: ${{ env.BAO_TREE_REV }} path: bao-tree persist-credentials: false @@ -62,8 +65,10 @@ jobs: - uses: extractions/setup-just@v2 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master with: + toolchain: "1.98.0" + components: rustfmt, clippy targets: wasm32-unknown-unknown - uses: Swatinem/rust-cache@v2 @@ -78,11 +83,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (in-workspace for path dep) + - name: Checkout bao-tree (PR 78 merge, in-workspace for path dep) uses: actions/checkout@v4 with: repository: n0-computer/bao-tree - ref: keyed-bao + ref: ${{ env.BAO_TREE_REV }} path: bao-tree persist-credentials: false @@ -91,7 +96,10 @@ jobs: - uses: extractions/setup-just@v2 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master + with: + toolchain: "1.98.0" + components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 @@ -101,13 +109,11 @@ jobs: RUST_BACKTRACE: 1 - name: Test (serial FEC path without parallel) - # Mutual exclusion: must name backend-rust under --no-default-features. run: cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path env: RUST_BACKTRACE: 1 - - name: Test (backend-rust + optional features; never --all-features) - # --all-features enables both backend-rust and backend-lean → compile_error!. + - name: Test (backend-rust + optional features) run: cargo test --features "async,async-tokio,man-gen" env: RUST_BACKTRACE: 1 @@ -139,25 +145,26 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (in-workspace for path dep) + - name: Checkout bao-tree (PR 78 merge, in-workspace for path dep) uses: actions/checkout@v4 with: repository: n0-computer/bao-tree - ref: keyed-bao + ref: ${{ env.BAO_TREE_REV }} path: bao-tree persist-credentials: false - name: Patch cargo to use workspace bao-tree uses: ./.github/actions/ci-patch-bao-tree - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master with: + toolchain: "1.98.0" + components: rustfmt, clippy targets: ${{ matrix.target }} - uses: Swatinem/rust-cache@v2 - - name: Check ${{ matrix.target }} (backend-rust + pqc/optional; never --all-features) - # --all-features enables both backends → compile_error!. + - name: Check ${{ matrix.target }} (backend-rust + pqc/optional) run: cargo check --target ${{ matrix.target }} --features "async,async-tokio,man-gen" - name: Check ${{ matrix.target }} (backend-rust only, no pqc) @@ -173,19 +180,21 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Checkout bao-tree keyed fork (in-workspace for path dep) + - name: Checkout bao-tree (PR 78 merge, in-workspace for path dep) uses: actions/checkout@v4 with: repository: n0-computer/bao-tree - ref: keyed-bao + ref: ${{ env.BAO_TREE_REV }} path: bao-tree persist-credentials: false - name: Patch cargo to use workspace bao-tree uses: ./.github/actions/ci-patch-bao-tree - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master with: + toolchain: "1.98.0" + components: rustfmt, clippy targets: wasm32-unknown-unknown - uses: Swatinem/rust-cache@v2 @@ -193,36 +202,18 @@ jobs: - name: Check wasm32 (backend-rust only, no pqc) run: cargo check --target wasm32-unknown-unknown --no-default-features --features "backend-rust" - # Dual-backend Phase 5 / G11 + R7 G8 full close: Linux lean full dual suite freeze. - # Normative command: `just test-lean-ci` = unfiltered cargo test under lean features - # (builds libcarbonado via nix if needed; fail-closed if .so missing). - # backend-rust full suite remains the `desktop` job above — never regress. - dual-backend-lean: + # Lean proofs + AOT demo. No libcarbonado / carbonado-sys / Cargo backend-lean. + lean-proofs: runs-on: ubuntu-latest needs: lint timeout-minutes: 180 steps: - uses: actions/checkout@v4 - - - name: Checkout bao-tree keyed fork (in-workspace for path dep) - uses: actions/checkout@v4 with: - repository: n0-computer/bao-tree - ref: keyed-bao - path: bao-tree - persist-credentials: false + submodules: false - - name: Patch cargo to use workspace bao-tree - uses: ./.github/actions/ci-patch-bao-tree - - - uses: extractions/setup-just@v2 - - - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 - - - name: Free disk for nix + cargo + - name: Free disk for nix run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true df -h @@ -234,21 +225,10 @@ jobs: extra_nix_config: | experimental-features = nix-command flakes - - name: Build libcarbonado (Lean AOT) - run: nix build .#libcarbonado -o result-libcarbonado - - - name: Fail-closed if libcarbonado missing + - name: Lean no-sorry + demo run: | set -euo pipefail - test -f result-libcarbonado/lib/libcarbonado.so - test -d result-libcarbonado/include - ls -la result-libcarbonado/lib/ - ls -la result-libcarbonado/include/ - - - name: Dual-backend lean freeze matrix (just test-lean-ci) - run: just test-lean-ci - env: - RUST_BACKTRACE: 1 - CARBONADO_LEAN_LIB: ${{ github.workspace }}/result-libcarbonado/lib - CARBONADO_LEAN_INCLUDE: ${{ github.workspace }}/result-libcarbonado/include - LD_LIBRARY_PATH: ${{ github.workspace }}/result-libcarbonado/lib + nix build .#checks.x86_64-linux.no-sorry + nix build .#checks.x86_64-linux.tooling-purity + nix build .#checks.x86_64-linux.carbonado + nix build .#checks.x86_64-linux.demo diff --git a/AGENTS.md b/AGENTS.md index c816e6a..c4072c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,41 +3,38 @@ **Project:** Carbonado (bitmask-stack/carbonado) **Mission:** Apocalypse-resistant archival format for consensus-critical data, with a focus on Bitcoin quantum resistance. -## Dual-backend product model (SSOT for parity) +## Product model (Rust engine + Lean proofs) | Concern | Role | |---------|------| -| **Rust** (`src/`, default `backend-rust`) | First-class engine; production library + CLI | -| **Rust tests** (`tests/`) | **Normative behavioral contract** — both backends must pass | -| **Lean 4** (`Carbonado/`, AOT `libcarbonado`) | Second engine: proofs + wire/C ABI compatible implementation | -| **Nix flakes** | Build Lean AOT, purity/no-sorry, package `libcarbonado` | +| **Rust** (`src/`, default `backend-rust`) | Production library + CLI | +| **Rust tests** (`tests/`) | **Normative behavioral contract** for the Rust engine | +| **Lean 4** (`Carbonado/`, `CarbonadoTest/`) | Spec + machine-checked proofs + AOT demo binary | +| **Nix flakes** | Lean compile, no-sorry, demo, Rust quality packages | | **`ref/`** | Pinned oracles (bao-tree, RustCrypto, zstd, …) | -**Parity bar (G8):** same tests on both engines — not Lean-only demos. See [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md), [docs/ABI.md](docs/ABI.md), [docs/PARITY.md](docs/PARITY.md), [docs/GAPS.md](docs/GAPS.md). +There is **no** `carbonado-sys` crate, **no** Cargo `backend-lean`, and **no** product C ABI (`libcarbonado.so` is gone). Do **not** claim G8 C-ABI parity. Lean remains proofs; Rust `tests/` remain the Rust contract. ```bash -cargo test # backend-rust (default features) -just test-lean-ci # backend-lean full dual suite (G8 closed at R7 / G11) -# Equivalent unfiltered lean suite: -# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +cargo test # Rust engine (default features) +just test-lean-ci # Lean no-sorry + AOT demo (nix); not cargo --features backend-lean ``` -Default `Cargo.toml` features already enable `backend-rust`. Adding `--features backend-lean` without `--no-default-features` enables **both** engines and hits `compile_error!` in `src/backend/mod.rs`. +`backend-rust` is an empty default marker so `--no-default-features --features "backend-rust,pqc,ots,cli"` still names the Rust engine. | Concern | Allowed | |---------|---------| -| Lean product logic, proofs, AOT | `Carbonado/`, `CarbonadoTest/` (not `Tests/` — collides with Rust `tests/` on Darwin) | -| Rust product + contract tests | `src/`, `tests/` (stay; do not delete for Lean purity) | +| Lean product logic, proofs, AOT demo | `Carbonado/`, `CarbonadoTest/` (not `Tests/` — collides with Rust `tests/` on Darwin) | +| Rust product + contract tests | `src/`, `tests/` | | Build Lean / packaging | Nix flakes (`flake.nix`, `nix/`) | +| Tiny C for Lean AOT **demo only** | `nix/native/carbonado_zstd.c`, `nix/native/carbonado_slh.c` (`@[extern]` into the Lean executable). Not a Rust-link target. | | Oracles / pins | `ref/` | -**Prove everything:** machine-checked Lean theorems **and** bit-match via the Rust suite on `backend-lean`. +**Prove everything:** machine-checked Lean theorems **and** the Rust suite on the Rust engine. Historical Lean AOT goldens under `tests/fixtures/g9/lean/` are decoded by Rust (`just test-g9`); they are not a live second Cargo engine. -**Agents:** implement and verify (`nix build`, `nix flake check`, `cargo test`, `just test-lean-ci` when touching dual-backend). Do **not** create commits or push; agents do not own git history. **Never regress `backend-rust` `cargo test`.** Phase 0–**5 closed** for dual-backend **allowlist + CI freeze** (G11). **Full-suite G8 closed at R7** (measured green; freeze = full dual suite). **G9 closed at R8** — no-compress body/headered/outboard both directions (`tests/g9_cross_backend.rs` + `tests/fixtures/g9/`). **W2 closed:** **W2d** codecode/decodec shipped (`tests/determinism_roundtrip.rs`); **W2a/W2b permanent** cross-engine Compression / directory encode residuals (same-engine re-encode green; decode interop SSOT). **R9 pure Lean depth closed** for G10 SLH FFI + seekable outboard slice C + rkyv dual-decode; **W3 closed** pure Lean rkyv encode + Directory/CLI (dual-suite composition may still use Rust bitcoinpqc / Rust rkyv as product SSOT — never claim dual-suite *requires* pure Lean). **R10 async dual policy closed** — dual freeze **never** requires `async`; optional `stream_decode_async` is dual-aware under `backend-lean`+`async` via R5 E1 `stream_decode` (disk O(encoded) staging; lean peak RAM **O(encoded + logical)**; not E2). **W5a/G1 closed** — permanent no `ref/carbonado-rust` product pin; live `src/`/`tests/` dual-suite SSOT; third-party `ref/` oracles only. Post-G8 residuals remaining: feature-gated `streaming_async` / `parallel_determinism` (permanent freeze exclusion); ~~W1a+W1b dual honesty~~ **closed** (public outboard stream E2 = S4 composition under lean; pure Lean chunked C residual); ~~W2d codecode/decodec~~ **closed**; ~~W2a/W2b~~ **permanent** (cross-engine compress/dir encode); ~~Lean rkyv encode/CLI CFP2~~ **W3 closed**; ~~W4a inboard O(slice) retain~~ **closed**; **W4b** permanent full-buffer C outboard slice; **W4c** permanent buffer-only zstd under lean; **W4d** permanent FEC O(body) + async encoded spool. +**Agents:** implement and verify (`nix build`, `nix flake check`, `cargo test`, `just test-lean-ci` when touching Lean). Do **not** create commits or push; agents do not own git history. **Never regress default `cargo test`.** Do not invent a fargo / Systems Lean integration. Do not re-add `backend-lean` as a stub. -**Gates:** `nix flake check` (Lean); `cargo test` (Rust default / CI `desktop`); `just test-lean-ci` (Lean full dual suite / CI `dual-backend-lean`); `just test-g9` for cross-backend matrix only. Phase 0–**5** + **R7 G8 full** + **R8 G9 matrix** + **R9 pure Lean depth** + **R10 async dual policy** + **W2 determinism** + **W4 memory** + **W5a G1** closed (permanent no `ref/carbonado-rust` product pin; live `src/`/`tests/` SSOT; W4b–d permanent residuals) — see GAPS R7–R10 + W2 + W4 + W5a + post-G8 residuals. - -**C ABI:** normative surface in `include/carbonado.h` + [docs/ABI.md](docs/ABI.md). Phase 2: body/headered/outboard/scrub/verify_slice live via Lean AOT `libcarbonado`. Phase 3 directory: **composition** (no new directory C symbols) — `just test-lean-phase3`. Phase 4: SLH/OTS dual-suite composition (Rust bitcoinpqc + CBOTS; no new SLH C symbols); CLI dual-engine for **directory** + buffer APIs — `just test-lean-phase4`. **R5 stream E1:** `stream_encode_inboard` / `stream_decode` / **encrypted** `stream_*_outboard` spool→Lean under `backend-lean` (O(logical); not E2 chunked). **W1a:** `file::decode_stream` MAC-before-body then Lean `decode_headered`. **W1b:** public **non-Compression** outboard stream S4 O(chunk/stripe) composition under lean (c4/c12 MVP; Compression under lean O(logical) bulk zstd); encrypted/inboard remain Lean E1. **R6:** outboard FEC erasure for truncated main (`decodeOutboardFec` ≡ Rust `fec_with_parity`). **R7:** full G8 close — `just test-lean-ci` = unfiltered lean suite (includes `bin_*`) / job `dual-backend-lean`. **R8:** G9 full cross-backend matrix — `just test-g9` / fixtures under `tests/fixtures/g9/`. **R10:** async dual policy — freeze excludes `async`; lean+async `stream_decode_async` → dual-aware `stream_decode` (E1). +**Gates:** `just check` / `just check-remote` (sequential fmt, clippy, nextest, then Lean, on the Nix remote builder); `just check-local` / `nix flake check` (same named checks, no force-remote; `nix flake check` is parallel); `cargo test` (Rust default / CI `desktop`); `just test-lean-ci` (Lean no-sorry + demo / CI `lean-proofs`); `just test-g9` for Rust decode of Lean goldens. GitHub Actions must not call `check-remote`. --- @@ -47,7 +44,7 @@ Default `Cargo.toml` features already enable `backend-rust`. Adding `--features | Axis | Status | |------|--------| | **Streaming / memory** | Phase 1 fused sync path shipped (`SeekableSpool`, streaming EtM, stripe FEC). **M1:** non-FEC verification (c6) uses `SeekWriteAt` (O(chunk) RAM); FEC verification retains O(FEC body) shard buffers under segment-wide RS geometry (`finish_into` avoids a second full logical `Vec`). Residuals: FEC O(segment body), O(sidecar) outboard verify, async encoded-body spool. See [doc/STREAMING_PARALLELISM.md](doc/STREAMING_PARALLELISM.md). **Not** the same as Bao slice/stream verification. | -| **Concurrency** | Phase 2 optional `async` / `stream_decode_async` (disk spool bridge; **R10:** dual-aware via R5 E1 under lean+async; freeze never requires `async`; WASM `NotImplemented`). | +| **Concurrency** | Phase 2 optional `async` / `stream_decode_async` (disk spool bridge; WASM `NotImplemented`). | | **Parallelism** | Phase 3 `parallel` feature (default on): `std::thread::scope` RS parity; WASM serial at runtime. No rayon; Tokio is not the CPU-parallel story. | **PQC:** `bitcoinpqc` 0.4, SLH-DSA-**SHA2**-128s sidecars only (`SLH_DSA_SHA2_128S`). Dev SHAKE-128s sidecars are incompatible — re-sign. @@ -113,7 +110,7 @@ These rules were added because the same misunderstandings have caused significan - Full documentation of every security-relevant decision (nonce scope, subkey labels, single-nonce behavior, sidecar signing rules, CTR counter management, etc.). - Real benchmarks proving hardware acceleration claims. - WASM support either works cleanly or has precise documented limitations. - - CI is strict: `cargo clippy --all-targets --features "async,async-tokio,man-gen" -D warnings` (never `--all-features` — that enables both `backend-rust` and `backend-lean` and hits `compile_error!`). Dual-backend lean gate: `just test-lean-ci`. + - CI is strict: `cargo clippy --all-targets --features "async,async-tokio,man-gen" -D warnings`. Lean proof gate: `just test-lean-ci` (nix no-sorry + demo). - Error handling is complete and specific; no lossy or generic errors hiding crypto failures. - Zeroization of secret material where practical. - Test coverage includes adversarial, large-payload, and cross-layer cases. @@ -167,12 +164,12 @@ The overarching principle is a **clean cryptographic break** (see §1). v1 ECIES | Header | ~160B with secp pubkey + Schnorr sig | 177B: MAGIC + payload_nonce + header_mac (64B) + bao hash + slh_pk + format + u32 chunk + lengths + meta | Separate header_mac (header-auth subkey) for integrity of public metadata. No secret key material. slh_pk moved to header (sig stays sidecar). | | Post-Quantum sigs | None (or ad-hoc) | SLH-DSA (SHA2-128s) **sidecars only** (`.cXX.slh`) | Sidecars preserve content-addressing and avoid bloat. bitcoinpqc 0.4 dogfooding per Surmount/BIP-360 mission. | | Forward Error Correction | zfec 4/8 (non-deterministic scrub for >~8KB, vulnerable to hits across all 8 chunks) | reed-solomon-erasure (RS 4/8): deterministic encode, reproducible scrub, better tolerance for distributed corruption ("chaos rays") while keeping 4/8 model | RS (BCH subclass) for pure determinism (critical for scrub re-encode + bao hash compare) and stronger erasure properties against partial corruption in every shard. Kept 4/8 for storage model ("application RAID"), alignment with 4 KiB slices/Bao leaves, and user intuition. | -| Verifiability (Bao) | bao 0.12/0.13 (1KB groups) | bao-tree fork: 4KB groups (BlockSize log=2) + keyed on format byte | 4KB aligns with disk sectors + reduces tree overhead. Keyed roots make Bao hash multi-dimensional (commits to Format pipeline for markets). | +| Verifiability (Bao) | bao 0.12/0.13 (1KB groups) | n0-computer/bao-tree (PR 78 keyed): 4KB groups (BlockSize log=2) + keyed on format byte | 4KB aligns with disk sectors + reduces tree overhead. Keyed roots make Bao hash multi-dimensional (commits to Format pipeline for markets). | | Slice / chunk counts | u16 limits (~64MiB FEC cap) | u32 (theoretical ~4GiB+ per segment) | Removed artificial caps for large archives. P1: `SLICE_LEN=4096` (one slice = one 4 KiB Bao leaf). | | Passphrase KDF | Argon2id wrapper inside library | Removed; caller responsibility (Argon2id recommended outside) | Keeps container security contract simple. Master key is 32/64B high-entropy material. | | Magic number | CARBONADO01 or similar (ECIES) | CARBONADO20\n (stable v2); 02 was dev transitional | Signals official stabilized 2.0 format. Old magic → clear external migration error. | | Version | Pre-0.7 (ECIES) | 2.0.0 (post-FEC + docs stabilization) | Marks end of fluid dev period. API now stable for semver. | -| Dependencies | ecies + secp + ... | aes+ctr+hmac+sha2 + reed-solomon-erasure + bao-tree fork + bitcoinpqc (optional pqc) | Clean break removal of ECIES-only crates. Hardware-accel friendly. | +| Dependencies | ecies + secp + ... | aes+ctr+hmac+sha2 + reed-solomon-erasure + n0-computer/bao-tree (keyed git pin) + bitcoinpqc (optional pqc) | Clean break removal of ECIES-only crates. Hardware-accel friendly. | | Optional hybrid layer | (the only encryption was the ECIES hybrid) | Pure symmetric is default. Added *optional* inner secp256k1-ECDH + ChaCha20-Poly1305 AEAD wrapped by outer AES-CTR + HMAC-EtM (via new hybrid_* and ecc_aead_* APIs) | "Maximal paranoia" defense-in-depth: different cipher families, different key-gen (ECDH+derive vs pure HMAC labels), HMAC + AEAD. See dedicated rationale below. Pure sym path and Encrypted bit semantics unchanged for normal use. secp here is *not* for the main container (no pubkeys in headers etc.). | #### Detailed Decision Rationales @@ -676,7 +673,7 @@ Current registered labels (must be kept in sync with code — full table in **Su - Suggestion: Use a keyed variant of the Bao tree (keyed on the format bitmask byte, or a small header prefix) so that the root hash cryptographically commits to which processing pipeline was used. - This would be extremely useful for data markets (see §9), because different format combinations (especially encrypted vs public) would produce distinguishable roots even for related data. - **Endianness for key material**: All integer fields in Carbonado (and in the Bao format itself) are little-endian. If a keyed Bao implementation derives a 32-byte key from header fields, those fields should be serialized in LE order for consistency. A minimal implementation that only keys on the single-byte `format` bitmask has no endianness issues at all. - - (Implemented) Original `bao` 0.13 lacked BlockSize and public keyed. Now using local SurmountSystems/bao-tree fork with BlockSize(2) for 4KB + keyed_hash on format byte (root commits to pipeline). See constants::BAO_BLOCK_SIZE and encoding::bao. Temporary fork pending upstream. + - (Implemented) Original `bao` 0.13 lacked BlockSize and public keyed. Now using n0-computer/bao-tree (PR 78 merge, git rev pin) with BlockSize(2) for 4KB + keyed_hash on format byte (root commits to pipeline). See constants::BAO_BLOCK_SIZE and stream::bao. Not published on crates.io yet. Because there are 16 possible format combinations, the same logical input can produce up to 16 different Bao hashes. In this sense the naming is **multi-dimensional**: - When the `Encrypted` bit is set (symmetric encryption), the hash primarily names an *encrypted+protected container*. @@ -957,7 +954,7 @@ This tension is acknowledged but not resolved in the current design. Carbonado i Remaining open (documented; active work called out): - **Pipeline memory residual (hard-break track):** fused encode/decode is O(chunk) spool + O(stripe) FEC encode; non-FEC verification decode is O(chunk) via `SeekWriteAt`; FEC verification decode retains O(FEC body) shard buffers (`FecInboardWriteAt`); outboard verify uses `PostOrderOutboard` + `ReadAt` (O(hash pair) per node); `stream_decode_async` fully spools encoded body to disk. Distinct from Bao **slice** verification (already O(slice) memory). See [doc/STREAMING_PARALLELISM.md](doc/STREAMING_PARALLELISM.md). - **WASM:** `cargo clippy --target wasm32-unknown-unknown --no-default-features --features "backend-rust"` is green (CI `lint-wasm`). **wasm32 + `pqc` probe (2026-07-08):** pointing global `CC_wasm32-unknown-unknown` at `libbitcoinpqc-bindings/wasm/clang-wasm32.sh` breaks **`zstd-sys`** (it tries to assemble `huf_decompress_amd64.S` with the wasm clang). Residual is build-env / dep CC scoping — not Carbonado crypto logic. Keep CI wasm lint **no-pqc** until bitcoinpqc (or zstd) wasm build is isolated. -- Bao crate: Surmount keyed bao-tree fork (`76-keyed-bao`), 4 KiB groups, `default-features = false` (no tokio/fs on wasm). Temporary until upstream. +- Bao crate: n0-computer/bao-tree git pin (PR 78 merge `dbc952e32cbda8ffd14c106b770e72987b01618e`), 4 KiB groups, `default-features = false` (no tokio/fs on wasm). Not a crates.io version yet. - reed-solomon-erasure: upstream "looking for maintainers"; periodic re-eval (no runtime issues). - (Perf: inboard `verify_slice` is O(slice) memory but O(N) encoded-byte I/O; outboard slice verify is O(slice) time+memory; scrub pre-check uses `verify_inboard_keyed` with O(1) retained decode memory (S5).) @@ -983,7 +980,7 @@ Major items completed in this session: - `chunk_index` widened u8 → u32 in Header + full auth coverage - `payload_nonce` semantics fully documented - All u16 slice bookkeeping (`EncodeInfo`, `extract_slice`, `verify_slice`, `scrub`) widened to u32, removing the ~64 MiB FEC segment cap -- Bao migrated from bao 0.13 (1KB fixed) to bao-tree fork with BAO_BLOCK_SIZE=from_chunk_log(2) for 4KB groups + keyed roots bound to format byte. P1: SLICE_LEN=4096, seekable slice module. Prefix+response for size in verifiable. +- Bao migrated from bao 0.13 (1KB fixed) to n0-computer/bao-tree with BAO_BLOCK_SIZE=from_chunk_log(2) for 4KB groups + keyed roots bound to format byte. P1: SLICE_LEN=4096, seekable slice module. Prefix+response for size in verifiable. - Theoretical max size calculation (≈17.18 billion GiB) - Extensive hardening of AGENTS.md, rustdocs, tests, CI, examples, and removal of all legacy ECIES/Nostr material - Full production verification gates passed repeatedly (strict clippy + tests) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8a05da..96e3833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,13 @@ All notable changes to the Carbonado crate and `carbonado` CLI are documented here. -## [Unreleased] — 2.1.0 (directory archive redesign) +## [0.7.0] — 2026-08-24 + +First crates.io release of the v2 format (`CARBONADO20\n`). Last published crate was **0.6.0** (v1/ECIES). In-tree `2.0.0` / `2.1.0` numbers were never published. + +### Removed + +- **`carbonado-sys` / product C ABI / Cargo `backend-lean`.** Lean remains proofs + AOT demo. Rust `tests/` remain the Rust contract. Do not claim G8 C-ABI parity. Tiny C remains only for Lean AOT demo `@[extern]` (zstd, SLH). ### Added @@ -17,6 +23,9 @@ All notable changes to the Carbonado crate and `carbonado` CLI are documented he ### Changed +- **Crate version 0.7.0** (crates.io next after 0.6.0). Format magic remains `CARBONADO20\n`. +- **Rust edition 2024 / rustc 1.98:** package `edition = "2024"`, `rust-version = "1.98.0"`, `rust-toolchain.toml` channel `1.98.0`. CI uses the same toolchain pin. Flake adds `oxalica/rust-overlay` for 1.98 in the dev shell and a `rustc-1_98` check (separate nixpkgs overlay so Lean AOT does not rebuild on Rust pin changes). +- **bao-tree upstream pin:** Cargo git dep is n0-computer/bao-tree at PR 78 merge `dbc952e32cbda8ffd14c106b770e72987b01618e` (keyed 4 KiB groups). Not a crates.io version. Replaces the `keyed-bao` branch pin and the earlier Surmount `76-keyed-bao` fork docs. - **M1 pipeline memory (hard break):** non-FEC verification decode (c6) uses `SeekWriteAt` over the post-preprocess spool (O(chunk) RAM; no full logical `Vec`). FEC verification uses `FecInboardWriteAt::finish_into` (stream logical bytes without a second full logical buffer; shard buffers remain O(FEC body) under segment-wide RS geometry). See `doc/STREAMING_PARALLELISM.md`. - **M2 outboard verify memory:** `stream_verification_outboard_verify` uses `PostOrderOutboard` + `ReadAt` (on-demand hash pairs) instead of copying the full sidecar into `PostOrderMemOutboard`. Streaming outboard decode keeps the sidecar on a disk spool. - **S5 scrub verify oracle:** `scrub` pre-check uses `verify_inboard_keyed` (`DiscardWriteAt` sink) instead of buffer `verification()` full-body staging; `scrub_outboard` pre-check uses `stream_verification_outboard_verify` with `io::sink()`. Memory tiers in `doc/STREAMING_PARALLELISM.md`. @@ -29,12 +38,10 @@ All notable changes to the Carbonado crate and `carbonado` CLI are documented he - **Error variant rename (breaking):** `InvalidPackIndex` → `InvalidFilepackManifest`. No enum alias is provided. - **Narrowed error taxonomy:** OTS proof size failures → `InvalidOtsProof`; Adamantine oversized `payload_len` → `InvalidAdamantinePayloadTooLarge`; directory decode integrity failures → `SegmentMainLenMismatch`, `ContentBlake3Mismatch`, `OutputPathEscape`, `OtsFeatureRequired`, `OtsProofRequired`. -### Deprecated (one release; `since = "2.1.0"`) +### Deprecated (one release; `since = "0.7.0"`) Crate-root type/const aliases: `PackIndex`, `PackEntry`, `PackSegmentRef`, `PACK_INDEX_VERSION`, `PACK_INDEX_FORMAT_LEVEL`, `PACK_INDEX_FORMAT_LEVEL_PUBLIC`, `PACK_INDEX_FORMAT_LEVEL_ENCRYPTED`, `MAX_PACK_ENTRIES`. The `carbonado::pack_index` module re-exports both new and deprecated names. -**Note:** Rust emits `deprecated` warnings only after the crate version reaches **2.1.0** (`Cargo.toml` is currently **2.0.0**). - ### Migration ```rust @@ -62,7 +69,7 @@ First public release. Symmetric v2 stack, streaming pipeline, seekable slices, s - **Symmetric v2 stack:** AES-256-CTR + full 64-byte HMAC-SHA512 Encrypt-then-MAC; HMAC-SHA512 BIP-32-style subkey derivation (`aes-ctr`, `etm-hmac`, `header-auth`). - **177-byte authenticated header:** `CARBONADO20\n` magic, `payload_nonce`, `header_mac`, Bao root, SLH-DSA public key slot, format bits, u32 `chunk_index`, lengths, metadata. -- **Keyed 4 KiB Bao groups** (local `bao-tree` fork): `SLICE_LEN=4096`; root commits to format pipeline byte. +- **Keyed 4 KiB Bao groups** (n0-computer/bao-tree keyed APIs; originally a local fork, now upstream PR 78): `SLICE_LEN=4096`; root commits to format pipeline byte. - **Seekable slice verification (P1):** `verify_slice_inboard_seekable`, `verify_slice_outboard` — O(slice) verified reads without full-stream materialization. - **Streaming-first encode/decode (P2):** `encode_stream` / `decode_stream`, `stream_encode_buffer`, `stream_encode_outboard`, `stream_decode_*`; buffer helpers in `encoding`/`decoding` delegate to `src/stream/`. - **Segment sharding (P3):** `encode_shard_stream` / `decode_shards_stream` for multi-segment logical files; `SHARDED` Adamantine flag when `PackEntry.segments.len() > 1`. diff --git a/Carbonado.lean b/Carbonado.lean index 63251b0..fe5ab72 100644 --- a/Carbonado.lean +++ b/Carbonado.lean @@ -21,7 +21,6 @@ import Carbonado.RkyvFilepack import Carbonado.Outboard import Carbonado.Directory import Carbonado.Cli -import Carbonado.Ffi /-- Library root namespace. -/ -def Carbonado.versionString : String := "lean-dual-backend-0" +def Carbonado.versionString : String := "lean-program-g-0" diff --git a/Carbonado/Compress.lean b/Carbonado/Compress.lean index 8bb4721..aba542a 100644 --- a/Carbonado/Compress.lean +++ b/Carbonado/Compress.lean @@ -2,8 +2,8 @@ Zstd compression for the Carbonado pipeline (Program F). Product AOT statically embeds libzstd (level 20) from the pinned `ref/zstd` - tree into `libcarbonado_native.a` via `nix/native` (`staticLibDeps`; **no** - shared `-lzstd`). + tree into the Lean AOT demo native archive via `nix/native` (`staticLibDeps`; + **no** shared `-lzstd`). This is not a Rust `-sys` product. **Evaluation model (LIMITS):** * `@[extern]` symbols are used by the **compiled** AOT binary. @@ -34,11 +34,227 @@ def zstdLevel : UInt32 := 20 /-- DoS cap on decompressed output (Rust `MAX_SEGMENT_MAIN_LEN` = 256 MiB). -/ def maxDecompressedLen : UInt64 := 256 * 1024 * 1024 -/-- Zstd frame magic (little-endian frame descriptor prefix). -/ +/-- Zstd frame magic (little-endian `0xFD2FB528`; RFC 8878 / `ref/zstd/doc/zstd_compression_format.md`). -/ def zstdMagic : List UInt8 := [0x28, 0xb5, 0x2f, 0xfd] theorem zstdMagic_length : zstdMagic.length = 4 := by native_decide +theorem zstdMagic_eq_literal : + zstdMagic = [0x28, 0xb5, 0x2f, 0xfd] := rfl + +/-- RFC `ZSTD_WINDOWLOG_ABSOLUTEMIN` (`ref/zstd/lib/common/zstd_internal.h`). -/ +def zstdWindowLogMin : Nat := 10 + +/-- Product one-shot and streaming frames: `Content_Checksum_flag` is clear. -/ +def zstdContentChecksum : Bool := false + +/-- Product frames: `Dictionary_ID_flag` is 0 (no dictionary). -/ +def zstdDictionaryIdFlag : UInt8 := 0 + +/-- Encoder-compliant unused bit (must be zero this spec version). -/ +def zstdUnusedBit : Bool := false + +/-- RFC reserved bit (decoder must reject if set). -/ +def zstdReservedBit : Bool := false + +/-- One-shot `ZSTD_compress` / `zstd::bulk` default: `contentSizeFlag = 1`. -/ +def zstdBufferContentSizeFlag : Bool := true + +/-- Streaming `copy_encode` with unknown size: `contentSizeFlag = 0`. -/ +def zstdStreamContentSizeFlag : Bool := false + +/-- Level-20 `windowLog` from `ref/zstd/lib/compress/clevels.h` `ZSTD_defaultCParameters[0][20]` + (srcSize > 256 KiB, and streaming with unknown size). -/ +def zstdLevel20WindowLogLarge : Nat := 25 + +/-- Level-20 `windowLog` for srcSize ≤ 256 KiB (`ZSTD_defaultCParameters[1][20]`). -/ +def zstdLevel20WindowLog256KiB : Nat := 18 + +/-- Level-20 `windowLog` for srcSize ≤ 128 KiB (`ZSTD_defaultCParameters[2][20]`). -/ +def zstdLevel20WindowLog128KiB : Nat := 17 + +/-- Level-20 `windowLog` for srcSize ≤ 16 KiB (`ZSTD_defaultCParameters[3][20]`). -/ +def zstdLevel20WindowLog16KiB : Nat := 14 + +/-- Frame-header parse errors (RFC reserved-bit + framing; not C status codes). -/ +inductive ZstdFrameError where + | truncatedHeader + | badMagic + | reservedBitSet + deriving DecidableEq, Repr + +/-- RFC `Frame_Header_Descriptor` (1 byte). Bit 7 is the high bit. -/ +structure FrameHeaderDescriptor where + contentSizeFlag : UInt8 + singleSegment : Bool + unusedBit : Bool + reservedBit : Bool + contentChecksum : Bool + dictionaryIdFlag : UInt8 + deriving DecidableEq, Repr + +/-- Parse the descriptor byte (total function). -/ +def parseFrameHeaderDescriptor (b : UInt8) : FrameHeaderDescriptor := + { + contentSizeFlag := (b >>> 6) &&& 3 + singleSegment := (b &&& 0x20) != 0 + unusedBit := (b &&& 0x10) != 0 + reservedBit := (b &&& 0x08) != 0 + contentChecksum := (b &&& 0x04) != 0 + dictionaryIdFlag := b &&& 3 + } + +/-- RFC `DID_Field_Size` from `Dictionary_ID_flag`. -/ +def didFieldSize (flag : UInt8) : Nat := + if flag == 0 then 0 + else if flag == 1 then 1 + else if flag == 2 then 2 + else if flag == 3 then 4 + else 0 + +/-- RFC `FCS_Field_Size` from `Frame_Content_Size_flag` + `Single_Segment_flag`. -/ +def fcsFieldSize (fcsFlag : UInt8) (singleSegment : Bool) : Nat := + if fcsFlag == 1 then 2 + else if fcsFlag == 2 then 4 + else if fcsFlag == 3 then 8 + else if fcsFlag == 0 then + if singleSegment then 1 else 0 + else 0 + +/-- `ref/zstd` `ZSTD_writeFrameHeader` FCS code when `contentSizeFlag` is set. -/ +def fcsCodeForSize (pledged : Nat) : Nat := + (if pledged >= 256 then 1 else 0) + + (if pledged >= 65536 + 256 then 1 else 0) + + (if pledged >= 4294967295 then 1 else 0) + +/-- Assemble the descriptor byte (`dictIDSizeCode + checksum<<2 + singleSegment<<5 + fcsCode<<6`). -/ +def frameHeaderDescriptionByte + (dictIdSizeCode checksumBit singleSegBit fcsCode : Nat) : UInt8 := + UInt8.ofNat (dictIdSizeCode + checksumBit * 4 + singleSegBit * 32 + fcsCode * 64) + +/-- RFC windowLog = 10 + Exponent (bits 7–3 of `Window_Descriptor`). -/ +def windowLogFromDescriptor (wd : UInt8) : Nat := + zstdWindowLogMin + (wd >>> 3).toNat + +/-- RFC `Window_Size = windowBase + (windowBase / 8) * Mantissa`. -/ +def windowSizeFromDescriptor (wd : UInt8) : Nat := + let exponent := (wd >>> 3).toNat + let mantissa := (wd &&& 7).toNat + let windowLog := zstdWindowLogMin + exponent + let windowBase := 2 ^ windowLog + windowBase + (windowBase / 8) * mantissa + +/-- Power-of-two window descriptor (mantissa 0): `(windowLog - 10) << 3`. -/ +def windowDescriptorByte (windowLog : Nat) : UInt8 := + UInt8.ofNat ((windowLog - zstdWindowLogMin) <<< 3) + +/-- `ZSTD_writeFrameHeader`: Single_Segment iff content size is present and window ≥ pledged. -/ +def singleSegment (contentSizeFlag : Bool) (windowSize pledgedSrcSize : Nat) : Bool := + contentSizeFlag && windowSize ≥ pledgedSrcSize + +/-- Parsed RFC `Frame_Header` (magic already consumed). `headerLen` counts magic. -/ +structure ParsedFrameHeader where + descriptor : FrameHeaderDescriptor + windowDescriptor : Option UInt8 + dictionaryId : Option UInt32 + contentSize : Option UInt64 + headerLen : Nat + deriving DecidableEq, Repr + +private def getUInt16LE (bs : ByteArray) (off : Nat) : UInt16 := + (bs.get! off).toUInt16 ||| ((bs.get! (off + 1)).toUInt16 <<< 8) + +/-- Parse magic + `Frame_Header`. Rejects RFC reserved bit. Unused bit is recorded, not rejected. -/ +def parseZstdFrameHeader (bs : ByteArray) : Except ZstdFrameError ParsedFrameHeader := + if bs.size < 5 then + .error .truncatedHeader + else if !(bs.get! 0 == 0x28 && bs.get! 1 == 0xb5 && bs.get! 2 == 0x2f && bs.get! 3 == 0xfd) then + .error .badMagic + else + let d := parseFrameHeaderDescriptor (bs.get! 4) + if d.reservedBit then + .error .reservedBitSet + else + let needWin := if d.singleSegment then 0 else 1 + let didSz := didFieldSize d.dictionaryIdFlag + let fcsSz := fcsFieldSize d.contentSizeFlag d.singleSegment + let headerLen := 5 + needWin + didSz + fcsSz + if bs.size < headerLen then + .error .truncatedHeader + else + let winOff := 5 + let windowDescriptor := + if d.singleSegment then none else some (bs.get! winOff) + let didOff := 5 + needWin + let dictionaryId := + if didSz == 0 then none + else if didSz == 1 then some (bs.get! didOff).toUInt32 + else if didSz == 2 then some (getUInt16LE bs didOff).toUInt32 + else if didSz == 4 then some (getUInt32LE bs didOff) + else none + let fcsOff := didOff + didSz + let contentSize := + if fcsSz == 0 then none + else if fcsSz == 1 then some (bs.get! fcsOff).toUInt64 + else if fcsSz == 2 then some ((getUInt16LE bs fcsOff).toUInt64 + 256) + else if fcsSz == 4 then some (getUInt32LE bs fcsOff).toUInt64 + else if fcsSz == 8 then some (getUInt64LE bs fcsOff) + else none + .ok { + descriptor := d + windowDescriptor := windowDescriptor + dictionaryId := dictionaryId + contentSize := contentSize + headerLen := headerLen + } + +/-- Shared product frame flags (both engines, both APIs). -/ +def productFrameHeaderOk (h : ParsedFrameHeader) : Bool := + !h.descriptor.unusedBit && + !h.descriptor.reservedBit && + !h.descriptor.contentChecksum && + h.descriptor.dictionaryIdFlag == 0 && + h.dictionaryId.isNone + +/-- One-shot / AOT frames for pledged size < 256 (1-byte FCS, Single_Segment, no window byte). + Matches AOT goldens empty/hello and G9 lean `outboard_c14` (26-byte plaintext). -/ +def productBufferSmallFrameOk (h : ParsedFrameHeader) (pledged : UInt64) : Bool := + productFrameHeaderOk h && + h.descriptor.singleSegment && + h.descriptor.contentSizeFlag == 0 && + h.windowDescriptor.isNone && + (match h.contentSize with + | some n => n == pledged + | none => false) + +/-- Rust `copy_encode` at level 20 with unknown size: no FCS, windowLog 25, mantissa 0. + Matches G9 rust `outboard_c14` (`28b52ffd0078…`). -/ +def productStreamUnknownSizeFrameOk (h : ParsedFrameHeader) : Bool := + productFrameHeaderOk h && + !h.descriptor.singleSegment && + h.descriptor.contentSizeFlag == 0 && + h.contentSize.isNone && + (match h.windowDescriptor with + | some wd => + windowLogFromDescriptor wd == zstdLevel20WindowLogLarge && (wd &&& 7) == 0 + | none => false) + +/-- AOT `ZSTD_compress` level-20 golden for `hello` (`Carbonado.Main` / Program F). -/ +def helloLevel20Golden : List UInt8 := + [0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x05, 0x29, 0x00, 0x00, 0x68, 0x65, 0x6c, 0x6c, 0x6f] + +/-- AOT `ZSTD_compress` level-20 golden for empty input. -/ +def emptyLevel20Golden : List UInt8 := + [0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00, 0x01, 0x00, 0x00] + +/-- Committed G9 lean `outboard_c14` main prefix (descriptor `0x20` + 1-byte FCS 26). -/ +def g9LeanC14Header : List UInt8 := + [0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x1a] + +/-- Committed G9 rust `outboard_c14` main prefix (descriptor `0x00` + window `0x78`). -/ +def g9RustC14Header : List UInt8 := + [0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x78] + /-- Pure status-prefix helper (identity payload). Used by extern Lean bodies. -/ def statusOkPayload (payload : ByteArray) : ByteArray := Id.run do @@ -167,4 +383,122 @@ theorem statusOk_payload_identity : | .error _ => false) = true := by native_decide +theorem zstdLevel_eq_20 : zstdLevel = 20 := rfl + +theorem zstdContentChecksum_off : zstdContentChecksum = false := rfl + +theorem zstdUnusedBit_off : zstdUnusedBit = false := rfl + +theorem zstdReservedBit_off : zstdReservedBit = false := rfl + +theorem zstdDictionaryIdFlag_none : zstdDictionaryIdFlag = 0 := rfl + +theorem zstdBufferContentSizeFlag_on : zstdBufferContentSizeFlag = true := rfl + +theorem zstdStreamContentSizeFlag_off : zstdStreamContentSizeFlag = false := rfl + +theorem zstdLevel20WindowLogLarge_eq : zstdLevel20WindowLogLarge = 25 := rfl + +theorem zstdLevel20WindowLog256KiB_eq : zstdLevel20WindowLog256KiB = 18 := rfl + +theorem zstdLevel20WindowLog128KiB_eq : zstdLevel20WindowLog128KiB = 17 := rfl + +theorem zstdLevel20WindowLog16KiB_eq : zstdLevel20WindowLog16KiB = 14 := rfl + +theorem zstdWindowLogMin_eq : zstdWindowLogMin = 10 := rfl + +/-- One-shot small frames: no dict, no checksum, Single_Segment, FCS code 0 → descriptor `0x20`. -/ +theorem aot_small_descriptor_byte : + frameHeaderDescriptionByte 0 0 1 0 = 0x20 := by native_decide + +/-- Streaming unknown size: no dict, no checksum, no Single_Segment, no FCS → descriptor `0x00`. -/ +theorem stream_unknown_descriptor_byte : + frameHeaderDescriptionByte 0 0 0 0 = 0x00 := by native_decide + +theorem level20_large_window_descriptor_byte : + windowDescriptorByte zstdLevel20WindowLogLarge = 0x78 := by native_decide + +theorem window_0x78_log : + windowLogFromDescriptor 0x78 = 25 := by native_decide + +theorem window_0x78_size : + windowSizeFromDescriptor 0x78 = 33554432 := by native_decide + +theorem fcs_code_below_256 : fcsCodeForSize 26 = 0 := by native_decide + +theorem fcs_field_small_single : fcsFieldSize 0 true = 1 := by native_decide + +theorem fcs_field_stream_none : fcsFieldSize 0 false = 0 := by native_decide + +theorem did_field_none : didFieldSize 0 = 0 := by native_decide + +theorem small_pledged_is_single_segment : + singleSegment true (2 ^ zstdLevel20WindowLogLarge) 26 = true := by native_decide + +theorem unknown_size_is_not_single_segment : + singleSegment false (2 ^ zstdLevel20WindowLogLarge) 26 = false := by native_decide + +theorem parse_empty_truncated : + (match parseZstdFrameHeader ByteArray.empty with + | .error .truncatedHeader => true + | _ => false) = true := by + native_decide + +theorem parse_four_bytes_truncated : + (match parseZstdFrameHeader (ofList [0x28, 0xb5, 0x2f, 0xfd]) with + | .error .truncatedHeader => true + | _ => false) = true := by + native_decide + +theorem parse_bad_magic : + (match parseZstdFrameHeader (ofList [0x00, 0x01, 0x02, 0x03, 0x20]) with + | .error .badMagic => true + | _ => false) = true := by + native_decide + +theorem parse_reserved_bit : + (match parseZstdFrameHeader (ofList [0x28, 0xb5, 0x2f, 0xfd, 0x08]) with + | .error .reservedBitSet => true + | _ => false) = true := by + native_decide + +theorem parse_hello_golden_header : + (match parseZstdFrameHeader (ofList helloLevel20Golden) with + | .ok h => productBufferSmallFrameOk h 5 + | .error _ => false) = true := by + native_decide + +theorem parse_empty_golden_header : + (match parseZstdFrameHeader (ofList emptyLevel20Golden) with + | .ok h => productBufferSmallFrameOk h 0 + | .error _ => false) = true := by + native_decide + +theorem parse_g9_lean_c14_header : + (match parseZstdFrameHeader (ofList g9LeanC14Header) with + | .ok h => productBufferSmallFrameOk h 26 + | .error _ => false) = true := by + native_decide + +theorem parse_g9_rust_c14_header : + (match parseZstdFrameHeader (ofList g9RustC14Header) with + | .ok h => productStreamUnknownSizeFrameOk h + | .error _ => false) = true := by + native_decide + +theorem hello_golden_has_magic : + hasZstdMagic (ofList helloLevel20Golden) = true := by native_decide + +theorem empty_golden_has_magic : + hasZstdMagic (ofList emptyLevel20Golden) = true := by native_decide + +theorem g9_headers_same_length : + g9LeanC14Header.length = g9RustC14Header.length := by native_decide + +theorem hello_golden_len : + helloLevel20Golden.length = 14 := by native_decide + +theorem empty_golden_len : + emptyLevel20Golden.length = 9 := by native_decide + end Carbonado.Compress diff --git a/Carbonado/Ffi.lean b/Carbonado/Ffi.lean deleted file mode 100644 index c7db416..0000000 --- a/Carbonado/Ffi.lean +++ /dev/null @@ -1,514 +0,0 @@ -/- - C ABI surface for dual-backend parity (docs/ABI.md). - - Pure helpers map Pipeline results to ABI error codes. `@[export]` entry points - return packed ByteArrays for the C glue in `nix/native/carbonado_abi.c`: - - status-prefixed: [u32 LE status][payload…] - encode body: [u32 LE status][u32 LE padding][u32 LE chunk_len] - [u32 LE bytes_ecc][u32 LE verifiable_slice_count] - [u32 LE bytes_compressed][u32 LE bytes_encrypted] - [32-byte hash][body…] - (success prefix 60 bytes; error = [status:4] only) - encode headered: [u32 LE status][u32 LE padding][u32 LE chunk_len] - [u32 LE bytes_ecc][u32 LE verifiable_slice_count] - [u32 LE bytes_compressed][u32 LE bytes_encrypted] - [archive…] - (success prefix 28 bytes; error = [status:4] only) - encode outboard: [u32 LE status][u32 LE padding][u32 LE chunk_len] - [u32 LE bytes_compressed][u32 LE bytes_encrypted] - [32 hash][u32 main_len][main][u32 ob_len][ob] - [u32 par_len][par] - (fixed prefix before segments: 52 bytes) - - C wrappers initialize the Lean runtime, convert buffers ↔ ByteArray, and expose - the stable `carbonado_*` symbols in `include/carbonado.h`. - - Phase 2 adds outboard / scrub / slice exports (additive on ABI version 1). - R3 adds compress/encrypt stage counters on encode packs (ABI version stays 1). --/ -import Carbonado.Constants -import Carbonado.Crypto.Util -import Carbonado.Bao.Product -import Carbonado.Header -import Carbonado.Pipeline -import Carbonado.Outboard -import Carbonado.Scrub - -namespace Carbonado.Ffi - -open Carbonado.Constants -open Carbonado.Crypto.Util -open Carbonado.Bao.Product -open Carbonado.Header -open Carbonado.Pipeline -open Carbonado.Outboard -open Carbonado.Scrub - -/-- ABI version (must match `include/carbonado.h` / docs/ABI.md). -/ -def abiVersion : UInt32 := 1 - -/-- Stable C error codes (docs/ABI.md). -/ -def ok : UInt32 := 0 -def errInvalidArgument : UInt32 := 1 -def errInvalidKeyLength : UInt32 := 2 -def errAuthentication : UInt32 := 3 -def errInvalidMagic : UInt32 := 4 -def errInvalidHeader : UInt32 := 5 -def errFec : UInt32 := 6 -def errBao : UInt32 := 7 -def errZstd : UInt32 := 8 -def errScrubUnnecessary : UInt32 := 9 -def errScrubFailed : UInt32 := 10 -def errNotImplemented : UInt32 := 11 -def errInternal : UInt32 := 12 -/-- Distinct from scrub recovery failure (docs/ABI.md Phase 2). -/ -def errScrubRequiresVerification : UInt32 := 13 - -/-- Collapse `PipelineError` into ABI codes (exhaustive; docs/ABI.md). - - R4 fidelity: Bao auth / short inboard prefix must not collapse into a single - `errBao` diagnostic — dual-suite `matches!` expects `AuthenticationFailed` and - `InvalidHeaderLength` respectively (same as pure Rust). `invalidSliceIndex` - stays `errBao` at the C boundary; Rust `lean::verify_slice` applies geometry - pre-checks to surface `InvalidSliceIndex { index, content_len }`. --/ -def ofPipelineError : PipelineError → UInt32 - | .invalidKeyLength => errInvalidKeyLength - | .payloadAuthenticationFailed | .headerAuthenticationFailed - | .baoAuthenticationFailed => errAuthentication - | .badMagic => errInvalidMagic - | .invalidHeaderLength | .truncatedBody | .invalidFieldLength - | .invalidPrefix => errInvalidHeader - | .unevenShards | .tooFewShards | .emptyShard | .incorrectShardSize - | .badGeometry | .paddingTooLarge | .singularMatrix => errFec - | .truncatedResponse | .trailingData - | .invalidRootLength | .invalidSliceIndex | .invalidSliceCount => errBao - | .compressionFailed | .decompressionFailed | .decompressOutputTooLarge - | .zstdInvalidInput => errZstd - | .unnecessaryScrub => errScrubUnnecessary - | .invalidScrubbedHash => errScrubFailed - | .scrubRequiresVerification => errScrubRequiresVerification - | .invalidCiphertextLength | .invalidNonceLength | .insufficientNonces => errInvalidArgument - | .invalidChunkSequence | .emptySegment => errInvalidArgument - -def masterOk (master : ByteArray) : Bool := - master.size == 32 || master.size == 64 - -/-- Append u32 little-endian. -/ -def pushU32LE (out : ByteArray) (x : UInt32) : ByteArray := - out.push (UInt8.ofNat (UInt32.toNat x % 256)) - |>.push (UInt8.ofNat (UInt32.toNat (x >>> 8) % 256)) - |>.push (UInt8.ofNat (UInt32.toNat (x >>> 16) % 256)) - |>.push (UInt8.ofNat (UInt32.toNat (x >>> 24) % 256)) - -/-- Pack `[u32 LE status][payload]`. -/ -def packStatus (code : UInt32) (payload : ByteArray) : ByteArray := - appendBA (pushU32LE ByteArray.empty code) payload - -/-- Pack encode body error: `[u32 LE status]` only (C parses status-first; see carbonado_abi.c). -/ -def packEncodeErr (code : UInt32) : ByteArray := - pushU32LE ByteArray.empty code - -/-- Encode metadata fields returned with body/headered/outboard success packs. -/ -structure EncodeMeta where - padding : UInt32 - chunkLen : UInt32 - bytesEcc : UInt32 - verifiableSliceCount : UInt32 - bytesCompressed : UInt32 - bytesEncrypted : UInt32 - deriving DecidableEq - -/-- Convert pipeline `EncodeInfo` length fields to u32 `EncodeMeta` (fail-closed on overflow). -/ -def encodeMetaOf (info : EncodeInfo) : Except UInt32 EncodeMeta := - match natToU32Field info.paddingLen with - | .error e => .error (ofPipelineError e) - | .ok pad => - match natToU32Field info.chunkLen with - | .error e => .error (ofPipelineError e) - | .ok cl => - match natToU32Field info.bytesEcc with - | .error e => .error (ofPipelineError e) - | .ok be => - match natToU32Field info.verifiableSliceCount with - | .error e => .error (ofPipelineError e) - | .ok vsc => - match natToU32Field info.bytesCompressed with - | .error e => .error (ofPipelineError e) - | .ok bc => - match natToU32Field info.bytesEncrypted with - | .error e => .error (ofPipelineError e) - | .ok be2 => - .ok { - padding := pad - chunkLen := cl - bytesEcc := be - verifiableSliceCount := vsc - bytesCompressed := bc - bytesEncrypted := be2 - } - -/-- Pack six u32 EncodeMeta fields after status (24 bytes). -/ -def pushEncodeMeta (out : ByteArray) (em : EncodeMeta) : ByteArray := - pushU32LE - (pushU32LE - (pushU32LE - (pushU32LE - (pushU32LE - (pushU32LE out em.padding) - em.chunkLen) - em.bytesEcc) - em.verifiableSliceCount) - em.bytesCompressed) - em.bytesEncrypted - -/-- Pack encode body success: - `[u32 LE status=0][padding:4][chunk_len:4][bytes_ecc:4][vsc:4] - [bytes_compressed:4][bytes_encrypted:4][32 hash][body]`. - - Requires `hash.size = 32`; otherwise packs `errInternal` (defensive layout guard). - Header size after status: 24 + 32 = 56; total prefix 60 bytes. --/ -def packEncodeOk (em : EncodeMeta) (hash body : ByteArray) : ByteArray := - if hash.size != 32 then - packEncodeErr errInternal - else - let hdr := pushEncodeMeta (pushU32LE ByteArray.empty ok) em - appendBA (appendBA hdr hash) body - -/-- Pack headered encode success: - `[status:4][pad:4][chunk:4][ecc:4][vsc:4][comp:4][enc:4][archive…]`. - - Total meta prefix 28 bytes (status + EncodeMeta). --/ -def packHeaderedOk (em : EncodeMeta) (archive : ByteArray) : ByteArray := - appendBA (pushEncodeMeta (pushU32LE ByteArray.empty ok) em) archive - -/-- Pack length-prefixed segment: `[u32 LE len][bytes]`, or `none` if len overflows u32. -/ -def packLenPrefixed? (payload : ByteArray) : Option ByteArray := - match natToU32Field payload.size with - | .error _ => none - | .ok n => some (appendBA (pushU32LE ByteArray.empty n) payload) - -/-- Pack outboard encode success: - `[status:4][padding:4][chunk_len:4][bytes_compressed:4][bytes_encrypted:4][hash:32] - [u32 main_len][main][u32 ob_len][ob][u32 par_len][par]`. - - Fixed prefix before segments: 52 bytes. Oversized segments → `errInternal`. --/ -def packOutboardOk (padding chunkLen bytesCompressed bytesEncrypted : UInt32) - (hash main ob par : ByteArray) : ByteArray := - if hash.size != 32 then - packEncodeErr errInternal - else - match packLenPrefixed? main with - | none => packEncodeErr errInternal - | some m => - match packLenPrefixed? ob with - | none => packEncodeErr errInternal - | some o => - match packLenPrefixed? par with - | none => packEncodeErr errInternal - | some p => - let hdr := - pushU32LE - (pushU32LE - (pushU32LE - (pushU32LE (pushU32LE ByteArray.empty ok) padding) - chunkLen) - bytesCompressed) - bytesEncrypted - let withHash := appendBA hdr hash - appendBA (appendBA (appendBA withHash m) o) p - -/-- Pure headered encode for FFI (explicit nonce; optional SLH pk + 8-byte metadata). - - Header always carries a 16-byte `payload_nonce` (Rust `file::encode`). Encrypted - formats require a caller-supplied 16-byte nonce; public formats use zeros when - nonce is empty/absent. - - `slhPublicKey` must be empty or 32 bytes (empty → zeros). `metadata` must be empty - or 8 bytes (empty → zeros). Wrong lengths → `errInvalidArgument`. - - Returns full archive + stage-counter `EncodeMeta` (R3). --/ -def encodeHeaderedBytes (master nonce plaintext slhPublicKey metadataBytes : ByteArray) - (format : UInt8) : Except UInt32 (ByteArray × EncodeMeta) := - if !masterOk master then .error errInvalidKeyLength - else if !(slhPublicKey.size == 0 || slhPublicKey.size == 32) then - .error errInvalidArgument - else if !(metadataBytes.size == 0 || metadataBytes.size == 8) then - .error errInvalidArgument - else - let fmt := FormatBits.ofUInt8 format - let n := - if fmt.encrypted then nonce - else if nonce.size == nonceLen then nonce - else replicate nonceLen 0 - let slh := if slhPublicKey.size == 32 then slhPublicKey else replicate 32 0 - let metaBytes := if metadataBytes.size == 8 then metadataBytes else replicate 8 0 - if n.size != nonceLen then - .error errInvalidArgument - else - match encodeHeadered master n plaintext fmt 0 slh metaBytes with - | .error e => .error (ofPipelineError e) - | .ok (_hdr, archive, info) => - match encodeMetaOf info with - | .error e => .error e - | .ok em => .ok (archive, em) - -/-- Pure headered decode for FFI. -/ -def decodeHeaderedBytes (master archive : ByteArray) : Except UInt32 ByteArray := - if !masterOk master then .error errInvalidKeyLength - else - match decodeHeadered master archive with - | .error e => .error (ofPipelineError e) - | .ok pt => .ok pt - -/-- Low-level body encode (embedded-nonce encrypt path; `headerPathEncrypt = false`). -/ -def encodeBodyBytes (master nonce plaintext : ByteArray) (format : UInt8) : - Except UInt32 (ByteArray × ByteArray × EncodeMeta) := - if !masterOk master then .error errInvalidKeyLength - else - let fmt := FormatBits.ofUInt8 format - if fmt.encrypted && nonce.size != nonceLen then - .error errInvalidArgument - else - let n := if fmt.encrypted then nonce else ByteArray.empty - match encodeBody master n plaintext fmt false with - | .error e => .error (ofPipelineError e) - | .ok enc => - match encodeMetaOf enc.info with - | .error e => .error e - | .ok em => .ok (enc.body, enc.baoHash, em) - -/-- Low-level body decode. -/ -def decodeBodyBytes (master hash body : ByteArray) (padding : UInt32) (format : UInt8) : - Except UInt32 ByteArray := - if !masterOk master then .error errInvalidKeyLength - else if hash.size != 32 then .error errInvalidArgument - else - let fmt := FormatBits.ofUInt8 format - -- Nonce unused for public / embedded-nonce decrypt (passed empty). - match decodeBody master ByteArray.empty hash body padding.toNat fmt false with - | .error e => .error (ofPipelineError e) - | .ok pt => .ok pt - -/-- Outboard encode. - - `headerPath ≠ 0` → encrypted bare main is `[tag|ct]` (nonce out-of-band). - `headerPath = 0` → encrypted bare main is `[nonce|tag|ct]` (embedded). - - Returns main/ob/par/hash + pad/chunk + compress/encrypt stage counters (R3). --/ -def encodeOutboardBytes (master nonce plaintext : ByteArray) (format headerPath : UInt8) : - Except UInt32 - (ByteArray × ByteArray × ByteArray × ByteArray × UInt32 × UInt32 × UInt32 × UInt32) := - if !masterOk master then .error errInvalidKeyLength - else - let fmt := FormatBits.ofUInt8 format - if fmt.encrypted && nonce.size != nonceLen then - .error errInvalidArgument - else - let n := if fmt.encrypted then nonce else ByteArray.empty - let hp := headerPath != 0 - match encodeOutboardBody master n plaintext fmt hp with - | .error e => .error (ofPipelineError e) - | .ok enc => - match natToU32Field enc.paddingLen with - | .error e => .error (ofPipelineError e) - | .ok pad => - match natToU32Field enc.chunkLen with - | .error e => .error (ofPipelineError e) - | .ok cl => - match natToU32Field enc.bytesCompressed with - | .error e => .error (ofPipelineError e) - | .ok bc => - match natToU32Field enc.bytesEncrypted with - | .error e => .error (ofPipelineError e) - | .ok be => - .ok (enc.main, enc.verificationOutboard, enc.fecParity, enc.baoHash, - pad, cl, bc, be) - -/-- Outboard decode. `headerPath`/`nonce` must match encode-time layout. -/ -def decodeOutboardBytes (master hash main verOutboard fecParity : ByteArray) - (padding : UInt32) (format headerPath : UInt8) (nonce : ByteArray) : - Except UInt32 ByteArray := - if !masterOk master then .error errInvalidKeyLength - else if hash.size != 32 then .error errInvalidArgument - else - let fmt := FormatBits.ofUInt8 format - let hp := headerPath != 0 - if hp && fmt.encrypted && nonce.size != nonceLen then - .error errInvalidArgument - else - let n := if hp && fmt.encrypted then nonce else ByteArray.empty - match decodeOutboardBody master hash main verOutboard fecParity padding.toNat fmt hp n with - | .error e => .error (ofPipelineError e) - | .ok pt => .ok pt - -/-- Inboard scrub (returns recovered body bytes). -/ -def scrubBytes (body hash : ByteArray) (padding : UInt32) (format : UInt8) : - Except UInt32 ByteArray := - if hash.size != 32 then .error errInvalidArgument - else - let fmt := FormatBits.ofUInt8 format - match scrubInboard body hash padding.toNat fmt with - | .error e => .error (ofPipelineError e) - | .ok recovered => .ok recovered - -/-- Outboard scrub (returns recovered bare main). -/ -def scrubOutboardBytes (main verOutboard fecParity hash : ByteArray) - (padding chunkLen : UInt32) (format : UInt8) : Except UInt32 ByteArray := - if hash.size != 32 then .error errInvalidArgument - else - let fmt := FormatBits.ofUInt8 format - match scrubOutboard main verOutboard fecParity hash padding.toNat chunkLen.toNat fmt with - | .error e => .error (ofPipelineError e) - | .ok bare => .ok bare - -/-- Inboard verify_slice / extract_slice (W4a: auth-first O(slice) retain; full body at C). - - Product path walks the full inboard artifact from offset 8 (no second response copy; - O(N) time) but retains only the requested slice bytes (O(slice) output). C ABI still - takes the full inboard body buffer as input. `count == 0` still authenticates first - (Lean C auth-first contract), unlike the Rust lean wrapper which short-circuits empty - success before C (parity with pure-Rust `verify_slice_inboard_seekable`). --/ -def verifySliceBytes (body hash : ByteArray) (index count : UInt32) (format : UInt8) : - Except UInt32 ByteArray := - if hash.size != 32 then .error errInvalidArgument - else if count.toNat == 0 then - match verifySliceInboardForFormat format hash body 0 0 with - | .error e => .error (ofPipelineError (ofBaoError e)) - | .ok data => .ok data - else - match verifySliceInboardForFormat format hash body index.toNat count.toNat with - | .error e => .error (ofPipelineError (ofBaoError e)) - | .ok data => .ok data - -/-- Seekable outboard verify_slice (O(slice + height) hash; full buffers at C ABI — W4b permanent). -/ -def verifySliceOutboardBytes (main outboard hash : ByteArray) - (index count : UInt32) (format : UInt8) : Except UInt32 ByteArray := - if hash.size != 32 then .error errInvalidArgument - else - match verifySliceOutboardForFormat format hash main outboard index.toNat count.toNat with - | .error e => .error (ofPipelineError (ofBaoError e)) - | .ok data => .ok data - -/-- Format verification key (32 bytes). -/ -def verificationKeyBytes (format : UInt8) : ByteArray := - carbonadoVerificationKey format - -/-- Round-trip self-check (public or encrypted with given nonce). -/ -def roundtripHeaderedOk (master nonce plaintext : ByteArray) (format : UInt8) : Bool := - match encodeHeaderedBytes master nonce plaintext ByteArray.empty ByteArray.empty format with - | .error _ => false - | .ok (arch, _em) => - match decodeHeaderedBytes master arch with - | .error _ => false - | .ok pt => ctEq pt plaintext - ---------------------------------------------------------------------------- --- @[export] surface for C glue (namespaced `l_` to avoid clashing with C ABI) ---------------------------------------------------------------------------- - -/-- Packed: always 32-byte key (status implied OK). -/ -@[export l_carbonado_verification_key] -def l_carbonado_verification_key (format : UInt8) : ByteArray := - verificationKeyBytes format - -/-- Packed success: EncodeMeta + archive; error: `[status:4]` only. - Optional `slhPublicKey` (0 or 32 B) and `metadataBytes` (0 or 8 B). -/ -@[export l_carbonado_encode_headered] -def l_carbonado_encode_headered (master nonce plaintext slhPublicKey metadataBytes : ByteArray) - (format : UInt8) : ByteArray := - match encodeHeaderedBytes master nonce plaintext slhPublicKey metadataBytes format with - | .error e => packEncodeErr e - | .ok (arch, em) => packHeaderedOk em arch - -/-- Packed: `[status:4][plaintext…]`. -/ -@[export l_carbonado_decode_headered] -def l_carbonado_decode_headered (master archive : ByteArray) : ByteArray := - match decodeHeaderedBytes master archive with - | .error e => packStatus e ByteArray.empty - | .ok pt => packStatus ok pt - -/-- Packed success: encode meta + hash + body; error: `[status:4]` only. -/ -@[export l_carbonado_encode] -def l_carbonado_encode (master nonce plaintext : ByteArray) (format : UInt8) : ByteArray := - match encodeBodyBytes master nonce plaintext format with - | .error e => packEncodeErr e - | .ok (body, hash, em) => - if hash.size != 32 then packEncodeErr errInternal - else packEncodeOk em hash body - -/-- Packed: `[status:4][plaintext…]`. -/ -@[export l_carbonado_decode] -def l_carbonado_decode (master hash body : ByteArray) (padding : UInt32) (format : UInt8) : ByteArray := - match decodeBodyBytes master hash body padding format with - | .error e => packStatus e ByteArray.empty - | .ok pt => packStatus ok pt - -/-- Packed outboard encode success layout; error `[status:4]`. - - `headerPath ≠ 0` → header-path `[tag|ct]` encrypt; `0` → embedded-nonce. --/ -@[export l_carbonado_encode_outboard] -def l_carbonado_encode_outboard (master nonce plaintext : ByteArray) - (format headerPath : UInt8) : ByteArray := - match encodeOutboardBytes master nonce plaintext format headerPath with - | .error e => packEncodeErr e - | .ok (main, ob, par, hash, pad, cl, bc, be) => - packOutboardOk pad cl bc be hash main ob par - -/-- Packed: `[status:4][plaintext…]`. - - `headerPath ≠ 0` requires 16-byte `nonce` for encrypted formats. --/ -@[export l_carbonado_decode_outboard] -def l_carbonado_decode_outboard (master hash main verOutboard fecParity : ByteArray) - (padding : UInt32) (format headerPath : UInt8) (nonce : ByteArray) : ByteArray := - match decodeOutboardBytes master hash main verOutboard fecParity padding format headerPath nonce with - | .error e => packStatus e ByteArray.empty - | .ok pt => packStatus ok pt - -/-- Packed: `[status:4][recovered…]`. -/ -@[export l_carbonado_scrub] -def l_carbonado_scrub (body hash : ByteArray) (padding : UInt32) (format : UInt8) : ByteArray := - match scrubBytes body hash padding format with - | .error e => packStatus e ByteArray.empty - | .ok rec => packStatus ok rec - -/-- Packed: `[status:4][recovered bare…]`. -/ -@[export l_carbonado_scrub_outboard] -def l_carbonado_scrub_outboard (main verOutboard fecParity hash : ByteArray) - (padding chunkLen : UInt32) (format : UInt8) : ByteArray := - match scrubOutboardBytes main verOutboard fecParity hash padding chunkLen format with - | .error e => packStatus e ByteArray.empty - | .ok bare => packStatus ok bare - -/-- Packed: `[status:4][slice bytes…]`. -/ -@[export l_carbonado_verify_slice] -def l_carbonado_verify_slice (body hash : ByteArray) (index count : UInt32) (format : UInt8) : - ByteArray := - match verifySliceBytes body hash index count format with - | .error e => packStatus e ByteArray.empty - | .ok data => packStatus ok data - -/-- Packed: `[status:4][slice bytes…]` from bare main + post-order outboard. -/ -@[export l_carbonado_verify_slice_outboard] -def l_carbonado_verify_slice_outboard (main outboard hash : ByteArray) - (index count : UInt32) (format : UInt8) : ByteArray := - match verifySliceOutboardBytes main outboard hash index count format with - | .error e => packStatus e ByteArray.empty - | .ok data => packStatus ok data - -theorem abiVersion_eq : abiVersion = 1 := by native_decide - -theorem masterOk_32 : masterOk (replicate 32 0) = true := by native_decide - -theorem masterOk_31 : masterOk (replicate 31 0) = false := by native_decide - -end Carbonado.Ffi diff --git a/Carbonado/Filepack.lean b/Carbonado/Filepack.lean index 781ba6a..44e6189 100644 --- a/Carbonado/Filepack.lean +++ b/Carbonado/Filepack.lean @@ -1,11 +1,8 @@ /- FilepackManifest v2 for Adamantine catalogs (Program G). - **Wire note (LIMITS / dual-suite):** Dual-suite directory archives use Rust - **rkyv** `FilepackManifestWire` as the normative Adamantine payload body. Under - `backend-lean`, `file::encode_directory` / `decode_directory` keep rkyv in Rust - and dispatch segment/catalog *crypto* through Lean C ABI (composition). Dual-suite - does **not** require pure Lean rkyv. + **Wire note:** Production directory archives use Rust **rkyv** + `FilepackManifestWire` as the Adamantine payload body. **W3 pure Lean product wire:** `Carbonado/RkyvFilepack.lean` provides bit-exact rkyv **encode** (`encodeRkyvManifest` / `encodeCatalogBody`) and **decode** diff --git a/Carbonado/Main.lean b/Carbonado/Main.lean index 8972689..b37ffdf 100644 --- a/Carbonado/Main.lean +++ b/Carbonado/Main.lean @@ -1053,6 +1053,9 @@ def runDemo : IO Unit := do | .ok ct => expectTrue "zstd hello magic" (hasZstdMagic ct) expectHex "zstd hello level20" ct "28b52ffd200529000068656c6c6f" + match parseZstdFrameHeader ct with + | .error e => fail s!"zstd hello frame header: {repr e}" + | .ok h => expectTrue "zstd hello product buffer frame" (productBufferSmallFrameOk h 5) match decompress ct with | .error e => fail s!"zstd decompress hello: {repr e}" | .ok pt => expectTrue "zstd hello roundtrip" (ctEq pt hello) @@ -1061,9 +1064,22 @@ def runDemo : IO Unit := do | .error e => fail s!"zstd empty compress: {repr e}" | .ok ct => expectHex "zstd empty level20" ct "28b52ffd2000010000" + match parseZstdFrameHeader ct with + | .error e => fail s!"zstd empty frame header: {repr e}" + | .ok h => expectTrue "zstd empty product buffer frame" (productBufferSmallFrameOk h 0) match decompress ct with | .error e => fail s!"zstd empty decompress: {repr e}" | .ok pt => expectTrue "zstd empty roundtrip" (ctEq pt ByteArray.empty) + -- Committed G9 c14 prefixes (W2a residual is descriptor 0x20 vs 0x00; not a bitstream proof) + match parseZstdFrameHeader (ofList g9LeanC14Header) with + | .error e => fail s!"g9 lean c14 header: {repr e}" + | .ok h => expectTrue "g9 lean c14 buffer frame" (productBufferSmallFrameOk h 26) + match parseZstdFrameHeader (ofList g9RustC14Header) with + | .error e => fail s!"g9 rust c14 header: {repr e}" + | .ok h => expectTrue "g9 rust c14 stream frame" (productStreamUnknownSizeFrameOk h) + expectTrue "zstd checksum flag off" (!zstdContentChecksum) + expectTrue "zstd dict id flag none" (zstdDictionaryIdFlag == 0) + expectTrue "zstd level20 windowLog large" (zstdLevel20WindowLogLarge == 25) -- Corrupt frame → decompressionFailed (not lumped) match decompress (ofList [0x00, 0x01, 0x02, 0x03]) with | .error .decompressionFailed => pure () @@ -1087,6 +1103,7 @@ def runDemo : IO Unit := do | .error e => fail s!"zstd zeros dec: {repr e}" | .ok pt => expectTrue "zstd zeros roundtrip" (ctEq pt zeros) IO.println "zstd goldens + roundtrip + error paths ok" + IO.println "zstd frame header params ok" -- Pipeline c2 (compression only) roundtrip under AOT zstd match roundtripBody master42 nonce11 hello (FormatBits.ofUInt8 2) with diff --git a/Carbonado/Pipeline.lean b/Carbonado/Pipeline.lean index 231bb14..6ba608b 100644 --- a/Carbonado/Pipeline.lean +++ b/Carbonado/Pipeline.lean @@ -325,7 +325,7 @@ def natToU32Field (n : Nat) : Except PipelineError UInt32 := /-- Headered encode: body + authenticated 177-byte Header (header-path encrypt). - Third component is pipeline `EncodeInfo` (stage counters for C ABI / dual-backend). + Third component is pipeline `EncodeInfo` (stage counters). -/ def encodeHeadered (master nonce plaintext : ByteArray) (format : FormatBits) (chunkIndex : UInt32) (slhPublicKey metadata : ByteArray) : diff --git a/Carbonado/RkyvFilepack.lean b/Carbonado/RkyvFilepack.lean index 4d0446e..e27204c 100644 --- a/Carbonado/RkyvFilepack.lean +++ b/Carbonado/RkyvFilepack.lean @@ -22,9 +22,8 @@ 2. Write contiguous ArchivedFilepackEntry records. 3. Write root at end. - Dual-suite directory wire remains **Rust rkyv SSOT** via composition for product - encode under `backend-lean`. Pure Lean directory/CLI (W3b) emits this rkyv body - so Rust dual-suite `decode_directory` can consume Lean-made catalogs. + Production directory encode remains **Rust rkyv**. Pure Lean directory/CLI emits + this rkyv body so Rust `decode_directory` can consume Lean-made catalogs. CFP2 remains available for pure-Lean demos only (`FilepackManifest.toWireBytes`); it is **not** byte-identical to rkyv. -/ diff --git a/Carbonado/Slh.lean b/Carbonado/Slh.lean index 582baea..133aa4e 100644 --- a/Carbonado/Slh.lean +++ b/Carbonado/Slh.lean @@ -6,13 +6,10 @@ * Public key (32 B) lives in Header.slh_public_key, not the sidecar * Signature is over the 32-byte Bao root of the target container - **G10 (R9):** real SLH-DSA via `@[extern]` into libbitcoinpqc objects linked - in `libcarbonado_native.a` (`nix/native/carbonado_slh.c`). Lean elaborator - bodies are fail-closed fallbacks (do **not** `native_decide` over live crypto). - AOT `Main` / C ABI / dual-suite composition exercise the real oracle. - - Dual-suite product SLH may still use Rust `bitcoinpqc` composition; pure Lean - is for `libcarbonado` purity. Composition remains SSOT for dual-suite wire. + Real SLH-DSA via `@[extern]` into libbitcoinpqc objects linked in the Lean AOT + demo native archive (`nix/native/carbonado_slh.c`). Lean elaborator bodies are + fail-closed fallbacks (do **not** `native_decide` over live crypto). The AOT + demo exercises the real oracle. Product SLH on the Rust library uses `bitcoinpqc`. -/ import Carbonado.Constants import Carbonado.Crypto.Util diff --git a/CarbonadoTest/Compress.lean b/CarbonadoTest/Compress.lean index 75b39b1..a396a75 100644 --- a/CarbonadoTest/Compress.lean +++ b/CarbonadoTest/Compress.lean @@ -93,6 +93,56 @@ theorem decompress_bit_clear : native_decide /-- Level constant is 20. -/ -theorem level_20 : zstdLevel = 20 := by native_decide +theorem level_20 : zstdLevel = 20 := zstdLevel_eq_20 + +theorem magic_literal : zstdMagic = [0x28, 0xb5, 0x2f, 0xfd] := zstdMagic_eq_literal + +theorem checksum_off : zstdContentChecksum = false := zstdContentChecksum_off + +theorem no_dictionary : zstdDictionaryIdFlag = 0 := zstdDictionaryIdFlag_none + +theorem buffer_content_size_on : zstdBufferContentSizeFlag = true := zstdBufferContentSizeFlag_on + +theorem stream_content_size_off : zstdStreamContentSizeFlag = false := zstdStreamContentSizeFlag_off + +theorem window_log_large : zstdLevel20WindowLogLarge = 25 := zstdLevel20WindowLogLarge_eq + +theorem aot_small_fd : frameHeaderDescriptionByte 0 0 1 0 = 0x20 := aot_small_descriptor_byte + +theorem stream_unknown_fd : frameHeaderDescriptionByte 0 0 0 0 = 0x00 := stream_unknown_descriptor_byte + +theorem window_byte_25 : windowDescriptorByte 25 = 0x78 := level20_large_window_descriptor_byte + +theorem window_0x78 : windowLogFromDescriptor 0x78 = 25 := window_0x78_log + +theorem hello_frame : + (match parseZstdFrameHeader (ofList helloLevel20Golden) with + | .ok h => productBufferSmallFrameOk h 5 + | .error _ => false) = true := parse_hello_golden_header + +theorem empty_frame : + (match parseZstdFrameHeader (ofList emptyLevel20Golden) with + | .ok h => productBufferSmallFrameOk h 0 + | .error _ => false) = true := parse_empty_golden_header + +theorem g9_lean_c14_frame : + (match parseZstdFrameHeader (ofList g9LeanC14Header) with + | .ok h => productBufferSmallFrameOk h 26 + | .error _ => false) = true := parse_g9_lean_c14_header + +theorem g9_rust_c14_frame : + (match parseZstdFrameHeader (ofList g9RustC14Header) with + | .ok h => productStreamUnknownSizeFrameOk h + | .error _ => false) = true := parse_g9_rust_c14_header + +theorem reserved_rejected : + (match parseZstdFrameHeader (ofList [0x28, 0xb5, 0x2f, 0xfd, 0x08]) with + | .error .reservedBitSet => true + | _ => false) = true := parse_reserved_bit + +theorem bad_magic : + (match parseZstdFrameHeader (ofList [0x00, 0x01, 0x02, 0x03, 0x20]) with + | .error .badMagic => true + | _ => false) = true := parse_bad_magic end CarbonadoTest.Compress diff --git a/CarbonadoTest/Pipeline.lean b/CarbonadoTest/Pipeline.lean index eb6081e..5c5e543 100644 --- a/CarbonadoTest/Pipeline.lean +++ b/CarbonadoTest/Pipeline.lean @@ -20,7 +20,6 @@ import Carbonado.Scrub import Carbonado.Shard import Carbonado.Fec.Inboard import Carbonado.Bao.Product -import Carbonado.Ffi import CarbonadoTest.Scaffold namespace CarbonadoTest.Pipeline @@ -35,7 +34,6 @@ open Carbonado.Scrub open Carbonado.Shard open Carbonado.Fec.Inboard open Carbonado.Bao.Product -open Carbonado.Ffi private def master42 : ByteArray := replicate 32 0x42 private def nonce11 : ByteArray := replicate 16 0x11 @@ -409,29 +407,10 @@ theorem c15_odd : formatC15.toUInt8 % 2 = 1 := by native_decide theorem c14_even : formatC14.toUInt8 % 2 = 0 := by native_decide -/-! ## R2 headered FFI: wrong-length SLH/meta fail-closed (encodeHeaderedBytes) - - C ABI is length-implicit (null → empty BA; non-null always copies fixed 32/8). - Wrong sizes are only expressible on the Lean pure/export ByteArray surface. --/ - -/-- SLH pk size ∉ {0, 32} → errInvalidArgument (before encode). -/ -theorem encode_headered_bytes_bad_slh_len : - (match encodeHeaderedBytes master42 nonce11 (utf8 "x") (replicate 16 0) ByteArray.empty 0 with - | .error e => e == errInvalidArgument - | .ok _ => false) = true := by - native_decide - -/-- Metadata size ∉ {0, 8} → errInvalidArgument (before encode). -/ -theorem encode_headered_bytes_bad_meta_len : - (match encodeHeaderedBytes master42 nonce11 (utf8 "x") ByteArray.empty (replicate 4 0) 0 with - | .error e => e == errInvalidArgument - | .ok _ => false) = true := by - native_decide - -/-- Empty SLH + empty meta accepted (zeros on wire) — control for length gates. -/ -theorem encode_headered_bytes_empty_slh_meta_ok : - (match encodeHeaderedBytes master42 nonce11 (utf8 "x") ByteArray.empty ByteArray.empty 0 with +/-- Zero SLH pk + zero metadata accepted on the headered path. -/ +theorem encode_headered_zero_slh_meta_ok : + (match encodeHeadered master42 nonce11 (utf8 "x") (FormatBits.ofUInt8 0) 0 + zeroSlhPk zeroMeta with | .ok _ => true | .error _ => false) = true := by native_decide diff --git a/Cargo.toml b/Cargo.toml index 246f435..9e8f8d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "carbonado" -edition = "2021" -version = "2.0.0" +edition = "2024" +rust-version = "1.98.0" +version = "0.7.0" license = "MIT" description = "An apocalypse-resistant data storage format for the truly paranoid. Fully symmetric AES-256-CTR + HMAC-SHA512 (EtM) container. Callers supply high-entropy master keys; passphrase KDF (e.g. Argon2id) is caller responsibility. SLH-DSA signatures are provided only as sidecars (via `bitcoinpqc`). Clean cryptographic break from ECIES — v1 encrypted archives require external migration." documentation = "https://docs.rs/carbonado" @@ -14,17 +15,18 @@ include = ["src/**/*", "LICENSE", "README.md", "doc/man/*.1", "doc/man/README.md [dependencies] # bao kept for Hash re-export (blake3::Hash), pub re-export in lib, and legacy error type bridging. -# All Bao logic now uses the local keyed fork for 4KB chunk groups + format-keyed roots. +# All Bao logic uses n0-computer/bao-tree keyed 4KB groups + format-keyed roots. bao = "0.13" -# Keyed Bao from n0-computer/bao-tree PR 78 (rklaehn `keyed-bao`, builds on #77). +# Keyed Bao: n0-computer/bao-tree PR 78 merged to main (not on crates.io yet). +# Pin the merge commit, not a branch and not a crates.io version. # Default 4KB groups (BlockSize::from_chunk_log(2)) via BAO_BLOCK_SIZE. Keyed mode # makes root = keyed_hash(key_from_format, data) so the Bao hash commits to the # exact format pipeline (multi-dimensional naming). See AGENTS.md and # constants::BAO_BLOCK_SIZE. # default-features = false: crate defaults pull in tokio/fs and break some # cross/wasm targets. We only need sync keyed + validate. -bao_tree = { package = "bao-tree", git = "https://github.com/n0-computer/bao-tree.git", branch = "keyed-bao", default-features = false, features = ["validate"] } +bao_tree = { package = "bao-tree", git = "https://github.com/n0-computer/bao-tree.git", rev = "dbc952e32cbda8ffd14c106b770e72987b01618e", default-features = false, features = ["validate"] } futures-lite = { version = "2", optional = true, default-features = false, features = ["std"] } tokio = { version = "1", features = ["rt"], optional = true } bitmask-enum = "2.1.0" @@ -72,20 +74,16 @@ getrandom = { version = "0.2", features = ["js"] } # bitcoinpqc 0.4 (WASM-capable build.rs); SLH-DSA-SHA2-128s sidecars. # Local mirror lag: `.cargo/config.toml` patches 0.4 until 2026-07-18 (see lift checklist there). bitcoinpqc = { version = "0.4", optional = true } -# Dual-backend: Lean AOT C library (docs/ABI.md, docs/TEST_CONTRACT.md). -carbonado-sys = { path = "carbonado-sys", optional = true } # Note: ecies + libsecp256k1-core + nostr + secp256k1 removed (clean break from old ECIES design and Nostr bech32/npub helpers). # Future quantum-resistant key formats (qpub etc.) will be handled separately. [features] -# Default: pure Rust engine + existing features. Dual-backend: see backend-rust / backend-lean. +# Default: Rust engine + existing features. There is no Cargo Lean backend +# (no carbonado-sys / libcarbonado). Lean lives in Carbonado/ as proofs + AOT demo. default = ["backend-rust", "pqc", "ots", "cli", "parallel"] -# Pure Rust implementation (default engine). +# Empty marker kept so `--features backend-rust` / serial-FEC recipes stay valid. backend-rust = [] -# Lean AOT engine via carbonado-sys / libcarbonado (G8). Requires CARBONADO_LEAN_LIB. -# `require-lib` fail-closes the build when the AOT shared library is missing. -backend-lean = ["dep:carbonado-sys", "carbonado-sys/require-lib"] pqc = ["dep:bitcoinpqc"] # OpenTimestamps stub stamping (offline/testable; no network calendar in default build). ots = [] diff --git a/README.md b/README.md index f98379d..3e0370a 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ Here is what changed and why we made each decision: Use `carbonado::crypto::{hybrid_encrypt, hybrid_decrypt, SecpPublicKey, SecpSecretKey, ...}` (and the lower `ecc_aead_*` if desired). **Composition**: hybrid replaces the encryption step. For full archives with header/FEC/Bao, run hybrid on (optionally compressed) data first, then continue with zfec/bao using a format that does *not* have the Encrypted bit set (standard decode paths will not attempt a pure-symmetric decrypt). On read, recover the hybrid-blob then call hybrid_decrypt with master + recipient secret. The outer EtM of the hybrid still uses your master key for the wrap. Pure symmetric (Encrypted bit) stays the default single-layer path. This is deliberate defense-in-depth for the truly paranoid. -- **Magic number and versioning**: We bumped the crate to version 2.0.0 and changed the magic number at the start of every file to `CARBONADO20\n`. - The old development magic (`CARBONADO02\n`) will be rejected with a clear error. This marks the point where the format is considered stabilized. +- **Magic number and versioning**: The crate version on crates.io is **0.7.0** (after 0.6.0 v1). The on-disk format magic is `CARBONADO20\n`. + The old development magic (`CARBONADO02\n`) will be rejected with a clear error. This marks the point where the v2 format is considered stabilized. - **Clean break on old files**: The library will not read or write files created with the old ECIES design. If you have old encrypted archives, you must extract them with an older version of the tools and re-encode them with a fresh master key. We made this decision so the code stays simple and we don't have to carry security baggage from the old design forever. @@ -345,24 +345,26 @@ Code, dependencies, and programs can be vendored and preserved wherever they are ## Development -Requires [just](https://github.com/casey/just), [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`), and a keyed `bao-tree` sibling at `../bao-tree` (branch `76-keyed-bao`): +Requires [just](https://github.com/casey/just), [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`), and **Rust 1.98** (edition 2024; see `rust-toolchain.toml`). Cargo fetches keyed `bao-tree` from n0-computer at the PR 78 merge SHA. An optional sibling checkout at `../bao-tree` speeds clean builds (`just setup-bao-tree`): ```bash -just setup-bao-tree # once, if ../bao-tree is missing +just setup-bao-tree # optional; pins n0-computer/bao-tree at the merge SHA just # list recipes just all # everything (fmt, lint, tests, release build, source grep) ``` | Recipe | What it does | |--------|----------------| -| `just fmt` | Formatting | +| `just check` / `just check-remote` | Full sequential gate on the Nix remote builder: fmt, clippy, nextest, then Lean proofs/demo. | +| `just check-local` | Same sequence as host Nix flake checks (this machine may run rustc) | +| `just fmt` | `cargo fmt --all -- --check` (CI `lint` job; does not write) | | `just lint` | Clippy **and** source checks (no v1 ECIES, prod `unwrap`, magic string, etc.) | | `just test` | Full test suite (`backend-rust` default) | | `just test-smoke` | Slice/streaming/sharding/bao contract tests | -| `just test-lean-ci` | Dual-backend lean **full suite** freeze (G8 closed at R7; needs Nix + `libcarbonado`; CI `dual-backend-lean`) | +| `just test-lean-ci` | Lean no-sorry + AOT demo (`nix`); not a Cargo Lean engine. CI job `lean-proofs`. | | `just build` + `just test-cli` | Release binary + CLI tests | -**Dual-backend (Rust + Lean AOT):** default `cargo test` is pure Rust. Lean engine tests require `nix build .#libcarbonado -o result-libcarbonado`, then `CARBONADO_LEAN_LIB` / `CARBONADO_LEAN_INCLUDE` / `LD_LIBRARY_PATH` (or just `just test-lean-ci`, which builds and fail-closes if the shared library is missing). CI runs both: job `desktop` (`backend-rust`) and job `dual-backend-lean` (`just test-lean-ci`). Full lean suite parity (**G8**) is **closed at R7** — freeze = full dual suite under lean features; see [docs/GAPS.md](docs/GAPS.md) and [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md). +**Rust engine + Lean proofs:** default `cargo test` is the production library. Lean is spec + theorems + an AOT demo (`nix build .#carbonado`). There is no `carbonado-sys` / `libcarbonado` Rust link and no Cargo `backend-lean`. Do not claim G8 C-ABI parity. See [docs/GAPS.md](docs/GAPS.md) and [docs/TEST_CONTRACT.md](docs/TEST_CONTRACT.md). CI runs the same recipes — see `.github/workflows/rust.yaml`. @@ -446,7 +448,7 @@ Running scrub on an input that has no errors in it actually returns an error; th The 4/8 RS parameters mean only 4 valid shards are needed while 8 are stored — half can fail. This roughly doubles payload size (on top of encryption and Bao overhead). Shard size aligns with 4 KiB Bao slice/leaf geometry (`SLICE_LEN=4096`). -Carbonado now uses 4 KiB chunk groups for Bao trees (via the local keyed bao-tree fork at BlockSize log=2). Slices for verification are 4 KiB content units (`SLICE_LEN=4096`, one slice = one Bao leaf). This aligns with 4 KiB SSD/HDD sectors and reduces tree overhead for small and large files. The root hash is keyed on the format bitmask for multi-dimensional naming. +Carbonado now uses 4 KiB chunk groups for Bao trees (n0-computer/bao-tree keyed hashing, BlockSize log=2). Slices for verification are 4 KiB content units (`SLICE_LEN=4096`, one slice = one Bao leaf). This aligns with 4 KiB SSD/HDD sectors and reduces tree overhead for small and large files. The root hash is keyed on the format bitmask for multi-dimensional naming. Storage providers will not need to use RAID to protect storage volumes so long as `carbonadod` is configured to store archive chunks on 8 separate storage volumes. In case a volume fails, scrubbing will recover the missing data. When data is served, only 4 of the chunks are needed. This results in a sort of user-level "application RAID", which is inline with Carbonado's design principles of being a flexible format with user-friendly configuration options. It's designed to be as approachable for "Uncle Jim" hobbyists to use as it is for professional mining datacenters bagged in FIL or XCH. diff --git a/benches/crypto_bench.rs b/benches/crypto_bench.rs index 0cf5944..c1cf32f 100644 --- a/benches/crypto_bench.rs +++ b/benches/crypto_bench.rs @@ -16,7 +16,7 @@ use carbonado::crypto::{slh_dsa_generate_keypair, slh_dsa_sign, slh_dsa_verify}; use carbonado::file::encode_directory; use carbonado::{decode, decode_outboard, encode, encode_outboard, scrub_outboard}; -use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; +use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main}; use getrandom::getrandom; use std::fs; use std::path::PathBuf; diff --git a/benches/parallel_bench.rs b/benches/parallel_bench.rs index 2c7ce67..79cfef9 100644 --- a/benches/parallel_bench.rs +++ b/benches/parallel_bench.rs @@ -14,9 +14,9 @@ use std::io::Cursor; use carbonado::constants::FEC_M; use carbonado::stream::fec::FecInboardEncoder; use carbonado::stream::parallel::{ - encode_rs_parity_serial, encode_rs_parity_with_config, ParallelConfig, + ParallelConfig, encode_rs_parity_serial, encode_rs_parity_with_config, }; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; use reed_solomon_erasure::galois_8::ReedSolomon; fn patterned(len: usize) -> Vec { diff --git a/build.rs b/build.rs deleted file mode 100644 index 5c5de10..0000000 --- a/build.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Propagate libcarbonado rpath onto final binaries/tests (backend-lean). -//! -//! `carbonado-sys` sets `rustc-link-search` / `rustc-link-lib`, but `rustc-link-arg` -//! rpath from a dependency build script is not applied to dependents' final links. - -use std::env; -use std::path::PathBuf; - -fn main() { - println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_LIB"); - println!("cargo:rerun-if-cfg=feature=\"backend-lean\""); - - let lean = env::var("CARGO_FEATURE_BACKEND_LEAN").is_ok(); - if !lean { - return; - } - if let Ok(lib) = env::var("CARBONADO_LEAN_LIB") { - let lib_dir = PathBuf::from(lib); - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); - } -} diff --git a/carbonado-sys/Cargo.toml b/carbonado-sys/Cargo.toml deleted file mode 100644 index 6e53b21..0000000 --- a/carbonado-sys/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "carbonado-sys" -version = "0.1.0" -edition = "2021" -license = "MIT" -description = "FFI bindings to Lean AOT libcarbonado (dual-backend parity)" -publish = false - -[dependencies] - -# When enabled (by carbonado `backend-lean`), fail the build if CARBONADO_LEAN_LIB is unset -# or does not contain libcarbonado shared/static artifacts. -[features] -require-lib = [] - -[build-dependencies] diff --git a/carbonado-sys/build.rs b/carbonado-sys/build.rs deleted file mode 100644 index 533d99d..0000000 --- a/carbonado-sys/build.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Link against Nix-built `libcarbonado` when `CARBONADO_LEAN_LIB` / `CARBONADO_LEAN_INCLUDE` -//! are set (or after `nix build .#libcarbonado` + env). -//! -//! ```bash -//! nix build .#libcarbonado -o result-libcarbonado -//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} -//! # Freeze allowlist (Phase 5 / G11): just test-lean-ci -//! cargo test -p carbonado --no-default-features --features "backend-lean,pqc,ots,cli" -//! ``` -//! -//! Prefers the shared library (`libcarbonado.so`) produced by leanc (Lean runtime -//! already linked). Falls back to static `libcarbonado.a` when only the archive -//! is present (requires a full Lean link line — not the default path). -//! -//! With the `require-lib` feature (enabled by carbonado `backend-lean`), a missing -//! `CARBONADO_LEAN_LIB` or missing library file is a **hard build error** (fail-closed). - -use std::env; -use std::path::PathBuf; - -fn main() { - println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_LIB"); - println!("cargo:rerun-if-env-changed=CARBONADO_LEAN_INCLUDE"); - - let require_lib = env::var_os("CARGO_FEATURE_REQUIRE_LIB").is_some(); - let lib = env::var_os("CARBONADO_LEAN_LIB").map(PathBuf::from); - let include = env::var_os("CARBONADO_LEAN_INCLUDE").map(PathBuf::from); - - if let Some(inc) = include { - println!("cargo:include={}", inc.display()); - } - - if let Some(lib_dir) = lib { - let so = lib_dir.join("libcarbonado.so"); - let dylib = lib_dir.join("libcarbonado.dylib"); - let archive = lib_dir.join("libcarbonado.a"); - if !so.exists() && !dylib.exists() && !archive.exists() { - let msg = format!( - "CARBONADO_LEAN_LIB={} has no libcarbonado.so/.dylib/.a — run: nix build .#libcarbonado -o result-libcarbonado", - lib_dir.display() - ); - if require_lib { - panic!("{msg}"); - } - println!("cargo:warning={msg}"); - return; - } - - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - - if so.exists() || dylib.exists() { - println!("cargo:rustc-link-lib=dylib=carbonado"); - // Runtime resolution for tests without installing into system paths. - println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); - } else { - println!("cargo:rustc-link-lib=static=carbonado"); - println!( - "cargo:warning=libcarbonado shared object missing; linking static (may need Lean runtime libs)" - ); - } - println!("cargo:rustc-link-lib=pthread"); - println!("cargo:rustc-link-lib=m"); - println!("cargo:rustc-link-lib=dl"); - } else if require_lib { - panic!( - "CARBONADO_LEAN_LIB unset while carbonado-sys/require-lib is enabled (backend-lean). \ - Build: nix build .#libcarbonado -o result-libcarbonado && \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib \ - CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include" - ); - } else { - // Allow standalone carbonado-sys docs/check without the AOT lib. - println!( - "cargo:warning=CARBONADO_LEAN_LIB unset; carbonado-sys will not link libcarbonado" - ); - } -} diff --git a/carbonado-sys/src/lib.rs b/carbonado-sys/src/lib.rs deleted file mode 100644 index cfb1475..0000000 --- a/carbonado-sys/src/lib.rs +++ /dev/null @@ -1,220 +0,0 @@ -//! Low-level FFI to Lean AOT `libcarbonado` (see `include/carbonado.h`, `docs/ABI.md`). - -#![allow(non_camel_case_types)] - -use std::os::raw::{c_int, c_void}; - -pub const CARBONADO_ABI_VERSION: u32 = 1; - -pub const CARBONADO_OK: c_int = 0; -pub const CARBONADO_ERR_INVALID_ARGUMENT: c_int = 1; -pub const CARBONADO_ERR_INVALID_KEY_LENGTH: c_int = 2; -pub const CARBONADO_ERR_AUTHENTICATION: c_int = 3; -pub const CARBONADO_ERR_INVALID_MAGIC: c_int = 4; -pub const CARBONADO_ERR_INVALID_HEADER: c_int = 5; -pub const CARBONADO_ERR_FEC: c_int = 6; -pub const CARBONADO_ERR_BAO: c_int = 7; -pub const CARBONADO_ERR_ZSTD: c_int = 8; -pub const CARBONADO_ERR_SCRUB_UNNECESSARY: c_int = 9; -pub const CARBONADO_ERR_SCRUB_FAILED: c_int = 10; -pub const CARBONADO_ERR_NOT_IMPLEMENTED: c_int = 11; -pub const CARBONADO_ERR_INTERNAL: c_int = 12; -pub const CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION: c_int = 13; - -extern "C" { - pub fn carbonado_abi_version() -> u32; - pub fn carbonado_free(p: *mut c_void); - pub fn carbonado_encode( - master: *const u8, - master_len: usize, - plaintext: *const u8, - plaintext_len: usize, - format: u8, - nonce: *const u8, - nonce_len: usize, - out: *mut *mut u8, - out_len: *mut usize, - hash_out: *mut u8, - padding_out: *mut u32, - chunk_len_out: *mut u32, - bytes_ecc_out: *mut u32, - verifiable_slice_count_out: *mut u32, - bytes_compressed_out: *mut u32, - bytes_encrypted_out: *mut u32, - ) -> c_int; - pub fn carbonado_decode( - master: *const u8, - master_len: usize, - hash: *const u8, - hash_len: usize, - body: *const u8, - body_len: usize, - padding: u32, - format: u8, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_encode_headered( - master: *const u8, - master_len: usize, - plaintext: *const u8, - plaintext_len: usize, - format: u8, - nonce: *const u8, - nonce_len: usize, - slh_pk: *const u8, - metadata: *const u8, - out: *mut *mut u8, - out_len: *mut usize, - padding_out: *mut u32, - chunk_len_out: *mut u32, - bytes_ecc_out: *mut u32, - verifiable_slice_count_out: *mut u32, - bytes_compressed_out: *mut u32, - bytes_encrypted_out: *mut u32, - ) -> c_int; - pub fn carbonado_decode_headered( - master: *const u8, - master_len: usize, - archive: *const u8, - archive_len: usize, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_verification_key(format: u8, key_out: *mut u8) -> c_int; - pub fn carbonado_encode_outboard( - master: *const u8, - master_len: usize, - plaintext: *const u8, - plaintext_len: usize, - format: u8, - nonce: *const u8, - nonce_len: usize, - header_path: u8, - main_out: *mut *mut u8, - main_len: *mut usize, - outboard_out: *mut *mut u8, - outboard_len: *mut usize, - parity_out: *mut *mut u8, - parity_len: *mut usize, - hash_out: *mut u8, - padding_out: *mut u32, - chunk_len_out: *mut u32, - bytes_compressed_out: *mut u32, - bytes_encrypted_out: *mut u32, - ) -> c_int; - pub fn carbonado_decode_outboard( - master: *const u8, - master_len: usize, - hash: *const u8, - hash_len: usize, - main: *const u8, - main_len: usize, - outboard: *const u8, - outboard_len: usize, - parity: *const u8, - parity_len: usize, - padding: u32, - format: u8, - header_path: u8, - nonce: *const u8, - nonce_len: usize, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_scrub( - body: *const u8, - body_len: usize, - hash: *const u8, - hash_len: usize, - padding: u32, - format: u8, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_scrub_outboard( - main: *const u8, - main_len: usize, - outboard: *const u8, - outboard_len: usize, - parity: *const u8, - parity_len: usize, - hash: *const u8, - hash_len: usize, - padding: u32, - chunk_len: u32, - format: u8, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_verify_slice( - body: *const u8, - body_len: usize, - hash: *const u8, - hash_len: usize, - index: u32, - count: u32, - format: u8, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_verify_slice_outboard( - main: *const u8, - main_len: usize, - outboard: *const u8, - outboard_len: usize, - hash: *const u8, - hash_len: usize, - index: u32, - count: u32, - format: u8, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_slh_keygen( - entropy: *const u8, - entropy_len: usize, - pk_out: *mut u8, - sk_out: *mut u8, - ) -> c_int; - pub fn carbonado_slh_sign( - secret_key: *const u8, - secret_key_len: usize, - message: *const u8, - message_len: usize, - out: *mut *mut u8, - out_len: *mut usize, - ) -> c_int; - pub fn carbonado_slh_verify( - public_key: *const u8, - public_key_len: usize, - message: *const u8, - message_len: usize, - signature: *const u8, - signature_len: usize, - ) -> c_int; -} - -/// Safe wrapper: free a buffer returned by libcarbonado. -/// -/// # Safety -/// `p` must be null or a pointer returned by libcarbonado. -pub unsafe fn free(p: *mut u8) { - carbonado_free(p as *mut c_void); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn abi_version_matches_header() { - // Only runs when linked against libcarbonado (CARBONADO_LEAN_LIB set). - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - eprintln!("skip: CARBONADO_LEAN_LIB unset"); - return; - } - let v = unsafe { carbonado_abi_version() }; - assert_eq!(v, CARBONADO_ABI_VERSION); - } -} diff --git a/doc/TEST_STRATEGY.md b/doc/TEST_STRATEGY.md index 686562a..7bd53fa 100644 --- a/doc/TEST_STRATEGY.md +++ b/doc/TEST_STRATEGY.md @@ -133,15 +133,13 @@ Carbonado uses **reed-solomon-erasure 4/8**: any **4 of 8** shards reconstruct t ```bash # Full native gate (default + serial FEC + optional features) -# Never --all-features: enables both backend-rust and backend-lean → compile_error!. cargo test cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path cargo test --features "async,async-tokio,man-gen" cargo clippy --all-targets --features "async,async-tokio,man-gen" -- -D warnings -# Dual-backend lean freeze (G11 + R7 G8 full): just test-lean-ci -# = unfiltered cargo test --no-default-features --features "backend-lean,pqc,ots,cli" -# Feature-gated async/parallel suites are 0 tests under this feature set (not dual residual). +# Lean proofs + AOT demo (not a Cargo Lean engine) +just test-lean-ci # FEC-focused cargo test --test fec_chaos --test fec_scrub_matrix --test shard_fec_scrub @@ -175,5 +173,5 @@ just lint-wasm - **Optional rust matrix:** `cargo test --features "async,async-tokio,man-gen"` (never `--all-features`) - **Phase 3 determinism:** covered by default `cargo test --test parallel_determinism` (RS parity vs `encode_rs_parity_serial`, c12/c14 bytes + Bao root, scrub roundtrip) - **WASM `parallel`:** compile-only in `test-matrix` (`cargo check --target wasm32-unknown-unknown --features "async,async-tokio,man-gen"` and no-pqc `backend-rust` only); runtime serial fallback documented in `STREAMING_PARALLELISM.md` § Phase 3 WASM -- **Lean dual freeze (G11 + R7 G8 full closed):** job `dual-backend-lean` / `just test-lean-ci` = unfiltered full lean suite; `streaming_async` needs `async`, `parallel_determinism` needs `parallel` (not in dual feature set) +- **Lean proofs:** job `lean-proofs` / `just test-lean-ci` = nix no-sorry + AOT demo - Proptest cases capped at 32 for `fec_chaos` (raise when stable) \ No newline at end of file diff --git a/docs/ABI.md b/docs/ABI.md deleted file mode 100644 index 162858f..0000000 --- a/docs/ABI.md +++ /dev/null @@ -1,334 +0,0 @@ -# carbonado C ABI (dual-backend) - -Stable C interface for the **Lean AOT engine** (`libcarbonado`). Rust `backend-lean` links this library (`carbonado-sys`) and is required to expose the **same high-level Rust API** as `backend-rust` so that `tests/` is one contract. - -**Normative sources (must stay in sync):** - -| Artifact | Role | -|----------|------| -| [`include/carbonado.h`](../include/carbonado.h) | C declarations (v0 core + Phase 2 additive) | -| [`carbonado-sys/src/lib.rs`](../carbonado-sys/src/lib.rs) | Rust FFI bindings + error constants | -| [`nix/native/carbonado_abi.c`](../nix/native/carbonado_abi.c) | Strong C exports calling Lean `@[export] l_carbonado_*` | -| [`Carbonado/Ffi.lean`](../Carbonado/Ffi.lean) | Lean pure helpers + live `@[export]` surface | -| This document | Ownership, versioning, error codes, link instructions | - -**ABI version:** `1` (`CARBONADO_ABI_VERSION`). Bump major on breaking changes (symbol rename, error-code reuse, semantic change of successful outputs). - ---- - -## Memory ownership - -| Pattern | Rule | -|---------|------| -| Input buffers | Caller owns; not freed by libcarbonado | -| Output buffers | Returned via `uint8_t **out` + `size_t *out_len`; allocated with **`malloc`**. C callers free with **`carbonado_free`**. Rust `backend-lean` **copies** into a `Vec` then calls **`carbonado_free`** (allocator-agnostic; safe with jemalloc/mimalloc GlobalAlloc) | -| Errors | Integer codes only on the hot path; no heap error strings in v0 | -| Null | Null input pointers with non-zero lengths → `CARBONADO_ERR_INVALID_ARGUMENT` (when implemented) | - -```c -void carbonado_free(void *p); /* free(NULL) is a no-op */ -``` - ---- - -## Versioning - -```c -#define CARBONADO_ABI_VERSION 1u -uint32_t carbonado_abi_version(void); /* returns CARBONADO_ABI_VERSION */ -``` - -Lean: `Carbonado.Ffi.abiVersion` / `@[export carbonado_abi_version]`. - ---- - -## Error codes (v0) - -Stable integers shared by `include/carbonado.h`, `carbonado-sys`, and `Carbonado.Ffi`. Converted to `CarbonadoError` in `src/backend/mod.rs` (`lean::map_err`). Unknown codes → generic failure. - -Two columns matter for dual-backend work: - -- **Target mapping** — intended 1:1 (or documented multi-source) diagnostics for failure-mode `matches!` tests. -- **Current `map_err` (Phase 2)** — live mapping; scrub codes 9/10/13 are distinct. Residual: InvalidArgument still generic; FEC modes collapse to `UnevenFecChunks` (with targeted MissingFecParity remap on outboard scrub). - -| Code | Name | Meaning | Target Rust mapping | Current `lean::map_err` (Phase 2 + R4) | -|-----:|------|---------|---------------------|--------------------------------------| -| 0 | `CARBONADO_OK` | Success | `Ok` | `Ok` | -| 1 | `CARBONADO_ERR_INVALID_ARGUMENT` | Null/lengths/nonce size/sequence | dedicated arg/nonce/segment variants as needed | `InternalStateError("…invalid argument…")` (**P2 residual:** add `InvalidArgument` / reuse nonce variants before allowlist expands to nonce/sequence fails) | -| 2 | `CARBONADO_ERR_INVALID_KEY_LENGTH` | Master not 32 or 64 bytes | `InvalidKeyLength` | **`InvalidKeyLength`** | -| 3 | `CARBONADO_ERR_AUTHENTICATION` | Header MAC / payload EtM / **Bao auth** | `AuthenticationFailed` | **`AuthenticationFailed`** (R4: `baoAuthenticationFailed` joins header/payload auth) | -| 4 | `CARBONADO_ERR_INVALID_MAGIC` | Bad `CARBONADO20\n` (or related magic) | `InvalidMagicNumber` | `InvalidMagicNumber("lean-backend")` | -| 5 | `CARBONADO_ERR_INVALID_HEADER` | Truncated/malformed header, body bounds, **short inboard Bao prefix** | `InvalidHeaderLength` | **`InvalidHeaderLength`** (R4: Lean `invalidPrefix` maps here) | -| 6 | `CARBONADO_ERR_FEC` | RS geometry / shard errors | `UnevenFecChunks` / FEC failures | `UnevenFecChunks` | -| 7 | `CARBONADO_ERR_BAO` | Bao stream truncation / trailing / residual slice geometry | `BaoResponseTruncated` (not auth) | **`BaoResponseTruncated`** (R4: auth no longer collapses here) | -| 8 | `CARBONADO_ERR_ZSTD` | Compress/decompress failures | `ZstdError` | `ZstdError("lean-backend zstd")` | -| 9 | `CARBONADO_ERR_SCRUB_UNNECESSARY` | Scrub not needed | `UnnecessaryScrub` | `UnnecessaryScrub` | -| 10 | `CARBONADO_ERR_SCRUB_FAILED` | Scrub cannot recover | `InvalidScrubbedHash` | `InvalidScrubbedHash` | -| 11 | `CARBONADO_ERR_NOT_IMPLEMENTED` | Surface not exported or still stubbed | `NotImplemented` | **`NotImplemented`** | -| 12 | `CARBONADO_ERR_INTERNAL` | Unexpected / allocator / invariant | internal / dedicated | `InternalStateError` | -| 13 | `CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION` | Scrub without Verification bit | `ScrubRequiresVerification` | **`ScrubRequiresVerification`** (P2) | - -**Phase 2 + R4 mapping:** scrub requires-verification is distinct (code 13). **R4:** `ofPipelineError` sends `baoAuthenticationFailed` → code 3 and `invalidPrefix` → code 5 (no longer collapsed into code 7). `InvalidSliceIndex { index, content_len }` is produced by Rust-side geometry pre-checks in `lean::verify_slice` (C ABI carries no structured fields). Residual: no dedicated InvalidArgument; MissingFecParity may still surface via FEC path when parity absent after verify fail. - -**Collapse rule (C boundary):** Fine-grained Lean `PipelineError` variants map through `Carbonado.Ffi.ofPipelineError` into these **integer codes**. Distinct failure modes that tests assert via `matches!` must either keep distinct codes or get refined Rust-side mapping before those tests are on the lean allowlist. Do **not** permanently map unrelated failures to a single diagnostic variant. - -**Phase 2 status:** v0 body/headered **plus** outboard/scrub/verify_slice are **live**. Encode packs include chunk/ecc/vsc metadata; **R3** adds `bytes_compressed` / `bytes_encrypted` (nullable C out-params; ABI version remains 1 additive). - ---- - -## Core functions (in `include/carbonado.h`) - -Signatures must match the header byte-for-byte in meaning. - -### Lifecycle - -```c -uint32_t carbonado_abi_version(void); -void carbonado_free(void *p); -``` - -### Encode (low-level buffer ≈ Rust `encoding::encode` body) - -Low-level layout: when encrypted, the body uses the embedded-nonce blob shape Rust low-level paths use (`[nonce|tag|ct]` inside the encrypt stage as applicable). For **public** formats, `nonce` may be null / `nonce_len == 0`. For **encrypted** formats, `nonce` must be 16 bytes (tests use fixed nonces for determinism). - -```c -/* out: verifiable body only (no Carbonado Header). hash_out: 32-byte Bao root. - * Meta out-params optional (nullable). Skipped compress/encrypt stages report 0 (R3). - * Lean pack: success = [status:4][pad:4][chunk:4][ecc:4][vsc:4] - * [bytes_compressed:4][bytes_encrypted:4][hash:32][body…] (prefix 60); - * error = [status:4] only. C parses status first so encode failures return real ABI codes. */ -int carbonado_encode( - const uint8_t *master, size_t master_len, /* 32 or 64 */ - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, /* 16 if encrypted; else 0/null */ - uint8_t **out, size_t *out_len, - uint8_t hash_out[32], - uint32_t *padding_out, /* nullable */ - uint32_t *chunk_len_out, /* nullable */ - uint32_t *bytes_ecc_out, /* nullable */ - uint32_t *verifiable_slice_count_out, /* nullable */ - uint32_t *bytes_compressed_out, /* nullable (R3) */ - uint32_t *bytes_encrypted_out /* nullable (R3) */ -); -``` - -### Decode (low-level ≈ Rust `decoding::decode`) - -```c -int carbonado_decode( - const uint8_t *master, size_t master_len, - const uint8_t *hash, size_t hash_len, /* 32 */ - const uint8_t *body, size_t body_len, - uint32_t padding, - uint8_t format, - uint8_t **out, size_t *out_len -); -``` - -### Headered encode/decode (≈ Rust `file::encode` / `file::decode`) - -```c -/* Full file: Header (177 B) || body. Bao root lives in the header. - * slh_pk: NULL → zero-filled field; non-NULL must point to exactly 32 valid bytes. - * metadata: NULL → zero-filled field; non-NULL must point to exactly 8 valid bytes. - * Additive params (R2 SLH/meta + R3 stage counters): ABI version stays 1. - * Lean pack: success = [status:4][pad:4][chunk:4][ecc:4][vsc:4] - * [bytes_compressed:4][bytes_encrypted:4][archive…] (prefix 28); - * error = [status:4] only. - * C is length-implicit (non-null always copies fixed width). Wrong lengths are - * Lean/export ByteArray-only (encodeHeaderedBytes → errInvalidArgument). */ -int carbonado_encode_headered( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, /* 16 when Encrypted bit set */ - const uint8_t *slh_pk, /* nullable 32 B */ - const uint8_t *metadata, /* nullable 8 B */ - uint8_t **out, size_t *out_len, - uint32_t *padding_out, /* nullable (R3) */ - uint32_t *chunk_len_out, /* nullable */ - uint32_t *bytes_ecc_out, /* nullable */ - uint32_t *verifiable_slice_count_out, /* nullable */ - uint32_t *bytes_compressed_out, /* nullable */ - uint32_t *bytes_encrypted_out /* nullable */ -); - -int carbonado_decode_headered( - const uint8_t *master, size_t master_len, - const uint8_t *archive, size_t archive_len, - uint8_t **out, size_t *out_len -); -``` - -Lean pure + live C: `Carbonado.Ffi.encodeHeaderedBytes` / `decodeHeaderedBytes` via -`l_carbonado_encode_headered` / `l_carbonado_decode_headered`. -Lean takes `slhPublicKey` / `metadataBytes` as `ByteArray` (empty or exact length 32 / 8; -empty → zeros; other sizes → `errInvalidArgument`). Coverage: `CarbonadoTest.Pipeline` -`encode_headered_bytes_bad_*_len` theorems. Rust `lean::encode_headered` returns -`(archive, EncodeInfo)`; `file::encode` threads that info (R3). High-level `file::encode` -still passes `slh_public_key = None` (zeros); dual-suite SLH sets the field via `Header` -APIs / sidecars (G10-A). - -**R2 residual (not a dedicated `InvalidArgument` variant):** if code 1 reaches Rust -`map_err`, callers see `InternalStateError("lean-backend invalid argument")` — same -P2 residual as other `CARBONADO_ERR_INVALID_ARGUMENT` sources (table above). Typed -headered encode never surfaces wrong SLH/meta lengths through C. - -### Verification key - -```c -/* Format-keyed Bao key: blake3::derive_key("carbonado-v2/verification", &[format]). */ -int carbonado_verification_key(uint8_t format, uint8_t key_out[32]); -``` - -Lean pure: `Carbonado.Ffi.verificationKeyBytes`. - ---- - -### Outboard / scrub / slice (Phase 2 — live) - -```c -/* header_path != 0 → encrypted bare main [tag|ct] (file::encode_outboard); - * header_path == 0 → embedded [nonce|tag|ct] (encoding::encode_outboard). */ -/* Lean pack prefix 52: status+pad+chunk+comp+enc+hash; then len-prefixed main/ob/par (R3). */ -int carbonado_encode_outboard(/* master, pt, format, nonce, header_path → main/outboard/parity + hash + pad/chunk + compress/encrypt */); -int carbonado_decode_outboard(/* master, hash, main, outboard, parity, padding, format, header_path, nonce → plaintext */); -int carbonado_scrub(/* body, hash, padding, format → recovered body or SCRUB_* error */); -int carbonado_scrub_outboard(/* main, outboard, parity, hash, padding, chunk_len, format → bare */); -int carbonado_verify_slice(/* body, hash, index, count, format → slice bytes */); -/* R9: seekable outboard slice (O(slice+height) hash; full main+outboard buffers). */ -int carbonado_verify_slice_outboard(/* main, outboard, hash, index, count, format → slice */); -/* R9 / G10: SLH-DSA-SHA2-128s (libbitcoinpqc objects in libcarbonado_native.a). */ -int carbonado_slh_keygen(/* entropy≥128 → pk[32], sk[64] */); -int carbonado_slh_sign(/* sk[64], message → malloc 7856 B sig */); -int carbonado_slh_verify(/* pk[32], message, sig[7856] → OK or AUTHENTICATION */); -``` - -See `include/carbonado.h` for full signatures. `extract_slice` is verify_slice with `count == 1` (Rust-side). - -**`verify_slice` (inboard) — W4a closed (retained output):** Lean walks the full inboard bao response for authentication (O(N) time; inboard embeds full-range response) starting at offset 8 (no second full-response copy) but retains only the requested slice bytes (O(slice) output) via `decodeRecRetainRange` — same class as Rust `SliceRegionWriter` / `verify_slice_inboard_seekable`. Leaf hashing may use O(leaf) temps. C ABI still takes the **full inboard body buffer as input** (no streaming ReadAt). Do **not** claim O(slice) peak RSS when the caller already holds the full body `Vec`. **`count == 0` split:** Lean C / pure `verifySliceInboard` is **auth-first**; dual Rust API / `lean::verify_slice` short-circuits empty success before C (parity with pure-Rust seekable). - -**`verify_slice_outboard` (R9 + W4b permanent full-buffer):** Lean walks only the requested leaf-group range (O(slice + height) hash work; W4b offset walk avoids recursive full half-extracts). C ABI still takes full main + full outboard buffers in memory — **permanent residual** (no additive `carbonado_verify_slice_outboard_at` / `ReadAt` callback ABI this wave). Under `backend-lean` the Rust dispatcher materializes `data_len` once when `data` is a generic `ReadAt`. - -**`count == 0` (outboard slice):** empty success **immediately** — no root/geometry/OOB/auth checks (matches Rust `stream/slice.rs::verify_slice_outboard`). Authentication and OOB apply only when `count > 0`. - -**SLH C error mapping (R9):** `carbonado_slh_keygen` / `_sign` library failure → `CARBONADO_ERR_INTERNAL`; `carbonado_slh_verify` reject → `CARBONADO_ERR_AUTHENTICATION`; bad args → `CARBONADO_ERR_INVALID_ARGUMENT`. Empty message (`NULL`, len 0) is accepted (non-NULL empty buffer passed to libbitcoinpqc). - -## Phase 3 directory (composition — no new C symbols) - -Directory dual-backend does **not** add `carbonado_encode_directory` / `decode_directory` C exports. -`file::encode_directory` / `decode_directory` under `backend-lean` compose existing ABI: - -| Directory stage | Lean C ABI used | -|-----------------|-----------------| -| Bare segment mains | `carbonado_encode_outboard` / `carbonado_decode_outboard` (embedded-nonce) | -| Catalog inboard c14/c15 | `carbonado_encode_headered` / `carbonado_decode_headered` | -| rkyv FilepackManifest v2 + Adamantine envelope/payload + FS | **Dual-suite composition:** Rust rkyv+FS (SSOT). **Pure Lean Directory/CLI** also emit rkyv (**W3**); CFP2 dual-decode fallback only | - -Dual-suite catalogs are **rkyv** (same as `backend-rust`; Rust composition SSOT for dual encode). Pure Lean Directory/CLI emit rkyv (**W3**); CFP2 is dual-decode fallback only (LIMITS). -Allowlist: `tests/lean_backend_phase3.rs` (`just test-lean-phase3`). - -**Bao error mapping (R4):** Lean `ofPipelineError` no longer collapses all Bao failures to -`CARBONADO_ERR_BAO`. Current fidelity: - -| Lean failure | ABI code | Rust mapping | -|--------------|----------|--------------| -| Bao auth (wrong key / root / leaf-parent mismatch) | 3 `AUTHENTICATION` | `AuthenticationFailed` | -| Short inboard prefix (`invalidPrefix`) | 5 `INVALID_HEADER` | `InvalidHeaderLength` | -| Truncation / trailing / residual geometry (no Rust pre-check) | 7 `BAO` | `BaoResponseTruncated` | -| OOB slice index | n/a (Rust pre-check in `lean::verify_slice`) | `InvalidSliceIndex { index, content_len }` | - -Directory catalog body-tamper under both backends surfaces `AuthenticationFailed` for keyed Bao -auth failure (see `tests/directory_archive.rs`). Residual: pure-Rust outboard/inboard entry -points may still use `OutboardVerificationFailed` in other paths — do not re-collapse auth to -code 7. - -## Phase 4: SLH / CLI / OTS (composition for dual-suite) - -**G10 strategy A (dual-suite, still valid):** product SLH under `backend-lean` may use Rust -`crypto::slh_dsa_*` + `bitcoinpqc` composition. Dual-suite does **not** require pure Lean SLH. - -**R9 / G10 full (pure Lean depth):** libbitcoinpqc SLH-DSA-SHA2-128s objects are linked into -`libcarbonado_native.a` (pinned `b309f444…`). Live symbols: - -| Symbol | Role | -|--------|------| -| `carbonado_slh_keygen` | entropy ≥128 → pk 32 + sk 64 | -| `carbonado_slh_sign` | sk 64 + message → malloc 7856 B signature | -| `carbonado_slh_verify` | pk + message + sig → OK / AUTHENTICATION | -| Lean `@[extern]` | `carbonado_slh_{keygen,sign,verify}_raw` → `Carbonado/Slh.lean` `signRoot` / `verifyRoot` | - -Dual-suite may keep Rust bitcoinpqc composition as product SSOT; pure Lean is for -`libcarbonado` purity. Fail-closed on bad signatures. - -Allowlist: `tests/lean_backend_phase4.rs` + `tests/slh_outboard.rs` (`just test-lean-phase4`). - -## R9 additive C surface (ABI version stays 1) - -| Symbol | Rust analogue | Status | -|--------|---------------|--------| -| `carbonado_verify_slice_outboard` | `verify_slice_outboard` | **live** (R9) | -| `carbonado_slh_keygen` / `_sign` / `_verify` | `crypto::slh_dsa_*` | **live** (R9 / G10) | -| Optional pure-buffer directory helpers | composition | optional (not required) | - ---- - -## Implementation status (Phase 2–4 close) - -| Symbol | Lean pure | C in `libcarbonado` | `carbonado-sys` | Rust `backend-lean` dispatch | -|--------|-----------|---------------------|-----------------|------------------------------| -| `carbonado_abi_version` | `abiVersion` (C-owned) | **live** | bound | `lean::abi_version` | -| `carbonado_free` | — | **live** | bound | C callers: `carbonado_free`. Rust `backend-lean`: copy via `take_buf` then `carbonado_free` (not `Vec::from_raw_parts`) | -| `carbonado_encode` | `l_carbonado_encode` | **live** (+ meta; R3 compress/encrypt) | bound | `encoding::encode` → `lean::encode` | -| `carbonado_decode` | `l_carbonado_decode` | **live** | bound | `decoding::decode` → `lean::decode` | -| `carbonado_encode_headered` | `l_carbonado_encode_headered` | **live** (+ SLH/meta R2; EncodeMeta R3) | bound | `file::encode` → `lean::encode_headered` | -| `carbonado_decode_headered` | `l_carbonado_decode_headered` | **live** | bound | `file::decode` → `lean::decode_headered` | -| `carbonado_verification_key` | `l_carbonado_verification_key` | **live** | bound | `crypto::carbonado_verification_key` | -| `carbonado_encode_outboard` | `l_carbonado_encode_outboard` | **live** (+ R3 compress/encrypt) | bound | `encoding::encode_outboard` | -| `carbonado_decode_outboard` | `l_carbonado_decode_outboard` | **live** | bound | `decoding::decode_outboard` | -| `carbonado_scrub` | `l_carbonado_scrub` | **live** | bound | `decoding::scrub` | -| `carbonado_scrub_outboard` | `l_carbonado_scrub_outboard` | **live** | bound | `decoding::scrub_outboard` | -| `carbonado_verify_slice` | `l_carbonado_verify_slice` | **live** | bound | `decoding::verify_slice` | -| `carbonado_verify_slice_outboard` | `l_carbonado_verify_slice_outboard` | **live** (R9) | bound | `stream::verify_slice_outboard` → `lean::verify_slice_outboard` | -| `carbonado_slh_keygen` | `@[extern]` raw | **live** (R9) | bound | optional; dual-suite may use Rust `bitcoinpqc` | -| `carbonado_slh_sign` | `@[extern]` raw | **live** (R9) | bound | optional; dual-suite may use Rust `bitcoinpqc` | -| `carbonado_slh_verify` | `@[extern]` raw | **live** (R9) | bound | optional; dual-suite may use Rust `bitcoinpqc` | - -**Phase 2 allowlist:** `tests/lean_backend_smoke.rs` + `tests/lean_backend_phase2.rs` (`just test-lean-phase2`). - -**Phase 3 allowlist:** + `tests/lean_backend_phase3.rs` + `tests/format_policy.rs` (`just test-lean-phase3`). -Directory composition (rkyv catalog, no new C symbols). - -**Phase 4 allowlist:** + `tests/lean_backend_phase4.rs` + `tests/slh_outboard.rs` (`just test-lean-phase4`; features include `cli` for subprocess smoke). -SLH/CLI/OTS composition (no new C symbols). - -**Phase 5 freeze (G11) + R7 G8 full close:** `just test-lean-ci` — full dual suite under lean features (unfiltered `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"`). See [GAPS.md](./GAPS.md) R7 / [TEST_CONTRACT.md](./TEST_CONTRACT.md) Phase 5. Full-suite G8 **closed** at R7; post-G8 residuals are purity/feature-policy only. - ---- - -## Linking - -```text -# After: nix build .#libcarbonado -o result-libcarbonado -export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} -# Phase 5 freeze (CI + humans): -just test-lean-ci -# Phase-scoped: -# just test-lean-phase4 -# link: -L $CARBONADO_LEAN_LIB -lcarbonado (+ rpath); NEEDED libleanshared from nix store -``` - -Exact `cargo` `rustc-link-*` flags live in [`carbonado-sys/build.rs`](../carbonado-sys/build.rs). With feature `require-lib` (enabled by carbonado `backend-lean`), a missing `CARBONADO_LEAN_LIB` or missing library file is a **hard build error**. Without `require-lib`, unset env only warns (docs/check). CI / `just test-lean-ci` **fail-closed** if `libcarbonado.so` (or `.dylib`) is missing under `CARBONADO_LEAN_LIB`. - -**Packaging:** `nix build .#libcarbonado -o result-libcarbonado` produces `lib/libcarbonado.so` (leanc + whole-archive Lean AOT + zstd/ABI glue + NEEDED absolute nix-store `libleanshared`) and `lib/libcarbonado.a`. Prefer the shared object from `carbonado-sys` (rpath set from `CARBONADO_LEAN_LIB`). Redistribution is nix-store-coupled until a bundled runtime story lands. - -**`EncodeInfo` on lean (R3):** full stage counters from Lean pack — `padding_len`, `chunk_len`, `bytes_ecc`, `verifiable_slice_count`, `bytes_compressed`, `bytes_encrypted`, body lengths. Skipped compress/encrypt stages report **0** (matches Rust stream path). `compression_factor` / `amplification_factor` computed in Rust from those fields. - ---- - -## Mutual exclusion of Cargo features - -Enable **exactly one** of `backend-rust` or `backend-lean` per build (`src/backend/mod.rs` `compile_error!`). Dual-backend CI runs two invocations, not one binary with both engines. diff --git a/docs/GAPS.md b/docs/GAPS.md index 4c08fc0..18decaa 100644 --- a/docs/GAPS.md +++ b/docs/GAPS.md @@ -2,26 +2,24 @@ Living inventory. IDs are durable; close only when theorems and/or parity gates are green. -## Dual-backend model (north star) +## Product model (north star) | Role | Location | |------|----------| -| First-class engine | **Rust** (`src/`, default `backend-rust`) — production library + CLI | -| Normative contract | **Rust `tests/`** — both backends must pass the same suite | -| Second engine | **Lean 4 AOT** (`Carbonado/`, `libcarbonado` via C ABI) — proofs + wire-compatible implementation | -| Build / proofs | Nix flakes (`nix flake check`, `libcarbonado` package) | +| Production engine | **Rust** (`src/`, default `backend-rust`) — library + CLI | +| Normative contract | **Rust `tests/`** — Rust engine only | +| Spec + proofs | **Lean 4** (`Carbonado/`, `CarbonadoTest/`) + AOT demo | +| Build / proofs | Nix flakes (`nix flake check`, no-sorry, demo) | | Oracles | `ref/` pins + parity drivers | -**Parity bar (G8):** same `tests/` on both engines (not Lean-only demos). Default features enable `backend-rust` only — do **not** pass `--features backend-lean` alone (both engines → `compile_error!`). +There is **no** `carbonado-sys`, **no** Cargo `backend-lean`, and **no** product C ABI. Do **not** claim G8 C-ABI parity. ```bash -cargo test # backend-rust (default) -just test-lean-ci # G8 freeze = full dual suite -# Equivalent unfiltered lean suite (includes bin_* via cli feature): -# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +cargo test # Rust engine (default) +just test-lean-ci # Lean no-sorry + AOT demo (nix) ``` -See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), AGENTS.md dual-backend block. +See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [PARITY.md](./PARITY.md), [LIMITS.md](./LIMITS.md), AGENTS.md product model. | ID | Gap | Status | |----|-----|--------| @@ -33,10 +31,10 @@ See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [PARITY.md](./PA | G5 | Pipeline / stream / scrub / shard | **closed** (Program E) | | G6 | zstd link + SLH product | **closed** (zstd AOT closed; dual-suite SLH composition **P4**; pure Lean SLH FFI **R9/G10** — dual-suite may still use Rust `bitcoinpqc` composition by design) | | G7 | Adamantine + CLI | **partial** (dual directory + stream E1 + **W1a/W1b** closed; **W3** pure Lean rkyv encode/CLI closed; dual-suite rkyv encode remains Rust composition SSOT; pure Lean chunked stream residual) | -| **G8** | **Dual-backend: C ABI + full `cargo test --no-default-features --features "backend-lean,pqc,ots[,cli]"` suite** | **closed** (2026-07 R7: full suite green under lean; freeze = full suite via `just test-lean-ci`; post-G8 purity/feature residuals below) | -| G9 | Cross-backend encode/decode matrix (Rust↔Lean) | **closed** (2026-07 R8: no-compress body/headered/outboard both directions + fixed-nonce encrypted; `tests/g9_cross_backend.rs` + `tests/fixtures/g9/`; **W2d** codecode/decodec shipped; **W2a/W2b** permanent cross-engine compress/dir encode residuals) | -| G10 | libbitcoinpqc real SLH sign/verify in libcarbonado | **closed** (R9: pin `b309f444…` into `libcarbonado_native.a`; `carbonado_slh_*` C ABI + Lean `@[extern]`; AOT `signRoot`/`verifyRoot` live; dual-suite may still use Rust composition) | -| G11 | Live CI matrix both backends | **closed** (2026-07 P5: Linux job `dual-backend-lean` runs `just test-lean-ci`; `desktop` keeps `backend-rust` full suite) | +| **G8** | Dual-backend via C ABI (`carbonado-sys` / `libcarbonado` / Cargo `backend-lean`) | **removed** (2026-08-24: trampoline deleted; do not claim C-ABI parity) | +| G9 | Cross-backend encode/decode matrix (Rust↔Lean) | **partial** (2026-08-24: Rust still decodes committed Lean AOT goldens in `tests/fixtures/g9/lean/`; live rust→lean encode via C is gone) | +| G10 | libbitcoinpqc real SLH sign/verify in the Lean AOT demo | **closed** (Lean `@[extern]` into pinned SLH objects; not a Rust `-sys` product) | +| G11 | Live CI matrix both backends | **superseded** (2026-08-24: CI `desktop` is Rust; CI `lean-proofs` is nix no-sorry + demo) | ## Dual-backend phases (G8 breakdown) @@ -76,7 +74,11 @@ See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [PARITY.md](./PA **P4 honest residuals (historical; superseded in part at R9/R10/W1a):** ~~pure Lean `signRoot` / libbitcoinpqc~~ **closed R9**; ~~seekable outboard slice C~~ **closed R9**; rkyv dual-decode **closed R9** (encode residual remains); stream dual E1 is **spool-to-buffer** (not true chunked stream — O(logical) RAM); ~~`file::decode_stream` pure-Rust~~ **W1a closed**; ~~residual full files `sharding` / `fec_chaos`~~ **green at R6**; ~~async dual~~ **closed R10**. -### P5 deliverables (evidence of close) +### Removed 2026-08-24 + +Cargo `backend-lean`, `carbonado-sys`, `include/carbonado.h`, `docs/ABI.md`, and `libcarbonado` as a Rust-link target are **gone**. Sections below that describe P1–R7 C ABI dual-suite freeze are **historical**. Live gates: `cargo test` (Rust) and `just test-lean-ci` (Lean proofs + AOT demo). + +### P5 deliverables (historical) | Deliverable | Location | |-------------|----------| @@ -135,7 +137,7 @@ just test-lean-ci | Pure Lean chunked stream C ABI | No streaming C symbols; inboard/encrypted stream remain E1; public outboard E2 is **composition** | residual after W1b | | ~~codecode / decodec determinism suite~~ | **W2d closed** — `tests/determinism_roundtrip.rs` (no-compress body/headered/outboard; same-engine compress + directory) | **closed** | | Compression encode bit-match (cross-engine) | **permanent residual (W2a)** — Lean AOT zstd frames ≠ Rust `zstd`; decode interop only; same-engine codecode green | permanent | -| Directory encode bit-match (cross-engine) | **permanent residual (W2b)** — live rust `0b119f12…` ≠ live lean `f67b6f49…` (pinned); `phase3_g9_directory` decode-only SSOT (not re-encode golden); same-engine codecode green | permanent | +| Directory encode bit-match (cross-engine) | **permanent residual (W2b)** — live rust `16e2369f…` ≠ live lean `d468ea7a…` (pinned); rust pin equals `phase3_g9_directory` seed (catalog bundle follows sorted `rel_path`, not `read_dir`); same-engine codecode green | permanent | ### W1 — Dual-suite honesty (closed 2026-07) @@ -182,8 +184,8 @@ just test-lean-ci | Item | Status | Detail | |------|--------|--------| | **W2d** codecode / decodec | **closed** | `tests/determinism_roundtrip.rs` — EDE + DED under G9 MASTER/NONCE/`g9_matrix_v1`. Matrix: body c0/c1/c4/c5/c8/c9/c12/c13; headered c4/c5/c12/c13; outboard c4/c5/c12/c13. Both engines (default rust + lean freeze). Asserts `pt' == pt` and `A' == A` / `B == A`. | -| **W2a** Compression cross-engine | **permanent residual** | Same-engine body/headered/outboard compress codecode/decodec green. Cross-engine: G9 `outboard_c14` mains differ (frame descriptor `00` vs `20`; roots `0abe5781…` vs `129b4518…`); hard-asserted fail-closed. Decode interop only. See [LIMITS.md](./LIMITS.md). | -| **W2b** Directory cross-engine | **permanent residual** | Same-engine public directory codecode/decodec green. Cross-engine residual is **live rust `0b119f12…` ≠ live lean `f67b6f49…`** (pinned hard asserts). `phase3_g9_directory` (`16e2369f…`) is **decode-only SSOT**, not a live re-encode golden. | +| **W2a** Compression cross-engine | **permanent residual** | Same-engine body/headered/outboard compress codecode/decodec green. Cross-engine: G9 `outboard_c14` mains differ (frame descriptor `00` vs `20`; roots `0abe5781…` vs `129b4518…`); hard-asserted fail-closed. **Header parameters specified** (level 20, magic, checksum off, no dict, rust windowLog 25 vs lean Single_Segment+FCS) in `Carbonado.Compress` + `tests/zstd_frame_params.rs`. Full compressed-block identity still unproved. Decode interop only. See [LIMITS.md](./LIMITS.md). | +| **W2b** Directory cross-engine | **permanent residual** | Same-engine public directory codecode/decodec green. Cross-engine residual is **live rust `16e2369f…` ≠ live lean `d468ea7a…`** (pinned hard asserts). `phase3_g9_directory` (`16e2369f…`) matches live rust encode after catalog bundle append follows sorted `rel_path`. | | **W2c** Full c0–c15 G9 matrix | **skipped** (optional) | Not required after W2a permanent residual. | ### W3 — Pure Lean product wire (closed 2026-07) @@ -225,7 +227,7 @@ just test-lean-ci | Residual | Notes | |----------|--------| | Compression encode bit-match | **permanent (W2a)** — Lean AOT zstd ≠ Rust `zstd`; c14 outboard **decode** interop only; same-engine codecode green in `determinism_roundtrip` | -| Directory encode bit-match | **permanent (W2b)** — live rust≠lean catalog roots (pinned); `phase3_g9_directory` is **decode-only SSOT** (not live re-encode golden); same-engine directory codecode green | +| Directory encode bit-match | **permanent (W2b)** — live rust≠lean catalog roots (pinned `16e2369f…` vs `d468ea7a…`); rust pin equals `phase3_g9_directory` seed (sorted `rel_path` bundle); same-engine directory codecode green | | Full c0–c15 with Compression | deferred / not required (W2c optional skipped) | | ~~codecode/decodec suite~~ | **W2d closed** — `tests/determinism_roundtrip.rs` | diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 2f46a24..f09de73 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -2,16 +2,18 @@ ## Current product surface -### Dual-backend (normative product model) +### Product model -| Backend | Status | +| Layer | Status | |---------|--------| -| **Rust** (`src/`, default `backend-rust`) | First-class production library + CLI; full `cargo test` | -| **Lean AOT** (`Carbonado/`, `libcarbonado`, optional `backend-lean`) | Second engine: proofs + wire/C ABI; dual-suite G8 closed at R7 | -| **Rust `tests/`** | Normative behavioral contract for **both** engines | +| **Rust** (`src/`, default `backend-rust`) | Production library + CLI; full `cargo test` | +| **Lean** (`Carbonado/`, AOT demo) | Spec + proofs; tiny C `@[extern]` for zstd/SLH in the demo only | +| **Rust `tests/`** | Normative behavioral contract for the Rust engine | + +There is no `carbonado-sys`, no Cargo `backend-lean`, and no product C ABI. Do not claim G8 C-ABI parity. - AOT CLI (`packages.carbonado` / `nix run`) runs **Programs A–G**: constants, EtM, FEC, keyed Bao, full pipeline (c0–c15), Header wire, scrub, stream bounds, multi-segment shards, **zstd-20 compression (linked)**, **SLH1 sidecar wire + bind-to-root model**, **Adamantine 1.0 directories**, **encode/decode/slh CLI**. -- Rust tree (`src/`, `tests/`, …) **stays** first-class (AGENTS dual-backend). **G1/W5a closed:** permanent policy — **no** `ref/carbonado-rust` product pin; live tree is dual-suite SSOT. Not a license to delete `src/` or `tests/`. +- Rust tree (`src/`, `tests/`, …) **stays** first-class. **G1/W5a closed:** permanent policy — **no** `ref/carbonado-rust` product pin. Not a license to delete `src/` or `tests/`. - Lean theorem/test tree is **`CarbonadoTest/`** (not `Tests/`) so it does not collide with Rust `tests/` on case-insensitive filesystems (Darwin APFS). - Dependency direction is **CarbonadoTest → Carbonado** only. @@ -121,16 +123,17 @@ - Encode rejects `requireOts` (`otsFeatureRequired`); does not mint undecodeable archives. - CLI encode rejects symlink source entries (`symlinkNotAllowed`); decode refuses write-through symlinks when detectible. -## Dual-backend (G8 full closed at R7; P0–P5 closed for allowlist + CI freeze) +## Engines (2026-08-24) -| Backend | Status | +| Layer | Status | |---------|--------| | `backend-rust` (default) | Full Rust engine; full `cargo test` (must never regress); CI job **`desktop`** | -| `backend-lean` | **G8 full closed (R7)** — body/headered/outboard/scrub/verify_slice C ABI + directory composition + SLH/OTS composition; CLI dual-engine for **directory** (+ buffer APIs) + **single-file stream E1** (inboard/encrypted spool→Lean; O(logical) RAM) + **W1b public non-compress outboard S4 composition E2** (c0/c4/c8/c12; Compression under lean O(logical) bulk zstd; not pure Lean stream); freeze = **full dual suite** via `just test-lean-ci` / CI job **`dual-backend-lean`** (G11 **closed**) | -| Cross encode/decode Rust↔Lean | **G9 closed (R8)** — no-compress body/headered/outboard both directions (`tests/g9_cross_backend.rs` + `tests/fixtures/g9/`); **W2d** codecode/decodec shipped (`tests/determinism_roundtrip.rs`); **W2a/W2b permanent:** cross-engine Compression / directory encode not bit-identical (decode interop + same-engine re-encode only); directory decode seed remains `phase3_g9_directory` | -| Docs / inventory (Phase 0) | **Closed** — [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md) | +| Lean proofs + AOT demo | `just test-lean-ci` / CI job **`lean-proofs`**: nix no-sorry + demo. No Cargo `backend-lean`. | +| Lean AOT goldens | Rust still decodes `tests/fixtures/g9/lean/` (`just test-g9`). Live rust→lean encode via C is gone. | + +G8 C-ABI dual-backend (`carbonado-sys` / `libcarbonado` / Cargo `backend-lean`) was **removed**. Historical R7 freeze language below is archaeology, not a live gate. -**R7 freeze command (measured green):** `just test-lean-ci` runs unfiltered `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"` (lib units + all integration tests including `bin_*`). Never `--features backend-lean` alone — defaults already enable `backend-rust`. +**Lean proof command:** `just test-lean-ci` builds nix `no-sorry`, `tooling-purity`, `carbonado`, and `demo`. **Permanent feature-gated exclusions from dual freeze** (lean features stay `"backend-lean,pqc,ots,cli"` — **never** add `async` / `async-tokio` / `parallel` to freeze): @@ -155,8 +158,8 @@ - ~~Stream dual under lean is E1-only~~ **W1b closed (MVP):** see **Stream E1/E2 API matrix** below. Pure Lean chunked stream C ABI residual remains (no streaming C symbols). - ~~`file::decode_stream` pure-Rust residual~~ **W1a closed** — under lean, spools header+`encoded_len` body → Lean `decode_headered` (peak O(archive+plaintext); not E2) - ~~Full **codecode** / **decodec** matrix~~ **W2d closed** — `tests/determinism_roundtrip.rs` (no-compress + same-engine compress/directory) -- **W2a permanent residual — Compression cross-engine encode:** Lean AOT zstd frames are **not** bit-identical to Rust `zstd` even at level 20 / same pin rev. Measured evidence (G9 `outboard_c14`): mains both 35 B; frame descriptor byte differs (`28b5 2ffd **00**…` rust vs `28b5 2ffd **20**…` lean); Bao roots and FEC parity diverge. **Policy:** decode interop only across engines; re-encode not bit-identical; same-engine codecode/decodec still requires `A' == A` (green). Do not claim rust↔lean compress wire identity. -- **W2b permanent residual — Directory cross-engine encode:** compare **live rust vs live lean** catalog roots under identical pins (phase3 seed tree, zero master, default options) — **not** lean-vs-stale-seed. Pinned in `tests/determinism_roundtrip.rs`: live rust `0b119f12…`, live lean `f67b6f49…` (hard `assert_ne!`). `phase3_g9_directory` catalog `16e2369f…` is **decode-only SSOT** (lags live rust catalog packaging while segment mains may still match). Same-engine directory codecode/decodec green. +- **W2a permanent residual — Compression cross-engine encode:** Lean AOT zstd frames are **not** bit-identical to Rust `zstd` even at level 20 / same pin rev. Measured evidence (G9 `outboard_c14`): mains both 35 B; frame descriptor byte differs (`28b5 2ffd **00**…` rust vs `28b5 2ffd **20**…` lean); Bao roots and FEC parity diverge. **Specified and proved (header bits only):** level 20, magic `28b52ffd`, checksum off, no dictionary, reserved/unused 0; AOT/`ZSTD_compress` small frames use Single_Segment + 1-byte FCS (`0x20`); rust `copy_encode` unknown-size uses windowLog 25 (`0x00` `0x78`). Lean `Carbonado.Compress`; Rust `tests/zstd_frame_params.rs` parses frames. **Not proved:** compressed-block identity for arbitrary payloads (G9 26-byte c14 happens to share the raw last-block after the 6-byte header). **Policy:** decode interop only across engines; re-encode not bit-identical; same-engine codecode/decodec still requires `A' == A` (green). Do not claim rust↔lean compress wire identity. +- **W2b permanent residual — Directory cross-engine encode:** compare **live rust vs live lean** catalog roots under identical pins (phase3 seed tree, zero master, default options). Pinned in `tests/determinism_roundtrip.rs`: live rust `16e2369f…`, live lean `d468ea7a…` (hard `assert_ne!`). The rust pin equals the `phase3_g9_directory` catalog: encode sorts by `rel_path` before appending verification outboard / FEC (not `read_dir` order). Same-engine directory codecode/decodec green. The remaining residual is catalog packaging across engines (zstd), not filesystem listing order. - **W4c permanent residual — streaming zstd under lean:** buffer-only bulk zstd for Lean frame parity; public Compression outboard under lean stays O(logical) — not E2 (see matrix). - **W4d permanent residual — FEC / async spool:** FEC verify O(FEC body) shards (segment-wide RS); async always disk-stages O(encoded); lean+async peak RAM O(encoded+logical). See [STREAMING_PARALLELISM.md](../doc/STREAMING_PARALLELISM.md). - ~~Live Nix product-matrix vs optional frozen `ref/carbonado-rust`~~ **W5a / G1 closed** — permanent no product pin; live `src/` + `tests/` SSOT; third-party `ref/` pins only diff --git a/docs/PARITY.md b/docs/PARITY.md index 1eae627..9b04544 100644 --- a/docs/PARITY.md +++ b/docs/PARITY.md @@ -1,18 +1,18 @@ # carbonado — parity and `ref/` pins -## Method (dual-backend) +## Method -1. **Primary parity bar (G8):** the same Rust tests under `tests/` pass on **`backend-rust`** and **`backend-lean`** (Lean AOT `libcarbonado` via C ABI). See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [ABI.md](./ABI.md), [GAPS.md](./GAPS.md). **G8 full closed at R7** — `just test-lean-ci` / CI `dual-backend-lean` runs the **full** dual suite under lean features; permanent feature-gated / purity residuals only (`streaming_async` / `parallel_determinism` feature-gated off freeze — **R10** closed async dual policy; ~~pure Lean rkyv encode~~ **W3 closed** — dual-suite encode remains Rust rkyv composition SSOT; ~~W1a+W1b~~ dual honesty closed — public outboard stream E2 is S4 composition under lean; pure Lean chunked C residual). +1. **Rust contract:** `tests/` on the Rust engine (`cargo test`). There is no Cargo Lean backend and no product C ABI. Do not claim G8 C-ABI parity. See [TEST_CONTRACT.md](./TEST_CONTRACT.md), [GAPS.md](./GAPS.md). 2. **Component oracles:** pin reference trees under `ref/`; keep offline drivers (`etm-vectors`, `rs-vectors`, `bao-vectors`) for fast regression against Lean goldens / AOT demos. -3. **Cross-backend tests (G9):** **closed at R8** — Rust encode → Lean decode and reverse for no-compress body/headered/outboard (public + fixed-nonce encrypted); fixtures under `tests/fixtures/g9/`; contract `tests/g9_cross_backend.rs`. **W2d** same-engine codecode/decodec in `tests/determinism_roundtrip.rs`. **Permanent residuals (W2a/W2b):** cross-engine Compression / directory encode bit-match (decode interop only). +3. **Lean goldens (G9 remainder):** Rust still decodes committed Lean AOT bytes under `tests/fixtures/g9/lean/` (`just test-g9`). Live rust→lean encode via C is gone. **W2d** same-engine codecode/decodec in `tests/determinism_roundtrip.rs`. Compression / directory encode vs Lean AOT frames remains a measured residual. **SSOT roles (do not invert):** | Layer | Role | |-------|------| -| Rust `src/` + default `backend-rust` | First-class production engine | -| Rust `tests/` | Normative behavioral contract for **both** backends | -| Lean `Carbonado/` + AOT `libcarbonado` | Second engine: proofs + wire/C-ABI compatible implementation | +| Rust `src/` + default `backend-rust` | Production engine | +| Rust `tests/` | Normative behavioral contract for the Rust engine | +| Lean `Carbonado/` + AOT demo | Spec, proofs, demo binary (tiny C `@[extern]` for zstd/SLH only) | | `ref/` | Pinned third-party oracles and vector drivers | Pin the exact **third-party** trees the Rust product used (Bao, RS, crypto crates, zstd, bitcoinpqc, …); Lean AOT must remain wire-compatible with that contract. Rust product under `src/` + `tests/` is **first-class SSOT** — not demoted to “oracle only.” **G1 closed (W5a):** no `ref/carbonado-rust` product pin (permanent policy; see freeze strategy below). @@ -21,7 +21,7 @@ Pin the exact **third-party** trees the Rust product used (Bao, RS, crypto crate | ref path | Source | Pin (commit / tag) | |----------|--------|--------------------| -| `ref/bao-tree` | `https://github.com/SurmountSystems/bao-tree.git` | **`02916e784bb0afe0fd5a73c291c8c5335865e166`** (Cargo.lock; branch `76-keyed-bao`) | +| `ref/bao-tree` | `https://github.com/SurmountSystems/bao-tree.git` | Oracle snapshot **`02916e784bb0afe0fd5a73c291c8c5335865e166`** (keyed work before upstream squash). **Product** cargo dep is n0-computer/bao-tree **`dbc952e32cbda8ffd14c106b770e72987b01618e`** (PR 78 merge; git rev, not crates.io). | | `ref/reed-solomon-erasure` | `https://github.com/darrenldl/reed-solomon-erasure.git` | tag **`v5.0.3`** → **`9f974918f8c598eee351406c36fa0295f4bb4d69`** | | `ref/rustcrypto-block-ciphers` | `https://github.com/RustCrypto/block-ciphers.git` | tag **`aes-v0.8.4`** → **`f2dbee516b4d0cf4cb4f3045d09e35b5fd80087b`** | | `ref/rustcrypto-macs` | `https://github.com/RustCrypto/MACs.git` | tag **`hmac-v0.12.1`** → **`46797e3b44973a30edb9d7f3a3ebb41810061d90`** | @@ -104,7 +104,7 @@ cd ref/parity-harness/drivers/bao-vectors && cargo run --quiet ## Program F zstd + SLH parity 1. **zstd pin / product SSOT:** commit **`f8745da6…`** (tag v1.5.7). Checked out as `ref/zstd` submodule for oracle/review; flake **fetches the same rev+hash** into `nix/native` (`zstdPinned` in `flake.nix`) and **statically** compiles `lib/common|compress|decompress` + FFI into `libcarbonado_native.a` (level 20, single-threaded, no shared `-lzstd`). nixpkgs is only for the host toolchain / Lean headers — **not** the zstd source pin. Updating zstd requires: submodule checkout, `flake.nix` rev/hash, and PARITY table together. -2. **Goldens (AOT `demo`):** API frames for empty (`28b52ffd2000010000`) and `hello` (`28b52ffd200529000068656c6c6f`); corrupt frame → `decompressionFailed`; tight `maxOut` → `outputTooLarge`; zeros shrink; pipeline c2/c6 + **headered c3/c7**; full format matrix incl. compression at runtime. +2. **Goldens (AOT `demo`):** API frames for empty (`28b52ffd2000010000`) and `hello` (`28b52ffd200529000068656c6c6f`); frame-header parse (Single_Segment + 1-byte FCS, checksum off, no dict); G9 c14 prefixes (`20 1a` lean vs `00 78` rust); corrupt frame → `decompressionFailed`; tight `maxOut` → `outputTooLarge`; zeros shrink; pipeline c2/c6 + **headered c3/c7**; full format matrix incl. compression at runtime. Rust `tests/zstd_frame_params.rs` parses the same bits (not a full bitstream proof; W2a). 3. **Interpreter residual:** Lean `@[extern]` bodies are identity for elaborator; real frames only in AOT (LIMITS). CarbonadoTest `native_decide` covers non-compression formats + status maps. 4. **SLH1 wire:** magic `SLH1`, signature 7856 B, sidecar 7860 B — matches `src/crypto.rs` `SLH1_*` and AGENTS §2.3. Pure suite: length errors + `parseMagicAtExactLen` / `badSlhMagic` gate theorems; full-length build/parse + all-zero magic in `demo`. 5. **Bind-to-root + live SLH (R9 / G10):** Lean model requires signature message = Bao root (`verifyBoundToExpected`); wrong root → `verificationFailed`. Real SLH-DSA-SHA2-128s sign/verify is **linked** via flake pin `b309f444…` (same SSOT as `ref/bitcoinpqc` submodule target) into `libcarbonado_native.a` — C `carbonado_slh_*` + Lean `@[extern]`. Nested worktree emptiness is not a product residual when the flake fetch pin is present. Dual-suite product SLH may still use Rust `bitcoinpqc` composition. @@ -120,7 +120,7 @@ cd ref/parity-harness/drivers/bao-vectors && cargo run --quiet ```bash git submodule update --init --recursive -# bao-tree must be at the lock commit: +# Oracle bao-tree (Lean goldens / bao-vectors). Product cargo uses n0-computer PR 78 merge. git -C ref/bao-tree checkout 02916e784bb0afe0fd5a73c291c8c5335865e166 ``` diff --git a/docs/PROOFS.md b/docs/PROOFS.md index e58b6ac..5e77315 100644 --- a/docs/PROOFS.md +++ b/docs/PROOFS.md @@ -2,7 +2,7 @@ **Policy:** product Lean under `Carbonado/` and `CarbonadoTest/` must contain **no** proof holes. -**Dual-backend:** Lean theorems prove properties of the Lean engine and wire model. **Bit-match / behavioral parity** with production is additionally enforced by the Rust suite on `backend-lean` (G8) and `ref/` oracles ([PARITY.md](./PARITY.md)). Proofs do **not** replace `tests/`; they complement them. +Lean theorems prove properties of the Lean wire model. **Bit-match / behavioral contract** for the production library is the Rust suite (`tests/`) plus `ref/` oracles ([PARITY.md](./PARITY.md)). Proofs do **not** replace `tests/`; they complement them. There is no Cargo Lean backend and no G8 C-ABI parity claim. **Hole vocabulary (forbidden):** `sorry`, `admit` (Lean alias for `sorry`). Enforced by `nix build .#checks.x86_64-linux.no-sorry` / `nix flake check` (and explicit check builds). The gate fails closed if those directories or their `*.lean` roots are missing. @@ -24,14 +24,14 @@ Enforced by `nix build .#checks.x86_64-linux.no-sorry` / `nix flake check` (and | `Carbonado/Stream.lean` | `full_stripe_inboard_len`, `full_stripe_retain`, `empty_stripe_retain`, `one_byte_stripe_retain`, `chunk_eq_slice`, `stripe_eq_k_slices` | | `Carbonado/Scrub.lean` | Pure RS mask search + Bao root oracle (AOT + CarbonadoTest) | | `Carbonado/Shard.lean` | `split_empty_budget`, `split_hello_budget_2`, `split_empty_plaintext` | -| `Carbonado/Compress.lean` | `zstdMagic_length`; `ofStatus_*`; `decode_status_*` for every status code; `statusOk_payload_identity` (pure framing helper; not an `@[extern]` decide) | +| `Carbonado/Compress.lean` | `zstdMagic_length`; `ofStatus_*`; `decode_status_*` for every status code; `statusOk_payload_identity` (pure framing helper; not an `@[extern]` decide). **Frame header (2026-08-24):** RFC descriptor/window/FCS parse; product flags (level 20, checksum off, no dict, reserved/unused 0); AOT small-frame descriptor `0x20`; streaming unknown-size descriptor `0x00` + windowLog 25; hello/empty/G9 prefix parse theorems. Not a full bitstream proof (W2a). | | `Carbonado/Slh.lean` | Wire: `parse_short_length`, `parse_empty`, `build_bad_sig_len`, `slh1_magic_bytes`, **`parse_magic_bad` / `zeros_not_slh1_magic` / `parse_bad_magic_when_exact`** (`badSlhMagic` path). Binding: `bind_bad_{pk,root,sig}`, **`wrong_root_fails`**, `sign_unavailable`, `sign_bad_root`. Full 7856 B wire: AOT Main | | `CarbonadoTest/Scaffold.lean` | Re-exports / restates wire invariants | | `CarbonadoTest/EtM.lean` | Re-exports MAC + guard theorems; **`native_decide` matrix** for crypto goldens | | `CarbonadoTest/Fec.lean` | Geometry + RS; all 7 `FecError` paths | | `CarbonadoTest/Bao.lean` | Geometry; BLAKE3; stream slice; **exact** every `BaoError` | | `CarbonadoTest/Pipeline.lean` | Non-compression format matrix + path tests; **exact maps** incl. `ofZstdError` → `zstdInvalidInput` | -| `CarbonadoTest/Compress.lean` | Every `ZstdError` status map; bit-clear compress/decompress; pipeline maps | +| `CarbonadoTest/Compress.lean` | Every `ZstdError` status map; bit-clear compress/decompress; pipeline maps; restates frame-header product theorems (hello/empty/G9 prefixes, reserved/magic reject) | | `CarbonadoTest/Slh.lean` | Short-path every `SlhError` **except** full-length AOT-only wire (length/magic/sig/pk/root/unavailable/verification via prefix+gate theorems; full 7860 B `badSlhMagic` + roundtrip in Main) | | `Carbonado/Adamantine.lean` | `adamantineMagic_*`, `adamantineHeaderLen_eq`; encode/decode empty public; `invalid_flags_bit1`; `invalid_fmt_c0`; `short_header`; `dev_v2_rejected` | | `Carbonado/Filepack.lean` | `cfp2Magic_length`; path: `rel_empty`, `rel_traversal`, `rel_absolute`, `rel_backslash`, `rel_ok`, `rel_empty_component` | diff --git a/docs/SPEC-MATRIX.md b/docs/SPEC-MATRIX.md index 3d573d8..5c51452 100644 --- a/docs/SPEC-MATRIX.md +++ b/docs/SPEC-MATRIX.md @@ -14,9 +14,9 @@ Every product capability maps to Lean module(s), parity gate(s), and proof statu | Outboard | `Carbonado.Bao` create/verify; `Carbonado.Outboard` product body (bare main + FEC parity + verification sidecar) | bao-vectors + `demo` outboard segment roundtrip | **Program D+G**: post-order Bao outboard; directory segments via `encodeOutboardBody` / `decodeOutboardBody` | | Streaming bounds | `Carbonado.Stream` | demo greps + theorems | **Program E:** O(stripe) FEC retain theorems (`maxFecStripeRetain`); pure stripe transducer model | | Sharding | `Carbonado.Shard` | demo multi-segment roundtrip | **Program E:** budget split + `chunk_index` sequence + headered segments | -| Zstd-20 compress | `Carbonado.Compress`, `CarbonadoTest.Compress` | `demo` API goldens (empty/hello); pipeline c2/c6 | **Program F closed**: linked zstd; status taxonomy; interpreter identity fallback (LIMITS) | +| Zstd-20 compress | `Carbonado.Compress`, `CarbonadoTest.Compress` | `demo` API goldens (empty/hello) + frame-header parse; `tests/zstd_frame_params.rs` reads frames; pipeline c2/c6 | **Program F closed**: linked zstd; status taxonomy; interpreter identity fallback (LIMITS). **Frame parameters specified:** level 20, magic, checksum off, no dict, AOT small-frame `0x20` + FCS, rust streaming `0x00` + windowLog 25. Full compressed-block identity still unproved (W2a). | | SLH1 sidecars | `Carbonado.Slh`, `CarbonadoTest.Slh` | `demo` wire + bind-to-root + live sign/verify | **Program F + R9/G10 closed**: wire/binding theorems; live SLH-DSA-SHA2-128s via libbitcoinpqc pin + `carbonado_slh_*` (LIMITS: elaborator fail-closed; dual-suite may keep Rust composition) | -| Adamantine directory | `Carbonado.Adamantine`, `Filepack`, `RkyvFilepack`, `Outboard`, `Directory`, `CarbonadoTest.Directory` | `demo` Program G greps; pure roundtrip AOT; dual-suite `lean_backend_phase3` | **Program G + W3 closed**: pure Lean directory/CLI emit **rkyv** FilepackManifestWire v2 (goldens). **P3 dual-suite closed**: rkyv catalog via Rust composition + Lean segment/catalog crypto (dual-suite encode SSOT; not pure Lean required). CFP2 dual-decode fallback only | +| Adamantine directory | `Carbonado.Adamantine`, `Filepack`, `RkyvFilepack`, `Outboard`, `Directory`, `CarbonadoTest.Directory` | `demo` Program G greps; pure roundtrip AOT | **Program G + W3 closed**: pure Lean directory/CLI emit **rkyv** FilepackManifestWire v2 (goldens). Rust directory encode remains the production path. | | CLI | `Carbonado.Cli`, `Carbonado.Main` | `demo` + CLI subcommands | **Program G + R9:** encode/decode file+dir; single-file default `{bao_root_hex}.c{fmt:02x}`; dir default `{input}-archive/`; `slh parse` wire; `slh verify` live oracle (exit 0 accept / exit 1 reject) | -Expand rows until full product parity with dual-backend G8 (`backend-rust` + `backend-lean` on `tests/`). **G1/W5a closed:** no optional `ref/carbonado-rust` product pin (live tree SSOT). Component rows above track Lean+Nix proof/oracle gates; dual-suite status is [GAPS.md](./GAPS.md) G8 / [TEST_CONTRACT.md](./TEST_CONTRACT.md). +Component rows track Lean+Nix proof/oracle gates. The Rust behavioral contract is [TEST_CONTRACT.md](./TEST_CONTRACT.md). **G1/W5a closed:** no optional `ref/carbonado-rust` product pin. G8 C-ABI dual-backend was **removed** 2026-08-24. diff --git a/docs/TEST_CONTRACT.md b/docs/TEST_CONTRACT.md index 13c4911..d5f1b95 100644 --- a/docs/TEST_CONTRACT.md +++ b/docs/TEST_CONTRACT.md @@ -1,33 +1,23 @@ -# carbonado — Rust test contract (dual-backend) +# carbonado — Rust test contract -The Rust integration suite under `tests/` is the **normative behavioral contract**. -Both `backend-rust` (default) and `backend-lean` (Lean AOT `libcarbonado` via C ABI) must pass the **same** tests, growing from a Phase 1 allowlist to the full suite. +The Rust integration suite under `tests/` is the **normative behavioral contract for the Rust engine**. -See [ABI.md](./ABI.md), [PARITY.md](./PARITY.md), [GAPS.md](./GAPS.md) G8, [LIMITS.md](./LIMITS.md). +Lean (`Carbonado/`, `CarbonadoTest/`) is spec + proofs + an AOT demo. There is **no** Cargo `backend-lean`, **no** `carbonado-sys`, and **no** product C ABI. Do **not** claim G8 C-ABI parity. -**Invariant:** never regress `backend-rust` `cargo test` while landing Lean paths. +See [PARITY.md](./PARITY.md), [GAPS.md](./GAPS.md), [LIMITS.md](./LIMITS.md), AGENTS.md product model. -## How backends relate to this suite +**Invariant:** never regress default `cargo test`. + +## Engines | Feature flag | Engine | Expected of this suite | |--------------|--------|------------------------| -| `backend-rust` (default) | Pure Rust (`src/encoding`, `src/decoding`, `src/file`, …) | Full green (always) | -| `backend-lean` | Lean AOT via `carbonado-sys` / `libcarbonado` | Full green (G8 closed at R7; freeze = unfiltered dual suite) | +| `backend-rust` (default, empty marker) | Pure Rust (`src/encoding`, `src/decoding`, `src/file`, …) | Full green | ```bash -# Normative default (backend-rust) cargo test - -# Dual-backend freeze (Phase 5 / G11 + R7 G8 full close — shared CI + humans) -# Prefer the single recipe (builds libcarbonado if needed; fail-closed if .so missing): -just test-lean-ci - -# Manual equivalent (R7: freeze = full unfiltered dual suite): -# nix build .#libcarbonado -o result-libcarbonado -# export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -# export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -# export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} -# cargo test --no-default-features --features "backend-lean,pqc,ots,cli" +just test-lean-ci # Lean no-sorry + AOT demo (nix), not cargo --features backend-lean +just test-g9 # Rust decode of committed Lean AOT goldens under tests/fixtures/g9/lean/ ``` Helpers under `tests/common/` are not separate contract files; they support the files below. @@ -36,274 +26,43 @@ Helpers under `tests/common/` are not separate contract files; they support the ## Classification (every `tests/*.rs` file) -| Class | File | lean-backend target phase | Notes | -|-------|------|---------------------------|-------| -| **core** | `codec.rs` | Phase 1–2 | Low-level `encode`/`decode`, slice, scrub, header layout, samples | -| **core** | `format.rs` | Phase 1–2 | Full format matrix (inboard + outboard + scrub) | -| **core** | `format_amplification.rs` | Phase 2 | Size/geometry amplification via `file::encode` | -| **core** | `header_tamper.rs` | Phase 1–2 | Header field flips → auth / layout failures | -| **core** | `bao_keyed_contract.rs` | Phase 1–2 | Verification key, keyed roots, slice verify paths | -| **core** | `adversarial_proptest.rs` | Phase 2 | Proptest outboard/header adversarial | -| **core** | `deprecation_aliases.rs` | Phase 3+ | Type/const aliases only (no encode path) | -| **fec_scrub** | `fec_chaos.rs` | Phase 2 | Distributed knockouts inboard/outboard | -| **fec_scrub** | `fec_scrub_matrix.rs` | Phase 2 | Scrub matrix public/encrypted FEC | -| **fec_scrub** | `shard_fec_scrub.rs` | Phase 2 | Per-segment scrub after sharding | -| **fec_scrub** | `udp_fec_sim.rs` | Phase 2 | Datagram FEC sim + directory scrub path | -| **fec_scrub** | `apocalypse.rs` | Phase 2 | Large-sample encode/scrub chaos | -| **stream** | `streaming.rs` | Phase 2 | Stream encode/decode buffer + outboard | -| **stream** | `streaming_limits.rs` | Phase 2 | Bounds, FEC encoder, crypto stream, scrub | -| **stream** | `seekable_slices.rs` | Phase 2 | O(slice) retain/hash; full body/main at lean C (W4a/W4b; see LIMITS slice table) | -| **shard** | `sharding.rs` | Phase 2 | `encode_shard_stream` / `decode_shards_stream` | -| **directory** | `directory_archive.rs` | Phase 3 (core); OTS cases Phase 4 closed | Adamantine 1.0 + rkyv catalog + scrub_outboard; OTS via CBOTS composition | -| **directory** | `filepack_interop.rs` | Phase 3 | Filepack / CBOR interop + directory decode | -| **directory** | `format_policy.rs` | Phase 3 | Segment format policy (no I/O encode) | -| **cli** | `bin_cli.rs` | Phase 4 | Prebuilt `carbonado` binary CLI (default rust-engine; lean when rebuilt with `backend-lean,cli`) | -| **cli** | `bin_smoke.rs` | Phase 4 | CLI smoke encode/decode | -| **cli** | `bin_heuristics.rs` | Phase 4 | Filename heuristics + CLI | -| **pqc** | `slh_outboard.rs` | Phase 4 closed | SLH-DSA sidecars + header `slh_public_key` (Rust bitcoinpqc under both backends) | -| **async** | `streaming_async.rs` | **feature-gated permanent** (needs `async`; **R10 closed**) | Freeze excludes `async` → 0 tests under dual suite; optional lean+async dual-aware via R5 E1 | -| **parallel** | `parallel_determinism.rs` | **feature-gated** (needs `parallel`; not dual residual) | RS parallel vs serial determinism; 0 tests under dual feature set | -| **parallel** | `serial_fec_path.rs` | Phase 2 (serial) | Serial FEC encoder vs buffer path | -| **lean_allowlist** | `lean_backend_smoke.rs` | Phase 1 closed | Dual-backend body/headered/auth smoke (`just test-lean-smoke`) | -| **lean_allowlist** | `lean_backend_phase2.rs` | Phase 2 closed | Outboard/scrub/slice/stream + G9 seed (`just test-lean-phase2`) | -| **lean_allowlist** | `lean_backend_phase3.rs` | Phase 3 closed | Directory composition + G9 dir fixture (`just test-lean-phase3`) | -| **lean_allowlist** | `lean_backend_phase4.rs` | Phase 4 closed | SLH composition + CLI dual (directory dual-engine + buffer APIs; stream E1 dual closed at R5; **W1a** `decode_stream` dual closed; **W1b** public non-compress outboard composition E2 closed; pure Lean chunked stream C residual) + directory OTS (`just test-lean-phase4`) | -| **g9_matrix** | `g9_cross_backend.rs` | **R8 / G9 closed** | Full cross-backend no-compress matrix both directions; fixtures `tests/fixtures/g9/` (`just test-g9`) | -| **determinism** | `determinism_roundtrip.rs` | **W2d closed** | codecode (EDE) + decodec (DED) no-compress matrix; same-engine compress (body/headered/outboard) + directory; W2a/W2b hard residual asserts (live-vs-live dir roots; G9 c14 mains) | -| **lean_freeze (P5+R7)** | see Phase 5 / R7 section | Phase 5 + R7 closed | Freeze = full dual suite: `just test-lean-ci` (unfiltered lean features; includes `g9_cross_backend` + `determinism_roundtrip`) | - -**Inventory count:** 32 integration test files under `tests/*.rs` (R8: `g9_cross_backend`; W2: `determinism_roundtrip`). - -**Directory vs OTS:** Core Adamantine / rkyv dual-suite green is **Phase 3**. Entry/catalog OTS proof cases (feature `ots`) are **Phase 4 closed** via pure-Rust CBOTS composition over Lean container crypto (no Lean-native stamping). - ---- - -## Primary public APIs used by tests - -Mapped from actual `use carbonado::…` imports in `tests/*.rs`. C ABI column is the dual-backend export target ([ABI.md](./ABI.md)). - -| API / type | Typical tests | C ABI priority | -|------------|---------------|----------------| -| `encode` / `decode` / `encode_with_nonce` (crate root = `encoding`/`decoding`) | codec, format, header_tamper, fec_*, apocalypse, udp_fec_sim, parallel_determinism, **g9_cross_backend** | **v0** (`carbonado_encode` / `carbonado_decode`; fixed nonce via optional C arg) | -| `encode_outboard` / `decode_outboard` | format, fec_*, bao_keyed, streaming*, directory, adversarial | **P2 live** (`carbonado_encode_outboard` / `carbonado_decode_outboard`) | -| `scrub` / `scrub_outboard` | codec, format, fec_*, apocalypse, streaming_limits, shard_fec_scrub, directory | **P2 live** (`carbonado_scrub` / `carbonado_scrub_outboard`) | -| `verify_slice` / `extract_slice` | codec, seekable_slices, bao_keyed | **P2 + W4a** (`carbonado_verify_slice`; extract = count 1; O(slice) retain; full body input) | -| `verify_slice_inboard_seekable` / `verify_slice_outboard` | bao_keyed, seekable_slices | **v1+**; **R9** outboard C live (`carbonado_verify_slice_outboard` + lean dispatch) | -| `crypto::slh_*` (pure Lean path) | AOT demo / optional | **R9** `carbonado_slh_*` live; dual-suite may keep Rust bitcoinpqc | -| rkyv catalog encode+decode (Lean) | AOT demo goldens | **W3** `Carbonado/RkyvFilepack` encode/decode; dual-suite encode still Rust rkyv composition SSOT | -| `carbonado_verification_key` | bao_keyed_contract | **v0** | -| `file::encode` / `file::decode` / `Header` | format, format_amplification, header_tamper, streaming_limits, slh_outboard, adversarial | **v0** (`carbonado_encode_headered` / `carbonado_decode_headered`) | -| `file::encode_stream` / `decode_stream` | streaming, streaming_limits | **R5 E1** encode_stream → Lean; **W1a** `decode_stream` spool→Lean `decode_headered` (E1 RAM; not E2) | -| `file::encode_directory` / `encode_directory_with_options` / `decode_directory` | directory_archive, filepack_interop, udp_fec_sim | **P3 live** (composition: rkyv+FS Rust; segment/catalog crypto via outboard/headered Lean ABI) | -| `stream_encode_buffer` / `stream_decode_buffer` (+ outboard buffer variants) | streaming*, bao_keyed, parallel_determinism | Phase 2 + **R5** stream I/O E1 over same Lean buffer ABI | -| `stream_encode_inboard` / `stream_decode` | streaming, streaming_limits | **R5 E1** spool-to-buffer → Lean (O(logical); not E2) | -| `stream_encode_outboard` / `stream_decode_outboard` | streaming, streaming_limits | **W1b:** public non-compress → S4 O(chunk/stripe) composition E2 under lean; public+Compression under lean O(logical) bulk zstd; encrypted → Lean E1 | -| `stream::fec::*` / `stream::parallel::*` | streaming_limits, serial_fec_path, parallel_determinism | rust-internal / serial lean | -| `encode_shard_stream` / `decode_shards_stream` | sharding, shard_fec_scrub | Phase 2 | -| Adamantine / filepack_manifest / format_policy | directory_*, filepack_interop, format_policy | **P3 live** (rkyv wire; format_policy pure Rust) | -| `crypto::slh_*` / sidecar helpers (dual-suite product) | slh_outboard, lean_backend_phase4 | **P4 live** (G10-A: Rust `bitcoinpqc` composition under both backends) | -| `carbonado_slh_*` C ABI (pure Lean path) | AOT demo / optional C consumers | **R9 live** optional purity; dual-suite need not switch from composition | -| `ots::*` | directory_archive OTS + lean_backend_phase4 (feature `ots`) | **P4 live** (Rust CBOTS; no Lean OTS engine) | -| Deprecation aliases (`PackIndex`, …) | deprecation_aliases | n/a (API surface only) | -| `stream_decode_async` | streaming_async | **R10:** optional adapter; dual freeze never requires `async`; under lean+async → dual-aware `stream_decode` (disk O(encoded); lean peak RAM O(encoded+logical); not E2) | -| CLI binary (`src/bin/carbonado`) | bin_*, lean_backend_phase4 | **P4 + R5 + W1:** directory CLI + buffer APIs + stream encode E1 + `decode_stream` W1a + public outboard W1b composition; rebuild with `cli`+`backend-lean` | - -### Error-contract note (both backends) - -Tests that `matches!` ultra-specific `CarbonadoError` variants require a stable C-code → Rust mapping ([ABI.md](./ABI.md) error table). Phase 1 may collapse some Lean `PipelineError` variants into broader ABI codes; refine mapping before claiming full-suite green on failure-mode tests (`header_tamper`, scrub unnecessary vs failed, etc.). - ---- - -## Phase 1 allowlist (first green `backend-lean` gate) - -**Phase 1 closed:** live C ABI via Lean AOT (`l_carbonado_*` + `carbonado_abi.c`), `nix build .#libcarbonado`, Rust `backend-lean` dispatch for `encode`/`decode`/`file::{encode,decode}`/`carbonado_verification_key`, allowlist `tests/lean_backend_smoke.rs` (`just test-lean-smoke`). - -### Phase 1 scope (concrete) - -1. **Public (even) formats only** for first green: e.g. c0, c2, c4, c6, c12, c14 — no encryption / no random nonce dependency until headered encrypted path is deterministic under test nonces. -2. **Buffer / headered APIs only** (match C ABI v0): - - `carbonado_abi_version` / `carbonado_free` - - `carbonado_verification_key` - - `carbonado_encode` / `carbonado_decode` (low-level body; Rust `encoding::encode` / `decoding::decode` shape) - - `carbonado_encode_headered` / `carbonado_decode_headered` (Rust `file::encode` / `file::decode` shape) -3. **Suggested first test targets** (grow in CI / justfile as green): - - Subset of `tests/codec.rs` (roundtrip + basic failure) **or** a dedicated `tests/lean_backend_smoke.rs` reusing `tests/common` helpers - - `tests/bao_keyed_contract.rs` cases that only need verification key + comparable encode roots - - Selected `tests/header_tamper.rs` auth-fail cases once headered encode is real (not stub) -4. **Explicitly out of Phase 1:** outboard, scrub, seekable slice C exports, directory/rkyv, CLI, SLH FFI, async, parallel RS. - -### Phase 1 non-goals - -- Full `tests/` green on `backend-lean` -- Changing normative wire format -- Replacing or deleting the Rust engine - -Document the live freeze in CI / justfile. Full suite is the G8 bar — **closed at R7** (see Phase 5 / R7). - ---- - -## Phase 2 allowlist (outboard / scrub / slice + G9 start) - -**Phase 2 closed:** additive C ABI for outboard encode/decode, scrub / scrub_outboard, verify_slice; richer encode metadata (`chunk_len`, `bytes_ecc`, `verifiable_slice_count`); Lean geometry-peel inboard scrub + outboard scrub; Rust dispatch under `backend-lean` for those APIs + stream buffer composition over body/outboard ABI; allowlist `tests/lean_backend_smoke.rs` + `tests/lean_backend_phase2.rs` (`just test-lean-phase2`). - -### Phase 2 scope (concrete) - -1. **Public formats** c0/c4/c12/c14 (+ fixed-nonce encrypted helpers for c5) on body, headered, outboard. -2. **New C symbols (ABI v1 additive):** - - `carbonado_encode_outboard` / `carbonado_decode_outboard` - - `carbonado_scrub` / `carbonado_scrub_outboard` - - `carbonado_verify_slice` - - `CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION` (13) — distinct from scrub recovery failure - - encode body pack extended with chunk/ecc/vsc fields (nullable C out-params) -3. **G9 start:** rust-engine golden buffers (c0/c4 body + headered c4) decoded under lean; lean re-encode bit-matches rust body. -4. **Stream buffers:** `stream_encode_buffer` / `stream_decode_buffer` / outboard buffer helpers compose over Lean under `backend-lean` (not silent pure-Rust). - -### Phase 2 residuals (historical; full suite green at R7) - -- ~~Residual sharding/fec_chaos~~ **green under lean (R6)** — **`format` (R1)** / **`header_tamper` (R2)** / **`format_amplification` (R3)** / **`codec` + `seekable_slices` (R4)** / **`streaming` + `streaming_limits` (R5)** / **`sharding` + `fec_chaos` (R6)** / `bao_keyed_contract` / `fec_scrub_matrix` in freeze -- Seekable outboard slice C **R9 live** (`carbonado_verify_slice_outboard` + lean dispatch; **W4b permanent** full buffers at C ABI — no ReadAt callback) -- **Inboard `verify_slice` under lean (W4a closed):** auth-first O(slice) retain via `decodeRecRetainRange` (O(N) time over full response; full body input at C) — parity with Rust `SliceRegionWriter` class; `seekable_slices` freeze-green -- CI freeze (Phase 5); pure Lean SLH FFI **closed at R9** (dual-suite may keep composition) -- ~~Pure Lean rkyv encode residual (**W3**)~~ **closed** — pure Lean encode/decode + Directory/CLI rkyv; dual-suite still uses Rust rkyv via composition -- Rust-root directory checksum goldens under lean encode (**W2b permanent residual** — same-engine directory codecode green in `determinism_roundtrip`; cross-engine roots may differ; `phase3_g9_directory` decode SSOT) - -### Determinism contracts (**W2d shipped**) - -| Contract | Steps | Assert | -|----------|-------|--------| -| **codecode** (EDE) | encode → decode → encode | `pt' == pt` and `A' == A` under fixed params (nonce pinned when Encrypted) | -| **decodec** (DED) | decode archive → encode → decode | `pt' == pt` and `B == A` when encode is deterministic under same pins | - -**Shipped:** `tests/determinism_roundtrip.rs` (auto-included in lean freeze). Pins: G9 MASTER/NONCE/`g9_matrix_v1`. - -| Matrix | Coverage | Wire equality | -|--------|----------|---------------| -| No-compress body | c0/c1/c4/c5/c8/c9/c12/c13 | full `A' == A` both engines | -| No-compress headered | c4/c5/c12/c13 | full `A' == A` both engines | -| No-compress outboard | c4/c5/c12/c13 | full wire (main/out/par/header) both engines | -| Compress body (same-engine) | c2/c3/c6/c7/c10/c11/c14/c15 | same-engine `A' == A`; **cross-engine permanent residual (W2a)** | -| Compress headered (same-engine) | c6/c7/c14/c15 | same-engine `A' == A` | -| Compress outboard (same-engine) | c6/c7/c14/c15 | same-engine wire equality | -| Directory public (same-engine) | phase3 seed tree, zero master | same-engine catalog+segments | -| DED from G9 body goldens | body no-compress, active-engine fixtures | re-encode matches committed golden | -| W2a residual | G9 `outboard_c14` rust vs lean mains | hard `assert_ne!` + frame descriptor (fail-closed if fixtures missing) | -| W2b residual | live rust vs live lean catalog roots | hard pins `0b119f12…` ≠ `f67b6f49…`; seed `16e2369f…` decode-only | - -Encrypted without fixed nonce: wire identity out of scope. - ---- - -## Phase 3 allowlist (directory dual-backend) - -**Phase 3 closed:** directory encode/decode under `backend-lean` via composition (Rust rkyv FilepackManifest v2 + Adamantine framing + FS; Lean outboard/headered crypto). Allowlist `tests/lean_backend_phase3.rs` (+ `format_policy`); G9 rust-encode fixture → lean decode (`tests/fixtures/phase3_g9_directory/`). `just test-lean-phase3`. - -Core `directory_archive` and non-golden `filepack_interop` pass under lean in practice; dedicated allowlist remains the gate. OTS dual-backend / CLI dual → Phase 4 (closed). - ---- - -## Phase 4 allowlist (PQC + CLI + directory OTS) - -**Phase 4 closed:** dual-suite SLH via **G10 strategy A** (Rust `bitcoinpqc` `crypto::slh_*` under both backends). **R9:** pure Lean `signRoot`/`verifyRoot` live via libbitcoinpqc in `libcarbonado` (optional purity path; dual-suite need not switch). Directory OTS: offline CBOTS composition (no Lean-native stamping). - -**CLI dual (honest scope):** -- **Dual-engine:** directory encode/decode (library + subprocess), buffer APIs (`file::encode` / `file::encode_outboard` / headered decode), inboard/encrypted stream **E1** → Lean buffer ABI, **W1a** `decode_stream` → Lean `decode_headered`. -- **W1b:** public `stream_*_outboard` under lean is S4 O(chunk/stripe) **composition** (not pure Lean stream). Pure Lean chunked C residual remains. -- **Lean-linked binary:** `cli` + `backend-lean` proves link/run; directory subprocess + stream dual paths are CLI evidence. - -```bash -just test-lean-phase4 -# or: -cargo test --no-default-features --features "backend-lean,pqc,ots,cli" \ - --test lean_backend_smoke --test lean_backend_phase2 --test lean_backend_phase3 \ - --test format_policy --test slh_outboard --test lean_backend_phase4 -``` - -Allowlist: `lean_backend_phase4.rs` + `slh_outboard.rs` (+ Phase 1–3). Full `bin_*` matrix optional under lean-linked binary. - ---- - -## Phase 5 — CI freeze both backends (G11 closed; G8 allowlist freeze → R7 full) - -**Phase 5 closed** as dual-backend CI freeze of the allowlist. **R7 expanded freeze to full dual suite and closed full-suite G8.** - -### Normative commands - -| Backend | CI job (`.github/workflows/rust.yaml`) | Local / shared command | -|---------|----------------------------------------|-------------------------| -| `backend-rust` | **`desktop`** | `cargo test`; serial: `--no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path`; optional: `--features "async,async-tokio,man-gen"` (never `--all-features`); smoke + CLI | -| `backend-lean` | **`dual-backend-lean`** | `just test-lean-ci` (full dual suite as of R7) | - -### Lean env + build (fail-closed) - -```bash -nix build .#libcarbonado -o result-libcarbonado -export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} -# Fail-closed: libcarbonado.so (or .dylib) must exist under CARBONADO_LEAN_LIB -just test-lean-ci -``` - -- **Never** enable both engines: must use `--no-default-features --features "backend-lean,pqc,ots,cli"` (not `--features backend-lean` alone). -- If `CARBONADO_LEAN_LIB` is unset, `just test-lean-ci` runs `nix build .#libcarbonado -o result-libcarbonado` then exports env. -- If the shared library is still missing after build, the recipe **exits non-zero** (fail-closed). -- `carbonado-sys` with feature `require-lib` (enabled by `backend-lean`) **hard-errors** if `CARBONADO_LEAN_LIB` is unset or the library file is missing; CI always sets env after nix build. -- Lean-only integration crates (`tests/lean_backend_*.rs`) use `#![cfg(feature = "backend-lean")]`. Under default/`backend-rust` builds they compile as empty harnesses (0 tests) — expected, not a silent skip of freeze coverage (freeze always uses `backend-lean`). - -### Freeze contents (R7) - -Unfiltered full dual suite under features `"backend-lean,pqc,ots,cli"`: lib units + all integration tests (phase gates, measured-green files from P5/R1–R6, **`bin_cli` / `bin_heuristics` / `bin_smoke`**, etc.). Historical P5 explicit `--test` allowlist documented in [GAPS.md](./GAPS.md) for archaeology. - -### G8 status (honest) - -| Claim | Status | -|-------|--------| -| G11 live CI both backends (Linux) | **closed** | -| G8 **allowlist** dual-backend bar (P5 freeze) | **closed** (this phase) | -| G8 **full** `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"` | **closed** (R7 2026-07) — freeze equals full suite | - -**Post-G8 residuals (purity / feature-policy — not dual-suite red):** ~~pure Lean SLH FFI~~ **R9 closed**; ~~seekable outboard slice C~~ **R9 closed**; ~~rkyv dual-decode~~ **R9 closed**; ~~pure Lean rkyv encode + Directory/CLI~~ **W3 closed** (dual-suite Rust rkyv encode composition SSOT; never claim dual-suite *requires* pure Lean); ~~async dual policy~~ **R10 closed** (freeze never requires `async`; lean+async dual-aware via E1); `streaming_async` / `parallel_determinism` permanently feature-gated off freeze; ~~W1a+W1b dual honesty / public outboard E2~~ **closed** (pure Lean chunked C residual); ~~W2d codecode/decodec~~ **closed** (`determinism_roundtrip`); ~~W2a/W2b~~ **permanent residuals** (cross-engine compress/dir encode; same-engine green); ~~W4a inboard O(slice) retain~~ **closed**; **W4b** permanent full-buffer C outboard slice; **W4c** permanent buffer-only zstd under lean; **W4d** permanent FEC O(body) + async encoded spool. - ---- - -## Later phases (test-suite coverage) - -| Phase | Test classes unlocked | Depends on | -|-------|----------------------|------------| -| **2** | outboard/scrub/slice smoke + G9 seed + stream buffer compose | **closed** — see Phase 2 allowlist above | -| **3** | directory (core Adamantine/rkyv), format_policy, filepack_interop (non-golden), deprecation_aliases | **closed** — rkyv dual-suite via composition | -| **4** | cli dual, pqc (slh_outboard), ots directory paths | **closed** — G10-A composition; pure Lean SLH FFI **R9 closed** | -| **5** | CI freeze both backends; G11 closed; G8 allowlist freeze | **closed** — `just test-lean-ci` + `dual-backend-lean` job | -| **R7** | Full G8 close; freeze = unfiltered lean suite (incl. `bin_*`) | **closed** (2026-07) — see [GAPS.md](./GAPS.md) R7 | -| **R8** | G9 full cross-backend matrix (body/headered/outboard, both directions) | **closed** (2026-07) — `tests/g9_cross_backend.rs` + `tests/fixtures/g9/`; see [GAPS.md](./GAPS.md) R8 | - -### R8 / G9 classification - -| Concern | Detail | -|---------|--------| -| Contract file | `tests/g9_cross_backend.rs` | -| Fixtures | `tests/fixtures/g9/{rust,lean}/` (manifest JSON + binary blobs) | -| lean→rust | default `backend-rust` decodes `lean/*` (no libcarbonado required) | -| rust→lean | `backend-lean` + `CARBONADO_LEAN_LIB` decodes `rust/*`; public + fixed-nonce encrypted body re-encode bit-match | -| Matrix | body c0/c1/c4/c5/c8/c9/c12/c13; headered c4/c5/c12/c13; outboard c4/c5/c12/c13/c14 | -| Residual | **W2a/W2b permanent:** cross-engine Compression / directory encode bit-match; same-engine codecode green; `phase3_g9_directory` decode seed remains SSOT | -| Regen | `just g9-gen-fixtures` or `G9_WRITE_FIXTURES=1` on ignored `write_fixtures` | - -### W2d / determinism classification - -| Concern | Detail | -|---------|--------| -| Contract file | `tests/determinism_roundtrip.rs` | -| Contracts | **codecode** (EDE) + **decodec** (DED) — shipped for claimed matrix | -| Pins | same MASTER/NONCE/`g9_matrix_v1` as G9 | -| Engines | default `backend-rust` + lean freeze (auto-include) | -| Cross-link | [GAPS.md](./GAPS.md) W2 table; [LIMITS.md](./LIMITS.md) W2a/W2b permanent residuals | +| Class | File | Notes | +|-------|------|-------| +| **core** | `codec.rs` | Low-level `encode`/`decode`, slice, scrub, header layout, samples | +| **core** | `format.rs` | Full format matrix (inboard + outboard + scrub) | +| **core** | `format_amplification.rs` | Size/geometry amplification via `file::encode` | +| **core** | `header_tamper.rs` | Header field flips → auth / layout failures | +| **core** | `bao_keyed_contract.rs` | Verification key, keyed roots, slice verify paths | +| **core** | `adversarial_proptest.rs` | Proptest outboard/header adversarial | +| **core** | `deprecation_aliases.rs` | Type/const aliases only (no encode path) | +| **fec_scrub** | `fec_chaos.rs` | Distributed knockouts inboard/outboard | +| **fec_scrub** | `fec_scrub_matrix.rs` | Scrub matrix public/encrypted FEC | +| **fec_scrub** | `shard_fec_scrub.rs` | Per-segment scrub after sharding | +| **fec_scrub** | `udp_fec_sim.rs` | Datagram FEC sim + directory scrub path | +| **fec_scrub** | `apocalypse.rs` | Large-sample encode/scrub chaos | +| **stream** | `streaming.rs` | Stream encode/decode buffer + outboard | +| **stream** | `streaming_limits.rs` | Bounds, FEC encoder, crypto stream, scrub | +| **stream** | `seekable_slices.rs` | O(slice) retain/hash | +| **shard** | `sharding.rs` | `encode_shard_stream` / `decode_shards_stream` | +| **directory** | `directory_archive.rs` | Adamantine 1.0 + rkyv catalog + scrub_outboard | +| **directory** | `filepack_interop.rs` | Filepack / CBOR interop + directory decode | +| **directory** | `format_policy.rs` | Segment format policy (no I/O encode) | +| **cli** | `bin_cli.rs` | Prebuilt `carbonado` binary CLI | +| **cli** | `bin_smoke.rs` | CLI smoke encode/decode | +| **cli** | `bin_heuristics.rs` | Filename heuristics + CLI | +| **pqc** | `slh_outboard.rs` | SLH-DSA sidecars + header `slh_public_key` | +| **async** | `streaming_async.rs` | `#![cfg(feature = "async")]` | +| **parallel** | `parallel_determinism.rs` | RS parallel vs serial determinism | +| **parallel** | `serial_fec_path.rs` | Serial FEC encoder vs buffer path | +| **g9_goldens** | `g9_cross_backend.rs` | Rust decode of committed Lean AOT goldens + rust self-roundtrip (`just test-g9`) | +| **determinism** | `determinism_roundtrip.rs` | codecode (EDE) + decodec (DED); same-engine compress + directory | +| **zstd** | `zstd_frame_params.rs` | Frame flags vs Lean AOT goldens (honest descriptor residual) | +| **rkyv** | `rkyv_golden_lock.rs` | Directory catalog rkyv goldens | + +Removed 2026-08-24: `lean_backend_smoke.rs`, `lean_backend_phase2.rs`, `lean_backend_phase3.rs`, `lean_backend_phase4.rs` (they existed only for Cargo `backend-lean` via C). --- -## Maintenance +## Historical note (not current product) -- New `tests/*.rs` files **must** be added to the classification table above in the same PR. -- New public encode/decode surfaces used by tests must be listed in the API table and, if dual-backend-relevant, in [ABI.md](./ABI.md). -- Prefer strict `matches!` on specific `CarbonadoError` variants for failure-mode tests; when ABI collapse prevents 1:1 mapping, document backend-aware expectations rather than loosening asserts permanently. -- **R7 freeze is unfiltered** under lean features (`just test-lean-ci` = full `cargo test --no-default-features --features "backend-lean,pqc,ots,cli"`). New contract integration tests **auto-enter** CI `dual-backend-lean` — no allowlist edit required. Rust-only or feature-gated suites **must** use `#![cfg(...)]` (as `streaming_async` / `parallel_determinism` do) and be documented under post-G8 residuals in [GAPS.md](./GAPS.md) / [LIMITS.md](./LIMITS.md); otherwise lean CI will compile and run them. +G8 dual-backend via C ABI (`carbonado-sys` / `libcarbonado` / `backend-lean`) was **removed** 2026-08-24. Earlier GAPS/TEST_CONTRACT text that claimed full-suite parity through that C trampoline is archaeology, not a live gate. Lean proofs and the AOT demo remain. diff --git a/docs/VISION.md b/docs/VISION.md index 4b14c26..1d291f2 100644 --- a/docs/VISION.md +++ b/docs/VISION.md @@ -1,25 +1,25 @@ -# carbonado — vision (dual-backend: Rust + Lean 4 AOT + Nix) +# carbonado — vision (Rust engine + Lean 4 proofs + Nix) **Mission:** Apocalypse-resistant archival format for consensus-critical data. -**Product model:** Rust remains a **first-class** production engine (`src/`, default `backend-rust`). Lean 4 AOT (`Carbonado/`, `libcarbonado`) is a **second engine**: machine-checked proofs plus a wire- and C-ABI-compatible implementation. Both must pass the same Rust `tests/` (G8 dual-backend parity). +**Product model:** Rust is the production engine (`src/`, default `backend-rust`). Lean 4 (`Carbonado/`) is spec + machine-checked proofs plus an AOT demo. There is no product C ABI and no Cargo Lean backend. Do not claim G8 C-ABI parity. ## Prove everything Each product claim is either: -- machine-checked in Lean (no `sorry` in product), and/or -- bit-matched via the Rust suite on `backend-lean` and/or pinned `ref/` oracles (CI parity gates). +- machine-checked in Lean (no `sorry` in product), and/or +- covered by the Rust suite (`tests/`) and/or pinned `ref/` oracles (CI gates). Lean covers encode/decode, EtM, FEC, keyed Bao, scrub, streaming geometry, sharding, Adamantine directories, outboard, SLH sidecars, CLI — **in addition to**, not as a deletion of, the Rust engine. ## Method -1. Pin references under `ref/` (submodules, exact commits from Cargo.lock / Surmount forks). -2. Lean algorithms + theorems; keep Rust `src/`/`tests/` first-class. -3. AOT to C via Lean’s backend (never hand-edit generated C); expose C ABI for `backend-lean`. -4. Nix links AOT objects and allowed external C (zstd, libbitcoinpqc) until replaced. -5. Parity: `ref/` drivers + dual-backend `cargo test` ([TEST_CONTRACT.md](./TEST_CONTRACT.md), [PARITY.md](./PARITY.md)). +1. Pin references under `ref/` (submodules, exact commits from Cargo.lock / third-party oracles). +2. Lean algorithms + theorems; keep Rust `src/`/`tests/` first-class. +3. Lean AOT demo may link tiny C `@[extern]` shims (zstd, SLH) for goldens. That is not a Rust `-sys` product. +4. Nix builds Lean proofs/demo and Rust quality packages. +5. Parity: `ref/` drivers + Rust `cargo test` ([TEST_CONTRACT.md](./TEST_CONTRACT.md), [PARITY.md](./PARITY.md)). ## Precedents diff --git a/examples/slh_dsa_sidecar.rs b/examples/slh_dsa_sidecar.rs index 2e45bf6..fbe796b 100644 --- a/examples/slh_dsa_sidecar.rs +++ b/examples/slh_dsa_sidecar.rs @@ -10,7 +10,7 @@ use carbonado::crypto::{ read_slh_sidecar, slh_dsa_generate_keypair, slh_dsa_sign, slh_dsa_verify, write_slh_sidecar, }; -use carbonado::file::{encode, Header}; +use carbonado::file::{Header, encode}; use getrandom::getrandom; fn main() -> Result<(), Box> { @@ -85,7 +85,9 @@ fn main() -> Result<(), Box> { assert!(!still_valid); println!("\nSLH-DSA sidecar signing example completed successfully."); - println!("Remember: SLH-DSA public key is stored in the Carbonado Header; only the signature is handled in the sidecar. Never embed signatures inside the container."); + println!( + "Remember: SLH-DSA public key is stored in the Carbonado Header; only the signature is handled in the sidecar. Never embed signatures inside the container." + ); Ok(()) } diff --git a/flake.lock b/flake.lock index 39782de..49fd51e 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1787326676, + "narHash": "sha256-lWhBbBvC05/xwivKBBiM2YNizpmgqCgyOIzomvRuwxs=", + "owner": "ipetkov", + "repo": "crane", + "rev": "692f7e9ef2ece8125b466f66f2af532b3edaed0d", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" @@ -103,12 +118,34 @@ }, "root": { "inputs": { + "crane": "crane", "flake-parts": "flake-parts", "lean4-nix": "lean4-nix", "nixpkgs": [ "lean4-nix", "nixpkgs" + ], + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" ] + }, + "locked": { + "lastModified": 1787540965, + "narHash": "sha256-48/4bbmK3W3Av3YPnrFb/tVQXPvBwU6kyM3Pb13VVD8=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "ab450d47a3f906d19de1b332915bfc6e5b29c853", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 97ca92c..80f9628 100644 --- a/flake.nix +++ b/flake.nix @@ -1,16 +1,23 @@ { - description = "carbonado — apocalypse-resistant archival format (Lean 4 AOT + Nix)"; + description = "carbonado — apocalypse-resistant archival format (Rust engine + Lean 4 proofs + Nix)"; inputs = { nixpkgs.follows = "lean4-nix/nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; lean4-nix.url = "github:lenianiva/lean4-nix"; + # rustc 1.98 matching Cargo.toml rust-version / rust-toolchain.toml. + # Overlay only; Lean AOT still uses lean4-nix's nixpkgs + toolchain file. + rust-overlay.url = "github:oxalica/rust-overlay"; + rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; + crane.url = "github:ipetkov/crane"; }; outputs = inputs @ { nixpkgs, flake-parts, lean4-nix, + rust-overlay, + crane, ... }: flake-parts.lib.mkFlake {inherit inputs;} { @@ -43,6 +50,7 @@ && !(base == "examples" && type == "directory") && !(base == ".git" && type == "directory") && !(base == "result" || pkgs.lib.hasPrefix "result-" base) + && !(base == "rust-toolchain.toml") && pkgs.lib.cleanSourceFilter path type; }; @@ -50,18 +58,18 @@ holePattern = ''(^|[^a-zA-Z_])(sorry|admit)([^a-zA-Z_]|$)''; # Program F: static zstd from the **same pin as ref/zstd** (v1.5.7 / - # f8745da6…) + FFI glue → libcarbonado_native.a. Fetched by fixed rev/hash - # so flake purity does not require the submodule worktree to be git-tracked - # in the parent tree; SHA must stay in lockstep with docs/PARITY.md. + # f8745da6…) + Lean @[extern] glue → carbonado-native archive (AOT demo). + # Fetched by fixed rev/hash so flake purity does not require the submodule + # worktree to be git-tracked; SHA must stay in lockstep with docs/PARITY.md. zstdPinned = pkgs.fetchFromGitHub { owner = "facebook"; repo = "zstd"; rev = "f8745da6ff1ad1e7bab384bd1f9d742439278e99"; hash = "sha256-tNFWIT9ydfozB8dWcmTMuZLCQmQudTFJIkSr0aG7S44="; }; - # R9 / G10: libbitcoinpqc pin matching ref/bitcoinpqc submodule + # libbitcoinpqc pin matching ref/bitcoinpqc submodule # (b309f444… / branch 27-slh-dsa-sha-2-128s). SLH-DSA-SHA2-128s only - # (no secp/ML-DSA) is compiled into libcarbonado_native.a. + # (no secp/ML-DSA) is compiled into the Lean AOT demo native archive. bitcoinpqcPinned = pkgs.fetchFromGitHub { owner = "cryptoquick"; repo = "libbitcoinpqc"; @@ -73,18 +81,14 @@ leanAll = pkgs.lean.lean-all; zstdSrc = zstdPinned; bitcoinpqcSrc = bitcoinpqcPinned; - carbonadoInclude = ./include; }; leanPkg = pkgs.lean.buildLeanPackage { name = "carbonado"; # Separate roots so CarbonadoTest compiles without product → test imports. # lean4-nix only discovers modules under the root name of each entry. - # Carbonado.Ffi is a root so `@[export] l_carbonado_*` AOT objects land in - # staticLib even when Main does not import Ffi. roots = [ "Carbonado.Main" - "Carbonado.Ffi" "Carbonado.RkyvFilepack" "CarbonadoTest.Scaffold" "CarbonadoTest.EtM" @@ -98,72 +102,11 @@ src = productSrc; debug = false; leancFlags = ["-O3" "-DNDEBUG"]; - # Static zstd + C ABI glue (no shared libzstd — avoids lld shlib-undefined/pthread). + # Static zstd + SLH @[extern] glue for the Lean AOT demo (no shared libzstd). staticLibDeps = [carbonadoNative]; linkFlags = []; }; - # Dual-backend product archive: Lean AOT objects + native zstd/ABI glue, - # packaged as a shared library (leanc links Lean runtime) plus a static - # archive for `nm` / partial static consumers. - libcarbonado = - pkgs.runCommand "libcarbonado" { - nativeBuildInputs = [pkgs.binutils pkgs.stdenv.cc pkgs.lean.leanc]; - } '' - set -euo pipefail - mkdir -p $out/lib $out/include - - LEAN_A="${leanPkg.staticLib}/libcarbonado.a" - NATIVE_A="${carbonadoNative}/libcarbonado_native.a" - test -f "$LEAN_A" - test -f "$NATIVE_A" - - # Shared library via leanc + Lean shared stdlib (Init/runtime). - # lean4-nix staticLib is a *thin* archive; leanc/ld accept it with whole-archive. - # --whole-archive keeps @[export] + C ABI symbols from being GC'd. - # Pass libleanshared the same way buildLeanPackage.executable does (withSharedStdlib). - ${pkgs.lean.leanc}/bin/leanc -shared -fPIC \ - -Wl,--whole-archive "$LEAN_A" "$NATIVE_A" -Wl,--no-whole-archive \ - ${pkgs.lean.leanshared}/* \ - -o $out/lib/libcarbonado.so - - # Regular static archive for `nm` / consumers: thin member paths + native objects. - WORK=$(mktemp -d) - cd "$WORK" - mapfile -t LEAN_OBJS < <(${pkgs.binutils}/bin/ar t "$LEAN_A") - ${pkgs.binutils}/bin/ar x "$NATIVE_A" - ${pkgs.binutils}/bin/ar rcs $out/lib/libcarbonado.a "''${LEAN_OBJS[@]}" ./*.o - - cp ${./include}/carbonado.h $out/include/ - echo "libcarbonado: packaged static + shared" >&2 - ''; - - leanAbiCheck = - pkgs.runCommand "carbonado-lean-abi" { - nativeBuildInputs = [pkgs.binutils]; - } '' - set -euo pipefail - test -f ${libcarbonado}/include/carbonado.h - test -f ${libcarbonado}/lib/libcarbonado.a - test -f ${libcarbonado}/lib/libcarbonado.so - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_abi_version - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_free - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_encode - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_decode - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_encode_headered - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_decode_headered - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_verification_key - nm ${libcarbonado}/lib/libcarbonado.a | grep -q l_carbonado_encode_headered - nm ${libcarbonado}/lib/libcarbonado.a | grep -q l_carbonado_verification_key - # R9 / G10 SLH + seekable outboard slice - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_slh_keygen - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_slh_sign - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_slh_verify - nm ${libcarbonado}/lib/libcarbonado.a | grep -q carbonado_verify_slice_outboard - nm ${libcarbonado}/lib/libcarbonado.a | grep -q l_carbonado_verify_slice_outboard - echo ok > $out - ''; - noSorry = pkgs.runCommand "carbonado-no-sorry" { src = productSrc; @@ -204,6 +147,66 @@ src = productSrc; }; + # Separate nixpkgs + rust-overlay so Lean AOT does not rebuild when + # the Rust toolchain pin moves. Crane runs fmt/clippy/nextest with + # named feature sets (never --all-features). + pkgsRust = import nixpkgs { + inherit system; + overlays = [rust-overlay.overlays.default]; + }; + rustToolchain = pkgsRust.rust-bin.stable."1.98.0".default.override { + extensions = ["rust-src" "clippy" "rustfmt"]; + }; + craneLib = (crane.mkLib pkgsRust).overrideToolchain rustToolchain; + + # bitcoinpqc CMake FetchContent pin (libbitcoinpqc CMakeLists.txt GIT_TAG v0.5.0). + secp256k1Src = pkgsRust.fetchFromGitHub { + owner = "bitcoin-core"; + repo = "secp256k1"; + rev = "e3a885d42a7800c1ccebad94ad1e2b82c4df5c65"; + hash = "sha256-XcxBzOJngrm1szs48bBS6pcH2yaLfLKPUtyQ51eItaw="; + }; + + mkCargoSrc = root: + pkgsRust.lib.fileset.toSource { + root = root; + fileset = pkgsRust.lib.fileset.unions [ + (root + "/Cargo.toml") + (root + "/Cargo.lock") + (root + "/src") + (root + "/tests") + (root + "/benches") + (root + "/examples") + (root + "/.cargo") + ]; + }; + + # --impure just check reads the working tree so uncommitted Rust is in + # the crate source. Pure eval (nix flake check, GHA) uses the flake copy. + worktreePwd = builtins.getEnv "PWD"; + worktreeIsCarbonado = + worktreePwd + != "" + && builtins.pathExists (worktreePwd + "/flake.nix") + && builtins.pathExists (worktreePwd + "/Cargo.toml") + && builtins.pathExists (worktreePwd + "/src"); + cargoRoot = + if worktreeIsCarbonado + then /. + worktreePwd + else ./.; + cargoSrc = mkCargoSrc cargoRoot; + + mkCargoQuality = remote: + import ./nix/cargo-quality.nix { + inherit secp256k1Src craneLib rustToolchain; + lib = pkgsRust.lib; + pkgs = pkgsRust; + src = cargoSrc; + inherit remote; + }; + cargoChecks = mkCargoQuality false; + cargoQuality = mkCargoQuality true; + # Run AOT binary as a check (constants + EtM + FEC + Bao + pipeline + Program F). demo = pkgs.runCommand "carbonado-demo" { @@ -294,6 +297,7 @@ grep -q "zstd status mapping ok" $out grep -q "PipelineError zstd maps ok" $out grep -q "zstd goldens + roundtrip + error paths ok" $out + grep -q "zstd frame header params ok" $out grep -q "pipeline compression formats c2/c6 + headered c3/c7 ok" $out grep -q "SLH1 wire framing ok" $out grep -q "SLH live sign/verify ok" $out @@ -343,13 +347,17 @@ default = leanPkg.executable; carbonado = leanPkg.executable; carbonado-release = carbonadoRelease; - libcarbonado = libcarbonado; + # Remote-builder rustc (just check-remote). Not used by GHA. + fmt-quality = cargoQuality.fmt; + clippy-rust-quality = cargoQuality.clippy-rust; + nextest-rust-quality = cargoQuality.nextest-rust; + nextest-rust-cargo-on-builder-quality = cargoQuality.nextest-rust-cargo-on-builder; }; apps.default = { type = "app"; program = "${leanPkg.executable}/bin/carbonado"; - meta.description = "Carbonado Lean 4 AOT product binary (Programs A–G: Adamantine dirs + CLI)"; + meta.description = "Carbonado Lean 4 AOT demo (proofs + goldens; not a Rust -sys engine)"; }; checks = { @@ -358,7 +366,18 @@ demo = demo; # Building the package is itself a check of Lean compile (includes CarbonadoTest roots). carbonado = leanPkg.executable; - lean-abi = leanAbiCheck; + rustc-1_98 = pkgs.runCommand "carbonado-rustc-1.98" { + nativeBuildInputs = [rustToolchain]; + } '' + set -euo pipefail + rustc --version | tee "$out" + grep -q '^rustc 1\.98' "$out" + ''; + # Cargo trio (never --all-features). GHA may realize these; they do + # not require surmount-remote. just check-remote uses *-quality packages. + fmt = cargoChecks.fmt; + clippy-rust = cargoChecks.clippy-rust; + nextest-rust = cargoChecks.nextest-rust; }; devShells.default = pkgs.mkShell { @@ -368,6 +387,7 @@ gnupg ripgrep scc + rustToolchain # Host elan/lake may be used; lean4-nix provides leanc via package builds. ]; shellHook = '' @@ -377,14 +397,11 @@ echo "carbonado Lean 4 + Nix dev shell" echo "Lean toolchain pin: $(cat lean-toolchain)" echo "Build: nix build .#carbonado" - echo "Lib: nix build .#libcarbonado" - echo "Check: nix flake check" + echo "Check: just check # sequential fmt/clippy/nextest/Lean on the remote builder" + echo " nix flake check" echo "Run: nix run" - echo "Lean backend tests:" - echo " nix build .#libcarbonado -o result-libcarbonado" - echo " export CARBONADO_LEAN_LIB=\$PWD/result-libcarbonado/lib CARBONADO_LEAN_INCLUDE=\$PWD/result-libcarbonado/include" - echo " export LD_LIBRARY_PATH=\$CARBONADO_LEAN_LIB" - echo " cargo test --no-default-features --features \"backend-lean,pqc,ots\" --test lean_backend_smoke" + echo "Lean proofs: nix build .#checks.$(nix eval --impure --raw --expr builtins.currentSystem).no-sorry" + echo "Lean demo: nix build .#checks.$(nix eval --impure --raw --expr builtins.currentSystem).demo" fi ''; }; diff --git a/include/carbonado.h b/include/carbonado.h deleted file mode 100644 index dc6faa3..0000000 --- a/include/carbonado.h +++ /dev/null @@ -1,240 +0,0 @@ -/** - * carbonado C ABI — Lean AOT engine (libcarbonado) - * - * See docs/ABI.md for ownership, error codes, and versioning. - * ABI version 1 (v0 core + Phase 2 additive outboard/scrub/slice). - */ -#ifndef CARBONADO_H -#define CARBONADO_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define CARBONADO_ABI_VERSION 1u - -#define CARBONADO_OK 0 -#define CARBONADO_ERR_INVALID_ARGUMENT 1 -#define CARBONADO_ERR_INVALID_KEY_LENGTH 2 -#define CARBONADO_ERR_AUTHENTICATION 3 -#define CARBONADO_ERR_INVALID_MAGIC 4 -#define CARBONADO_ERR_INVALID_HEADER 5 -#define CARBONADO_ERR_FEC 6 -#define CARBONADO_ERR_BAO 7 -#define CARBONADO_ERR_ZSTD 8 -#define CARBONADO_ERR_SCRUB_UNNECESSARY 9 -#define CARBONADO_ERR_SCRUB_FAILED 10 -#define CARBONADO_ERR_NOT_IMPLEMENTED 11 -#define CARBONADO_ERR_INTERNAL 12 -/** Scrub called without Verification bit (distinct from recovery failure). */ -#define CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION 13 - -/** Returns CARBONADO_ABI_VERSION. */ -uint32_t carbonado_abi_version(void); - -/** Free a buffer returned by libcarbonado (malloc family). */ -void carbonado_free(void *p); - -/** - * Low-level encode (Rust encoding::encode body shape). - * On success: *out is malloc'd body, hash_out is 32-byte Bao root. - * Encrypted formats require nonce_len == 16. - * padding/chunk/ecc/vsc/compressed/encrypted out-params may be NULL. - * bytes_compressed / bytes_encrypted are 0 when those stages are skipped (R3). - */ -int carbonado_encode( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, - uint8_t **out, size_t *out_len, - uint8_t hash_out[32], - uint32_t *padding_out, - uint32_t *chunk_len_out, - uint32_t *bytes_ecc_out, - uint32_t *verifiable_slice_count_out, - uint32_t *bytes_compressed_out, - uint32_t *bytes_encrypted_out); - -/** - * Low-level decode of a verifiable body (hash + padding + format). - */ -int carbonado_decode( - const uint8_t *master, size_t master_len, - const uint8_t *hash, size_t hash_len, - const uint8_t *body, size_t body_len, - uint32_t padding, - uint8_t format, - uint8_t **out, size_t *out_len); - -/** - * Headered encode: full file Header || body (Rust file::encode shape). - * slh_pk: NULL → zero-filled 32 B field; non-NULL must point to exactly 32 valid bytes - * (C always copies 32 when non-NULL; wrong lengths are Lean ByteArray-only). - * metadata: NULL → zero-filled 8 B field; non-NULL must point to exactly 8 valid bytes - * (C always copies 8 when non-NULL; wrong lengths are Lean ByteArray-only). - * Stage-counter out-params (nullable): padding/chunk/ecc/vsc/compressed/encrypted (R3). - */ -int carbonado_encode_headered( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, - const uint8_t *slh_pk, - const uint8_t *metadata, - uint8_t **out, size_t *out_len, - uint32_t *padding_out, - uint32_t *chunk_len_out, - uint32_t *bytes_ecc_out, - uint32_t *verifiable_slice_count_out, - uint32_t *bytes_compressed_out, - uint32_t *bytes_encrypted_out); - -/** - * Headered decode: full file archive → plaintext. - */ -int carbonado_decode_headered( - const uint8_t *master, size_t master_len, - const uint8_t *archive, size_t archive_len, - uint8_t **out, size_t *out_len); - -/** Format-keyed verification key (32 bytes). */ -int carbonado_verification_key(uint8_t format, uint8_t key_out[32]); - -/** - * Outboard encode: bare main + optional verification outboard + FEC parity sidecars. - * Any of main_out / outboard_out / parity_out must be non-NULL with matching len ptr. - * Empty sidecars return *out=NULL, *out_len=0. - * - * header_path != 0: encrypted bare main is [tag|ct] (nonce out-of-band; file::encode_outboard). - * header_path == 0: encrypted bare main is [nonce|tag|ct] (encoding::encode_outboard). - * Encrypted formats require nonce_len == 16. - * bytes_compressed_out / bytes_encrypted_out may be NULL (0 when stage skipped; R3). - */ -int carbonado_encode_outboard( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, - uint8_t header_path, - uint8_t **main_out, size_t *main_len, - uint8_t **outboard_out, size_t *outboard_len, - uint8_t **parity_out, size_t *parity_len, - uint8_t hash_out[32], - uint32_t *padding_out, - uint32_t *chunk_len_out, - uint32_t *bytes_compressed_out, - uint32_t *bytes_encrypted_out); - -/** - * Outboard decode: bare main + optional sidecars → plaintext. - * outboard/parity may be null with len 0 when the format does not require them. - * - * header_path / nonce must match encode-time layout (see carbonado_encode_outboard). - */ -int carbonado_decode_outboard( - const uint8_t *master, size_t master_len, - const uint8_t *hash, size_t hash_len, - const uint8_t *main, size_t main_len, - const uint8_t *outboard, size_t outboard_len, - const uint8_t *parity, size_t parity_len, - uint32_t padding, - uint8_t format, - uint8_t header_path, - const uint8_t *nonce, size_t nonce_len, - uint8_t **out, size_t *out_len); - -/** - * Inboard scrub: recover damaged Bao+FEC body (or SCRUB_UNNECESSARY / REQUIRES_VERIFICATION). - */ -int carbonado_scrub( - const uint8_t *body, size_t body_len, - const uint8_t *hash, size_t hash_len, - uint32_t padding, - uint8_t format, - uint8_t **out, size_t *out_len); - -/** - * Outboard scrub: recover damaged bare main using outboard + FEC parity. - */ -int carbonado_scrub_outboard( - const uint8_t *main, size_t main_len, - const uint8_t *outboard, size_t outboard_len, - const uint8_t *parity, size_t parity_len, - const uint8_t *hash, size_t hash_len, - uint32_t padding, - uint32_t chunk_len, - uint8_t format, - uint8_t **out, size_t *out_len); - -/** - * Inboard verify_slice / extract_slice: authenticated slice bytes from inboard body. - * - * W4a: Lean retains O(slice) output while walking the full inboard response for - * auth (O(N) time). Caller still supplies the full body buffer (input). - * count==0 returns empty after full auth (Lean auth-first path). - */ -int carbonado_verify_slice( - const uint8_t *body, size_t body_len, - const uint8_t *hash, size_t hash_len, - uint32_t index, - uint32_t count, - uint8_t format, - uint8_t **out, size_t *out_len); - -/** - * Seekable outboard verify_slice: authenticated slice bytes from bare main + - * post-order outboard sidecar (keyed Bao, 4 KiB groups). - * - * Time/hash work is O(slice + tree height) over the requested ranges (not full - * re-encode). C ABI still takes full main + outboard buffers in memory — W4b - * permanent residual (no streaming ReadAt / callback ABI); see docs/LIMITS.md. - * - * count==0 → empty success immediately (no auth / geometry / OOB checks) — - * matches Rust `verify_slice_outboard` extract semantics. OOB index and - * authentication apply only when count > 0. - */ -int carbonado_verify_slice_outboard( - const uint8_t *main, size_t main_len, - const uint8_t *outboard, size_t outboard_len, - const uint8_t *hash, size_t hash_len, - uint32_t index, - uint32_t count, - uint8_t format, - uint8_t **out, size_t *out_len); - -/** - * SLH-DSA-SHA2-128s keygen (G10). entropy_len must be ≥ 128. - * pk_out: 32 bytes; sk_out: 64 bytes (caller-owned stack/heap buffers). - */ -int carbonado_slh_keygen( - const uint8_t *entropy, size_t entropy_len, - uint8_t pk_out[32], - uint8_t sk_out[64]); - -/** - * SLH-DSA-SHA2-128s sign. secret_key_len must be 64. - * On success: *out is malloc'd 7856-byte signature (free with carbonado_free). - */ -int carbonado_slh_sign( - const uint8_t *secret_key, size_t secret_key_len, - const uint8_t *message, size_t message_len, - uint8_t **out, size_t *out_len); - -/** - * SLH-DSA-SHA2-128s verify. public_key_len 32; signature_len 7856. - * Returns CARBONADO_OK on accept, CARBONADO_ERR_AUTHENTICATION on reject. - */ -int carbonado_slh_verify( - const uint8_t *public_key, size_t public_key_len, - const uint8_t *message, size_t message_len, - const uint8_t *signature, size_t signature_len); - -#ifdef __cplusplus -} -#endif - -#endif /* CARBONADO_H */ diff --git a/justfile b/justfile index 0fb025f..16bef48 100644 --- a/justfile +++ b/justfile @@ -1,33 +1,50 @@ # Carbonado development tasks. Run `just` to list recipes. # Before a release: `just all` +# +# Full quality gate (fmt, clippy, nextest, then Lean proof/demo checks) on +# the Nix remote builder: `just check` / `just check-remote`. Sequential. +# There is no Cargo Lean backend. GitHub Actions must not call check-remote; +# GHA keeps `just fmt` / `just lint` / `just test` / Lean nix checks. +# Force-remote nix: caller max-jobs 0, --store ssh-ng (machines file), +# --eval-store auto, --cores 64. rustc requires surmount-remote. set shell := ["bash", "-euo", "pipefail", "-c"] +# Host system for flake check attributes. Prefer CI_SYSTEM. Do not call nix +# at just parse time. +system := env_var_or_default("CI_SYSTEM", `case "$(uname -s)-$(uname -m)" in Linux-x86_64) echo x86_64-linux;; Linux-aarch64|Linux-arm64) echo aarch64-linux;; Darwin-x86_64) echo x86_64-darwin;; Darwin-arm64) echo aarch64-darwin;; *) echo "unsupported $(uname -s)-$(uname -m); set CI_SYSTEM=..." >&2; exit 1;; esac`) + default: @just --list -# Clone the keyed bao-tree sibling (../bao-tree, branch 76-keyed-bao). +# Clone n0-computer/bao-tree at the PR 78 merge SHA (optional sibling path patch). +bao_tree_rev := "dbc952e32cbda8ffd14c106b770e72987b01618e" + setup-bao-tree: #!/usr/bin/env bash set -euo pipefail + PIN="{{bao_tree_rev}}" if [[ -f ../bao-tree/Cargo.toml ]]; then echo "../bao-tree already present" else - git clone -b 76-keyed-bao https://github.com/SurmountSystems/bao-tree.git ../bao-tree + git clone https://github.com/n0-computer/bao-tree.git ../bao-tree fi - rg -q 'keyed_hash_subtree|KeyedHash|create_keyed' ../bao-tree/src - echo "bao-tree OK (keyed fork)" + git -C ../bao-tree fetch --all --tags + git -C ../bao-tree checkout "$PIN" + rg -q 'create_keyed|keyed_outboard_post_order' ../bao-tree/src + echo "bao-tree OK (n0-computer $PIN)" # Optional: verify sibling bao-tree when using `.cargo/config.toml` path patch. require-bao-tree: #!/usr/bin/env bash set -euo pipefail + PIN="{{bao_tree_rev}}" if [[ ! -f ../bao-tree/Cargo.toml ]]; then echo "Missing ../bao-tree. Run: just setup-bao-tree (optional path patch for faster local builds)" exit 1 fi - if ! rg -q 'keyed_hash_subtree|KeyedHash|create_keyed' ../bao-tree/src 2>/dev/null; then - echo "Wrong bao-tree at ../bao-tree — need SurmountSystems branch 76-keyed-bao" + if ! rg -q 'create_keyed|keyed_outboard_post_order' ../bao-tree/src 2>/dev/null; then + echo "Wrong bao-tree at ../bao-tree — need n0-computer PR 78 merge ($PIN)" exit 1 fi @@ -40,25 +57,356 @@ dev-local-bao: cp -f .cargo/config.toml.example .cargo/config.toml echo "Local bao-tree path patch enabled (.cargo/config.toml)" +# Host cargo fmt (CI `lint` job). Check gate uses --all -- --check, not a write. fmt: - cargo fmt --check + cargo fmt --all -- --check fmt-fix: - cargo fmt + cargo fmt --all -# rustfmt has no `-W`; `--check` is the fail-if-unformatted equivalent of `cargo fmt --all -W`. -# Never `--all-features` on clippy: that enables both `backend-rust` and `backend-lean` -# and hits `compile_error!`. This is the rust-compatible stand-in (same as `just lint` / CI). -# Rust-only gate: fmt --check, clippy (rust features), nextest. Stops on first failure. -check: - cargo fmt --all -- --check - cargo clippy --all-targets --features "async,async-tokio,man-gen" -- -D warnings - cargo nextest run +# Fail loud before force-remote nix. Reuses the trusted-user machines file +# (default $HOME/.config/nix/machines). Does not bake a host address. Does +# not fall back to local Nix store builds. Override: GROK_NIX_BUILDERS_FILE. +[private] +require_remote_builder: + #!/usr/bin/env bash + set -euo pipefail + file="${GROK_NIX_BUILDERS_FILE:-$HOME/.config/nix/machines}" + known_hosts="${GROK_NIX_KNOWN_HOSTS:-$HOME/.ssh/known_hosts}" + extra_ssh="-o UserKnownHostsFile=${known_hosts} -o StrictHostKeyChecking=yes" + if [[ -n "${NIX_SSHOPTS:-}" ]]; then + export NIX_SSHOPTS="${NIX_SSHOPTS} ${extra_ssh}" + else + export NIX_SSHOPTS="${extra_ssh}" + fi + if [[ ! -s "${file}" ]]; then + echo "The Nix builders file is missing or empty: ${file}." >&2 + echo "just check-remote reuses the trusted-user machines file already named in the user Nix config (override with GROK_NIX_BUILDERS_FILE)." >&2 + echo "Host cargo recipes (just fmt, just lint, just test) do not need this file." >&2 + exit 2 + fi + if ! grep -q 'ssh-ng://' "${file}"; then + echo "The Nix builders file ${file} has no ssh-ng:// builder line." >&2 + echo "just check-remote will not fall back to local Nix store builds." >&2 + exit 2 + fi + ssh_ng_host() { + local u="${1#ssh-ng://}" + u="${u%%\?*}" + u="${u#*@}" + u="${u%%/*}" + if [[ "${u}" == \[* ]]; then + u="${u#\[}" + u="${u%%]*}" + else + u="${u%%:*}" + fi + printf '%s' "${u}" + } + host_key_present() { + local host="$1" + [[ -s "${known_hosts}" ]] || return 1 + ssh-keygen -F "${host}" -f "${known_hosts}" 2>/dev/null | awk '!/^#/ && $2 ~ /^ssh-/ { found=1; exit } END { exit !found }' + } + while IFS= read -r line || [[ -n "${line}" ]]; do + [[ "${line}" == ssh-ng://* ]] || continue + set -- ${line} + host="$(ssh_ng_host "${1}")" + if [[ -z "${host}" ]] || ! host_key_present "${host}"; then + echo "This account's known_hosts has no host key for the machines-file builder." >&2 + echo "User ssh to Host surmount-1 is not the nix build SSH path (nix-daemon opens ssh-ng)." >&2 + echo "just check-remote sets NIX_SSHOPTS to this account's known_hosts and will not fall back to a local rustc." >&2 + exit 2 + fi + done < "${file}" + inject_feats="${GROK_NIX_REMOTE_SYSTEM_FEATURES-}" + if [[ -z "${inject_feats}" ]]; then + if ! ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=yes surmount-1 true; then + echo "SSH BatchMode to Host surmount-1 failed." >&2 + echo "just check-remote requires that existing remote builder and will not fall back to local Nix store builds." >&2 + exit 2 + fi + fi + remote_feats="" + if [[ -n "${inject_feats}" ]]; then + remote_feats="${inject_feats}" + else + set +e + feats_out="$(ssh -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=yes surmount-1 'nix config show' 2>/dev/null)" + feats_status=$? + set -e + if [[ "${feats_status}" -ne 0 ]]; then + echo "Could not read the remote builder nix-daemon system-features over SSH BatchMode." >&2 + echo "just check-remote will not start the long quality build until that query works." >&2 + exit 2 + fi + remote_feats="$(awk -F' = ' '/^system-features / { print $2; exit }' <<<"${feats_out}")" + if [[ -z "${remote_feats}" ]]; then + echo "The remote builder SSH reply had no system-features line." >&2 + echo "just check-remote will not start the long quality build until the remote nix-daemon reports its feature list." >&2 + exit 2 + fi + fi + if ! grep -Eq '(^|[[:space:],{])surmount-remote($|[[:space:],}])' <<<"${remote_feats}"; then + echo "The remote nix-daemon does not list surmount-remote in its system-features." >&2 + echo "The client machines file advertises that feature, so Nix will schedule rustc on the remote, then the daemon will refuse: missing system features." >&2 + echo "Add surmount-remote to the builder daemon (NixOS extra-system-features / nix.conf) and restart or switch. just check-remote will not start the long quality build until that feature is present." >&2 + exit 2 + fi + echo "==> just check-remote: using builders file ${file}" + echo "==> just check-remote: NIX_SSHOPTS uses this account's known_hosts (host-key checks stay on)" + echo "==> just check-remote: rustc, clippy, and nextest require the remote builder surmount-remote feature (fallback=false). This laptop does not advertise that feature, so local nixbld cannot take the rustc job." + echo "==> just check-remote: force-remote nix sets max-jobs 0. This laptop must not build. Fixed-output derivations and toolchain downloads go to the remote builder." + echo "==> just check-remote: force-remote nix uses --store ssh-ng (same machines-file builder) and --eval-store auto. -L logs still stream. nix build --no-link skips a local result symlink." + echo "==> just check-remote: force-remote nix uses --cores 64. Host machines max-jobs should advertise that many jobs on the builder." + +# Retry a nix command. When GROK_NIX_FORCE_REMOTE=1, append force-remote +# flags (max-jobs 0, ssh-ng --store, --eval-store auto). Hard SSH / +# missing-system-features / rustfmt Diff-in / clippy could-not-compile / +# nextest fail exit on attempt 1. +[private] +[positional-arguments] +nix_retry +cmd: + #!/usr/bin/env bash + set -euo pipefail + raw_attempts="${NIX_RETRY_ATTEMPTS:-4}" + if [[ ! "${raw_attempts}" =~ ^[1-9][0-9]*$ ]]; then + echo "==> nix_retry: NIX_RETRY_ATTEMPTS must be a positive integer, got: ${raw_attempts}" >&2 + exit 2 + fi + attempts="${raw_attempts}" + backoff=5 + n=1 + attempt_log="$(mktemp)" + enriched_builders="" + cleanup_nix_retry_log() { rm -f "${attempt_log}" "${enriched_builders}"; } + trap cleanup_nix_retry_log EXIT + force_remote_opts=() + if [[ "${GROK_NIX_FORCE_REMOTE:-}" == "1" ]]; then + builders_file="${GROK_NIX_BUILDERS_FILE:-$HOME/.config/nix/machines}" + known_hosts="${GROK_NIX_KNOWN_HOSTS:-$HOME/.ssh/known_hosts}" + extra_ssh="-o UserKnownHostsFile=${known_hosts} -o StrictHostKeyChecking=yes" + if [[ -n "${NIX_SSHOPTS:-}" ]]; then + export NIX_SSHOPTS="${NIX_SSHOPTS} ${extra_ssh}" + else + export NIX_SSHOPTS="${extra_ssh}" + fi + ssh_ng_host() { + local u="${1#ssh-ng://}" + u="${u%%\?*}" + u="${u#*@}" + u="${u%%/*}" + if [[ "${u}" == \[* ]]; then + u="${u#\[}" + u="${u%%]*}" + else + u="${u%%:*}" + fi + printf '%s' "${u}" + } + host_key_b64() { + local host="$1" + local line typ key + [[ -s "${known_hosts}" ]] || return 1 + line="$(ssh-keygen -F "${host}" -f "${known_hosts}" 2>/dev/null | awk '!/^#/ && $2=="ssh-ed25519" {print; exit}')" + if [[ -z "${line}" ]]; then + line="$(ssh-keygen -F "${host}" -f "${known_hosts}" 2>/dev/null | awk '!/^#/ && $2 ~ /^ssh-/ {print; exit}')" + fi + [[ -n "${line}" ]] || return 1 + typ="$(awk '{print $2}' <<<"${line}")" + key="$(awk '{print $3}' <<<"${line}")" + printf '%s' "${typ} ${key}" | base64 -w0 + } + max_conn="${GROK_NIX_SSH_NG_MAX_CONNECTIONS:-8}" + if [[ ! "${max_conn}" =~ ^[1-9][0-9]*$ ]]; then + echo "==> nix_retry: GROK_NIX_SSH_NG_MAX_CONNECTIONS must be a positive integer, got: ${max_conn}" >&2 + exit 2 + fi + enriched_builders="$(mktemp)" + chmod 600 "${enriched_builders}" + while IFS= read -r line || [[ -n "${line}" ]]; do + if [[ "${line}" != ssh-ng://* ]]; then + printf '%s\n' "${line}" >>"${enriched_builders}" + continue + fi + uri="" systems="" ssh_key="" max_jobs="" speed="" supported="" mandatory="" host_key="" + read -r uri systems ssh_key max_jobs speed supported mandatory host_key _rest <<<"${line}" || true + if [[ "${uri}" != *"max-connections="* ]]; then + if [[ "${uri}" == *\?* ]]; then + uri="${uri}&max-connections=${max_conn}" + else + uri="${uri}?max-connections=${max_conn}" + fi + fi + if [[ -n "${host_key:-}" && "${host_key}" != "-" ]]; then + printf '%s %s %s %s %s %s %s %s\n' \ + "${uri}" "${systems:--}" "${ssh_key:--}" "${max_jobs:--}" "${speed:--}" "${supported:--}" "${mandatory:--}" "${host_key}" >>"${enriched_builders}" + continue + fi + host="$(ssh_ng_host "${uri}")" + if ! b64="$(host_key_b64 "${host}")"; then + echo "==> nix_retry: this account's known_hosts has no host key for the machines-file builder. User ssh to Host surmount-1 is not the nix build SSH path." >&2 + exit 2 + fi + printf '%s %s %s %s %s %s %s %s\n' \ + "${uri}" "${systems:--}" "${ssh_key:--}" "${max_jobs:--}" "${speed:--}" "${supported:--}" "${mandatory:--}" "${b64}" >>"${enriched_builders}" + done < "${builders_file}" + builders_file="${enriched_builders}" + store_uri="" + while IFS= read -r bline || [[ -n "${bline}" ]]; do + if [[ "${bline}" == ssh-ng://* ]]; then + read -r store_uri _ <<<"${bline}" || true + break + fi + done < "${builders_file}" + if [[ -z "${store_uri}" || "${store_uri}" != ssh-ng://* ]]; then + echo "==> nix_retry: GROK_NIX_FORCE_REMOTE needs an ssh-ng:// builder URI in the machines file so nix can use --store on that builder. This laptop must not realize the graph into the local store." >&2 + exit 2 + fi + force_remote_opts=( + --option builders "@${builders_file}" + --option builders-use-substitutes true + --option fallback false + --option system-features "kvm nixos-test uid-range" + --option max-jobs 0 + --cores 64 + --store "${store_uri}" + --eval-store auto + ) + if [[ "${2:-}" == "build" ]]; then + force_remote_opts+=(--no-link) + fi + fi + if [[ "${1:-}" == ssh-ng://* ]]; then + echo "==> nix_retry: the first argument is a machines-file line, not the nix command. Pass --option builders @file after the command; do not put the machines line in \"\$@\"." >&2 + exit 2 + fi + while true; do + if ((${#force_remote_opts[@]})); then + banner_opts=() + skip_store_uri=0 + for opt in "${force_remote_opts[@]}"; do + if [[ "${skip_store_uri}" -eq 1 ]]; then + banner_opts+=("") + skip_store_uri=0 + continue + fi + if [[ "${opt}" == "--store" ]]; then + banner_opts+=(--store) + skip_store_uri=1 + continue + fi + banner_opts+=("${opt}") + done + echo "==> nix attempt ${n}/${attempts}: $* ${banner_opts[*]}" + else + echo "==> nix attempt ${n}/${attempts}: $*" + fi + set +e + set +o pipefail + "$@" "${force_remote_opts[@]}" 2>&1 | tee "${attempt_log}" + status="${PIPESTATUS[0]}" + set -o pipefail + set -e + if [[ "${status}" -eq 0 ]]; then + exit 0 + fi + if grep -qE 'failed to start SSH connection|Failed to find a machine for remote build' "${attempt_log}"; then + echo "==> nix_retry: the builder is listed, but SSH did not start. rustc was not run locally. Not retrying this hard remote miss." >&2 + exit "${status}" + fi + if grep -qE 'missing system features' "${attempt_log}"; then + echo "==> nix_retry: the remote builder refused this derivation: missing system features. Add surmount-remote to the builder daemon and retry. Not retrying this hard remote miss." >&2 + exit "${status}" + fi + if grep -qE 'Diff in ' "${attempt_log}"; then + echo "==> nix_retry: cargo fmt / rustfmt check failed (Diff in). Format the listed files and retry. Not retrying this hard quality miss." >&2 + exit "${status}" + fi + if grep -qE 'error: could not compile|clippy::' "${attempt_log}"; then + echo "==> nix_retry: cargo clippy / rustc quality failed (could not compile). Fix the listed errors and retry. Not retrying this hard quality miss." >&2 + exit "${status}" + fi + if grep -qE 'cannot update the lock file|--locked was passed' "${attempt_log}"; then + echo "==> nix_retry: cargo lockfile / --locked mismatch. Not retrying this hard quality miss." >&2 + exit "${status}" + fi + if grep -qE 'hash mismatch in fixed-output derivation' "${attempt_log}"; then + echo "==> nix_retry: nix fixed-output hash mismatch. Update the listed sha256 and retry. Not retrying this hard quality miss." >&2 + exit "${status}" + fi + if grep -qE 'error: test run failed|test run failed' "${attempt_log}"; then + echo "==> nix_retry: cargo nextest / test run failed. Fix the listed tests and retry. Not retrying this hard quality miss." >&2 + exit "${status}" + fi + if [[ "${status}" -eq 127 ]] && grep -qE 'ssh-ng://.*No such file or directory' "${attempt_log}"; then + echo "==> nix_retry: the command was a machines-file line (exit 127). Force-remote builders belong in --option builders @file after nix. Not retrying this hard recipe miss." >&2 + exit "${status}" + fi + if [[ "${n}" -ge "${attempts}" ]]; then + echo "==> nix FAILED after ${n} attempt(s) (exit ${status}): $*" >&2 + exit "${status}" + fi + echo "==> nix attempt ${n} failed (exit ${status}); retrying in ${backoff}s..." >&2 + sleep "${backoff}" + backoff=$((backoff * 3)) + n=$((n + 1)) + done + +# Sequential host-Nix flake checks (fmt, clippy, nextest, then Lean proofs). +# Does not force the remote builder; this laptop may rustc. +check-local: + #!/usr/bin/env bash + set -euo pipefail + sys="{{ system }}" + echo "==> just check-local: fmt" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.fmt" + echo "==> just check-local: clippy (backend-rust + async,async-tokio,man-gen)" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.clippy-rust" + echo "==> just check-local: nextest (backend-rust)" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.nextest-rust" + echo "==> just check-local: Lean gates" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.no-sorry" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.tooling-purity" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.carbonado" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.demo" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.rustc-1_98" + +# Sequential force-remote gate. rustc requires surmount-remote. Quote +# .#attr; unquoted # is a bash comment. +check-remote: require_remote_builder + #!/usr/bin/env bash + set -euo pipefail + export GROK_NIX_FORCE_REMOTE=1 + export GROK_NIX_BUILDERS_FILE="${GROK_NIX_BUILDERS_FILE:-$HOME/.config/nix/machines}" + known_hosts="${GROK_NIX_KNOWN_HOSTS:-$HOME/.ssh/known_hosts}" + extra_ssh="-o UserKnownHostsFile=${known_hosts} -o StrictHostKeyChecking=yes" + if [[ -n "${NIX_SSHOPTS:-}" ]]; then + export NIX_SSHOPTS="${NIX_SSHOPTS} ${extra_ssh}" + else + export NIX_SSHOPTS="${extra_ssh}" + fi + sys="{{ system }}" + echo "==> just check-remote: fmt" + just nix_retry nix build --impure -L --print-out-paths "path:.#fmt-quality" + echo "==> just check-remote: clippy (backend-rust + async,async-tokio,man-gen)" + just nix_retry nix build --impure -L --print-out-paths "path:.#clippy-rust-quality" + echo "==> just check-remote: nextest (backend-rust)" + just nix_retry nix build --impure -L --print-out-paths "path:.#nextest-rust-quality" + echo "==> just check-remote: Lean gates" + just nix_retry nix build --impure -L --print-out-paths "path:.#checks.${sys}.no-sorry" + just nix_retry nix build --impure -L --print-out-paths "path:.#checks.${sys}.tooling-purity" + just nix_retry nix build --impure -L --print-out-paths "path:.#checks.${sys}.carbonado" + just nix_retry nix build --impure -L --print-out-paths "path:.#checks.${sys}.demo" + just nix_retry nix build --impure -L --print-out-paths "path:.#checks.${sys}.rustc-1_98" + +# Full gate on the remote builder (Surmount split: check-remote is the +# builder path; check is the name operators type). +check: check-remote # Clippy + project-specific source checks (things clippy does not know about). lint: _clippy _lint-source -# Never use `--all-features` here: that enables both `backend-rust` and `backend-lean` → compile_error!. # Cover optional features mutually compatible with default `backend-rust`. [private] _clippy: @@ -153,9 +501,8 @@ _lint-source: fi echo "" echo "--- 4. NotImplemented only on intentional residual / map sites ---" - # Allowed (documented dual-backend / platform residuals — not silent crypto stubs): + # Allowed (documented residuals — not silent crypto stubs): # - error.rs enum variant definition - # - backend lean ABI code → CarbonadoError map arm # - stream_decode_async on wasm32 (documented NotImplemented residual) # - doc comments mentioning the variant # (R2: file::encode metadata/SLH are plumbed — no longer NotImplemented) @@ -167,7 +514,7 @@ _lint-source: if echo "$line" | rg -q '^\S+:\d+:[[:space:]]*(//|///|\*)'; then continue; fi # enum variant if echo "$line" | rg -q 'src/error\.rs:'; then continue; fi - # match-arm mapping from C ABI + # match-arm mapping to the variant if echo "$line" | rg -q '=>[[:space:]]*CarbonadoError::NotImplemented'; then continue; fi # intentional wasm async residual if echo "$line" | rg -q 'src/stream/decode_async\.rs:'; then continue; fi @@ -175,7 +522,7 @@ _lint-source: done || true) fi if [[ -z "$notimpl_bad" ]]; then - pass "NotImplemented only at allowlisted residual/map sites (dual-backend + wasm async)" + pass "NotImplemented only at allowlisted residual/map sites (wasm async)" if [[ -n "$notimpl_hits" ]]; then echo " Allowlisted evidence:" echo "$notimpl_hits" | sed 's/^/ /' @@ -232,12 +579,11 @@ _lint-source: exit 1 fi -# wasm32: always name backend-rust under --no-default-features (mutual exclusion). +# wasm32: name backend-rust under --no-default-features (empty marker + lib). lint-wasm: cargo clippy --target wasm32-unknown-unknown --no-default-features --features "backend-rust" -- -D warnings -# Default features (includes `parallel`), serial FEC, then backend-rust + optional features. -# Never `--all-features` (enables both backends → compile_error!). +# Default features (includes `parallel`), serial FEC, then optional features. test: cargo test cargo test --no-default-features --features "backend-rust,pqc,ots,cli" --test serial_fec_path @@ -253,104 +599,23 @@ test-parallel: test-smoke: cargo test --test streaming --test seekable_slices --test sharding --test bao_keyed_contract -# Shared lean env: build libcarbonado if CARBONADO_LEAN_LIB unset; fail-closed if .so/.dylib missing. -# stdout: only `export …` lines (safe for `eval "$(just _lean-env)"`); diagnostics on stderr. -[private] -_lean-env: - #!/usr/bin/env bash - set -euo pipefail - if [[ -z "${CARBONADO_LEAN_LIB:-}" ]]; then - # Dedicated symlink so other `nix build` targets do not clobber `result/`. - nix build .#libcarbonado -o result-libcarbonado - export CARBONADO_LEAN_LIB="$PWD/result-libcarbonado/lib" - export CARBONADO_LEAN_INCLUDE="$PWD/result-libcarbonado/include" - fi - if [[ ! -f "${CARBONADO_LEAN_LIB}/libcarbonado.so" && ! -f "${CARBONADO_LEAN_LIB}/libcarbonado.dylib" ]]; then - echo "FATAL: libcarbonado shared library missing under CARBONADO_LEAN_LIB=${CARBONADO_LEAN_LIB}" >&2 - echo " Build: nix build .#libcarbonado -o result-libcarbonado" >&2 - echo " Then: export CARBONADO_LEAN_LIB=\$PWD/result-libcarbonado/lib" >&2 - echo " export CARBONADO_LEAN_INCLUDE=\$PWD/result-libcarbonado/include" >&2 - exit 1 - fi - if [[ -z "${CARBONADO_LEAN_INCLUDE:-}" ]]; then - if [[ -d "$(dirname "${CARBONADO_LEAN_LIB}")/include" ]]; then - export CARBONADO_LEAN_INCLUDE="$(dirname "${CARBONADO_LEAN_LIB}")/include" - else - echo "FATAL: CARBONADO_LEAN_INCLUDE unset and cannot infer from CARBONADO_LEAN_LIB" >&2 - exit 1 - fi - fi - export LD_LIBRARY_PATH="${CARBONADO_LEAN_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - echo "CARBONADO_LEAN_LIB=$CARBONADO_LEAN_LIB" >&2 - echo "CARBONADO_LEAN_INCLUDE=$CARBONADO_LEAN_INCLUDE" >&2 - printf 'export CARBONADO_LEAN_LIB=%q\n' "$CARBONADO_LEAN_LIB" - printf 'export CARBONADO_LEAN_INCLUDE=%q\n' "$CARBONADO_LEAN_INCLUDE" - printf 'export LD_LIBRARY_PATH=%q\n' "$LD_LIBRARY_PATH" - -# Dual-backend Phase 1: build libcarbonado and run lean allowlist smoke. -test-lean-smoke: - #!/usr/bin/env bash - set -euo pipefail - eval "$(just _lean-env)" - cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_smoke - -# Dual-backend Phase 2: outboard/scrub/slice + G9 buffer seeds (+ Phase 1 smoke). -test-lean-phase2: - #!/usr/bin/env bash - set -euo pipefail - eval "$(just _lean-env)" - cargo test --no-default-features --features "backend-lean,pqc,ots" \ - --test lean_backend_smoke --test lean_backend_phase2 - -# Dual-backend Phase 3: directory composition (rkyv catalog + Lean segment/catalog crypto). -test-lean-phase3: - #!/usr/bin/env bash - set -euo pipefail - eval "$(just _lean-env)" - cargo test --no-default-features --features "backend-lean,pqc,ots" \ - --test lean_backend_smoke --test lean_backend_phase2 --test lean_backend_phase3 \ - --test format_policy - -# Dual-backend Phase 4: SLH composition (G10-A) + CLI dual path + directory OTS. -# `cli` enables lean-linked binary: directory subprocess = dual-engine; single-file stream = link smoke. -test-lean-phase4: - #!/usr/bin/env bash - set -euo pipefail - eval "$(just _lean-env)" - cargo test --no-default-features --features "backend-lean,pqc,ots,cli" \ - --test lean_backend_smoke --test lean_backend_phase2 --test lean_backend_phase3 \ - --test format_policy --test slh_outboard --test lean_backend_phase4 - -# Dual-backend Phase 5 / G11 + R7 G8 full close: shared CI + human gate. -# Freeze = full dual suite under lean features (G8 closed 2026-07 R7). See docs/GAPS.md. -# Permanent feature-gated exclusions under this feature set (0 tests, not dual residual): -# streaming_async needs `async` (R10 closed: freeze never requires async; lean+async dual-aware); -# parallel_determinism needs `parallel` (Lean RS serial). -# Post-G8 residuals (not dual-suite failures): stream E2, file::decode_stream pure-Rust, -# pure Lean rkyv encode residual — composition paths remain SSOT for those layers. +# Lean proof + AOT demo gates (no Rust -sys / libcarbonado). test-lean-ci: #!/usr/bin/env bash set -euo pipefail - eval "$(just _lean-env)" - # Full dual suite (lib units + all integration tests, including bin_*). Never add async. - cargo test --no-default-features --features "backend-lean,pqc,ots,cli" + sys="{{ system }}" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.no-sorry" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.tooling-purity" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.carbonado" + nix build --impure -L --print-out-paths "path:.#checks.${sys}.demo" -# G9 / R8: cross-backend matrix both directions (lean fixtures → rust; rust fixtures → lean). +# Rust decode of committed Lean AOT goldens under tests/fixtures/g9/lean/. test-g9: - #!/usr/bin/env bash - set -euo pipefail cargo test --test g9_cross_backend - eval "$(just _lean-env)" - cargo test --no-default-features --features "backend-lean,pqc,ots" --test g9_cross_backend -# Regenerate G9 goldens under tests/fixtures/g9/{rust,lean}/ (requires libcarbonado for lean). +# Regenerate rust goldens under tests/fixtures/g9/rust/. g9-gen-fixtures: - #!/usr/bin/env bash - set -euo pipefail G9_WRITE_FIXTURES=1 cargo test --test g9_cross_backend write_fixtures -- --ignored --nocapture - eval "$(just _lean-env)" - G9_WRITE_FIXTURES=1 cargo test --no-default-features --features "backend-lean,pqc,ots" \ - --test g9_cross_backend write_fixtures -- --ignored --nocapture build: cargo build --bin carbonado --release diff --git a/nix/cargo-quality.nix b/nix/cargo-quality.nix new file mode 100644 index 0000000..c23fc96 --- /dev/null +++ b/nix/cargo-quality.nix @@ -0,0 +1,217 @@ +# Crane fmt / clippy / nextest for Carbonado. +# +# Never pass cargo --all-features. There is no Cargo Lean backend. +# +# Feature sets (must stay in sync with justfile comments and CI): +# rust clippy/nextest extras: async,async-tokio,man-gen +# (on top of default backend-rust,pqc,ots,cli,parallel) +# rust serial FEC: --no-default-features --features backend-rust,pqc,ots,cli +{ + lib, + pkgs, + craneLib, + rustToolchain, + src, + secp256k1Src, + remote ? false, +}: let + rustExtraFeatures = "async,async-tokio,man-gen"; + rustSerialFeatures = "backend-rust,pqc,ots,cli"; + + remoteSystemFeatures = [ + "big-parallel" + "surmount-remote" + ]; + + remoteAttrs = lib.optionalAttrs remote { + preferLocalBuild = false; + requiredSystemFeatures = remoteSystemFeatures; + }; + + pnameSuffix = + if remote + then "-quality" + else ""; + + # bitcoinpqc's CMake FetchContent clones bitcoin-core/secp256k1. The Nix + # sandbox has no network, so point FetchContent at a pinned source. + cmakeWithSecp = pkgs.writeShellScriptBin "cmake" '' + case "''${1:-}" in + --build|--install|--help|-E|--version) + exec ${pkgs.cmake}/bin/cmake "$@" + ;; + esac + exec ${pkgs.cmake}/bin/cmake \ + -DFETCHCONTENT_SOURCE_DIR_SECP256K1=${secp256k1Src} \ + -DFETCHCONTENT_FULLY_DISCONNECTED=ON \ + "$@" + ''; + + # zstd-sys 2.0.16 build.rs uses pkg-config when the *presence* of + # ZSTD_SYS_USE_PKG_CONFIG is Some (including the string "0") or when the + # crate feature `pkg-config` is on. Host cargo leaves the var unset and + # compiles bundled zstd 1.5.7. Do not put nixpkgs zstd or pkg-config on + # this compile PATH (that is how zstd-sys would pick system libzstd). + # buildDepsOnly must see the same hook so it does not cache a pkg-config + # libzstd.a into cargoArtifacts. + forceBundledZstdHook = pkgs.makeSetupHook {name = "carbonado-force-bundled-zstd";} ( + pkgs.writeText "carbonado-force-bundled-zstd.sh" '' + unset ZSTD_SYS_USE_PKG_CONFIG + unset PKG_CONFIG PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR + '' + ); + + nativeBuildInputs = [ + cmakeWithSecp + pkgs.rustPlatform.bindgenHook + forceBundledZstdHook + ]; + + cargoJobsFromCores = '' + cargoJobs="''${NIX_BUILD_CORES:-32}" + case "$cargoJobs" in + "" | *[!0-9]*) cargoJobs=32 ;; + esac + if [ "$cargoJobs" -gt 32 ]; then + cargoJobs=32 + fi + if [ "$cargoJobs" -lt 2 ]; then + cargoJobs=32 + fi + export CARGO_BUILD_JOBS="$cargoJobs" + unset MAKEFLAGS MFLAGS CARGO_MAKEFLAGS + unset ZSTD_SYS_USE_PKG_CONFIG + unset PKG_CONFIG PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR + echo "carbonado cargo jobs=$CARGO_BUILD_JOBS NIX_BUILD_CORES=''${NIX_BUILD_CORES:-unset}" + ''; + + commonArgs = { + inherit src nativeBuildInputs; + pname = "carbonado"; + version = (craneLib.crateNameFromCargoToml {cargoToml = src + "/Cargo.toml";}).version; + strictDeps = true; + enableParallelBuilding = true; + CARGO_BUILD_JOBS = "32"; + CARGO_PROFILE = "dev"; + hardeningDisable = ["all"]; + # Presence of ZSTD_SYS_USE_PKG_CONFIG (even =0) makes zstd-sys probe + # nixpkgs libzstd. Unset on the deps layer and the test layer. + preConfigure = '' + unset ZSTD_SYS_USE_PKG_CONFIG + unset PKG_CONFIG PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR + echo "carbonado zstd-sys: bundled (ZSTD_SYS_USE_PKG_CONFIG unset; pkg-config not on compile PATH)" + ''; + }; + + # Do not put --all-targets here: crane buildDepsOnly already adds it. + rustCargoExtraArgs = "--features ${rustExtraFeatures} --locked"; + + rustArtifacts = craneLib.buildDepsOnly (commonArgs + // remoteAttrs + // { + pname = "carbonado-clippy-rust${pnameSuffix}"; + cargoExtraArgs = rustCargoExtraArgs; + doCheck = false; + }); + + applyRemote = drv: drv.overrideAttrs (_: remoteAttrs); + + fmt = applyRemote (craneLib.cargoFmt { + inherit src; + pname = "carbonado-fmt${pnameSuffix}"; + cargoExtraArgs = "--all"; + }); + + clippy-rust = applyRemote (craneLib.cargoClippy (commonArgs + // { + cargoArtifacts = rustArtifacts; + pname = "carbonado-clippy-rust${pnameSuffix}"; + cargoExtraArgs = "--locked"; + cargoClippyExtraArgs = "--all-targets --features ${rustExtraFeatures} -- -D warnings"; + doInstallCargoArtifacts = false; + })); + + nextest-rust = applyRemote (craneLib.mkCargoDerivation (commonArgs + // { + cargoArtifacts = rustArtifacts; + pname = "carbonado-nextest-rust${pnameSuffix}"; + pnameSuffix = ""; + doCheck = false; + doInstallCargoArtifacts = false; + nativeBuildInputs = nativeBuildInputs ++ [pkgs.cargo-nextest]; + buildPhaseCargoCommand = '' + ${cargoJobsFromCores} + echo "carbonado nextest (backend-rust + ${rustExtraFeatures})" + cargo nextest run --locked --features ${rustExtraFeatures} + echo "carbonado nextest serial FEC (--no-default-features --features ${rustSerialFeatures})" + cargo nextest run --locked --no-default-features --features ${rustSerialFeatures} --test serial_fec_path + echo "carbonado doctests (backend-rust + ${rustExtraFeatures})" + cargo test --locked --doc --features ${rustExtraFeatures} --profile "$CARGO_PROFILE" --jobs "$CARGO_BUILD_JOBS" + ''; + })); + + # grok-build cargo-on-builder style: rust-overlay cargo/nextest in this + # derivation. Not crane mkCargoDerivation (that always adds nixpkgs zstd + # for artifact compression, which can put libzstd headers on CPATH). + # zstd-sys compiles its bundled C here (no buildDepsOnly cache). + nextest-rust-cargo-on-builder = applyRemote ( + pkgs.stdenv.mkDerivation { + pname = "carbonado-nextest-rust-cargo-on-builder${pnameSuffix}"; + inherit (commonArgs) version src; + strictDeps = true; + enableParallelBuilding = true; + CARGO_BUILD_JOBS = "32"; + CARGO_PROFILE = "dev"; + hardeningDisable = ["all"]; + cargoVendorDir = craneLib.vendorCargoDeps {inherit src;}; + nativeBuildInputs = [ + rustToolchain + pkgs.cargo-nextest + cmakeWithSecp + pkgs.rustPlatform.bindgenHook + forceBundledZstdHook + craneLib.configureCargoCommonVarsHook + craneLib.configureCargoVendoredDepsHook + ]; + preConfigure = commonArgs.preConfigure; + buildPhase = '' + runHook preBuild + ${cargoJobsFromCores} + echo "carbonado cargo-on-builder: rust-overlay cargo/nextest, no crane zstd package" + rustc --version + cargo --version + echo "CC=''${CC:-unset}" + if command -v pkg-config >/dev/null 2>&1; then + echo "pkg-config=$(command -v pkg-config)" + else + echo "pkg-config=absent" + fi + set +e + cargo nextest run --locked --features ${rustExtraFeatures} -- directory_cross_engine_live_roots_residual golden_directory_interop_checksums_and_manifest_wire directory_encode_independent_of_readdir_order + nextest_status=$? + set -e + find target -name 'libzstd.a' -print -exec sha256sum {} \; -exec wc -c {} \; || true + if [ "$nextest_status" -ne 0 ]; then + exit "$nextest_status" + fi + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out" + echo ok > "$out/result" + runHook postInstall + ''; + } + ); + +in { + inherit + fmt + clippy-rust + nextest-rust + nextest-rust-cargo-on-builder + rustExtraFeatures + rustSerialFeatures + ; +} diff --git a/nix/native/carbonado_abi.c b/nix/native/carbonado_abi.c deleted file mode 100644 index f8042e4..0000000 --- a/nix/native/carbonado_abi.c +++ /dev/null @@ -1,725 +0,0 @@ -/** - * C ABI surface for libcarbonado (docs/ABI.md, include/carbonado.h). - * - * Phase 1+2 + R3: strong symbols for encode/decode/headered/verification_key plus - * outboard/scrub/slice that call Lean `@[export]` helpers (`l_carbonado_*` - * from Carbonado/Ffi.lean). Lean runtime is initialized once on first use. - * - * Lean pack layouts are internal to libcarbonado (co-versioned with this C glue); - * the public C API stays additive at ABI version 1 (nullable out-params). - * - * Packed Lean success layouts (errors are status-first: [u32 LE status] only): - * status payload: [u32 LE status][bytes…] - * encode body: [u32 LE status][pad:4][chunk:4][ecc:4][vsc:4] - * [comp:4][enc:4][32 hash][body…] (prefix 60) - * encode headered: [u32 LE status][pad:4][chunk:4][ecc:4][vsc:4] - * [comp:4][enc:4][archive…] (prefix 28) - * encode outboard: [u32 LE status][pad:4][chunk:4][comp:4][enc:4] - * [32 hash][u32 main_len][main][u32 ob_len][ob] - * [u32 par_len][par] (fixed prefix 52) - */ -#include -#include -#include -#include -#include - -#include "carbonado.h" - -/* Lean runtime (symbols in Lean Init / leanrt). */ -extern void lean_initialize_runtime_module(void); -extern void lean_io_mark_end_initialization(void); -extern bool lean_io_result_is_ok(b_lean_obj_arg r); -extern void lean_io_result_show_error(b_lean_obj_arg r); - -/* Module initializer generated for Carbonado.Ffi (chains Pipeline deps). */ -extern lean_obj_res initialize_Carbonado_Ffi(uint8_t builtin); - -/* @[export] helpers from Carbonado/Ffi.lean — callee owns arguments. */ -extern lean_obj_res l_carbonado_verification_key(uint8_t format); -extern lean_obj_res l_carbonado_encode_headered(lean_obj_arg master, lean_obj_arg nonce, - lean_obj_arg plaintext, lean_obj_arg slh_pk, - lean_obj_arg metadata, uint8_t format); -extern lean_obj_res l_carbonado_decode_headered(lean_obj_arg master, lean_obj_arg archive); -extern lean_obj_res l_carbonado_encode(lean_obj_arg master, lean_obj_arg nonce, - lean_obj_arg plaintext, uint8_t format); -extern lean_obj_res l_carbonado_decode(lean_obj_arg master, lean_obj_arg hash, - lean_obj_arg body, uint32_t padding, uint8_t format); -extern lean_obj_res l_carbonado_encode_outboard(lean_obj_arg master, lean_obj_arg nonce, - lean_obj_arg plaintext, uint8_t format, - uint8_t header_path); -extern lean_obj_res l_carbonado_decode_outboard(lean_obj_arg master, lean_obj_arg hash, - lean_obj_arg main, lean_obj_arg ver_outboard, - lean_obj_arg fec_parity, uint32_t padding, - uint8_t format, uint8_t header_path, - lean_obj_arg nonce); -extern lean_obj_res l_carbonado_scrub(lean_obj_arg body, lean_obj_arg hash, uint32_t padding, - uint8_t format); -extern lean_obj_res l_carbonado_scrub_outboard(lean_obj_arg main, lean_obj_arg ver_outboard, - lean_obj_arg fec_parity, lean_obj_arg hash, - uint32_t padding, uint32_t chunk_len, - uint8_t format); -extern lean_obj_res l_carbonado_verify_slice(lean_obj_arg body, lean_obj_arg hash, - uint32_t index, uint32_t count, uint8_t format); -extern lean_obj_res l_carbonado_verify_slice_outboard(lean_obj_arg main, lean_obj_arg outboard, - lean_obj_arg hash, uint32_t index, - uint32_t count, uint8_t format); - -static pthread_once_t g_lean_once = PTHREAD_ONCE_INIT; -static int g_lean_init_rc = -1; - -static void lean_init_once(void) { - lean_initialize_runtime_module(); - lean_obj_res res = initialize_Carbonado_Ffi(1); - if (!lean_io_result_is_ok(res)) { - lean_io_result_show_error(res); - lean_dec(res); - g_lean_init_rc = -1; - return; - } - lean_dec_ref(res); - lean_io_mark_end_initialization(); - g_lean_init_rc = 0; -} - -static int ensure_lean(void) { - (void)pthread_once(&g_lean_once, lean_init_once); - return g_lean_init_rc; -} - -uint32_t carbonado_abi_version(void) { - return CARBONADO_ABI_VERSION; -} - -void carbonado_free(void *p) { - free(p); -} - -/* Build a Lean ByteArray; takes a copy of `data` (may be NULL when len==0). */ -static lean_object *mk_byte_array(const uint8_t *data, size_t len) { - lean_object *ba = lean_alloc_sarray(1, len, len); - if (len > 0) { - if (data == NULL) { - lean_dec(ba); - return NULL; - } - memcpy(lean_sarray_cptr(ba), data, len); - } - return ba; -} - -static uint32_t read_u32_le(const uint8_t *p) { - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | - ((uint32_t)p[3] << 24); -} - -/* Copy Lean ByteArray payload into malloc'd buffer. */ -static int copy_sarray_payload(lean_object *ba, size_t offset, uint8_t **out, size_t *out_len) { - size_t n = lean_sarray_size(ba); - if (offset > n) { - return CARBONADO_ERR_INTERNAL; - } - size_t len = n - offset; - if (len == 0) { - *out = NULL; - *out_len = 0; - return CARBONADO_OK; - } - uint8_t *buf = (uint8_t *)malloc(len); - if (buf == NULL) { - return CARBONADO_ERR_INTERNAL; - } - memcpy(buf, lean_sarray_cptr(ba) + offset, len); - *out = buf; - *out_len = len; - return CARBONADO_OK; -} - -/* Unpack `[status:4][payload…]` → malloc payload on OK. */ -static int unpack_status_payload(lean_object *packed, uint8_t **out, size_t *out_len) { - size_t n = lean_sarray_size(packed); - if (n < 4) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - const uint8_t *p = lean_sarray_cptr(packed); - uint32_t status = read_u32_le(p); - if (status != CARBONADO_OK) { - lean_dec(packed); - return (int)status; - } - int rc = copy_sarray_payload(packed, 4, out, out_len); - lean_dec(packed); - return rc; -} - -/* Read length-prefixed segment; advances *off. */ -static int read_len_prefixed(const uint8_t *p, size_t n, size_t *off, uint8_t **out, - size_t *out_len) { - if (*off + 4 > n) { - return CARBONADO_ERR_INTERNAL; - } - uint32_t len = read_u32_le(p + *off); - *off += 4; - if (*off + len > n) { - return CARBONADO_ERR_INTERNAL; - } - if (len == 0) { - *out = NULL; - *out_len = 0; - return CARBONADO_OK; - } - uint8_t *buf = (uint8_t *)malloc(len); - if (buf == NULL) { - return CARBONADO_ERR_INTERNAL; - } - memcpy(buf, p + *off, len); - *off += len; - *out = buf; - *out_len = len; - return CARBONADO_OK; -} - -int carbonado_verification_key(uint8_t format, uint8_t key_out[32]) { - if (key_out == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - lean_obj_res ba = l_carbonado_verification_key(format); - if (lean_sarray_size(ba) != 32) { - lean_dec(ba); - return CARBONADO_ERR_INTERNAL; - } - memcpy(key_out, lean_sarray_cptr(ba), 32); - lean_dec(ba); - return CARBONADO_OK; -} - -int carbonado_encode_headered( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, - const uint8_t *slh_pk, - const uint8_t *metadata, - uint8_t **out, size_t *out_len, - uint32_t *padding_out, - uint32_t *chunk_len_out, - uint32_t *bytes_ecc_out, - uint32_t *verifiable_slice_count_out, - uint32_t *bytes_compressed_out, - uint32_t *bytes_encrypted_out) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (padding_out) *padding_out = 0; - if (chunk_len_out) *chunk_len_out = 0; - if (bytes_ecc_out) *bytes_ecc_out = 0; - if (verifiable_slice_count_out) *verifiable_slice_count_out = 0; - if (bytes_compressed_out) *bytes_compressed_out = 0; - if (bytes_encrypted_out) *bytes_encrypted_out = 0; - if (master == NULL || (plaintext == NULL && plaintext_len != 0)) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (nonce == NULL && nonce_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *m = mk_byte_array(master, master_len); - lean_object *n = mk_byte_array(nonce, nonce_len); - lean_object *pt = mk_byte_array(plaintext, plaintext_len); - /* Empty ByteArray when null → Lean zeros SLH/meta fields. */ - lean_object *slh = mk_byte_array(slh_pk, slh_pk != NULL ? 32 : 0); - lean_object *meta = mk_byte_array(metadata, metadata != NULL ? 8 : 0); - if (m == NULL || n == NULL || pt == NULL || slh == NULL || meta == NULL) { - if (m) lean_dec(m); - if (n) lean_dec(n); - if (pt) lean_dec(pt); - if (slh) lean_dec(slh); - if (meta) lean_dec(meta); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_encode_headered(m, n, pt, slh, meta, format); - size_t nlen = lean_sarray_size(packed); - /* Status-first: errors are packed as [status:4] only (see packEncodeErr). */ - if (nlen < 4) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - const uint8_t *p = lean_sarray_cptr(packed); - uint32_t status = read_u32_le(p); - if (status != CARBONADO_OK) { - lean_dec(packed); - return (int)status; - } - /* Success: status(4)+pad(4)+chunk(4)+ecc(4)+vsc(4)+comp(4)+enc(4)+archive = 28 + archive. */ - if (nlen < 28) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - if (padding_out) *padding_out = read_u32_le(p + 4); - if (chunk_len_out) *chunk_len_out = read_u32_le(p + 8); - if (bytes_ecc_out) *bytes_ecc_out = read_u32_le(p + 12); - if (verifiable_slice_count_out) *verifiable_slice_count_out = read_u32_le(p + 16); - if (bytes_compressed_out) *bytes_compressed_out = read_u32_le(p + 20); - if (bytes_encrypted_out) *bytes_encrypted_out = read_u32_le(p + 24); - int rc = copy_sarray_payload(packed, 28, out, out_len); - lean_dec(packed); - return rc; -} - -int carbonado_decode_headered( - const uint8_t *master, size_t master_len, - const uint8_t *archive, size_t archive_len, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (master == NULL || (archive == NULL && archive_len != 0)) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *m = mk_byte_array(master, master_len); - lean_object *a = mk_byte_array(archive, archive_len); - if (m == NULL || a == NULL) { - if (m) lean_dec(m); - if (a) lean_dec(a); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_decode_headered(m, a); - return unpack_status_payload(packed, out, out_len); -} - -int carbonado_encode( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, - uint8_t **out, size_t *out_len, - uint8_t hash_out[32], - uint32_t *padding_out, - uint32_t *chunk_len_out, - uint32_t *bytes_ecc_out, - uint32_t *verifiable_slice_count_out, - uint32_t *bytes_compressed_out, - uint32_t *bytes_encrypted_out) { - if (out == NULL || out_len == NULL || hash_out == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (padding_out) *padding_out = 0; - if (chunk_len_out) *chunk_len_out = 0; - if (bytes_ecc_out) *bytes_ecc_out = 0; - if (verifiable_slice_count_out) *verifiable_slice_count_out = 0; - if (bytes_compressed_out) *bytes_compressed_out = 0; - if (bytes_encrypted_out) *bytes_encrypted_out = 0; - if (master == NULL || (plaintext == NULL && plaintext_len != 0)) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (nonce == NULL && nonce_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *m = mk_byte_array(master, master_len); - lean_object *n = mk_byte_array(nonce, nonce_len); - lean_object *pt = mk_byte_array(plaintext, plaintext_len); - if (m == NULL || n == NULL || pt == NULL) { - if (m) lean_dec(m); - if (n) lean_dec(n); - if (pt) lean_dec(pt); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_encode(m, n, pt, format); - size_t nlen = lean_sarray_size(packed); - /* Status-first: errors are packed as [status:4] only (see packEncodeErr). */ - if (nlen < 4) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - const uint8_t *p = lean_sarray_cptr(packed); - uint32_t status = read_u32_le(p); - if (status != CARBONADO_OK) { - lean_dec(packed); - return (int)status; - } - /* Success: status(4)+pad(4)+chunk(4)+ecc(4)+vsc(4)+comp(4)+enc(4)+hash(32)+body = 60 + body. */ - if (nlen < 60) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - if (padding_out) *padding_out = read_u32_le(p + 4); - if (chunk_len_out) *chunk_len_out = read_u32_le(p + 8); - if (bytes_ecc_out) *bytes_ecc_out = read_u32_le(p + 12); - if (verifiable_slice_count_out) *verifiable_slice_count_out = read_u32_le(p + 16); - if (bytes_compressed_out) *bytes_compressed_out = read_u32_le(p + 20); - if (bytes_encrypted_out) *bytes_encrypted_out = read_u32_le(p + 24); - memcpy(hash_out, p + 28, 32); - int rc = copy_sarray_payload(packed, 60, out, out_len); - lean_dec(packed); - return rc; -} - -int carbonado_decode( - const uint8_t *master, size_t master_len, - const uint8_t *hash, size_t hash_len, - const uint8_t *body, size_t body_len, - uint32_t padding, - uint8_t format, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (master == NULL || hash == NULL || hash_len != 32) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (body == NULL && body_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *m = mk_byte_array(master, master_len); - lean_object *h = mk_byte_array(hash, hash_len); - lean_object *b = mk_byte_array(body, body_len); - if (m == NULL || h == NULL || b == NULL) { - if (m) lean_dec(m); - if (h) lean_dec(h); - if (b) lean_dec(b); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_decode(m, h, b, padding, format); - return unpack_status_payload(packed, out, out_len); -} - -int carbonado_encode_outboard( - const uint8_t *master, size_t master_len, - const uint8_t *plaintext, size_t plaintext_len, - uint8_t format, - const uint8_t *nonce, size_t nonce_len, - uint8_t header_path, - uint8_t **main_out, size_t *main_len, - uint8_t **outboard_out, size_t *outboard_len, - uint8_t **parity_out, size_t *parity_len, - uint8_t hash_out[32], - uint32_t *padding_out, - uint32_t *chunk_len_out, - uint32_t *bytes_compressed_out, - uint32_t *bytes_encrypted_out) { - if (main_out == NULL || main_len == NULL || outboard_out == NULL || outboard_len == NULL || - parity_out == NULL || parity_len == NULL || hash_out == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *main_out = NULL; - *main_len = 0; - *outboard_out = NULL; - *outboard_len = 0; - *parity_out = NULL; - *parity_len = 0; - if (padding_out) *padding_out = 0; - if (chunk_len_out) *chunk_len_out = 0; - if (bytes_compressed_out) *bytes_compressed_out = 0; - if (bytes_encrypted_out) *bytes_encrypted_out = 0; - if (master == NULL || (plaintext == NULL && plaintext_len != 0)) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (nonce == NULL && nonce_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *m = mk_byte_array(master, master_len); - lean_object *n = mk_byte_array(nonce, nonce_len); - lean_object *pt = mk_byte_array(plaintext, plaintext_len); - if (m == NULL || n == NULL || pt == NULL) { - if (m) lean_dec(m); - if (n) lean_dec(n); - if (pt) lean_dec(pt); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_encode_outboard(m, n, pt, format, header_path); - size_t nlen = lean_sarray_size(packed); - if (nlen < 4) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - const uint8_t *p = lean_sarray_cptr(packed); - uint32_t status = read_u32_le(p); - if (status != CARBONADO_OK) { - lean_dec(packed); - return (int)status; - } - /* status(4)+pad(4)+chunk(4)+comp(4)+enc(4)+hash(32) = 52 */ - if (nlen < 52) { - lean_dec(packed); - return CARBONADO_ERR_INTERNAL; - } - if (padding_out) *padding_out = read_u32_le(p + 4); - if (chunk_len_out) *chunk_len_out = read_u32_le(p + 8); - if (bytes_compressed_out) *bytes_compressed_out = read_u32_le(p + 12); - if (bytes_encrypted_out) *bytes_encrypted_out = read_u32_le(p + 16); - memcpy(hash_out, p + 20, 32); - size_t off = 52; - int rc = read_len_prefixed(p, nlen, &off, main_out, main_len); - if (rc != CARBONADO_OK) { - lean_dec(packed); - return rc; - } - rc = read_len_prefixed(p, nlen, &off, outboard_out, outboard_len); - if (rc != CARBONADO_OK) { - free(*main_out); - *main_out = NULL; - *main_len = 0; - lean_dec(packed); - return rc; - } - rc = read_len_prefixed(p, nlen, &off, parity_out, parity_len); - if (rc != CARBONADO_OK) { - free(*main_out); - free(*outboard_out); - *main_out = NULL; - *main_len = 0; - *outboard_out = NULL; - *outboard_len = 0; - lean_dec(packed); - return rc; - } - lean_dec(packed); - return CARBONADO_OK; -} - -int carbonado_decode_outboard( - const uint8_t *master, size_t master_len, - const uint8_t *hash, size_t hash_len, - const uint8_t *main, size_t main_len, - const uint8_t *outboard, size_t outboard_len, - const uint8_t *parity, size_t parity_len, - uint32_t padding, - uint8_t format, - uint8_t header_path, - const uint8_t *nonce, size_t nonce_len, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (master == NULL || hash == NULL || hash_len != 32) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (main == NULL && main_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (outboard == NULL && outboard_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (parity == NULL && parity_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (nonce == NULL && nonce_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *m = mk_byte_array(master, master_len); - lean_object *h = mk_byte_array(hash, hash_len); - lean_object *mn = mk_byte_array(main, main_len); - lean_object *ob = mk_byte_array(outboard, outboard_len); - lean_object *pr = mk_byte_array(parity, parity_len); - lean_object *n = mk_byte_array(nonce, nonce_len); - if (m == NULL || h == NULL || mn == NULL || ob == NULL || pr == NULL || n == NULL) { - if (m) lean_dec(m); - if (h) lean_dec(h); - if (mn) lean_dec(mn); - if (ob) lean_dec(ob); - if (pr) lean_dec(pr); - if (n) lean_dec(n); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = - l_carbonado_decode_outboard(m, h, mn, ob, pr, padding, format, header_path, n); - return unpack_status_payload(packed, out, out_len); -} - -int carbonado_scrub( - const uint8_t *body, size_t body_len, - const uint8_t *hash, size_t hash_len, - uint32_t padding, - uint8_t format, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (hash == NULL || hash_len != 32) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (body == NULL && body_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *b = mk_byte_array(body, body_len); - lean_object *h = mk_byte_array(hash, hash_len); - if (b == NULL || h == NULL) { - if (b) lean_dec(b); - if (h) lean_dec(h); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_scrub(b, h, padding, format); - return unpack_status_payload(packed, out, out_len); -} - -int carbonado_scrub_outboard( - const uint8_t *main, size_t main_len, - const uint8_t *outboard, size_t outboard_len, - const uint8_t *parity, size_t parity_len, - const uint8_t *hash, size_t hash_len, - uint32_t padding, - uint32_t chunk_len, - uint8_t format, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (hash == NULL || hash_len != 32) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (main == NULL && main_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (outboard == NULL && outboard_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (parity == NULL && parity_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *mn = mk_byte_array(main, main_len); - lean_object *ob = mk_byte_array(outboard, outboard_len); - lean_object *pr = mk_byte_array(parity, parity_len); - lean_object *h = mk_byte_array(hash, hash_len); - if (mn == NULL || ob == NULL || pr == NULL || h == NULL) { - if (mn) lean_dec(mn); - if (ob) lean_dec(ob); - if (pr) lean_dec(pr); - if (h) lean_dec(h); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = - l_carbonado_scrub_outboard(mn, ob, pr, h, padding, chunk_len, format); - return unpack_status_payload(packed, out, out_len); -} - -int carbonado_verify_slice( - const uint8_t *body, size_t body_len, - const uint8_t *hash, size_t hash_len, - uint32_t index, - uint32_t count, - uint8_t format, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (hash == NULL || hash_len != 32) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (body == NULL && body_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *b = mk_byte_array(body, body_len); - lean_object *h = mk_byte_array(hash, hash_len); - if (b == NULL || h == NULL) { - if (b) lean_dec(b); - if (h) lean_dec(h); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = l_carbonado_verify_slice(b, h, index, count, format); - return unpack_status_payload(packed, out, out_len); -} - -int carbonado_verify_slice_outboard( - const uint8_t *main, size_t main_len, - const uint8_t *outboard, size_t outboard_len, - const uint8_t *hash, size_t hash_len, - uint32_t index, - uint32_t count, - uint8_t format, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (hash == NULL || hash_len != 32) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (main == NULL && main_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (outboard == NULL && outboard_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (ensure_lean() != 0) { - return CARBONADO_ERR_INTERNAL; - } - - lean_object *mn = mk_byte_array(main, main_len); - lean_object *ob = mk_byte_array(outboard, outboard_len); - lean_object *h = mk_byte_array(hash, hash_len); - if (mn == NULL || ob == NULL || h == NULL) { - if (mn) lean_dec(mn); - if (ob) lean_dec(ob); - if (h) lean_dec(h); - return CARBONADO_ERR_INVALID_ARGUMENT; - } - - lean_obj_res packed = - l_carbonado_verify_slice_outboard(mn, ob, h, index, count, format); - return unpack_status_payload(packed, out, out_len); -} diff --git a/nix/native/carbonado_slh.c b/nix/native/carbonado_slh.c index b2fbfc0..20b2612 100644 --- a/nix/native/carbonado_slh.c +++ b/nix/native/carbonado_slh.c @@ -2,8 +2,8 @@ * Carbonado SLH-DSA-SHA2-128s FFI (R9 / G10). * * Links libbitcoinpqc SLH sources (sphincsplus + slh_dsa wrappers) only — - * no secp256k1 / ML-DSA. Dual-suite product SLH may still use Rust bitcoinpqc; - * this path makes pure Lean AOT / libcarbonado self-contained. + * no secp256k1 / ML-DSA. This path is for the Lean AOT demo binary only + * (`@[extern]`), not a Rust `-sys` / C ABI product. * * Lean @[extern] wire (status-prefixed ByteArray, like zstd): * carbonado_slh_keygen_raw : @& ByteArray → ByteArray @@ -19,9 +19,6 @@ * 2 other bad argument (wrong sk/pk/sig sizes) * 3 crypto failure (keygen/sign library error) * - * Public C ABI (include/carbonado.h): carbonado_slh_keygen / _sign / _verify. - * Keygen library failure → CARBONADO_ERR_INTERNAL (not AUTHENTICATION). - * Verify reject → CARBONADO_ERR_AUTHENTICATION. */ #include #include @@ -30,7 +27,6 @@ #include #include "libbitcoinpqc/slh_dsa.h" -#include "carbonado.h" enum { SLH_ST_OK = 0, @@ -129,71 +125,3 @@ LEAN_EXPORT uint8_t carbonado_slh_verify_raw(b_lean_obj_arg pk, b_lean_obj_arg m const uint8_t *msg = msg_ptr(m_p, m_len); return slh_dsa_sha2_128s_verify(sig_p, sig_len, msg, m_len, pk_p) == 0 ? 1 : 0; } - -/* ── Public C ABI ─────────────────────────────────────────────────────────── */ - -int carbonado_slh_keygen( - const uint8_t *entropy, size_t entropy_len, - uint8_t pk_out[32], - uint8_t sk_out[64]) { - if (entropy == NULL || entropy_len < 128 || pk_out == NULL || sk_out == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (slh_dsa_sha2_128s_keygen(pk_out, sk_out, entropy, entropy_len) != 0) { - memset(sk_out, 0, 64); - /* Keygen failure is not an auth reject — map to INTERNAL. */ - return CARBONADO_ERR_INTERNAL; - } - return CARBONADO_OK; -} - -int carbonado_slh_sign( - const uint8_t *secret_key, size_t secret_key_len, - const uint8_t *message, size_t message_len, - uint8_t **out, size_t *out_len) { - if (out == NULL || out_len == NULL) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - *out = NULL; - *out_len = 0; - if (secret_key == NULL || secret_key_len != SLH_DSA_SHA2_128S_SECRET_KEY_SIZE) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (message == NULL && message_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - const uint8_t *msg = msg_ptr(message, message_len); - uint8_t *sig = (uint8_t *)malloc(SLH_DSA_SHA2_128S_SIGNATURE_SIZE); - if (sig == NULL) { - return CARBONADO_ERR_INTERNAL; - } - size_t siglen = 0; - if (slh_dsa_sha2_128s_sign(sig, &siglen, msg, message_len, secret_key) != 0 || - siglen != SLH_DSA_SHA2_128S_SIGNATURE_SIZE) { - free(sig); - return CARBONADO_ERR_INTERNAL; - } - *out = sig; - *out_len = SLH_DSA_SHA2_128S_SIGNATURE_SIZE; - return CARBONADO_OK; -} - -int carbonado_slh_verify( - const uint8_t *public_key, size_t public_key_len, - const uint8_t *message, size_t message_len, - const uint8_t *signature, size_t signature_len) { - if (public_key == NULL || public_key_len != SLH_DSA_SHA2_128S_PUBLIC_KEY_SIZE) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (signature == NULL || signature_len != SLH_DSA_SHA2_128S_SIGNATURE_SIZE) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - if (message == NULL && message_len != 0) { - return CARBONADO_ERR_INVALID_ARGUMENT; - } - const uint8_t *msg = msg_ptr(message, message_len); - if (slh_dsa_sha2_128s_verify(signature, signature_len, msg, message_len, public_key) != 0) { - return CARBONADO_ERR_AUTHENTICATION; - } - return CARBONADO_OK; -} diff --git a/nix/native/carbonado_zstd.c b/nix/native/carbonado_zstd.c index da2a66c..ecc4fc8 100644 --- a/nix/native/carbonado_zstd.c +++ b/nix/native/carbonado_zstd.c @@ -8,8 +8,8 @@ * status 3 = decompressed output exceeds max * status 4 = invalid input / size error * - * Linked into the AOT product via flake staticLibDeps: - * libcarbonado_native.a = this FFI + static libzstd objects from ref/zstd. + * Linked into the Lean AOT demo via flake staticLibDeps (not a Rust `-sys` product): + * carbonado-native archive = this FFI + static libzstd objects from ref/zstd. * No shared -lzstd. Lean elaborator uses the `@[extern]` body (identity fallback). */ #include diff --git a/nix/native/default.nix b/nix/native/default.nix index 65ccd60..4b8bd5f 100644 --- a/nix/native/default.nix +++ b/nix/native/default.nix @@ -1,5 +1,6 @@ -# Static FFI glue for Carbonado AOT (zstd + SLH-DSA + C ABI). +# Static FFI glue for the Lean AOT demo (zstd + SLH-DSA @[extern]). # Output: $out/libcarbonado_native.a (linked via buildLeanPackage.staticLibDeps). +# This is not a Rust -sys / C ABI product. # # Embeds: # * single-threaded libzstd from the **pinned** `ref/zstd` tree (v1.5.7) @@ -10,8 +11,7 @@ pkgs, leanAll, # pkgs.lean.lean-all — provides lean/lean.h zstdSrc, # flake: pinned zstd fetch - bitcoinpqcSrc, # flake: pinned libbitcoinpqc fetch (R9 / G10) - carbonadoInclude ? ../.. + "/include", # repo include/carbonado.h (ABI) + bitcoinpqcSrc, # flake: pinned libbitcoinpqc fetch }: pkgs.stdenv.mkDerivation { pname = "carbonado-native"; @@ -30,11 +30,6 @@ pkgs.stdenv.mkDerivation { ZSTD_LIB="${zstdSrc}/lib" PQC="${bitcoinpqcSrc}" - ABI_INC="${carbonadoInclude}" - if [ ! -f "$ABI_INC/carbonado.h" ]; then - echo "carbonado-native: missing $ABI_INC/carbonado.h" >&2 - exit 1 - fi if [ ! -d "$ZSTD_LIB" ]; then echo "carbonado-native: missing zstd lib dir at $ZSTD_LIB (init ref/zstd submodule)" >&2 exit 1 @@ -103,21 +98,13 @@ pkgs.stdenv.mkDerivation { compile_pqc "$PQC/src/slh_dsa/sign.c" slh_sign compile_pqc "$PQC/src/slh_dsa/verify.c" slh_verify - echo "carbonado-native: compiling carbonado_slh.c (Lean extern + C ABI)" + echo "carbonado-native: compiling carbonado_slh.c (Lean @[extern] only)" $CC -c -O2 -fPIC \ -I${leanAll}/include \ - -I"$ABI_INC" \ -I"$PQC/include" \ carbonado_slh.c \ -o carbonado_slh.o - echo "carbonado-native: compiling carbonado_abi.c (C ABI v1 + Lean glue)" - $CC -c -O2 -fPIC \ - -I${leanAll}/include \ - -I"$ABI_INC" \ - carbonado_abi.c \ - -o carbonado_abi.o - # Fail-closed: must have more than just the FFI object. ocount=$(ls -1 ./*.o 2>/dev/null | wc -l) if [ "$ocount" -lt 20 ]; then @@ -134,15 +121,13 @@ pkgs.stdenv.mkDerivation { installPhase = '' runHook preInstall # lean4-nix staticLibDeps expects $out/libcarbonado_native.a (archive root). - mkdir -p $out/lib $out/include + mkdir -p $out/lib cp libcarbonado_native.a $out/ cp libcarbonado_native.a $out/lib/ - ln -sf libcarbonado_native.a $out/lib/libcarbonado.a - cp "${carbonadoInclude}/carbonado.h" $out/include/ runHook postInstall ''; meta = { - description = "Carbonado Lean AOT native glue (static zstd + SLH-DSA + C ABI Lean bridge)"; + description = "Carbonado Lean AOT demo native glue (static zstd + SLH-DSA @[extern])"; }; } diff --git a/nix/tooling-purity.nix b/nix/tooling-purity.nix index 3e2c735..70faa09 100644 --- a/nix/tooling-purity.nix +++ b/nix/tooling-purity.nix @@ -1,7 +1,7 @@ -# checks.tooling-purity — dual-backend product tree purity constraints. -# Dual-backend SSOT (AGENTS / G1 W5a): Rust under src/, tests/, benches/, examples/ -# is permanent first-class product + dual-suite contract — NOT transitional and NOT -# moved to ref/carbonado-rust (permanent no product pin). ref/ is third-party oracles only. +# checks.tooling-purity — product tree purity constraints. +# SSOT: Rust under src/, tests/, benches/, examples/ is the production engine. +# Lean under Carbonado/ + CarbonadoTest/ is proofs + AOT demo. ref/ is third-party +# oracles only. There is no carbonado-sys crate and no product C ABI. # This check: # * requires product Lean roots to exist and be Lean-only # * bans product shell/python glue outside nix/ and ref/ @@ -70,7 +70,7 @@ pkgs.runCommand "carbonado-tooling-purity" { .git|.cargo|.github|.vscode|.gitignore|.gitmodules) return 0 ;; Carbonado|CarbonadoTest|nix|docs|doc|ref|src|tests|benches|examples|target) return 0 ;; Carbonado.lean|CarbonadoTest.lean) return 0 ;; - flake.nix|flake.lock|lean-toolchain|justfile) return 0 ;; + flake.nix|flake.lock|lean-toolchain|rust-toolchain.toml|justfile|build.rs) return 0 ;; # Optional Lake manifest for local Lean IDE/`lake build` (Nix remains SSOT package). lakefile.toml|lakefile.lean|lake-manifest.json) return 0 ;; AGENTS.md|README.md|LICENSE|CHANGELOG.md|Cargo.toml|Cargo.lock) return 0 ;; diff --git a/ref/README.md b/ref/README.md index 488cd77..a0e2c27 100644 --- a/ref/README.md +++ b/ref/README.md @@ -6,8 +6,8 @@ Trees under `ref/` are **third-party oracles / vendors only** — not product en | Engine | Location | |--------|----------| -| **Rust** (first-class + dual-suite SSOT) | live `src/`, `tests/` (also benches/examples/CLI) | -| **Lean 4** (proofs + AOT `libcarbonado`) | `Carbonado/`, `CarbonadoTest/`; built via Nix flakes | +| **Rust** (production engine) | live `src/`, `tests/` (also benches/examples/CLI) | +| **Lean 4** (proofs + AOT demo) | `Carbonado/`, `CarbonadoTest/`; built via Nix flakes | **G1/W5a permanent policy:** no `ref/carbonado-rust` product pin. Do not invent a submodule that freezes or demotes live Rust. @@ -17,7 +17,7 @@ See [docs/PARITY.md](../docs/PARITY.md) for pin table and [docs/SPEC-MATRIX.md]( | Path | Purpose | Status | |------|---------|--------| -| `bao-tree` | Surmount keyed Bao fork | **pinned** | +| `bao-tree` | Keyed Bao oracle (Surmount snapshot of the work now in n0-computer PR 78) | **pinned** | | `reed-solomon-erasure` | RS 4/8 | **pinned** | | `rustcrypto-block-ciphers` | AES 0.8.4 | **pinned** | | `rustcrypto-macs` | HMAC 0.12.1 | **pinned** | diff --git a/ref/parity-harness/drivers/bao-vectors/src/main.rs b/ref/parity-harness/drivers/bao-vectors/src/main.rs index d073d43..b23fd1f 100644 --- a/ref/parity-harness/drivers/bao-vectors/src/main.rs +++ b/ref/parity-harness/drivers/bao-vectors/src/main.rs @@ -1,4 +1,4 @@ -//! Golden vectors for Carbonado keyed Bao (bao-tree 76-keyed-bao, 4 KiB groups). +//! Golden vectors for Carbonado keyed Bao (n0-computer/bao-tree PR 78 keyed APIs, 4 KiB groups). use std::io::Cursor; use bao_tree::{ diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1bdda9c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,8 @@ +[toolchain] +channel = "1.98.0" +components = ["rustfmt", "clippy", "rust-src"] +targets = [ + "wasm32-unknown-unknown", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", +] diff --git a/src/backend/mod.rs b/src/backend/mod.rs deleted file mode 100644 index b07e09b..0000000 --- a/src/backend/mod.rs +++ /dev/null @@ -1,801 +0,0 @@ -//! Dual-backend dispatch (docs/TEST_CONTRACT.md, docs/ABI.md). -//! -//! - `backend-rust` (default): pure Rust implementation in this crate. -//! - `backend-lean`: Lean AOT `libcarbonado` via `carbonado-sys` (G8 dual-backend). -//! -//! Both features must not be enabled together for a single build that links both -//! engines into conflicting paths; prefer one engine per `cargo test` invocation. -//! -//! ## Phase 2–3 dispatch surface (`backend-lean`) -//! -//! Lean C ABI is used for: low-level [`crate::encode`]/crate::decode`], -//! [`crate::encode_outboard`]/crate::decode_outboard`], -//! [`crate::scrub`]/crate::scrub_outboard`], [`crate::verify_slice`]/crate::extract_slice`], -//! headered [`crate::file::encode`]/crate::file::decode`], stream buffer helpers -//! (`stream_encode_buffer` / `stream_decode_buffer` / outboard buffer), **R5/W1 E1** -//! stream I/O (`stream_encode_inboard` / `stream_decode` / **encrypted** -//! `stream_*_outboard` via spool-to-buffer; **W1b public** `stream_*_outboard` is rust S4 -//! composition — not Lean spool), and [`lean::verification_key`] for AOT parity checks. -//! -//! ### Phase 3 directory (composition — no new directory C symbols) -//! -//! [`crate::file::encode_directory`] / [`crate::file::decode_directory`] keep: -//! - **Rust:** FS walk, path fail-closed checks, **rkyv** `FilepackManifest` v2 wire -//! (normative Adamantine payload body), Adamantine envelope/payload framing, -//! centralized Bao+FEC bundle assembly, catalog COTS trailer. -//! - **Lean (via existing ABI):** segment bare mains via `encode_outboard` / -//! `decode_outboard` (embedded-nonce layout); catalog container via -//! `encode_headered` / `decode_headered` (`file::encode` / `file::decode`). -//! -//! Dual-suite catalogs are therefore **rkyv** (same bytes as `backend-rust`), not -//! Lean-only CFP2. **W3:** pure Lean Directory/CLI also emit rkyv (wire-compatible); -//! dual-suite encode remains Rust rkyv composition SSOT. CFP2 is dual-decode fallback only. -//! -//! ### Phase 4 PQC / CLI / OTS (composition) -//! -//! - **SLH-DSA (G10-A dual-suite + R9 pure Lean):** dual-suite product path may use -//! Rust `crypto::slh_*` + `bitcoinpqc` under both backends (composition). Lean holds -//! wire parse/build + bind-to-root (`Carbonado/Slh.lean`). **R9/G10:** pure Lean -//! `signRoot` / `verifyRoot` are live via `carbonado_slh_*` + libbitcoinpqc pin in -//! `libcarbonado` — dual-suite need not switch from composition. -//! - **CLI dual path (honest):** -//! - **Dual-engine:** directory CLI → `encode_directory` / `decode_directory` -//! (Lean segment/catalog crypto); buffer single-file APIs (`file::encode`, -//! `encode_outboard`, headered decode) dispatch to Lean under `backend-lean`. -//! - **R5 E1 + W1 stream dual:** `stream_encode_inboard` / `stream_decode` and -//! **encrypted** `stream_*_outboard` spool-to-buffer → Lean C ABI (O(logical) E1). -//! **W1b public** `stream_encode_outboard` / `stream_decode_outboard` use rust S4 -//! geometric composition under lean (**E2 O(chunk/stripe)** when !Compression; -//! Compression under lean is O(logical) bulk zstd; G9 no-compress wire bit-match; -//! not pure Lean stream). `encode_stream` uses Lean via `stream_encode_inboard`. -//! **W1a:** `file::decode_stream` verifies `header_mac` then spools body → Lean -//! `decode_headered` (E1). -//! - Build with `--features "backend-lean,pqc,ots,cli"` + `CARBONADO_LEAN_*` for a -//! lean-**linked** binary. Default `cargo build --bin carbonado` remains rust-engine. -//! - **Directory OTS:** offline CBOTS stubs in Rust (`ots` feature); composition -//! over Lean container crypto — no Lean-native stamping. -//! -//! **G8 full closed at R7** under `backend-lean`: freeze = unfiltered -//! `just test-lean-ci` (`cargo test --no-default-features --features -//! "backend-lean,pqc,ots,cli"`). Former residual suites (`format`, `codec`, -//! `header_tamper`, `format_amplification`, `streaming`/`streaming_limits`, -//! `seekable_slices`, `sharding`, `fec_chaos`, `bin_*`) are freeze-green. -//! -//! **Post-G8 residuals** (purity / feature-policy / composition honesty — not dual-suite red): -//! ~~pure-Lean rkyv encode~~ **W3 closed** (Lean `encodeRkyvManifest` + Directory/CLI; dual-suite -//! catalog encode still Rust rkyv composition SSOT), ~~stream E2 / dual honesty~~ **W1a+W1b closed** (public outboard S4 -//! composition E2; pure Lean chunked C residual), G9 Compression/directory encode bit-match -//! (**W2** permanent W2a/W2b), ~~codecode/decodec~~ **W2d closed**, ~~W4a inboard O(slice) retain~~ -//! **closed**, **W4b** permanent full-buffer C outboard slice, **W4c** permanent buffer-only zstd -//! under lean, **W4d** permanent FEC O(body) + async encoded spool, -//! `streaming_async` / `parallel_determinism` permanently feature-gated off freeze (R10: -//! freeze never requires `async` / `parallel`; Lean RS is serial). -//! -//! **R9 closed (optional pure Lean depth):** G10 SLH-DSA live in `libcarbonado` -//! (`carbonado_slh_*` + Lean `@[extern]`); seekable outboard slice C -//! (`carbonado_verify_slice_outboard` / Lean range verify); rkyv dual-decode residual -//! documented (composition remains SSOT for dual-suite directory wire). -//! -//! **R10 closed (async dual policy):** dual freeze **never** requires `async` -//! (`streaming_async` stays feature-gated → 0 tests under lean freeze). Optional -//! `stream_decode_async` stages the encoded body then calls dual-aware -//! [`crate::stream::stream_decode`] (R5 E1 under `backend-lean`; S4 pipeline under -//! `backend-rust`) — no silent pure-Rust pipeline when lean+async are both enabled. -//! WASM async remains `NotImplemented`. -//! -//! **R1 fail-closed (outboard Option semantics):** `None` means missing sidecar and -//! must error when the format bit requires it (`MissingVerificationOutboard` / -//! `MissingFecParity`). `Some(&[])` is a present empty outboard (valid for single-leaf -//! Bao trees) and is allowed through. Guarded in [`lean::decode_outboard`] before C. - -#[cfg(all(feature = "backend-lean", feature = "backend-rust"))] -compile_error!( - "enable only one of `backend-lean` or `backend-rust` (dual-backend CI runs them separately)" -); - -#[cfg(not(any(feature = "backend-lean", feature = "backend-rust")))] -compile_error!("enable `backend-lean` or `backend-rust` (see docs/TEST_CONTRACT.md)"); - -#[cfg(feature = "backend-rust")] -#[allow(dead_code)] // dispatch hooks land as encode/decode call sites migrate -pub mod rust_engine { - //! Marker: pure Rust paths are the default implementation modules (`encoding`, `decoding`, …). - pub const NAME: &str = "rust"; -} - -#[cfg(feature = "backend-lean")] -pub mod lean { - //! Lean AOT backend via C ABI (`carbonado-sys` / `libcarbonado`). - use crate::error::CarbonadoError; - use crate::structs::{EncodeInfo, Encoded, OutboardEncoded}; - use carbonado_sys as sys; - - pub const NAME: &str = "lean"; - - /// ABI version from the linked libcarbonado (requires `CARBONADO_LEAN_LIB`). - pub fn abi_version() -> u32 { - unsafe { sys::carbonado_abi_version() } - } - - /// Map C ABI codes to `CarbonadoError` (docs/ABI.md). - pub fn map_err(code: i32) -> CarbonadoError { - match code { - // P2 residual: no dedicated InvalidArgument variant (docs/ABI.md error table). - // Includes Lean-only wrong-length SLH/meta on encodeHeaderedBytes if ever - // surfaced via C; typed Rust Option<&[u8; N]> cannot express those lengths. - sys::CARBONADO_ERR_INVALID_ARGUMENT => { - CarbonadoError::InternalStateError("lean-backend invalid argument".into()) - } - sys::CARBONADO_ERR_INVALID_KEY_LENGTH => CarbonadoError::InvalidKeyLength, - sys::CARBONADO_ERR_AUTHENTICATION => CarbonadoError::AuthenticationFailed, - sys::CARBONADO_ERR_INVALID_MAGIC => { - CarbonadoError::InvalidMagicNumber("lean-backend".into()) - } - // Truncated/malformed header, body bounds, or short inboard Bao prefix - // (`invalidPrefix` / `invalidHeaderLength` via ofPipelineError). - sys::CARBONADO_ERR_INVALID_HEADER => CarbonadoError::InvalidHeaderLength, - sys::CARBONADO_ERR_FEC => CarbonadoError::UnevenFecChunks, - // Stream truncation / trailing data / root-length / residual slice geometry - // that was not pre-checked in `lean::verify_slice`. Bao *auth* failures use - // CARBONADO_ERR_AUTHENTICATION (R4). - sys::CARBONADO_ERR_BAO => { - CarbonadoError::BaoResponseTruncated("lean-backend bao/verify".into()) - } - sys::CARBONADO_ERR_ZSTD => CarbonadoError::ZstdError("lean-backend zstd".into()), - sys::CARBONADO_ERR_SCRUB_UNNECESSARY => CarbonadoError::UnnecessaryScrub, - sys::CARBONADO_ERR_SCRUB_FAILED => CarbonadoError::InvalidScrubbedHash, - sys::CARBONADO_ERR_SCRUB_REQUIRES_VERIFICATION => { - CarbonadoError::ScrubRequiresVerification - } - sys::CARBONADO_ERR_NOT_IMPLEMENTED => CarbonadoError::NotImplemented, - sys::CARBONADO_ERR_INTERNAL => { - CarbonadoError::InternalStateError("lean-backend internal error".into()) - } - _ => CarbonadoError::InternalStateError(format!("lean-backend unknown error {code}")), - } - } - - /// Copy a libcarbonado `malloc` buffer into a Rust `Vec`, then free via C. - /// - /// Avoids `Vec::from_raw_parts` over foreign allocators (jemalloc/mimalloc-safe). - fn take_buf(out: *mut u8, out_len: usize) -> Result, CarbonadoError> { - if out.is_null() { - if out_len == 0 { - return Ok(Vec::new()); - } - return Err(map_err(sys::CARBONADO_ERR_INTERNAL)); - } - let mut v = Vec::with_capacity(out_len); - // SAFETY: `out` is a non-null malloc buffer of length `out_len` from libcarbonado. - unsafe { - v.extend_from_slice(std::slice::from_raw_parts(out, out_len)); - sys::carbonado_free(out as *mut _); - } - Ok(v) - } - - /// Free a raw libcarbonado buffer if non-null (best-effort cleanup on multi-out failures). - fn free_raw(p: *mut u8) { - if !p.is_null() { - unsafe { sys::carbonado_free(p as *mut _) }; - } - } - - /// Format-keyed Bao verification key (32 bytes). - pub fn verification_key(format: u8) -> Result<[u8; 32], CarbonadoError> { - let mut key = [0u8; 32]; - let rc = unsafe { sys::carbonado_verification_key(format, key.as_mut_ptr()) }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - Ok(key) - } - - /// Low-level body encode (≈ `encoding::encode`). Returns body, Bao root, padding + meta. - /// - /// # `EncodeInfo` (R3) - /// - /// Stage counters (`bytes_compressed`, `bytes_encrypted`) and FEC/Bao geometry - /// (`padding_len`, `chunk_len`, `bytes_ecc`, `verifiable_slice_count`) are filled - /// from the Lean pack. Skipped stages report **0** (matches Rust stream path). - pub fn encode( - master: &[u8], - plaintext: &[u8], - format: u8, - nonce: Option<&[u8; 16]>, - ) -> Result { - let (nonce_ptr, nonce_len) = match nonce { - Some(n) => (n.as_ptr(), 16usize), - None => (std::ptr::null(), 0usize), - }; - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let mut hash_out = [0u8; 32]; - let mut padding_len = 0u32; - let mut chunk_len = 0u32; - let mut bytes_ecc = 0u32; - let mut verifiable_slice_count = 0u32; - let mut bytes_compressed = 0u32; - let mut bytes_encrypted = 0u32; - let rc = unsafe { - sys::carbonado_encode( - master.as_ptr(), - master.len(), - plaintext.as_ptr(), - plaintext.len(), - format, - nonce_ptr, - nonce_len, - &mut out, - &mut out_len, - hash_out.as_mut_ptr(), - &mut padding_len, - &mut chunk_len, - &mut bytes_ecc, - &mut verifiable_slice_count, - &mut bytes_compressed, - &mut bytes_encrypted, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - let body = take_buf(out, out_len)?; - let hash = crate::utils::decode_bao_hash(&hash_out)?; - let chunk_slice_count = if verifiable_slice_count > 0 { - verifiable_slice_count / 8 - } else { - 0 - }; - let input_len = plaintext.len() as u32; - let info = EncodeInfo { - input_len, - output_len: body.len() as u32, - bytes_compressed, - compression_factor: bytes_compressed as f32 / input_len.max(1) as f32, - bytes_encrypted, - bytes_ecc, - bytes_verifiable: body.len() as u32, - // Match Rust stream: empty input → 0.0 (not 1.0). - amplification_factor: body.len() as f32 / input_len.max(1) as f32, - padding_len, - chunk_len, - verifiable_slice_count, - chunk_slice_count, - }; - Ok(Encoded(body, hash, info)) - } - - /// Low-level body decode (≈ `decoding::decode` buffer path). - pub fn decode( - master: &[u8], - hash: &[u8], - body: &[u8], - padding: u32, - format: u8, - ) -> Result, CarbonadoError> { - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_decode( - master.as_ptr(), - master.len(), - hash.as_ptr(), - hash.len(), - body.as_ptr(), - body.len(), - padding, - format, - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - take_buf(out, out_len) - } - - /// Headered encode via Lean AOT (explicit 16-byte nonce when encrypted). - /// - /// `slh_public_key` / `metadata`: when `None`, header fields are zeros (matches Rust). - /// - /// Returns the full archive (`Header || body`) and pipeline [`EncodeInfo`] stage - /// counters (R3: compress/encrypt + FEC/Bao geometry from Lean pack). - pub fn encode_headered( - master: &[u8], - plaintext: &[u8], - format: u8, - nonce: Option<&[u8; 16]>, - slh_public_key: Option<&[u8; 32]>, - metadata: Option<&[u8; 8]>, - ) -> Result<(Vec, EncodeInfo), CarbonadoError> { - let (nonce_ptr, nonce_len) = match nonce { - Some(n) => (n.as_ptr(), 16usize), - None => (std::ptr::null(), 0usize), - }; - let slh_ptr = slh_public_key - .map(|s| s.as_ptr()) - .unwrap_or(std::ptr::null()); - let meta_ptr = metadata.map(|m| m.as_ptr()).unwrap_or(std::ptr::null()); - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let mut padding_len = 0u32; - let mut chunk_len = 0u32; - let mut bytes_ecc = 0u32; - let mut verifiable_slice_count = 0u32; - let mut bytes_compressed = 0u32; - let mut bytes_encrypted = 0u32; - let rc = unsafe { - sys::carbonado_encode_headered( - master.as_ptr(), - master.len(), - plaintext.as_ptr(), - plaintext.len(), - format, - nonce_ptr, - nonce_len, - slh_ptr, - meta_ptr, - &mut out, - &mut out_len, - &mut padding_len, - &mut chunk_len, - &mut bytes_ecc, - &mut verifiable_slice_count, - &mut bytes_compressed, - &mut bytes_encrypted, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - let archive = take_buf(out, out_len)?; - let body_len = archive.len().saturating_sub(crate::file::Header::LEN) as u32; - let chunk_slice_count = if verifiable_slice_count > 0 { - verifiable_slice_count / 8 - } else { - 0 - }; - let input_len = plaintext.len() as u32; - let info = EncodeInfo { - input_len, - output_len: body_len, - bytes_compressed, - compression_factor: bytes_compressed as f32 / input_len.max(1) as f32, - bytes_encrypted, - bytes_ecc, - bytes_verifiable: body_len, - // Match Rust stream: empty input → 0.0 (not 1.0). - amplification_factor: body_len as f32 / input_len.max(1) as f32, - padding_len, - chunk_len, - verifiable_slice_count, - chunk_slice_count, - }; - Ok((archive, info)) - } - - /// Headered decode via Lean AOT → (Header, plaintext). - pub fn decode_headered( - master: &[u8], - archive: &[u8], - ) -> Result<(crate::file::Header, Vec), CarbonadoError> { - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_decode_headered( - master.as_ptr(), - master.len(), - archive.as_ptr(), - archive.len(), - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - let plaintext = take_buf(out, out_len)?; - // Reconstruct Header from archive prefix (already MAC-verified inside Lean). - if archive.len() < crate::file::Header::LEN { - return Err(CarbonadoError::InvalidHeaderLength); - } - let header = crate::file::Header::try_from(&archive[..crate::file::Header::LEN])?; - Ok((header, plaintext)) - } - - /// Outboard encode via Lean AOT. - /// - /// `header_path`: when true (and encrypted), bare main is `[tag|ct]` with nonce - /// out-of-band (matches `file::encode_outboard`). When false, embedded `[nonce|tag|ct]` - /// (matches low-level `encoding::encode_outboard`). - pub fn encode_outboard( - master: &[u8], - plaintext: &[u8], - format: u8, - nonce: Option<&[u8; 16]>, - header_path: bool, - ) -> Result { - let (nonce_ptr, nonce_len) = match nonce { - Some(n) => (n.as_ptr(), 16usize), - None => (std::ptr::null(), 0usize), - }; - let mut main_out: *mut u8 = std::ptr::null_mut(); - let mut main_len: usize = 0; - let mut ob_out: *mut u8 = std::ptr::null_mut(); - let mut ob_len: usize = 0; - let mut par_out: *mut u8 = std::ptr::null_mut(); - let mut par_len: usize = 0; - let mut hash_out = [0u8; 32]; - let mut padding_len = 0u32; - let mut chunk_len = 0u32; - let mut bytes_compressed = 0u32; - let mut bytes_encrypted = 0u32; - let rc = unsafe { - sys::carbonado_encode_outboard( - master.as_ptr(), - master.len(), - plaintext.as_ptr(), - plaintext.len(), - format, - nonce_ptr, - nonce_len, - u8::from(header_path), - &mut main_out, - &mut main_len, - &mut ob_out, - &mut ob_len, - &mut par_out, - &mut par_len, - hash_out.as_mut_ptr(), - &mut padding_len, - &mut chunk_len, - &mut bytes_compressed, - &mut bytes_encrypted, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - // Take all three buffers with cleanup on any failure (no leaked mallocs). - let main = match take_buf(main_out, main_len) { - Ok(v) => v, - Err(e) => { - free_raw(ob_out); - free_raw(par_out); - return Err(e); - } - }; - let ob_bytes = match take_buf(ob_out, ob_len) { - Ok(v) => v, - Err(e) => { - free_raw(par_out); - return Err(e); - } - }; - let par_bytes = take_buf(par_out, par_len)?; - let fmt = crate::constants::Format::from(format); - // Match Rust stream_encode_outboard_buffer: Verification ⇒ Some(ob) even when - // empty (valid single-leaf post-order outboard); Fec ⇒ Some(parity) similarly. - let verification_outboard = if fmt.contains(crate::constants::Format::Verification) { - Some(ob_bytes) - } else { - None - }; - let fec_parity = if fmt.contains(crate::constants::Format::Fec) { - Some(par_bytes) - } else { - None - }; - let hash = crate::utils::decode_bao_hash(&hash_out)?; - // Match Rust: outboard bytes_ecc is the FEC parity sidecar length, not main.len(). - let bytes_ecc = fec_parity.as_ref().map(|p| p.len() as u32).unwrap_or(0); - let verifiable_slice_count = if fmt.contains(crate::constants::Format::Fec) && chunk_len > 0 - { - // 8 shards × chunk_len / SLICE_LEN for inboard-equivalent bookkeeping. - (chunk_len * 8) / crate::constants::SLICE_LEN - } else { - 0 - }; - let input_len = plaintext.len() as u32; - let info = EncodeInfo { - input_len, - output_len: main.len() as u32, - bytes_compressed, - compression_factor: bytes_compressed as f32 / input_len.max(1) as f32, - bytes_encrypted, - bytes_ecc, - bytes_verifiable: main.len() as u32, - // Match Rust stream: empty input → 0.0 (not 1.0). - amplification_factor: main.len() as f32 / input_len.max(1) as f32, - padding_len, - chunk_len, - verifiable_slice_count, - chunk_slice_count: if verifiable_slice_count > 0 { - verifiable_slice_count / 8 - } else { - 0 - }, - }; - Ok(OutboardEncoded { - main, - verification_outboard, - fec_parity, - hash, - info, - }) - } - - /// Outboard decode via Lean AOT. - /// - /// `header_path` / `nonce` must match encode-time layout (see [`encode_outboard`]). - #[allow(clippy::too_many_arguments)] - pub fn decode_outboard( - master: &[u8], - hash: &[u8], - main: &[u8], - verification_outboard: Option<&[u8]>, - fec_parity: Option<&[u8]>, - padding: u32, - format: u8, - nonce: Option<&[u8; 16]>, - header_path: bool, - ) -> Result, CarbonadoError> { - // Mirror Rust stream_decode_outboard: `None` (missing sidecar) is distinct from - // `Some(&[])` (empty outboard for single-leaf trees). Fail closed before C. - let fmt = crate::constants::Format::from(format); - if fmt.contains(crate::constants::Format::Verification) && verification_outboard.is_none() { - return Err(CarbonadoError::MissingVerificationOutboard); - } - if fmt.contains(crate::constants::Format::Fec) && fec_parity.is_none() { - return Err(CarbonadoError::MissingFecParity); - } - let ob = verification_outboard.unwrap_or(&[]); - let par = fec_parity.unwrap_or(&[]); - let (nonce_ptr, nonce_len) = match nonce { - Some(n) => (n.as_ptr(), 16usize), - None => (std::ptr::null(), 0usize), - }; - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_decode_outboard( - master.as_ptr(), - master.len(), - hash.as_ptr(), - hash.len(), - main.as_ptr(), - main.len(), - ob.as_ptr(), - ob.len(), - par.as_ptr(), - par.len(), - padding, - format, - u8::from(header_path), - nonce_ptr, - nonce_len, - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - take_buf(out, out_len) - } - - /// Inboard scrub via Lean AOT. - pub fn scrub( - body: &[u8], - hash: &[u8], - padding: u32, - format: u8, - ) -> Result, CarbonadoError> { - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_scrub( - body.as_ptr(), - body.len(), - hash.as_ptr(), - hash.len(), - padding, - format, - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - take_buf(out, out_len) - } - - /// Outboard scrub via Lean AOT. - pub fn scrub_outboard( - main: &[u8], - verification_outboard: Option<&[u8]>, - fec_parity: Option<&[u8]>, - hash: &[u8], - padding: u32, - chunk_len: u32, - format: u8, - ) -> Result, CarbonadoError> { - let fmt = crate::constants::Format::from(format); - if !fmt.contains(crate::constants::Format::Verification) { - return Err(CarbonadoError::ScrubRequiresVerification); - } - let Some(ob) = verification_outboard else { - return Err(CarbonadoError::MissingVerificationOutboard); - }; - // Fec + None: do not fail closed here. Pristine path can still return - // UnnecessaryScrub without parity when verify ok; recovery needs parity. - // Call Lean with empty parity (`unwrap_or`); if FEC err on recovery, - // map MissingFecParity when parity was None (post-C remap below). - let par = fec_parity.unwrap_or(&[]); - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_scrub_outboard( - main.as_ptr(), - main.len(), - ob.as_ptr(), - ob.len(), - par.as_ptr(), - par.len(), - hash.as_ptr(), - hash.len(), - padding, - chunk_len, - format, - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - // Distinct missing-sidecar modes before collapsing to scrub/FEC. - if rc == sys::CARBONADO_ERR_FEC - && fmt.contains(crate::constants::Format::Fec) - && fec_parity.is_none() - { - return Err(CarbonadoError::MissingFecParity); - } - return Err(map_err(rc)); - } - take_buf(out, out_len) - } - - /// Inboard verify_slice via Lean AOT (W4a: O(slice) retained output in Lean). - /// - /// Geometry pre-checks mirror pure-Rust [`crate::stream::slice::verify_slice_inboard_seekable`] - /// order so dual-suite diagnostics keep `InvalidHeaderLength` / `HashDecodeError` / - /// `InvalidSliceIndex {..}` fields. Bao auth failures map via ABI - /// `CARBONADO_ERR_AUTHENTICATION` (R4 / docs/ABI.md). - /// - /// **Memory honesty:** Lean retains O(slice) after auth walk; this dispatcher still - /// passes the full `body` buffer to C (caller-owned input). `count == 0` returns - /// empty here without calling C (matches pure-Rust short-circuit). - pub fn verify_slice( - body: &[u8], - index: u32, - count: u32, - hash: &[u8], - format: u8, - ) -> Result, CarbonadoError> { - // Match pure-Rust order (stream/slice.rs::verify_slice_inboard_seekable): - // 1. count==0 → Ok([]) - // 2. content_len prefix (InvalidHeaderLength if < 8) - // 3. content_len==0 → InvalidSliceIndex - // 4. decode_bao_hash (HashDecodeError if len != 32) - // 5. OOB slice_byte_range → InvalidSliceIndex - // 6. C verify (auth / truncation / …) - if count == 0 { - return Ok(vec![]); - } - // Short inboard prefix (<8 B) → InvalidHeaderLength (Rust `inboard_bao_content_len_prefix`). - // Also enforced in Lean (`invalidPrefix` → ERR_INVALID_HEADER after R4 ofPipelineError). - if body.len() < 8 { - return Err(CarbonadoError::InvalidHeaderLength); - } - let content_len = u64::from_le_bytes( - body[0..8] - .try_into() - .map_err(|_| CarbonadoError::InvalidHeaderLength)?, - ); - // Empty-content slice index — structured fields the C ABI cannot carry. - if content_len == 0 { - return Err(CarbonadoError::InvalidSliceIndex { index, content_len }); - } - // Hash length before OOB (pure-Rust order: bad-hash+OOB → HashDecodeError first). - let _root = crate::utils::decode_bao_hash(hash)?; - // OOB slice index — need structured fields the C ABI cannot carry. - let slice_byte_start = u64::from(index) * u64::from(crate::constants::SLICE_LEN); - if slice_byte_start >= content_len { - return Err(CarbonadoError::InvalidSliceIndex { index, content_len }); - } - - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_verify_slice( - body.as_ptr(), - body.len(), - hash.as_ptr(), - hash.len(), - index, - count, - format, - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - take_buf(out, out_len) - } - - /// Seekable outboard verify_slice via Lean AOT (R9; W4b permanent full-buffer C). - /// - /// Geometry pre-checks mirror pure-Rust - /// [`crate::stream::slice::verify_slice_outboard`] for dual-suite diagnostics. - /// C ABI takes full main + outboard buffers (no ReadAt callback); Lean walks - /// O(slice + height) over the requested range. - pub fn verify_slice_outboard( - data: &[u8], - outboard_bytes: &[u8], - data_len: u64, - index: u32, - count: u32, - hash: &[u8], - format: u8, - ) -> Result, CarbonadoError> { - if count == 0 { - return Ok(vec![]); - } - if data_len == 0 { - return Err(CarbonadoError::InvalidSliceIndex { - index, - content_len: data_len, - }); - } - if data_len as usize != data.len() { - return Err(CarbonadoError::OutboardVerificationFailed(format!( - "data_len {data_len} != data buffer {}", - data.len() - ))); - } - let _root = crate::utils::decode_bao_hash(hash)?; - let slice_byte_start = u64::from(index) * u64::from(crate::constants::SLICE_LEN); - if slice_byte_start >= data_len { - return Err(CarbonadoError::InvalidSliceIndex { - index, - content_len: data_len, - }); - } - - let mut out: *mut u8 = std::ptr::null_mut(); - let mut out_len: usize = 0; - let rc = unsafe { - sys::carbonado_verify_slice_outboard( - data.as_ptr(), - data.len(), - outboard_bytes.as_ptr(), - outboard_bytes.len(), - hash.as_ptr(), - hash.len(), - index, - count, - format, - &mut out, - &mut out_len, - ) - }; - if rc != sys::CARBONADO_OK { - return Err(map_err(rc)); - } - take_buf(out, out_len) - } -} diff --git a/src/bin/carbonado/main.rs b/src/bin/carbonado/main.rs index 32c652e..2148591 100644 --- a/src/bin/carbonado/main.rs +++ b/src/bin/carbonado/main.rs @@ -19,12 +19,12 @@ use carbonado::cli_app::{Cli, Commands, KeyCommands}; use carbonado::constants::Format; use carbonado::file::{ - decode_directory, decode_stream, encode_directory_with_options, encode_stream, - DirectoryEncodeOptions, DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, + DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, DirectoryEncodeOptions, decode_directory, decode_stream, + encode_directory_with_options, encode_stream, }; use carbonado::paths::{ - detect_archive_layout, guess_format_from_filename, parse_bao_root_from_filename, - sidecar_sibling_path, ArchiveLayout, + ArchiveLayout, detect_archive_layout, guess_format_from_filename, parse_bao_root_from_filename, + sidecar_sibling_path, }; use carbonado::stream::decode::stream_decode_outboard; use carbonado::stream::encode::stream_encode_outboard; diff --git a/src/constants.rs b/src/constants.rs index 335b917..dfdbfb5 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -11,13 +11,31 @@ pub const MAGICNO: &[u8; 12] = b"CARBONADO20\n"; pub const SLICE_LEN: u32 = 4096; /// Default Bao tree block size for 4KB chunk groups (aligns with SSD/HDD sectors, -/// reduces tree overhead, improves max segment size). Uses the local keyed bao-tree fork. +/// reduces tree overhead, improves max segment size). Uses n0-computer/bao-tree keyed hashing. pub const BAO_BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(2); /// FEC data shards (k) pub const FEC_K: usize = 4; /// FEC total shards (m) pub const FEC_M: usize = 8; +/// Normative zstd compression level (AGENTS / Lean `Carbonado.Compress.zstdLevel`). +pub const ZSTD_LEVEL: i32 = 20; + +/// Zstandard frame magic, little-endian `0xFD2FB528` +/// (Lean `zstdMagic`; `ref/zstd/doc/zstd_compression_format.md`). +pub const ZSTD_MAGIC: [u8; 4] = [0x28, 0xb5, 0x2f, 0xfd]; + +/// Product frames do not set `Content_Checksum_flag` (Lean `zstdContentChecksum`). +pub const ZSTD_CONTENT_CHECKSUM: bool = false; + +/// Product frames do not emit a dictionary ID (Lean `zstdDictionaryIdFlag`). +pub const ZSTD_DICTIONARY_ID_FLAG: u8 = 0; + +/// Level-20 `windowLog` from `ref/zstd` `ZSTD_defaultCParameters[0][20]` +/// (srcSize > 256 KiB, and streaming `copy_encode` with unknown size). +/// Lean `zstdLevel20WindowLogLarge`. +pub const ZSTD_LEVEL20_WINDOW_LOG_LARGE: u32 = 25; + /// ## Bitmask for Carbonado formats c0-c15 /// /// | Format | Encryption | Compression | Verifiability | Error correction | Use-cases | diff --git a/src/crypto.rs b/src/crypto.rs index 80af737..26fa14b 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -81,13 +81,13 @@ use std::path::Path; -use aes::cipher::{KeyIvInit, StreamCipher}; use aes::Aes256; +use aes::cipher::{KeyIvInit, StreamCipher}; use blake3; -use chacha20poly1305::{aead::Aead, ChaCha20Poly1305, Key, Nonce}; +use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, aead::Aead}; use ctr::Ctr128BE; use hmac::{Hmac, Mac}; -use secp256k1::{ecdh::SharedSecret, Secp256k1}; +use secp256k1::{Secp256k1, ecdh::SharedSecret}; use sha2::Sha512; use crate::error::CarbonadoError; @@ -184,13 +184,12 @@ pub fn derive_subkey(master: &[u8], label: &str) -> Result<[u8; 64], CarbonadoEr /// See AGENTS.md §2.1.5 (Keyed Bao KDF) and `tests/bao_keyed_contract.rs`. pub fn carbonado_verification_key(format: u8) -> [u8; 32] { // Public KDF (not secret-key material). Always pure blake3 so the API is - // infallible on both backends. Lean AOT parity is checked via - // `backend::lean::verification_key` in `tests/lean_backend_smoke.rs`. + // infallible. Lean AOT demo checks the same domain string in Carbonado/Bao. blake3::derive_key("carbonado-v2/verification", &[format]) } /// Deprecated: use [`carbonado_verification_key`]. -#[deprecated(since = "2.1.0", note = "renamed to carbonado_verification_key")] +#[deprecated(since = "0.7.0", note = "renamed to carbonado_verification_key")] pub fn carbonado_bao_key(format: u8) -> [u8; 32] { carbonado_verification_key(format) } diff --git a/src/decoding.rs b/src/decoding.rs index 48cf3b5..0fb2d68 100644 --- a/src/decoding.rs +++ b/src/decoding.rs @@ -3,7 +3,6 @@ use std::io::Cursor; use log::trace; pub use crate::stream::compress::decompress_buffer as decompress; -#[cfg(feature = "backend-rust")] pub use crate::stream::decode::{stream_decode_buffer, stream_decode_outboard_buffer}; use crate::{ @@ -12,17 +11,15 @@ use crate::{ structs::EncodeInfo, }; -use reed_solomon_erasure::galois_8::Field; use reed_solomon_erasure::ReedSolomon; +use reed_solomon_erasure::galois_8::Field; -#[cfg(feature = "backend-rust")] use crate::{ constants::{Format, SLICE_LEN}, encoding, stream::{extract_slice_inboard_for_scrub, verify_slice_inboard_seekable}, utils::decode_bao_hash, }; -#[cfg(feature = "backend-rust")] use log::{debug, info, warn}; fn fec_chunks(chunked_bytes: &[(usize, &[u8])], padding: u32) -> Result, CarbonadoError> { @@ -55,7 +52,7 @@ fn fec_chunks(chunked_bytes: &[(usize, &[u8])], padding: u32) -> Result, let mut decoded = vec![]; for sh in shards.iter().take(data_shards) { - if let Some(ref s) = sh { + if let Some(s) = sh { decoded.extend_from_slice(s); } else { decoded.resize(decoded.len() + shard_size, 0); @@ -72,7 +69,6 @@ fn fec_chunks(chunked_bytes: &[(usize, &[u8])], padding: u32) -> Result, Ok(decoded) } -#[cfg_attr(feature = "backend-lean", allow(dead_code))] // used by rust scrub_outboard path pub fn verification_with_outboard( bare: &[u8], outboard: &[u8], @@ -144,7 +140,7 @@ pub fn fec_with_parity( let mut decoded = vec![]; for sh in shards.iter().take(FEC_K) { - if let Some(ref s) = sh { + if let Some(s) = sh { decoded.extend_from_slice(s); } else { decoded.resize(decoded.len() + shard_len, 0); @@ -207,14 +203,7 @@ pub fn decode( padding: u32, format: u8, ) -> Result, CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - crate::backend::lean::decode(master_key, hash, input, padding, format) - } - #[cfg(feature = "backend-rust")] - { - stream_decode_buffer(master_key, hash, input, padding, format) - } + stream_decode_buffer(master_key, hash, input, padding, format) } pub fn decode_outboard( @@ -226,34 +215,16 @@ pub fn decode_outboard( padding: u32, format: u8, ) -> Result, CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - // Low-level path: embedded-nonce when encrypted (header_path = false). - crate::backend::lean::decode_outboard( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - None, - false, - ) - } - #[cfg(feature = "backend-rust")] - { - stream_decode_outboard_buffer( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - None, - ) - } + stream_decode_outboard_buffer( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + None, + ) } pub fn extract_slice( @@ -273,14 +244,7 @@ pub fn verify_slice( format: u8, ) -> Result, CarbonadoError> { trace!("verify_slice seekable index={index} count={count} format=0x{format:02x}"); - #[cfg(feature = "backend-lean")] - { - crate::backend::lean::verify_slice(input, index, count, hash, format) - } - #[cfg(feature = "backend-rust")] - { - verify_slice_inboard_seekable(input, index, count, hash, format) - } + verify_slice_inboard_seekable(input, index, count, hash, format) } /// Recover a damaged inboard Bao+FEC archive via RS subset search and re-encode oracle. @@ -289,31 +253,11 @@ pub fn verify_slice( /// `InvalidHeaderLength`, `BaoResponseTruncated`, `StdIoError`, etc.) route into combinatorial /// FEC recovery — the API does not distinguish tamper from truncation before attempting recovery. /// Pristine archives return [`CarbonadoError::UnnecessaryScrub`]. -/// -/// Under `backend-lean`, uses C ABI `carbonado_scrub` (geometry peel + RS search). pub fn scrub( input: &[u8], hash: &[u8], encode_info: &EncodeInfo, format: u8, -) -> Result, CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - let _ = encode_info; // padding is the normative scrub input; chunk geometry from body - crate::backend::lean::scrub(input, hash, encode_info.padding_len, format) - } - #[cfg(feature = "backend-rust")] - { - scrub_rust(input, hash, encode_info, format) - } -} - -#[cfg(feature = "backend-rust")] -fn scrub_rust( - input: &[u8], - hash: &[u8], - encode_info: &EncodeInfo, - format: u8, ) -> Result, CarbonadoError> { let fmt = Format::from(format); if !fmt.contains(Format::Verification) { @@ -366,11 +310,11 @@ fn scrub_rust( } if let Ok((verif, got_h)) = encoding::verification_inboard_buffer(&scrubbed, format) + && got_h == hash + && verif.len() == input.len() { - if got_h == hash && verif.len() == input.len() { - recovered = Some(verif); - break; - } + recovered = Some(verif); + break; } } } @@ -390,40 +334,6 @@ pub fn scrub_outboard( encode_info: &EncodeInfo, format: u8, hash: &[u8], -) -> Result, CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - crate::backend::lean::scrub_outboard( - bare, - verification_outboard, - fec_parity, - hash, - encode_info.padding_len, - encode_info.chunk_len, - format, - ) - } - #[cfg(feature = "backend-rust")] - { - scrub_outboard_rust( - bare, - verification_outboard, - fec_parity, - encode_info, - format, - hash, - ) - } -} - -#[cfg(feature = "backend-rust")] -fn scrub_outboard_rust( - bare: &[u8], - verification_outboard: Option<&[u8]>, - fec_parity: Option<&[u8]>, - encode_info: &EncodeInfo, - format: u8, - hash: &[u8], ) -> Result, CarbonadoError> { let fmt = Format::from(format); if !fmt.contains(Format::Verification) { @@ -501,11 +411,11 @@ fn scrub_outboard_rust( sel.push((c.0, &c.1)); } } - if let Ok(cand_inner) = fec_chunks(&sel, padding) { - if verification_with_outboard(&cand_inner, ob, hash, format).is_ok() { - recovered = Some(cand_inner); - break; - } + if let Ok(cand_inner) = fec_chunks(&sel, padding) + && verification_with_outboard(&cand_inner, ob, hash, format).is_ok() + { + recovered = Some(cand_inner); + break; } } diff --git a/src/directory/format_policy.rs b/src/directory/format_policy.rs index 2fa691f..17997db 100644 --- a/src/directory/format_policy.rs +++ b/src/directory/format_policy.rs @@ -41,16 +41,16 @@ pub enum SegmentFormatPolicy { /// Force encrypted c15. ForceC15, /// Deprecated: use [`SegmentFormatPolicy::ForceC12`]. - #[deprecated(since = "2.1.0", note = "directory segments are c12–c15; use ForceC12")] + #[deprecated(since = "0.7.0", note = "directory segments are c12–c15; use ForceC12")] ForceC4, /// Deprecated: use [`SegmentFormatPolicy::ForceC14`]. - #[deprecated(since = "2.1.0", note = "directory segments are c12–c15; use ForceC14")] + #[deprecated(since = "0.7.0", note = "directory segments are c12–c15; use ForceC14")] ForceC6, /// Deprecated: use [`SegmentFormatPolicy::ForceC13`]. - #[deprecated(since = "2.1.0", note = "directory segments are c12–c15; use ForceC13")] + #[deprecated(since = "0.7.0", note = "directory segments are c12–c15; use ForceC13")] ForceC5, /// Deprecated: use [`SegmentFormatPolicy::ForceC15`]. - #[deprecated(since = "2.1.0", note = "directory segments are c12–c15; use ForceC15")] + #[deprecated(since = "0.7.0", note = "directory segments are c12–c15; use ForceC15")] ForceC7, } diff --git a/src/directory/mod.rs b/src/directory/mod.rs index f928add..564ad3a 100644 --- a/src/directory/mod.rs +++ b/src/directory/mod.rs @@ -3,6 +3,7 @@ pub mod format_policy; pub use format_policy::{ - resolve_catalog_format, SegmentFormatPolicy, SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, - SEGMENT_FORMAT_ENCRYPTED_RAW, SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, + SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, SEGMENT_FORMAT_ENCRYPTED_RAW, + SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, SegmentFormatPolicy, + resolve_catalog_format, }; diff --git a/src/encoding.rs b/src/encoding.rs index e2c13d3..31e987f 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -3,13 +3,9 @@ use log::trace; use crate::{error::CarbonadoError, structs::Encoded}; use crate::stream::encode::stream_encode_buffer_with_nonce; -#[cfg(feature = "backend-rust")] use crate::stream::encode::stream_encode_outboard_buffer; -/// Encode data into Carbonado format (delegates to streaming pipeline, or Lean AOT). -/// -/// Under `backend-lean`, uses C ABI `carbonado_encode`. See [`crate::backend::lean`] -/// for EncodeInfo stage counters (R3: compress/encrypt + FEC/Bao geometry). +/// Encode data into Carbonado format (delegates to the streaming pipeline). /// /// Encrypted formats use a CSPRNG nonce (embedded layout). For deterministic encrypted /// bodies (G9 fixtures), use [`encode_with_nonce`]. @@ -20,7 +16,7 @@ pub fn encode(master_key: &[u8], input: &[u8], format: u8) -> Result Result { trace!("encode_outboard format=0x{format:02x}"); - #[cfg(feature = "backend-lean")] - { - // Low-level path: embedded-nonce when encrypted (header_path = false). - let nonce = if format & 1 != 0 { - let mut n = [0u8; 16]; - getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; - Some(n) - } else { - None - }; - crate::backend::lean::encode_outboard(master_key, input, format, nonce.as_ref(), false) - } - #[cfg(feature = "backend-rust")] - { - stream_encode_outboard_buffer(master_key, input, format, None) - } + stream_encode_outboard_buffer(master_key, input, format, None) } -// Scrub recovery re-exports (Rust scrub path; lean scrub is in-engine) -#[cfg(feature = "backend-rust")] +// Scrub recovery re-exports pub use crate::stream::bao::verification_inboard_buffer; -#[cfg(feature = "backend-rust")] pub use crate::stream::fec::encode_inboard_buffer; diff --git a/src/error.rs b/src/error.rs index df00232..9c58e71 100644 --- a/src/error.rs +++ b/src/error.rs @@ -59,11 +59,15 @@ pub enum CarbonadoError { InvalidScrubbedHash, /// FEC padding should be zero when encoding (Carbonado adds its own) - #[error("Padding from FEC should always be zero, since Carbonado adds its own padding. Padding was {0}.")] + #[error( + "Padding from FEC should always be zero, since Carbonado adds its own padding. Padding was {0}." + )] EncodeFecPaddingError(usize), /// Invalid chunk length - #[error("Chunk length should be as calculated. Calculated chunk length was {0}, but actual chunk length was {1}")] + #[error( + "Chunk length should be as calculated. Calculated chunk length was {0}, but actual chunk length was {1}" + )] EncodeInvalidChunkLength(u32, usize), /// Invalid verifiable slice length @@ -71,7 +75,9 @@ pub enum CarbonadoError { InvalidVerifiableSliceCount(u32), /// Invalid magic number - #[error("File header lacks Carbonado magic number and may not be a proper Carbonado file. Magic number found was {0}.")] + #[error( + "File header lacks Carbonado magic number and may not be a proper Carbonado file. Magic number found was {0}." + )] InvalidMagicNumber(String), /// Invalid header length calculation @@ -277,9 +283,7 @@ pub enum CarbonadoError { MissingShardIndex { expected: u32, found: u32 }, /// Caller-supplied `ShardSource.chunk_index` does not match the authenticated header value. - #[error( - "Shard index mismatch: caller claimed {claimed}, header authenticated {authenticated}" - )] + #[error("Shard index mismatch: caller claimed {claimed}, header authenticated {authenticated}")] ShardIndexMismatch { claimed: u32, authenticated: u32 }, } diff --git a/src/file.rs b/src/file.rs index f1b0d8a..ff4c6af 100644 --- a/src/file.rs +++ b/src/file.rs @@ -9,34 +9,35 @@ use bao::Hash; // nom imports removed — legacy parse_bytes / old header parsing was deleted as part of the v2 replacement. // (secp256k1 imports removed - clean break, legacy Header parsing deleted) -#[cfg(feature = "backend-rust")] use crate::stream::decode::stream_decrypt_header_path; use crate::{ adamantine::{ - decode_adamantine, encode_adamantine, AdamantineHeader, ADAMANTINE_CARBONADO_FMT_ENCRYPTED, - ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_HEADER_LEN, + ADAMANTINE_CARBONADO_FMT_ENCRYPTED, ADAMANTINE_CARBONADO_FMT_PUBLIC, + ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_HEADER_LEN, AdamantineHeader, decode_adamantine, + encode_adamantine, }, adamantine_payload::{ - build_adamantine_payload, split_adamantine_payload, verification_slice_from_bundle, - MAX_ADAMANTINE_PAYLOAD_LEN, MAX_BAO_BUNDLE_LEN, + MAX_ADAMANTINE_PAYLOAD_LEN, MAX_BAO_BUNDLE_LEN, build_adamantine_payload, + split_adamantine_payload, verification_slice_from_bundle, }, constants::{Format, MAGICNO}, decoding, - directory::{format_policy::resolve_catalog_format, SegmentFormatPolicy}, + directory::{SegmentFormatPolicy, format_policy::resolve_catalog_format}, encoding, error::CarbonadoError, filepack_manifest::{ - FilepackEntry, FilepackManifest, SegmentRef, FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED, - FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, MAX_SEGMENT_MAIN_LEN, + FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + FILEPACK_MANIFEST_VERSION, FilepackEntry, FilepackManifest, MAX_SEGMENT_MAIN_LEN, + SegmentRef, }, paths::parse_bao_root_from_filename, - stream::{encode::stream_encode_outboard, DEFAULT_SEGMENT_PLAINTEXT_BUDGET}, + stream::{DEFAULT_SEGMENT_PLAINTEXT_BUDGET, encode::stream_encode_outboard}, structs::{EncodeInfo, OutboardEncoded}, utils::{calc_padding_len, decode_bao_hash, encode_bao_hash}, }; #[cfg(feature = "ots")] -use crate::ots::{stamp_bao_root, verify_stamp, OtsPolicy}; +use crate::ots::{OtsPolicy, stamp_bao_root, verify_stamp}; /// Default format for public directory archives (c14: public compressed + bao + fec). pub const DIRECTORY_ARCHIVE_FORMAT: u8 = FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC; @@ -311,8 +312,6 @@ impl Header { /// Reads the 177-byte [`Header`], verifies integrity, then reverses the body pipeline /// (Bao/FEC → decrypt → decompress as format bits require). /// -/// # `backend-rust` -/// /// Bao/FEC reverse pipes into a [`crate::stream::spool::SeekableSpool`] via /// [`crate::stream::decode::stream_decode_inboard_bao_fec_into`], bounded by /// [`Header::encoded_len`]. Encrypted segments use [`crate::stream::stream_decrypt_header_path`] @@ -325,164 +324,103 @@ impl Header { /// - **(C) Encrypted:** streaming EtM via spool two-pass MAC verify then CTR decrypt. /// - **(D) c4/c8:** bounded by `encoded_len` when known; incremental FEC/decompress otherwise. /// -/// # `backend-lean` (W1a) -/// -/// Reads the 177-byte header, verifies `header_mac` **before** body I/O (same fail-closed -/// order as rust), then spools `encoded_len` body into an archive buffer and calls Lean -/// [`crate::backend::lean::decode_headered`] for dual body pipeline. Peak RAM -/// **O(header + body + plaintext)** — E1 honesty, not stream E2. (W1b public **non-compress** -/// outboard stream uses S4 O(chunk) composition; this headered path remains Lean E1.) pub fn decode_stream( master_key: &[u8], mut input: R, output: &mut W, ) -> Result<(Header, u64), CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; - - let mut header_bytes = [0u8; Header::LEN]; - input - .read_exact(&mut header_bytes) - .map_err(CarbonadoError::StdIoError)?; - let header_probe = Header::try_from(&header_bytes[..])?; - // MAC-before-body (parity with rust path): reject unauthenticated peers before - // allocating/reading up to MAX_SEGMENT_MAIN_LEN body bytes. Lean re-verifies MAC - // inside decode_headered for dual body/pipeline honesty. - let auth_data = build_header_auth_data(&header_probe); - let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; - if !crate::crypto::ct_eq(&expected_mac, &header_probe.header_mac) { - return Err(CarbonadoError::AuthenticationFailed); - } - let body_len = header_probe.encoded_len as u64; - if body_len > MAX_SEGMENT_MAIN_LEN { - return Err(CarbonadoError::InternalStateError(format!( - "header encoded_len {body_len} exceeds MAX_SEGMENT_MAIN_LEN {MAX_SEGMENT_MAIN_LEN}" - ))); - } - let mut body = vec![0u8; body_len as usize]; - if let Err(e) = input.read_exact(&mut body) { - // Match rust pipeline taxonomy: short body after a valid header prefix is - // `InvalidHeaderLength` (not bare UnexpectedEof), e.g. truncated Bao bodies. - return Err(if e.kind() == std::io::ErrorKind::UnexpectedEof { - CarbonadoError::InvalidHeaderLength - } else { - CarbonadoError::StdIoError(e) - }); - } - let mut archive = Vec::with_capacity(Header::LEN + body.len()); - archive.extend_from_slice(&header_bytes); - archive.extend_from_slice(&body); - drop(body); // free body copy before Lean allocates plaintext - let (header, plaintext) = crate::backend::lean::decode_headered(master_key, &archive)?; - drop(archive); - output - .write_all(&plaintext) - .map_err(CarbonadoError::StdIoError)?; - Ok((header, plaintext.len() as u64)) - } - #[cfg(feature = "backend-rust")] - { - let mut header_bytes = [0u8; Header::LEN]; - input - .read_exact(&mut header_bytes) - .map_err(CarbonadoError::StdIoError)?; - let header = Header::try_from(&header_bytes[..])?; - - let auth_data = build_header_auth_data(&header); - let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; - if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { - return Err(CarbonadoError::AuthenticationFailed); - } - - let fmt = header.format; - let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; - let mut body_reader = std::io::Read::by_ref(&mut input).take(header.encoded_len as u64); - crate::stream::decode::stream_decode_inboard_bao_fec_into( - &mut body_reader, - header.hash.as_bytes(), - header.padding_len, - fmt, - Some(header.encoded_len as u64), + let mut header_bytes = [0u8; Header::LEN]; + input + .read_exact(&mut header_bytes) + .map_err(CarbonadoError::StdIoError)?; + let header = Header::try_from(&header_bytes[..])?; + + let auth_data = build_header_auth_data(&header); + let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; + if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { + return Err(CarbonadoError::AuthenticationFailed); + } + + let fmt = header.format; + let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; + let mut body_reader = std::io::Read::by_ref(&mut input).take(header.encoded_len as u64); + crate::stream::decode::stream_decode_inboard_bao_fec_into( + &mut body_reader, + header.hash.as_bytes(), + header.padding_len, + fmt, + Some(header.encoded_len as u64), + &mut post_preprocess, + )?; + post_preprocess.rewind()?; + let out_len = if fmt.contains(Format::Encryption) { + stream_decrypt_header_path( + master_key, + header.payload_nonce, &mut post_preprocess, - )?; - post_preprocess.rewind()?; - let out_len = if fmt.contains(Format::Encryption) { - stream_decrypt_header_path( - master_key, - header.payload_nonce, - &mut post_preprocess, - fmt.bits(), - output, - )? - } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(post_preprocess, output)? - } else { - std::io::copy(&mut post_preprocess, output).map_err(CarbonadoError::StdIoError)? - }; + fmt.bits(), + output, + )? + } else if fmt.contains(Format::Compression) { + crate::stream::compress::stream_decompress(post_preprocess, output)? + } else { + std::io::copy(&mut post_preprocess, output).map_err(CarbonadoError::StdIoError)? + }; - Ok((header, out_len)) - } + Ok((header, out_len)) } pub fn decode(master_key: &[u8], encoded: &[u8]) -> Result<(Header, Vec), CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - crate::backend::lean::decode_headered(master_key, encoded) - } - #[cfg(feature = "backend-rust")] - { - if encoded.len() < Header::LEN { - return Err(CarbonadoError::InvalidHeaderLength); - } - let (header_bytes, body) = encoded.split_at(Header::LEN); - let header = Header::try_from(header_bytes)?; - - // Verify header_mac - let auth_data = build_header_auth_data(&header); - let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; - - // Constant-time comparison for the header MAC to avoid timing side-channels. - // (See AGENTS.md for the constant-time review of EtM + header auth paths.) - if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { - return Err(CarbonadoError::AuthenticationFailed); - } - - // Same fused spool pipeline as decode_stream (streaming MAC-then-decrypt on header path). - // Body may include trailers (e.g. catalog COTS) after `encoded_len` bytes — limit the reader. - let fmt = header.format; - let body_len = header.encoded_len as usize; - if body.len() < body_len { - return Err(CarbonadoError::InvalidHeaderLength); - } - let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; - crate::stream::decode::stream_decode_inboard_bao_fec_into( - std::io::Cursor::new(&body[..body_len]), - header.hash.as_bytes(), - header.padding_len, - fmt, - Some(header.encoded_len as u64), + if encoded.len() < Header::LEN { + return Err(CarbonadoError::InvalidHeaderLength); + } + let (header_bytes, body) = encoded.split_at(Header::LEN); + let header = Header::try_from(header_bytes)?; + + // Verify header_mac + let auth_data = build_header_auth_data(&header); + let expected_mac = crate::crypto::compute_header_mac(master_key, &auth_data)?; + + // Constant-time comparison for the header MAC to avoid timing side-channels. + // (See AGENTS.md for the constant-time review of EtM + header auth paths.) + if !crate::crypto::ct_eq(&expected_mac, &header.header_mac) { + return Err(CarbonadoError::AuthenticationFailed); + } + + // Same fused spool pipeline as decode_stream (streaming MAC-then-decrypt on header path). + // Body may include trailers (e.g. catalog COTS) after `encoded_len` bytes — limit the reader. + let fmt = header.format; + let body_len = header.encoded_len as usize; + if body.len() < body_len { + return Err(CarbonadoError::InvalidHeaderLength); + } + let mut post_preprocess = crate::stream::spool::SeekableSpool::new()?; + crate::stream::decode::stream_decode_inboard_bao_fec_into( + std::io::Cursor::new(&body[..body_len]), + header.hash.as_bytes(), + header.padding_len, + fmt, + Some(header.encoded_len as u64), + &mut post_preprocess, + )?; + post_preprocess.rewind()?; + let mut decompressed = Vec::new(); + if fmt.contains(Format::Encryption) { + crate::stream::stream_decrypt_header_path( + master_key, + header.payload_nonce, &mut post_preprocess, + fmt.bits(), + &mut decompressed, )?; - post_preprocess.rewind()?; - let mut decompressed = Vec::new(); - if fmt.contains(Format::Encryption) { - crate::stream::stream_decrypt_header_path( - master_key, - header.payload_nonce, - &mut post_preprocess, - fmt.bits(), - &mut decompressed, - )?; - } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(post_preprocess, &mut decompressed)?; - } else { - std::io::copy(&mut post_preprocess, &mut decompressed) - .map_err(CarbonadoError::StdIoError)?; - } - - Ok((header, decompressed)) + } else if fmt.contains(Format::Compression) { + crate::stream::compress::stream_decompress(post_preprocess, &mut decompressed)?; + } else { + std::io::copy(&mut post_preprocess, &mut decompressed) + .map_err(CarbonadoError::StdIoError)?; } + + Ok((header, decompressed)) } /// High-level encode using the new v2 symmetric model (always inboard with Header prepended). @@ -495,26 +433,6 @@ pub fn decode(master_key: &[u8], encoded: &[u8]) -> Result<(Header, Vec), Ca /// Sidecar naming convention: .cXX.out (Bao), .cXX.par (FEC parity). /// See AGENTS §11.2 (completed) and low-level `encoding::encode_outboard`. /// -/// # `backend-lean` (Phase 2) -/// -/// Dispatches to Lean AOT via C ABI (`carbonado_encode_headered`). -/// -/// - **`metadata` / SLH pk:** plumbed through C ABI (nullable → zero fields). -/// - **`EncodeInfo`:** full stage counters from Lean pack (R3) — compress/encrypt when -/// those bits ran, FEC/Bao geometry, padding, and `output_len`/`bytes_verifiable` -/// from body length (matches header `encoded_len`). -/// - **Live dual under lean:** body/headered encode-decode, outboard (header-path when -/// `file::encode_outboard` supplies `Some(payload_nonce)`), scrub, verify_slice, -/// stream buffer + **R5 E1** stream I/O (inboard/encrypted outboard spool→Lean), -/// **W1a** `decode_stream` → Lean `decode_headered`, **W1b** public outboard stream -/// S4 O(chunk/stripe) composition, seekable outboard slice C (**R9**), optional **R10** -/// `stream_decode_async` under lean+`async` (dual-aware via E1; freeze never requires `async`). -/// - **Post-G8 residuals (honest):** pure Lean chunked stream residual (W1b public outboard -/// E2 is rust geometric composition under lean; encrypted/inboard stream remain E1); -/// dual-suite catalog encode remains Rust rkyv composition SSOT (**W3** pure Lean rkyv -/// also available); ~~W4a~~ O(slice) inboard retain closed; **W4b** full-buffer C outboard -/// slice permanent; **W4c** buffer-only zstd under lean; **W4d** FEC/async spool permanent. -/// Dual-suite SLH may use Rust `bitcoinpqc` composition. pub fn encode( master_key: &[u8], input: &[u8], @@ -526,7 +444,7 @@ pub fn encode( /// Headered inboard encode with optional fixed `payload_nonce` for encrypted formats. /// -/// When `explicit_nonce` is `Some(n)` and Encryption is set, **both backends** use `n` +/// When `explicit_nonce` is `Some(n)` and Encryption is set, this uses `n` /// literally (including all-zero). When `None`, encrypted formats draw a CSPRNG nonce /// (production default). Public formats use a zero `payload_nonce` field regardless. /// @@ -543,43 +461,12 @@ pub fn encode_with_nonce( metadata: Option<[u8; 8]>, explicit_nonce: Option<[u8; 16]>, ) -> Result<(Vec, EncodeInfo), CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - let format = level; - let nonce = if format & 1 != 0 { - match explicit_nonce { - Some(n) => Some(n), - None => { - let mut n = [0u8; 16]; - getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; - Some(n) - } - } - } else { - None - }; - let (archive, info) = crate::backend::lean::encode_headered( - master_key, - input, - format, - nonce.as_ref(), - None, // slh_public_key: dual-suite sets via sidecar path / Header APIs - metadata.as_ref(), - )?; - if archive.len() < Header::LEN { - return Err(CarbonadoError::InvalidHeaderLength); - } - Ok((archive, info)) - } - #[cfg(feature = "backend-rust")] - { - let mut out = Vec::new(); - let (header, info) = - encode_stream_with_nonce(master_key, input, level, metadata, &mut out, explicit_nonce)?; - let mut body = header.try_to_vec()?; - body.extend_from_slice(&out); - Ok((body, info)) - } + let mut out = Vec::new(); + let (header, info) = + encode_stream_with_nonce(master_key, input, level, metadata, &mut out, explicit_nonce)?; + let mut body = header.try_to_vec()?; + body.extend_from_slice(&out); + Ok((body, info)) } /// Headered inboard encode over [`Read`] / [`Write`]. Header is returned for staging; body @@ -883,6 +770,9 @@ pub fn decode_outboard( /// Segment Bao outboard data is centralized in the Adamantine payload bundle (no per-segment /// `.out`/`.par` sidecars). The catalog is always inboard c14/c15 with a `CARBONADO20\n` header. /// +/// Source files are collected first, sorted by `rel_path`, then encoded so verification outboard +/// and FEC blobs are appended in that order. `read_dir` listing order does not change the catalog. +/// /// Uses public c14 by default; `master_key` must be zeroed for public catalogs. pub fn encode_directory( master_key: &[u8], @@ -930,11 +820,15 @@ pub fn encode_directory_with_options( written_segment_paths: &mut rollback.segment_paths, }; - if let Err(err) = collect_and_encode_files(dir, Path::new(""), &mut state) { + let mut collected = Vec::new(); + if let Err(err) = collect_source_files(dir, Path::new(""), &mut collected) { + rollback_directory_encode_artifacts(&rollback); + return Err(err); + } + if let Err(err) = encode_collected_files(collected, &mut state) { rollback_directory_encode_artifacts(&rollback); return Err(err); } - entries.sort_by(|a, b| a.rel_path.cmp(&b.rel_path)); match write_catalog_artifact( master_key, @@ -1113,13 +1007,14 @@ pub fn decode_directory( Ok(()) } -/// Debug-only hook to inject catalog assembly failure (integration tests in debug builds). +/// Debug-only hooks for directory-encode integration tests. #[cfg(debug_assertions)] pub mod directory_encode_test_hooks { use std::cell::Cell; thread_local! { static FAIL_NEXT_CATALOG_WRITE: Cell = const { Cell::new(false) }; + static REVERSE_READDIR: Cell = const { Cell::new(false) }; } /// Arm the next [`encode_directory_with_options`] call on this thread to fail at catalog assembly. @@ -1134,9 +1029,31 @@ pub mod directory_encode_test_hooks { armed }) } + + /// Run `f` with each directory's `read_dir` listing reversed (nested walks included). + /// + /// Used to prove catalog bytes do not depend on filesystem listing order. + pub fn with_reverse_readdir(f: F) -> R + where + F: FnOnce() -> R, + { + struct Reset; + impl Drop for Reset { + fn drop(&mut self) { + REVERSE_READDIR.with(|flag| flag.set(false)); + } + } + REVERSE_READDIR.with(|flag| flag.set(true)); + let _reset = Reset; + f() + } + + pub(crate) fn reverse_readdir() -> bool { + REVERSE_READDIR.with(|flag| flag.get()) + } } -/// Mutable state shared while walking a source tree during directory encode. +/// Mutable state while encoding collected source files into segments and the catalog bundle. struct DirectoryEncodeState<'a> { master_key: &'a [u8], outdir: &'a Path, @@ -1180,13 +1097,27 @@ impl BaoBundleBuilder { } } -fn collect_and_encode_files( +/// One source file discovered during the directory walk, before encode. +struct CollectedSourceFile { + path: PathBuf, + rel_path: String, +} + +/// Walk `base` and collect regular files. Does not encode or append to the bundle. +fn collect_source_files( base: &Path, rel: &Path, - state: &mut DirectoryEncodeState<'_>, + files: &mut Vec, ) -> Result<(), CarbonadoError> { - for item in fs::read_dir(base).map_err(CarbonadoError::StdIoError)? { - let item = item.map_err(CarbonadoError::StdIoError)?; + let mut children: Vec = fs::read_dir(base) + .map_err(CarbonadoError::StdIoError)? + .collect::>>() + .map_err(CarbonadoError::StdIoError)?; + #[cfg(debug_assertions)] + if directory_encode_test_hooks::reverse_readdir() { + children.reverse(); + } + for item in children { let name = item.file_name().to_string_lossy().to_string(); if name == "." || name == ".." || name.contains('/') || name.contains('\\') { continue; @@ -1206,44 +1137,11 @@ fn collect_and_encode_files( let rel_str = child_rel.to_string_lossy().replace('\\', "/"); FilepackManifest::validate_rel_path(&rel_str)?; if file_type.is_dir() { - collect_and_encode_files(&path, &child_rel, state)?; + collect_source_files(&path, &child_rel, files)?; } else if file_type.is_file() { - let data = read_file(&path)?; - let content_blake3 = *blake3::hash(&data).as_bytes(); - let segment_format = state - .options - .segment_format_policy - .resolve_segment_format(state.catalog_format & 1 != 0, &data)?; - let segments = encode_file_segments( - state.master_key, - &data, - segment_format, - state.outdir, - state.options, - state.bao_bundle, - state.written_segment_paths, - )?; - #[cfg(feature = "ots")] - let ots_proof = if state - .options - .ots_policy - .as_ref() - .is_some_and(|p| p.stamp_entries) - { - let primary_root = segments[0].segment_bao_root; - Some(stamp_bao_root(&primary_root)?) - } else { - None - }; - state.entries.push(FilepackEntry { + files.push(CollectedSourceFile { + path, rel_path: rel_str, - content_blake3, - segment_format, - segments, - #[cfg(feature = "ots")] - ots_proof, - #[cfg(not(feature = "ots"))] - ots_proof: None, }); } else { return Err(CarbonadoError::UnsupportedFileType( @@ -1254,6 +1152,54 @@ fn collect_and_encode_files( Ok(()) } +/// Sort collected files by `rel_path`, then encode and append outboard/FEC in that order. +fn encode_collected_files( + mut files: Vec, + state: &mut DirectoryEncodeState<'_>, +) -> Result<(), CarbonadoError> { + files.sort_by(|a, b| a.rel_path.cmp(&b.rel_path)); + for file in files { + let data = read_file(&file.path)?; + let content_blake3 = *blake3::hash(&data).as_bytes(); + let segment_format = state + .options + .segment_format_policy + .resolve_segment_format(state.catalog_format & 1 != 0, &data)?; + let segments = encode_file_segments( + state.master_key, + &data, + segment_format, + state.outdir, + state.options, + state.bao_bundle, + state.written_segment_paths, + )?; + #[cfg(feature = "ots")] + let ots_proof = if state + .options + .ots_policy + .as_ref() + .is_some_and(|p| p.stamp_entries) + { + let primary_root = segments[0].segment_bao_root; + Some(stamp_bao_root(&primary_root)?) + } else { + None + }; + state.entries.push(FilepackEntry { + rel_path: file.rel_path, + content_blake3, + segment_format, + segments, + #[cfg(feature = "ots")] + ots_proof, + #[cfg(not(feature = "ots"))] + ots_proof: None, + }); + } + Ok(()) +} + /// Encode and write segment(s) for one file, sharding when over budget. fn encode_file_segments( master_key: &[u8], @@ -1774,12 +1720,12 @@ fn reject_symlink_components_under(base: &Path, target: &Path) -> Result<(), Car for component in rel.components() { if let std::path::Component::Normal(name) = component { current.push(name); - if let Ok(meta) = fs::symlink_metadata(¤t) { - if meta.file_type().is_symlink() { - return Err(CarbonadoError::SymlinkNotAllowed( - current.display().to_string(), - )); - } + if let Ok(meta) = fs::symlink_metadata(¤t) + && meta.file_type().is_symlink() + { + return Err(CarbonadoError::SymlinkNotAllowed( + current.display().to_string(), + )); } } } diff --git a/src/filepack.rs b/src/filepack.rs index c1de2d3..4fb9f73 100644 --- a/src/filepack.rs +++ b/src/filepack.rs @@ -600,10 +600,12 @@ mod tests { let parsed = parse_filepack_cbor(&packed.manifest).expect("parse"); assert!(!parsed.is_empty()); for (rel, size) in parsed.iter().map(|e| (e.rel_path.as_str(), e.size)) { - assert!(packed - .files - .iter() - .any(|(p, data)| p == rel && data.len() as u64 == size)); + assert!( + packed + .files + .iter() + .any(|(p, data)| p == rel && data.len() as u64 == size) + ); } } diff --git a/src/filepack_manifest.rs b/src/filepack_manifest.rs index 8b62e4b..f96727d 100644 --- a/src/filepack_manifest.rs +++ b/src/filepack_manifest.rs @@ -29,7 +29,7 @@ use std::collections::BTreeMap; use rkyv::rancor::Error as RkyvError; use rkyv::{Archive, Deserialize, Serialize}; -use crate::constants::{Format, FEC_K, FEC_M}; +use crate::constants::{FEC_K, FEC_M, Format}; use crate::directory::format_policy::validate_segment_format_for_catalog; use crate::error::CarbonadoError; use crate::filepack::{self, FilepackCborEntry, Packed}; @@ -279,12 +279,12 @@ impl FilepackManifest { "segment count exceeds maximum {MAX_SEGMENTS_PER_ENTRY}" ))); } - if let Some(proof) = entry.ots_proof.as_ref() { - if proof.len() > MAX_OTS_PROOF_LEN { - return Err(CarbonadoError::InvalidFilepackManifest(format!( - "ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" - ))); - } + if let Some(proof) = entry.ots_proof.as_ref() + && proof.len() > MAX_OTS_PROOF_LEN + { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" + ))); } validate_segment_format_for_catalog(entry.segment_format, catalog_encrypted).map_err( |e| match e { @@ -390,12 +390,12 @@ impl FilepackManifest { ))); } let catalog_encrypted = self.format_level & 1 != 0; - if let Some(proof) = &self.catalog_ots_proof { - if proof.len() > MAX_OTS_PROOF_LEN { - return Err(CarbonadoError::InvalidFilepackManifest(format!( - "catalog_ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" - ))); - } + if let Some(proof) = &self.catalog_ots_proof + && proof.len() > MAX_OTS_PROOF_LEN + { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "catalog_ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" + ))); } if self.entries.len() > MAX_FILEPACK_MANIFEST_ENTRIES { return Err(CarbonadoError::InvalidFilepackManifest(format!( @@ -418,19 +418,19 @@ impl FilepackManifest { for seg in &entry.segments { validate_segment_bundle_semantics(seg_fmt, seg, &entry.rel_path)?; } - if let Some(proof) = &entry.ots_proof { - if proof.len() > MAX_OTS_PROOF_LEN { - return Err(CarbonadoError::InvalidFilepackManifest(format!( - "ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" - ))); - } + if let Some(proof) = &entry.ots_proof + && proof.len() > MAX_OTS_PROOF_LEN + { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" + ))); } - if let Some(p) = prev { - if entry.rel_path.as_str() <= p { - return Err(CarbonadoError::InvalidFilepackManifest( - "entries must be strictly sorted by rel_path".into(), - )); - } + if let Some(p) = prev + && entry.rel_path.as_str() <= p + { + return Err(CarbonadoError::InvalidFilepackManifest( + "entries must be strictly sorted by rel_path".into(), + )); } prev = Some(entry.rel_path.as_str()); } diff --git a/src/lib.rs b/src/lib.rs index e0c102b..4828d95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,9 +58,6 @@ //////////////////////////////////////////////////////////////////////////////// -/// Dual-backend dispatch (`backend-rust` / `backend-lean`). See docs/TEST_CONTRACT.md. -pub mod backend; - /// For details on Carbonado formats and their uses, see the [Carbonado Format bitmask constant](constants::Format). pub mod constants; /// Symmetric cryptographic primitives for the v2 design. @@ -97,13 +94,13 @@ pub mod filepack_manifest; pub mod ots; /// Deprecated: use [`filepack_manifest`] instead. -#[deprecated(since = "2.1.0", note = "renamed to filepack_manifest")] +#[deprecated(since = "0.7.0", note = "renamed to filepack_manifest")] #[allow(deprecated)] pub mod pack_index { pub use crate::filepack_manifest::*; pub use crate::{ - PackEntry, PackIndex, PackSegmentRef, MAX_PACK_ENTRIES, PACK_INDEX_FORMAT_LEVEL, - PACK_INDEX_FORMAT_LEVEL_ENCRYPTED, PACK_INDEX_FORMAT_LEVEL_PUBLIC, PACK_INDEX_VERSION, + MAX_PACK_ENTRIES, PACK_INDEX_FORMAT_LEVEL, PACK_INDEX_FORMAT_LEVEL_ENCRYPTED, + PACK_INDEX_FORMAT_LEVEL_PUBLIC, PACK_INDEX_VERSION, PackEntry, PackIndex, PackSegmentRef, }; } /// Clap schema for the `carbonado` binary (`cli` feature). @@ -134,14 +131,14 @@ pub use decoding::scrub_outboard; #[doc(hidden)] pub use decoding::verify_inboard_keyed_oracle; -pub use paths::{detect_archive_layout, ArchiveLayout}; +pub use paths::{ArchiveLayout, detect_archive_layout}; #[cfg(feature = "async")] pub use stream::stream_decode_async; pub use stream::{ - decode_shards_stream, encode_shard_stream, stream_decode, stream_decode_buffer, - stream_decode_outboard, stream_decode_outboard_buffer, stream_encode_buffer, - stream_encode_buffer_with_nonce, stream_encode_outboard_buffer, verify_slice_inboard_seekable, - verify_slice_outboard, ShardEncodeResult, ShardSource, DEFAULT_SEGMENT_PLAINTEXT_BUDGET, + DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, decode_shards_stream, + encode_shard_stream, stream_decode, stream_decode_buffer, stream_decode_outboard, + stream_decode_outboard_buffer, stream_encode_buffer, stream_encode_buffer_with_nonce, + stream_encode_outboard_buffer, verify_slice_inboard_seekable, verify_slice_outboard, }; pub use bao; @@ -149,71 +146,71 @@ pub use bao; pub use structs::OutboardEncoded; pub use filepack::{ - pack_directory, parse_filepack_cbor, FilepackCborEntry, Packed, MAX_FILEPACK_CBOR_MANIFEST_LEN, - MAX_FILEPACK_PACKAGE_DEPTH, + FilepackCborEntry, MAX_FILEPACK_CBOR_MANIFEST_LEN, MAX_FILEPACK_PACKAGE_DEPTH, Packed, + pack_directory, parse_filepack_cbor, }; pub use adamantine::{ - decode_adamantine, encode_adamantine, AdamantineHeader, ADAMANTINE_CARBONADO_FMT_ENCRYPTED, - ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_HEADER_LEN, - ADAMANTINE_MAGIC, + ADAMANTINE_CARBONADO_FMT_ENCRYPTED, ADAMANTINE_CARBONADO_FMT_PUBLIC, + ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_HEADER_LEN, ADAMANTINE_MAGIC, AdamantineHeader, + decode_adamantine, encode_adamantine, }; pub use adamantine_payload::{ - build_adamantine_payload, fec_slice_from_bundle, split_adamantine_payload, - verification_slice_from_bundle, MAX_ADAMANTINE_PAYLOAD_LEN, MAX_BAO_BUNDLE_LEN, + MAX_ADAMANTINE_PAYLOAD_LEN, MAX_BAO_BUNDLE_LEN, build_adamantine_payload, + fec_slice_from_bundle, split_adamantine_payload, verification_slice_from_bundle, }; pub use directory::SegmentFormatPolicy; pub use filepack_manifest::{ - expected_fec_parity_len, FilepackEntry, FilepackManifest, FilepackSegmentMap, SegmentRef, FILEPACK_MANIFEST_FORMAT_LEVEL, FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED, - FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, - MAX_FILEPACK_MANIFEST_ENTRIES, MAX_SEGMENT_MAIN_LEN, + FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, FilepackEntry, + FilepackManifest, FilepackSegmentMap, MAX_FILEPACK_MANIFEST_ENTRIES, MAX_SEGMENT_MAIN_LEN, + SegmentRef, expected_fec_parity_len, }; /// Deprecated: renamed to [`FilepackManifest`]. -#[deprecated(since = "2.1.0", note = "renamed to FilepackManifest")] +#[deprecated(since = "0.7.0", note = "renamed to FilepackManifest")] pub type PackIndex = FilepackManifest; /// Deprecated: renamed to [`FilepackEntry`]. -#[deprecated(since = "2.1.0", note = "renamed to FilepackEntry")] +#[deprecated(since = "0.7.0", note = "renamed to FilepackEntry")] pub type PackEntry = FilepackEntry; /// Deprecated: renamed to [`SegmentRef`]. -#[deprecated(since = "2.1.0", note = "renamed to SegmentRef")] +#[deprecated(since = "0.7.0", note = "renamed to SegmentRef")] pub type PackSegmentRef = SegmentRef; /// Deprecated: renamed to [`FILEPACK_MANIFEST_VERSION`]. -#[deprecated(since = "2.1.0", note = "renamed to FILEPACK_MANIFEST_VERSION")] +#[deprecated(since = "0.7.0", note = "renamed to FILEPACK_MANIFEST_VERSION")] pub const PACK_INDEX_VERSION: u32 = FILEPACK_MANIFEST_VERSION; /// Deprecated: renamed to [`FILEPACK_MANIFEST_FORMAT_LEVEL`]. -#[deprecated(since = "2.1.0", note = "renamed to FILEPACK_MANIFEST_FORMAT_LEVEL")] +#[deprecated(since = "0.7.0", note = "renamed to FILEPACK_MANIFEST_FORMAT_LEVEL")] pub const PACK_INDEX_FORMAT_LEVEL: u8 = FILEPACK_MANIFEST_FORMAT_LEVEL; /// Deprecated: renamed to [`FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC`]. #[deprecated( - since = "2.1.0", + since = "0.7.0", note = "renamed to FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC" )] pub const PACK_INDEX_FORMAT_LEVEL_PUBLIC: u8 = FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC; /// Deprecated: renamed to [`FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED`]. #[deprecated( - since = "2.1.0", + since = "0.7.0", note = "renamed to FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED" )] pub const PACK_INDEX_FORMAT_LEVEL_ENCRYPTED: u8 = FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED; /// Deprecated: renamed to [`MAX_FILEPACK_MANIFEST_ENTRIES`]. -#[deprecated(since = "2.1.0", note = "renamed to MAX_FILEPACK_MANIFEST_ENTRIES")] +#[deprecated(since = "0.7.0", note = "renamed to MAX_FILEPACK_MANIFEST_ENTRIES")] pub const MAX_PACK_ENTRIES: usize = MAX_FILEPACK_MANIFEST_ENTRIES; #[cfg(feature = "ots")] -pub use ots::{stamp_bao_root, verify_stamp, OtsPolicy, OtsVerification}; +pub use ots::{OtsPolicy, OtsVerification, stamp_bao_root, verify_stamp}; pub use file::{ - decode_directory, encode_directory, encode_directory_with_options, DirectoryArchive, - DirectoryEncodeOptions, DIRECTORY_ARCHIVE_FORMAT, DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, - DIRECTORY_TEST_SEGMENT_BUDGET, + DIRECTORY_ARCHIVE_FORMAT, DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, DIRECTORY_TEST_SEGMENT_BUDGET, + DirectoryArchive, DirectoryEncodeOptions, decode_directory, encode_directory, + encode_directory_with_options, }; diff --git a/src/paths.rs b/src/paths.rs index dca5dec..62bb51f 100644 --- a/src/paths.rs +++ b/src/paths.rs @@ -206,19 +206,21 @@ fn is_segment_main_name(name: &str) -> bool { if is_adam_catalog_name(name) || name.contains(".adam.c") { return false; } - if let Some((stem, ext)) = name.rsplit_once('.') { - if (ext == "out" || ext == "par") && is_decimal_sidecar_stem(stem) { - return false; - } + if let Some((stem, ext)) = name.rsplit_once('.') + && (ext == "out" || ext == "par") + && is_decimal_sidecar_stem(stem) + { + return false; } if strip_decimal_suffix(name).is_some() { return true; } - if let Some((_, ext)) = name.rsplit_once('.') { - if ext.len() == 3 && ext.starts_with('c') && ext[1..].chars().all(|c| c.is_ascii_hexdigit()) - { - return !name.ends_with(".out") && !name.ends_with(".par"); - } + if let Some((_, ext)) = name.rsplit_once('.') + && ext.len() == 3 + && ext.starts_with('c') + && ext[1..].chars().all(|c| c.is_ascii_hexdigit()) + { + return !name.ends_with(".out") && !name.ends_with(".par"); } false } diff --git a/src/stream/bao.rs b/src/stream/bao.rs index e36846d..a9eea80 100644 --- a/src/stream/bao.rs +++ b/src/stream/bao.rs @@ -5,13 +5,13 @@ use std::io::{Cursor, Read, Seek, SeekFrom, Write}; use bao::Hash; use bao_tree::{ + BaoTree, ChunkRanges, io::{ outboard::{EmptyOutboard, PostOrderMemOutboard, PostOrderOutboard}, sync::{ - keyed_decode_ranges, keyed_encode_ranges_validated, keyed_outboard_post_order, ReadAt, + ReadAt, keyed_decode_ranges, keyed_encode_ranges_validated, keyed_outboard_post_order, }, }, - BaoTree, ChunkRanges, }; use crate::{ diff --git a/src/stream/compress.rs b/src/stream/compress.rs index 778dc5b..a2dc3ab 100644 --- a/src/stream/compress.rs +++ b/src/stream/compress.rs @@ -2,9 +2,9 @@ use std::io::{Read, Write}; -use crate::{error::CarbonadoError, filepack_manifest::MAX_SEGMENT_MAIN_LEN}; - -const ZSTD_LEVEL: i32 = 20; +use crate::{ + constants::ZSTD_LEVEL, error::CarbonadoError, filepack_manifest::MAX_SEGMENT_MAIN_LEN, +}; struct CountWriter { inner: W, @@ -15,13 +15,13 @@ struct CountWriter { impl Write for CountWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { let next = self.count.saturating_add(buf.len() as u64); - if let Some(max) = self.max { - if next > max { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "decompressed output exceeds maximum allowed size", - )); - } + if let Some(max) = self.max + && next > max + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "decompressed output exceeds maximum allowed size", + )); } let n = self.inner.write(buf)?; self.count += n as u64; @@ -33,51 +33,21 @@ impl Write for CountWriter { } } -/// Stream-compress `input` into `output` at level 20. Returns compressed bytes written. -/// -/// Under `backend-lean`, materializes the input and uses the zstd **buffer** API -/// (`ZSTD_compress` / `zstd::bulk`) so frames match Lean AOT (`Carbonado.Compress`). -/// Streaming `copy_encode` frames differ byte-for-byte from the buffer API at the same -/// level — that mismatch broke stream-vs-buffer parity under dual-engine (R5). +/// Stream-compress `input` into `output` at [`crate::constants::ZSTD_LEVEL`] (20). +/// Returns compressed bytes written. /// -/// **W4c permanent residual:** buffer-only under lean (no dual-safe multi-chunk streaming -/// frames). Cross-engine compress re-encode remains non-bit-identical (**W2a**); do not -/// invent streaming-frame bit-match claims. Peak: **O(logical)** RAM for compress under -/// lean — public outboard formats with the Compression bit (c2/c6/c10/c14) are **not** -/// W1b E2 under lean; E2 MVP is **non-compress** public outboard (c0/c4/c8/c12). -/// See docs/LIMITS.md Stream E1/E2 matrix. +/// Frame flags match Lean `Carbonado.Compress` where they can: checksum off, no dictionary. +/// Streaming `copy_encode` leaves content size unknown (descriptor differs from Lean AOT +/// one-shot `ZSTD_compress` frames). That is an honest frame-shape difference, not a bug. pub fn stream_compress(mut input: R, output: W) -> Result { - #[cfg(feature = "backend-lean")] - { - let mut plaintext = Vec::new(); - input - .read_to_end(&mut plaintext) - .map_err(CarbonadoError::StdIoError)?; - let compressed = zstd::bulk::Compressor::new(ZSTD_LEVEL) - .map_err(|e| CarbonadoError::ZstdError(e.to_string()))? - .compress(&plaintext) - .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; - let mut counter = CountWriter { - inner: output, - count: 0, - max: None, - }; - counter - .write_all(&compressed) - .map_err(CarbonadoError::StdIoError)?; - Ok(counter.count) - } - #[cfg(feature = "backend-rust")] - { - let mut counter = CountWriter { - inner: output, - count: 0, - max: None, - }; - zstd::stream::copy_encode(&mut input, &mut counter, ZSTD_LEVEL) - .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; - Ok(counter.count) - } + let mut counter = CountWriter { + inner: output, + count: 0, + max: None, + }; + zstd::stream::copy_encode(&mut input, &mut counter, ZSTD_LEVEL) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + Ok(counter.count) } /// Stream-decompress `input` into `output`. Returns decompressed bytes written. diff --git a/src/stream/crypto_stream.rs b/src/stream/crypto_stream.rs index f3f5233..c02f05e 100644 --- a/src/stream/crypto_stream.rs +++ b/src/stream/crypto_stream.rs @@ -14,8 +14,8 @@ use std::io::{ErrorKind, Read, Seek, SeekFrom, Write}; -use aes::cipher::{KeyIvInit, StreamCipher}; use aes::Aes256; +use aes::cipher::{KeyIvInit, StreamCipher}; use ctr::Ctr128BE; use hmac::{Hmac, Mac}; use sha2::Sha512; @@ -194,10 +194,10 @@ fn decrypt_ct_stream( } let n = input.read(&mut buf[..cap]).map_err(map_read_err)?; if n == 0 { - if let Some(r) = remaining { - if r > 0 { - return Err(CarbonadoError::InvalidCiphertextLength); - } + if let Some(r) = remaining + && r > 0 + { + return Err(CarbonadoError::InvalidCiphertextLength); } break; } @@ -255,10 +255,10 @@ pub fn stream_decrypt_with_nonce_bounded( if n == 0 { break; } - if let Some(limit) = ct_len { - if total.saturating_add(n as u64) > limit { - return Err(excess_ct_error(limit)); - } + if let Some(limit) = ct_len + && total.saturating_add(n as u64) > limit + { + return Err(excess_ct_error(limit)); } mac.update(&buf[..n]); spool @@ -311,10 +311,10 @@ pub fn stream_decrypt_with_nonce_seek( if n == 0 { break; } - if let Some(limit) = ct_len { - if total.saturating_add(n as u64) > limit { - return Err(excess_ct_error(limit)); - } + if let Some(limit) = ct_len + && total.saturating_add(n as u64) > limit + { + return Err(excess_ct_error(limit)); } mac.update(&buf[..n]); total += n as u64; diff --git a/src/stream/decode.rs b/src/stream/decode.rs index 537260b..7a109c3 100644 --- a/src/stream/decode.rs +++ b/src/stream/decode.rs @@ -1,21 +1,19 @@ //! Carbonado streaming decode pipelines (inboard + outboard). -use std::io::{copy, Cursor, Read, Seek, SeekFrom, Write}; +use std::io::{Cursor, Read, Seek, SeekFrom, Write, copy}; use crate::{ - constants::{Format, FEC_M}, + constants::{FEC_M, Format}, error::CarbonadoError, stream::{ bao::{read_inboard_bao_content_len_prefix, stream_verification_inboard_decode_with_len}, crypto_stream::{stream_decrypt_seek, stream_decrypt_with_nonce_seek}, - fec::{stream_decode_inboard, FecInboardWriteAt}, + fec::{FecInboardWriteAt, stream_decode_inboard}, spool::{SeekWriteAt, SeekableSpool}, }, }; /// Primary inboard decode (buffer). Used by [`crate::decoding::decode`]. -/// -/// Under `backend-lean`, composes over Lean C ABI body decode. pub fn stream_decode_buffer( master_key: &[u8], hash: &[u8], @@ -23,29 +21,21 @@ pub fn stream_decode_buffer( padding: u32, format: u8, ) -> Result, CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - crate::backend::lean::decode(master_key, hash, input, padding, format) - } - #[cfg(feature = "backend-rust")] - { - let mut out = Vec::new(); - stream_decode_inboard_pipeline( - master_key, - hash, - Cursor::new(input), - padding, - format, - None, - &mut out, - )?; - Ok(out) - } + let mut out = Vec::new(); + stream_decode_inboard_pipeline( + master_key, + hash, + Cursor::new(input), + padding, + format, + None, + &mut out, + )?; + Ok(out) } /// Primary outboard decode (buffer). Used by [`crate::decoding::decode_outboard`]. /// -/// Under `backend-lean`, composes over Lean C ABI outboard decode. /// `explicit_nonce.is_some()` → header-path decrypt (`[tag|ct]`); else embedded-nonce. #[allow(clippy::too_many_arguments)] pub fn stream_decode_outboard_buffer( @@ -58,37 +48,19 @@ pub fn stream_decode_outboard_buffer( format: u8, explicit_nonce: Option<[u8; 16]>, ) -> Result, CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - let header_path = explicit_nonce.is_some(); - crate::backend::lean::decode_outboard( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - explicit_nonce.as_ref(), - header_path, - ) - } - #[cfg(feature = "backend-rust")] - { - let mut out = Vec::new(); - stream_decode_outboard( - master_key, - hash, - Cursor::new(main), - verification_outboard.map(Cursor::new), - fec_parity.map(Cursor::new), - padding, - format, - explicit_nonce, - &mut out, - )?; - Ok(out) - } + let mut out = Vec::new(); + stream_decode_outboard( + master_key, + hash, + Cursor::new(main), + verification_outboard.map(Cursor::new), + fec_parity.map(Cursor::new), + padding, + format, + explicit_nonce, + &mut out, + )?; + Ok(out) } /// Stream inboard decode from `input` to `output`. @@ -102,12 +74,7 @@ pub fn stream_decode_outboard_buffer( /// Pass `encoded_body_len` when the reader may contain trailing bytes after the encoded /// body (FEC c8, compressed c4). When `Some`, excess or truncated input is rejected. /// -/// Under `backend-lean` (R5 E1 / W1b disk-backed): spool body (O(chunk) ingest) → Lean -/// [`crate::backend::lean::decode`] → write plaintext. Peak RAM **O(encoded + logical)** at -/// the Lean buffer boundary (not stream E2). See docs/LIMITS.md E1/E2 matrix. -/// -/// **R10:** [`super::stream_decode_async`] stages the encoded body then calls this function -/// (dual-aware; freeze never requires `async`). +/// [`super::stream_decode_async`] stages the encoded body then calls this function. pub fn stream_decode( master_key: &[u8], hash: &[u8], @@ -117,76 +84,18 @@ pub fn stream_decode( encoded_body_len: Option, output: &mut W, ) -> Result { - #[cfg(feature = "backend-lean")] - { - let mut input = input; - let body = read_encoded_body(&mut input, encoded_body_len)?; - let plaintext = crate::backend::lean::decode(master_key, hash, &body, padding, format)?; - // Free encoded body before writing plaintext (avoid simultaneous body+pt peak). - drop(body); - output - .write_all(&plaintext) - .map_err(CarbonadoError::StdIoError)?; - Ok(plaintext.len() as u64) - } - #[cfg(feature = "backend-rust")] - { - stream_decode_inboard_pipeline( - master_key, - hash, - input, - padding, - format, - encoded_body_len, - output, - ) - } -} - -/// Read a bounded or unbounded encoded body for Lean E1 spool decode. -/// -/// `encoded_body_len` must be a **trusted** length (typically header-derived -/// `encoded_len`). Declared lengths above [`crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN`] -/// are rejected before allocation (DoS soft cap; same order as segment main limits). -/// -/// **W1b:** unbounded path disk-spools first (O(chunk) ingest) then materializes once for -/// the Lean buffer ABI. Bounded path pre-sizes exactly `declared` (same as prior E1). -#[cfg(feature = "backend-lean")] -fn read_encoded_body( - input: &mut R, - encoded_body_len: Option, -) -> Result, CarbonadoError> { - use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; - - match encoded_body_len { - Some(declared) => { - if declared > MAX_SEGMENT_MAIN_LEN { - return Err(CarbonadoError::InternalStateError(format!( - "encoded_body_len {declared} exceeds MAX_SEGMENT_MAIN_LEN {MAX_SEGMENT_MAIN_LEN}" - ))); - } - let mut body = vec![0u8; declared as usize]; - input - .read_exact(&mut body) - .map_err(CarbonadoError::StdIoError)?; - // Reject trailing bytes beyond declared length (same contract as rust path). - let mut extra = [0u8; 1]; - match input.read(&mut extra) { - Ok(0) => Ok(body), - Ok(_) => Err(CarbonadoError::EncodedBodyExceedsDeclaredLength { declared }), - Err(e) => Err(CarbonadoError::StdIoError(e)), - } - } - None => { - // Unbounded: disk-spool with DoS cap, then materialize for Lean (W1b E1.5). - let body = SeekableSpool::spool_then_materialize(input, Some(MAX_SEGMENT_MAIN_LEN))?; - Ok(body) - } - } + stream_decode_inboard_pipeline( + master_key, + hash, + input, + padding, + format, + encoded_body_len, + output, + ) } /// Core inboard decode: Bao verify → FEC → decrypt → decompress. -#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_decode / buffer path only pub(crate) fn stream_decode_inboard_pipeline( master_key: &[u8], hash: &[u8], @@ -364,7 +273,6 @@ fn stream_decode_verified_inboard( Ok(()) } -#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_decode_inboard_pipeline only fn stream_decode_post_preprocess_seek( master_key: &[u8], mut input: R, @@ -393,20 +301,9 @@ fn stream_decode_post_preprocess_seek( } } -/// Stream outboard decode (incremental Bao/FEC/decrypt chain). -/// /// Stream outboard decode from main + optional sidecars. /// -/// # Memory / dual-backend matrix (W1b) -/// -/// | Backend | Path | Peak RAM | Engine | -/// |---------|------|----------|--------| -/// | `backend-rust` | all formats | **O(chunk/stripe)** S4 (FEC residual O(segment body)) | rust geometric + streaming EtM | -/// | `backend-lean` | **public non-Compression** (c0/c4/c8/c12) | **O(chunk/stripe) E2** | rust S4 geometric composition (G9 no-compress; c4/c12 evidenced); **not** pure-Lean stream | -/// | `backend-lean` | **public + Compression** (c2/c6/c10/c14) | **O(logical)** if decompress materializes | same composition; not advertised as E2 | -/// | `backend-lean` | **encrypted** | O(logical) E1 | Lean `decode_outboard` (crypto dual) | -/// -/// Buffer APIs remain Lean under `backend-lean` always. See docs/LIMITS.md. +/// Peak RAM is **O(chunk/stripe)** on the S4 path (FEC residual O(segment body)). #[allow(clippy::too_many_arguments)] pub fn stream_decode_outboard( master_key: &[u8], @@ -419,108 +316,20 @@ pub fn stream_decode_outboard( explicit_nonce: Option<[u8; 16]>, output: &mut W, ) -> Result { - #[cfg(feature = "backend-lean")] - { - let fmt = Format::from(format); - if !fmt.contains(Format::Encryption) { - // W1b: public S4 composition (E2 only when !Compression; see rustdoc matrix). - stream_decode_outboard_s4( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - explicit_nonce, - output, - ) - } else { - stream_decode_outboard_lean_e1( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - explicit_nonce, - output, - ) - } - } - #[cfg(feature = "backend-rust")] - { - stream_decode_outboard_s4( - master_key, - hash, - main, - verification_outboard, - fec_parity, - padding, - format, - explicit_nonce, - output, - ) - } -} - -/// Lean E1 outboard decode (encrypted dual crypto): disk-spool → buffer ABI → write. -#[cfg(feature = "backend-lean")] -#[allow(clippy::too_many_arguments)] -fn stream_decode_outboard_lean_e1( - master_key: &[u8], - hash: &[u8], - main: M, - verification_outboard: Option, - fec_parity: Option

, - padding: u32, - format: u8, - explicit_nonce: Option<[u8; 16]>, - output: &mut W, -) -> Result { - use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; - - let main_buf = SeekableSpool::spool_then_materialize(main, Some(MAX_SEGMENT_MAIN_LEN))?; - let ob_buf = match verification_outboard { - Some(r) => Some(SeekableSpool::spool_then_materialize( - r, - Some(MAX_SEGMENT_MAIN_LEN), - )?), - None => None, - }; - let par_buf = match fec_parity { - Some(r) => Some(SeekableSpool::spool_then_materialize( - r, - Some(MAX_SEGMENT_MAIN_LEN), - )?), - None => None, - }; - let header_path = explicit_nonce.is_some(); - let plaintext = crate::backend::lean::decode_outboard( + stream_decode_outboard_s4( master_key, hash, - &main_buf, - ob_buf.as_deref(), - par_buf.as_deref(), + main, + verification_outboard, + fec_parity, padding, format, - explicit_nonce.as_ref(), - header_path, - )?; - drop(main_buf); - drop(ob_buf); - drop(par_buf); - output - .write_all(&plaintext) - .map_err(CarbonadoError::StdIoError)?; - Ok(plaintext.len() as u64) + explicit_nonce, + output, + ) } -/// S4 outboard decode: O(chunk/stripe) peak (public geometric + encrypted EtM spool). -/// -/// Under `backend-lean` this is the **W1b public** composition path (caller gates Encryption). -/// Peak is E2 O(chunk/stripe) only when !Compression; see public matrix on `stream_decode_outboard`. +/// S4 outboard decode: O(chunk/stripe) peak (geometric + encrypted EtM spool). #[allow(clippy::too_many_arguments)] fn stream_decode_outboard_s4( master_key: &[u8], diff --git a/src/stream/decode_async.rs b/src/stream/decode_async.rs index 21991f9..9763289 100644 --- a/src/stream/decode_async.rs +++ b/src/stream/decode_async.rs @@ -1,12 +1,12 @@ -//! Async adapter for inboard decode — stages encoded body, then dual-aware sync decode (R10). +//! Async adapter for inboard decode — stages encoded body, then sync decode. use crate::{ constants::Format, error::CarbonadoError, stream::{ io::{ - async_copy_all, async_copy_bounded, async_reject_trailing, AsyncPipelineSink, - AsyncPipelineSource, BoundedCopyTruncation, + AsyncPipelineSink, AsyncPipelineSource, BoundedCopyTruncation, async_copy_all, + async_copy_bounded, async_reject_trailing, }, spool::SeekableSpool, stream_decode, @@ -18,45 +18,17 @@ use crate::{ /// Same high-level semantics as [`super::stream_decode`]: Bao verify → FEC reverse → decrypt → /// decompress (embedded-nonce layout). /// -/// ## Dual-backend policy (R10 closed) +/// Async is an optional concurrency adapter (disk spool bridge). WASM returns +/// [`CarbonadoError::NotImplemented`] (host temp spool). /// -/// | Concern | Policy | -/// |---------|--------| -/// | Dual freeze / `just test-lean-ci` | **Never requires `async`** — permanent. Features stay `"backend-lean,pqc,ots,cli"`. | -/// | `tests/streaming_async.rs` | `#![cfg(feature = "async")]` → **0 tests** under freeze (feature-gated; not dual-suite red). | -/// | Engine after spool | Calls dual-aware [`super::stream_decode`] (R5 E1), **not** pure-Rust-only `stream_decode_inboard_pipeline`. | -/// | `backend-rust` + `async` | Same S4 inboard pipeline as sync `stream_decode`. | -/// | `backend-lean` + `async` | Spool → E1 `stream_decode` → Lean `decode` (see costs; not stream E2). | -/// | WASM + `async` | [`CarbonadoError::NotImplemented`] (host temp spool). | -/// -/// Sync stream dual E1 remains the dual-suite contract for streaming. Async is an optional -/// concurrency adapter (disk spool bridge), not part of the freeze bar. -/// -/// Optional dual smoke (not freeze): -/// `cargo test --no-default-features --features "backend-lean,pqc,ots,async,async-tokio" --test streaming_async` -/// with `CARBONADO_LEAN_LIB` / `LD_LIBRARY_PATH` set. -/// -/// ## Phase 2 materialization tradeoff -/// -/// Unlike sync [`super::stream_decode`] under `backend-rust`, which streams incrementally from -/// [`std::io::Read`] into Bao/FEC (S4), this adapter **fully stages the encoded body** to a +/// Unlike sync [`super::stream_decode`], which streams incrementally from +/// [`std::io::Read`] into Bao/FEC, this adapter **fully stages the encoded body** to a /// disk-backed [`SeekableSpool`] before invoking the sync path. Every async decode therefore pays /// **O(encoded_body)** disk write + read for the input boundary, plus a plaintext spool before -/// [`async_copy_all`]. -/// -/// **Peak costs (honest):** -/// - **Disk (all engines):** O(encoded) staging + O(logical) plaintext spool traffic. -/// - **`backend-rust` peak RAM:** spool/chunk-oriented (FEC verification still O(FEC body) -/// shard buffers on the sync S4 path where applicable). -/// - **`backend-lean` peak RAM:** O(**encoded** + **logical**) — E1 `read_encoded_body` -/// materializes a full body `Vec` before Lean decode, then O(logical) plaintext. Do **not** -/// treat lean+async as O(logical) RAM only. +/// [`async_copy_all`]. Peak RAM stays spool/chunk-oriented (FEC verification still O(FEC body) +/// shard buffers on the sync path where applicable). /// -/// Not stream E2 / true chunked async Bao. -/// -/// ## Executor blocking -/// -/// The dual-aware sync path ([`super::stream_decode`]) runs as a **blocking** section inside +/// The sync path ([`super::stream_decode`]) runs as a **blocking** section inside /// this `async fn`. On Tokio/async-std this can starve the executor for large payloads. /// Integrators should either: /// - enable the `async-tokio` feature (uses `tokio::task::spawn_blocking` for the sync section), or @@ -65,18 +37,12 @@ use crate::{ /// Pass `encoded_body_len` when the reader may contain trailing bytes after the encoded body /// (FEC c8, compressed c4, verification c12/c14). When `Some`, excess or truncated input is rejected. /// -/// ## Truncation error taxonomy (spool bridge) -/// /// Non-verification formats (c4, c8) surface staging truncation as /// `StdIoError(UnexpectedEof, "truncated encoded body")` or `"truncated FEC body"` — aligned with -/// sync `take(limit)` paths. **Verification formats (c6/c12/c14/c15):** -/// - **`backend-rust`:** sync fails during incremental Bao (`BaoResponseTruncated`); this -/// adapter fails earlier at [`async_copy_bounded`] with the encoded-body staging message. -/// - **`backend-lean`:** both fail closed **before Bao**, but **not** at the same site/message — -/// async fails at adapter staging (`"truncated encoded body"`); sync E1 fails later in -/// `read_encoded_body` / `read_exact` as generic `UnexpectedEof` (`"failed to fill whole buffer"`). -/// -/// Callers must not assume identical error variants or messages across sync/async or engines. +/// sync `take(limit)` paths. Verification formats (c6/c12/c14/c15): sync fails during incremental +/// Bao (`BaoResponseTruncated`); this adapter fails earlier at [`async_copy_bounded`] with the +/// encoded-body staging message. Callers must not assume identical error variants or messages +/// across sync/async. #[cfg(all(feature = "async", not(target_arch = "wasm32")))] pub async fn stream_decode_async( master_key: &[u8], @@ -105,8 +71,7 @@ where } encoded_spool.rewind()?; - // Body length already enforced by staging; pass None so dual-aware stream_decode - // (R5 E1 under backend-lean, S4 pipeline under backend-rust) reads the whole spool. + // Body length already enforced by staging; pass None so stream_decode reads the whole spool. let (nbytes, mut plaintext_spool) = run_sync_stream_decode(master_key, hash, encoded_spool, padding, format).await?; async_copy_all(&mut plaintext_spool, output).await?; @@ -131,9 +96,7 @@ where Err(CarbonadoError::NotImplemented) } -/// Blocking dual-aware inboard decode after async staging. -/// -/// Uses [`stream_decode`] so `backend-lean` hits Lean E1 (no silent pure-Rust pipeline). +/// Blocking inboard decode after async staging. #[cfg(all(feature = "async", not(target_arch = "wasm32")))] async fn run_sync_stream_decode( master_key: &[u8], diff --git a/src/stream/encode.rs b/src/stream/encode.rs index 140430a..de1fd60 100644 --- a/src/stream/encode.rs +++ b/src/stream/encode.rs @@ -5,23 +5,20 @@ use std::io::{Read, Seek, SeekFrom, Write}; use bao::Hash; use crate::{ - constants::{Format, FEC_M, SLICE_LEN}, + constants::{FEC_M, Format, SLICE_LEN}, error::CarbonadoError, stream::{ compress::stream_compress, crypto_stream::{ stream_encrypt, stream_encrypt_embedded_with_nonce, stream_encrypt_with_nonce_seek, }, - fec::{feed_inboard_fec_stripe, write_inboard_stripe, FecStripeReadAt}, + fec::{FecStripeReadAt, feed_inboard_fec_stripe, write_inboard_stripe}, spool::SeekableSpool, }, structs::{EncodeInfo, OutboardEncoded}, }; -// Outboard S4 geometric pipeline (backend-rust always; backend-lean W1b public E2). use crate::stream::bao::stream_verification_outboard; -// Buffer-path helpers (rust engine encode_buffer only). -#[cfg(feature = "backend-rust")] use crate::stream::{ bao::{verification_inboard_buffer, verification_outboard_buffer}, compress::compress_buffer, @@ -57,7 +54,7 @@ pub struct PreprocessStats { /// in the sink as `[nonce|tag|ct]`. /// /// **`fixed_nonce`:** when `Some(n)` and Encryption is set, use `n` literally (including -/// all-zero — dual-backend identical). When `None`, draw a CSPRNG nonce (production). +/// all-zero). When `None`, draw a CSPRNG nonce (production). /// Prefer CSPRNG for live archives; fixed nonces are for tests/determinism only — see /// AGENTS §2.1.4 (nonce uniqueness; reuse under the same master is catastrophic). pub fn stream_preprocess( @@ -152,7 +149,6 @@ pub fn stream_preprocess( /// [`stream_preprocess`] for [`SeekableSpool`] sinks — encrypt replace uses /// [`SeekableSpool::overwrite_from`] so file size matches ciphertext (no stale tail bytes). -#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_encode_inboard only pub(crate) fn stream_preprocess_spool( master_key: &[u8], format: Format, @@ -259,7 +255,6 @@ fn encrypt_preprocess_sink( } /// Header-path encrypt for [`SeekableSpool`] preprocess sinks (uses [`SeekableSpool::overwrite_from`]). -#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_preprocess_spool only pub(crate) fn encrypt_preprocess_spool( master_key: &[u8], nonce: [u8; 16], @@ -298,9 +293,6 @@ fn reader_len(r: &mut R) -> Result { /// Primary inboard encode (buffer). Used by [`crate::encoding::encode`]. /// -/// Under `backend-lean`, composes over Lean C ABI body encode (same engine as -/// [`crate::encode`]) so streaming buffer tests do not silently use pure Rust. -/// /// Encrypted formats draw a random nonce (embedded layout). For a fixed nonce /// (G9 fixtures), use [`stream_encode_buffer_with_nonce`]. pub fn stream_encode_buffer( @@ -328,37 +320,6 @@ pub fn stream_encode_buffer_with_nonce( input: &[u8], format: u8, explicit_nonce: Option<[u8; 16]>, -) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - let nonce = if format & 1 != 0 { - match explicit_nonce { - Some(n) => Some(n), - None => { - let mut n = [0u8; 16]; - getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; - Some(n) - } - } - } else { - None - }; - let crate::structs::Encoded(body, hash, info) = - crate::backend::lean::encode(master_key, input, format, nonce.as_ref())?; - Ok((body, hash, info)) - } - #[cfg(feature = "backend-rust")] - { - stream_encode_buffer_rust(master_key, input, format, explicit_nonce) - } -} - -#[cfg(feature = "backend-rust")] -fn stream_encode_buffer_rust( - master_key: &[u8], - input: &[u8], - format: u8, - explicit_nonce: Option<[u8; 16]>, ) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { let fmt = Format::from(format); let input_len = input.len() as u32; @@ -445,49 +406,11 @@ fn stream_encode_buffer_rust( /// /// When `explicit_nonce` is `Some`, encrypted output is `[tag(64) | ct]` (header path). /// When `None`, encrypted output embeds the nonce (low-level path). -/// -/// Under `backend-lean`, composes over Lean C ABI with matching layout: -/// `explicit_nonce.is_some()` → `header_path=true` (`[tag|ct]`); else embedded-nonce. pub fn stream_encode_outboard_buffer( master_key: &[u8], input: &[u8], format: u8, explicit_nonce: Option<[u8; 16]>, -) -> Result { - #[cfg(feature = "backend-lean")] - { - let header_path = explicit_nonce.is_some(); - let nonce = if format & 1 != 0 { - if let Some(n) = explicit_nonce { - Some(n) - } else { - let mut n = [0u8; 16]; - getrandom::getrandom(&mut n).map_err(|_| CarbonadoError::RandomnessError)?; - Some(n) - } - } else { - None - }; - crate::backend::lean::encode_outboard( - master_key, - input, - format, - nonce.as_ref(), - header_path, - ) - } - #[cfg(feature = "backend-rust")] - { - stream_encode_outboard_buffer_rust(master_key, input, format, explicit_nonce) - } -} - -#[cfg(feature = "backend-rust")] -fn stream_encode_outboard_buffer_rust( - master_key: &[u8], - input: &[u8], - format: u8, - explicit_nonce: Option<[u8; 16]>, ) -> Result { let fmt = Format::from(format); let input_len = input.len() as u32; @@ -581,21 +504,11 @@ fn stream_encode_outboard_buffer_rust( /// Stream outboard encode to writers (public + encrypted). /// -/// # Memory / dual-backend matrix (W1b) -/// -/// | Backend | Path | Peak RAM | Engine | -/// |---------|------|----------|--------| -/// | `backend-rust` | all formats | **O(chunk/stripe)** S4 (compress streams) | rust geometric + streaming EtM | -/// | `backend-lean` | **public non-Compression** (c0/c4/c8/c12) | **O(chunk/stripe) E2** | rust S4 geometric composition (G9 **no-compress** wire bit-match; c4/c12 evidenced); **not** pure-Lean stream | -/// | `backend-lean` | **public + Compression** (c2/c6/c10/c14) | **O(logical)** at bulk zstd | same S4 composition; compress uses Lean-parity buffer zstd (not E2) | -/// | `backend-lean` | **encrypted** | O(logical) E1 | Lean `encode_outboard` (crypto dual) | +/// Peak RAM is **O(chunk/stripe)** on the S4 geometric path (compress streams). /// -/// Buffer APIs ([`stream_encode_outboard_buffer`]) remain Lean under `backend-lean` always. -/// Pure Lean chunked stream residual remains (no streaming C ABI). See docs/LIMITS.md. -/// -/// **Encrypted nonces:** both backends always draw a CSPRNG nonce for this stream API -/// (dual-identical). For a fixed nonce (tests/G9), use [`stream_encode_outboard_buffer`] -/// with `Some(nonce)` (header-path layout when `Some`). +/// Encrypted formats always draw a CSPRNG nonce for this stream API. For a fixed nonce +/// (tests/G9), use [`stream_encode_outboard_buffer`] with `Some(nonce)` (header-path +/// layout when `Some`). #[allow(clippy::too_many_arguments)] pub fn stream_encode_outboard( master_key: &[u8], @@ -607,136 +520,19 @@ pub fn stream_encode_outboard( payload_nonce: &mut [u8; 16], header_path_encrypt: bool, ) -> Result<(Hash, EncodeInfo), CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - let fmt = Format::from(format); - // W1b: public → S4 composition (E2 only when !Compression; see rustdoc matrix). - // Encrypted stays Lean E1 dual crypto. - if !fmt.contains(Format::Encryption) { - stream_encode_outboard_s4( - master_key, - input, - format, - main_out, - bao_out, - parity_out, - payload_nonce, - header_path_encrypt, - ) - } else { - stream_encode_outboard_lean( - master_key, - input, - format, - main_out, - bao_out, - parity_out, - payload_nonce, - header_path_encrypt, - ) - } - } - #[cfg(feature = "backend-rust")] - { - stream_encode_outboard_s4( - master_key, - input, - format, - main_out, - bao_out, - parity_out, - payload_nonce, - header_path_encrypt, - ) - } -} - -/// Lean E1: disk-spool plaintext (O(chunk) ingest) → `lean::encode_outboard` → write-all. -/// -/// Peak RAM remains O(logical) at the Lean buffer boundary (encrypted dual crypto). -#[cfg(feature = "backend-lean")] -#[allow(clippy::too_many_arguments)] -fn stream_encode_outboard_lean( - master_key: &[u8], - input: impl Read, - format: u8, - main_out: &mut M, - mut bao_out: Option<&mut O>, - mut parity_out: Option<&mut P>, - payload_nonce: &mut [u8; 16], - header_path_encrypt: bool, -) -> Result<(Hash, EncodeInfo), CarbonadoError> { - use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; - - let fmt = Format::from(format); - // Fail-closed before Lean work: required sidecar writers must be present when - // format bits demand them (decode-side lean::decode_outboard already enforces - // the symmetric Missing* contract). Rust stream path hard-requires Bao writer - // for Verification; FEC writer is fail-closed here for dual-backend symmetry - // with decode (silent drop of Lean-produced parity would diverge API contracts). - if fmt.contains(Format::Verification) && bao_out.is_none() { - return Err(CarbonadoError::MissingVerificationOutboard); - } - if fmt.contains(Format::Fec) && parity_out.is_none() { - return Err(CarbonadoError::MissingFecParity); - } - - // Disk-backed ingest (O(chunk) during copy); materialize once for Lean buffer ABI. - let plaintext = SeekableSpool::spool_then_materialize(input, Some(MAX_SEGMENT_MAIN_LEN))?; - - // Always CSPRNG when encrypted (matches rust `stream_preprocess(..., fixed_nonce=None)`). - // Fixed-nonce outboard: `stream_encode_outboard_buffer(..., Some(nonce))`. - let encrypted = fmt.contains(Format::Encryption); - if encrypted { - getrandom::getrandom(payload_nonce).map_err(|_| CarbonadoError::RandomnessError)?; - } else { - *payload_nonce = [0u8; 16]; - } - let nonce = if encrypted { - Some(*payload_nonce) - } else { - None - }; - - let oenc = crate::backend::lean::encode_outboard( + stream_encode_outboard_s4( master_key, - &plaintext, + input, format, - nonce.as_ref(), + main_out, + bao_out, + parity_out, + payload_nonce, header_path_encrypt, - )?; - // Free plaintext before writing outputs (avoid simultaneous pt + main peak). - drop(plaintext); - - main_out - .seek(SeekFrom::Start(0)) - .map_err(CarbonadoError::StdIoError)?; - main_out - .write_all(&oenc.main) - .map_err(CarbonadoError::StdIoError)?; - - if let Some(ob_writer) = bao_out.as_mut() { - if let Some(ref ob) = oenc.verification_outboard { - ob_writer - .write_all(ob) - .map_err(CarbonadoError::StdIoError)?; - } - } - if let Some(par_writer) = parity_out.as_mut() { - if let Some(ref par) = oenc.fec_parity { - par_writer - .write_all(par) - .map_err(CarbonadoError::StdIoError)?; - } - } - - Ok((oenc.hash, oenc.info)) + ) } -/// S4 outboard encode: O(chunk/stripe) peak RAM (public geometric + encrypted EtM spool). -/// -/// Under `backend-lean` this is the **W1b public** composition path (caller gates Encryption). -/// Peak is E2 O(chunk/stripe) only when !Compression; Compression under lean is O(logical) bulk zstd. +/// S4 outboard encode: O(chunk/stripe) peak RAM (geometric + encrypted EtM spool). #[allow(clippy::too_many_arguments)] fn stream_encode_outboard_s4( master_key: &[u8], @@ -749,8 +545,7 @@ fn stream_encode_outboard_s4( header_path_encrypt: bool, ) -> Result<(Hash, EncodeInfo), CarbonadoError> { let fmt = Format::from(format); - // Fail-closed: required sidecar writers when format bits demand them (matches - // lean E1 stream_encode_outboard_lean + decode Missing* contract). + // Fail-closed: required sidecar writers when format bits demand them. if fmt.contains(Format::Verification) && bao_out.is_none() { return Err(CarbonadoError::MissingVerificationOutboard); } @@ -988,12 +783,6 @@ fn stream_copy( /// Fused inboard encode: preprocess into a disk spool, then FEC/Bao directly to `output`. /// -/// Under `backend-lean` (R5 E1 / W1b residual): disk-spool plaintext → Lean body encode -/// (embedded-nonce via [`crate::backend::lean::encode`]) or headered encode when -/// `header_path_encrypt` (strip header, write body only). Peak RAM O(logical) at Lean -/// buffer boundary — not stream E2. Public **outboard** stream is W1b E2 (see -/// [`stream_encode_outboard`]). See docs/LIMITS.md. -/// /// Encrypted formats draw a CSPRNG nonce. For a fixed nonce (including all-zero), use /// [`stream_encode_inboard_with_nonce`]. pub fn stream_encode_inboard( @@ -1017,7 +806,7 @@ pub fn stream_encode_inboard( /// Like [`stream_encode_inboard`], with optional fixed AES-CTR nonce when encrypted. /// -/// When `fixed_nonce` is `Some(n)`, both backends use `n` literally (including all-zero). +/// When `fixed_nonce` is `Some(n)`, this uses `n` literally (including all-zero). /// When `None`, a CSPRNG nonce is drawn. **Test/determinism only** for fixed nonces — /// nonce reuse under the same master is catastrophic (AGENTS §2.1.4). Prefer /// [`stream_encode_inboard`] for production. @@ -1030,103 +819,17 @@ pub fn stream_encode_inboard_with_nonce( header_path_encrypt: bool, fixed_nonce: Option<[u8; 16]>, ) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { - #[cfg(feature = "backend-lean")] - { - stream_encode_inboard_lean( - master_key, - input, - format, - output, - payload_nonce, - header_path_encrypt, - fixed_nonce, - ) - } - #[cfg(feature = "backend-rust")] - { - let fmt = Format::from(format); - let mut spool = SeekableSpool::new()?; - let stats = stream_preprocess_spool( - master_key, - fmt, - input, - &mut spool, - payload_nonce, - header_path_encrypt, - fixed_nonce, - )?; - let (hash, info) = stream_encode_inboard_body(&mut spool, stats, format, output)?; - Ok((hash, info, stats)) - } -} - -/// Lean E1 fused inboard: disk-spool plaintext → lean encode / encode_headered → write body. -/// -/// Peak RAM O(logical) at Lean buffer boundary (W1b residual for inboard stream). -#[cfg(feature = "backend-lean")] -fn stream_encode_inboard_lean( - master_key: &[u8], - input: R, - format: u8, - output: &mut W, - payload_nonce: &mut [u8; 16], - header_path_encrypt: bool, - fixed_nonce: Option<[u8; 16]>, -) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { - use crate::filepack_manifest::MAX_SEGMENT_MAIN_LEN; - - let plaintext = SeekableSpool::spool_then_materialize(input, Some(MAX_SEGMENT_MAIN_LEN))?; - - let encrypted = format & 1 != 0; - if encrypted { - match fixed_nonce { - Some(n) => *payload_nonce = n, - None => { - getrandom::getrandom(payload_nonce).map_err(|_| CarbonadoError::RandomnessError)?; - } - } - } else { - *payload_nonce = [0u8; 16]; - } - - let nonce_ref: Option<&[u8; 16]> = if encrypted { Some(payload_nonce) } else { None }; - - let (body, hash, info) = if header_path_encrypt { - // Header-path: Lean builds Header||body; return body only + nonce from header. - let (archive, info) = crate::backend::lean::encode_headered( - master_key, &plaintext, format, nonce_ref, None, None, - )?; - drop(plaintext); - if archive.len() < crate::file::Header::LEN { - return Err(CarbonadoError::InvalidHeaderLength); - } - let header = crate::file::Header::try_from(&archive[..crate::file::Header::LEN])?; - *payload_nonce = header.payload_nonce; - let body = archive[crate::file::Header::LEN..].to_vec(); - drop(archive); - (body, header.hash, info) - } else { - let crate::structs::Encoded(body, hash, info) = - crate::backend::lean::encode(master_key, &plaintext, format, nonce_ref)?; - drop(plaintext); - (body, hash, info) - }; - - output - .write_all(&body) - .map_err(CarbonadoError::StdIoError)?; - - let bare_len = if encrypted { - info.bytes_encrypted as u64 - } else if info.bytes_compressed > 0 { - info.bytes_compressed as u64 - } else { - info.input_len as u64 - }; - let stats = PreprocessStats { - bare_len, - input_len: info.input_len as u64, - bytes_compressed: info.bytes_compressed, - }; + let fmt = Format::from(format); + let mut spool = SeekableSpool::new()?; + let stats = stream_preprocess_spool( + master_key, + fmt, + input, + &mut spool, + payload_nonce, + header_path_encrypt, + fixed_nonce, + )?; + let (hash, info) = stream_encode_inboard_body(&mut spool, stats, format, output)?; Ok((hash, info, stats)) } diff --git a/src/stream/fec.rs b/src/stream/fec.rs index 314bbfc..d4d97cb 100644 --- a/src/stream/fec.rs +++ b/src/stream/fec.rs @@ -2,8 +2,8 @@ use std::io::{Read, Write}; -use reed_solomon_erasure::galois_8::Field; use reed_solomon_erasure::ReedSolomon; +use reed_solomon_erasure::galois_8::Field; use crate::{ constants::{FEC_K, FEC_M, SLICE_LEN}, diff --git a/src/stream/mod.rs b/src/stream/mod.rs index 2a40d9c..0be486f 100644 --- a/src/stream/mod.rs +++ b/src/stream/mod.rs @@ -38,6 +38,6 @@ pub use encode::{ stream_encode_outboard_buffer, stream_preprocess, }; pub use shard::{ - decode_shards_stream, encode_shard_stream, ShardEncodeResult, ShardSource, - DEFAULT_SEGMENT_PLAINTEXT_BUDGET, + DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, decode_shards_stream, + encode_shard_stream, }; diff --git a/src/stream/parallel.rs b/src/stream/parallel.rs index e5b42ae..9b7190b 100644 --- a/src/stream/parallel.rs +++ b/src/stream/parallel.rs @@ -6,7 +6,7 @@ use std::sync::OnceLock; -use reed_solomon_erasure::galois_8::{mul_slice, mul_slice_xor, ReedSolomon}; +use reed_solomon_erasure::galois_8::{ReedSolomon, mul_slice, mul_slice_xor}; use crate::{ constants::{FEC_K, FEC_M}, diff --git a/src/stream/shard.rs b/src/stream/shard.rs index 4f72d30..69216a9 100644 --- a/src/stream/shard.rs +++ b/src/stream/shard.rs @@ -5,7 +5,7 @@ use std::io::{BufRead, Read, Write}; use crate::{ constants::Format, error::CarbonadoError, - file::{decode_stream, Header}, + file::{Header, decode_stream}, structs::EncodeInfo, }; diff --git a/src/stream/slice.rs b/src/stream/slice.rs index fe3f502..9e81abb 100644 --- a/src/stream/slice.rs +++ b/src/stream/slice.rs @@ -3,13 +3,13 @@ use std::io::{Cursor, Read}; #[cfg(feature = "backend-rust")] use bao_tree::io::{outboard::PostOrderMemOutboard, sync::keyed_valid_ranges}; use bao_tree::{ + BaoTree, ChunkNum, ChunkRanges, io::{ - outboard::EmptyOutboard, - sync::{keyed_decode_ranges, ReadAt, WriteAt}, DecodeError, + outboard::EmptyOutboard, + sync::{ReadAt, WriteAt, keyed_decode_ranges}, }, iter::BaoChunk, - BaoTree, ChunkNum, ChunkRanges, }; use crate::{ @@ -183,7 +183,6 @@ pub fn verify_slice_inboard_seekable( /// Does not perform keyed hash checks; RS + re-bao oracle in scrub filters bad candidates. /// Returns [`CarbonadoError::BaoResponseTruncated`] if the response ends before the slice /// window is fully populated. -#[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust scrub path only pub(crate) fn extract_slice_inboard_for_scrub( input: &[u8], index: u32, @@ -261,13 +260,8 @@ pub(crate) fn extract_slice_inboard_for_scrub( /// Verified read of `count` contiguous 4 KiB slices at `index` from bare data plus a /// post-order outboard sidecar. /// -/// **Memory and time (backend-rust):** O(slice) — validates only the requested chunk -/// ranges via `keyed_valid_ranges`, then reads the corresponding bare bytes. -/// -/// **backend-lean (R9 / W4b permanent):** dispatches to `carbonado_verify_slice_outboard` -/// (Lean range verify, O(slice+height) hash). The C ABI takes a full main buffer — when -/// `data` is not already contiguous in process memory this path materializes `data_len` -/// bytes once (honest LIMITS vs pure-Rust streaming `ReadAt`; no callback C ABI). +/// **Memory and time:** O(slice) — validates only the requested chunk ranges via +/// `keyed_valid_ranges`, then reads the corresponding bare bytes. pub fn verify_slice_outboard( data: D, outboard_bytes: &[u8], @@ -286,55 +280,34 @@ pub fn verify_slice_outboard( content_len: data_len, }); } - #[cfg(feature = "backend-lean")] - { - let len = usize::try_from(data_len).map_err(|_| { - CarbonadoError::OutboardVerificationFailed("data_len exceeds usize".into()) - })?; - let mut buf = vec![0u8; len]; - data.read_exact_at(0, &mut buf) - .map_err(map_valid_ranges_read_error)?; - crate::backend::lean::verify_slice_outboard( - &buf, - outboard_bytes, - data_len, - index, - count, - hash, - format, - ) - } - #[cfg(feature = "backend-rust")] - { - let root = decode_bao_hash(hash)?; - let tree = BaoTree::new(data_len, BAO_BLOCK_SIZE); - let ob = PostOrderMemOutboard { - root, - tree, - data: outboard_bytes, - }; - let key = carbonado_verification_key(format); - let ranges = slice_to_chunk_ranges(index, count); - // Cap expected chunks at content length (partial last leaf / short files). - let content_chunks = data_len.div_ceil(1024); - let expected_chunks = (u64::from(count) * CHUNKS_PER_SLICE) - .min(content_chunks.saturating_sub(u64::from(index) * CHUNKS_PER_SLICE)); + let root = decode_bao_hash(hash)?; + let tree = BaoTree::new(data_len, BAO_BLOCK_SIZE); + let ob = PostOrderMemOutboard { + root, + tree, + data: outboard_bytes, + }; + let key = carbonado_verification_key(format); + let ranges = slice_to_chunk_ranges(index, count); + // Cap expected chunks at content length (partial last leaf / short files). + let content_chunks = data_len.div_ceil(1024); + let expected_chunks = (u64::from(count) * CHUNKS_PER_SLICE) + .min(content_chunks.saturating_sub(u64::from(index) * CHUNKS_PER_SLICE)); - let mut validated = ChunkRanges::empty(); - for item in keyed_valid_ranges(&ob, &data, &ranges, &key) { - let range = item.map_err(map_valid_ranges_read_error)?; - validated |= ChunkRanges::from(range); - } - if chunk_count(&validated) < expected_chunks { - return Err(CarbonadoError::AuthenticationFailed); - } + let mut validated = ChunkRanges::empty(); + for item in keyed_valid_ranges(&ob, &data, &ranges, &key) { + let range = item.map_err(map_valid_ranges_read_error)?; + validated |= ChunkRanges::from(range); + } + if chunk_count(&validated) < expected_chunks { + return Err(CarbonadoError::AuthenticationFailed); + } - let (slice_byte_start, _slice_byte_end, actual_len) = - slice_byte_range(index, count, data_len)?; + let (slice_byte_start, _slice_byte_end, actual_len) = + slice_byte_range(index, count, data_len)?; - let mut out = vec![0u8; actual_len as usize]; - data.read_exact_at(slice_byte_start, &mut out) - .map_err(map_valid_ranges_read_error)?; - Ok(out) - } + let mut out = vec![0u8; actual_len as usize]; + data.read_exact_at(slice_byte_start, &mut out) + .map_err(map_valid_ranges_read_error)?; + Ok(out) } diff --git a/src/stream/spool.rs b/src/stream/spool.rs index c27be8e..9844fd1 100644 --- a/src/stream/spool.rs +++ b/src/stream/spool.rs @@ -60,7 +60,6 @@ impl SeekableSpool { } /// Truncate and replace contents from `src` (used after encrypt preprocess). - #[cfg_attr(feature = "backend-lean", allow(dead_code))] // rust stream_preprocess_spool only pub fn overwrite_from(&mut self, src: &mut Self) -> Result<(), CarbonadoError> { src.rewind()?; self.file.set_len(0).map_err(CarbonadoError::StdIoError)?; @@ -70,39 +69,6 @@ impl SeekableSpool { src.rewind()?; Ok(()) } - - /// Spool a reader to this temp file with O(chunk) RAM, then materialize for a buffer ABI. - /// - /// **W1b disk-backed E1.5:** ingest peak RAM is O(copy buffer), not O(N). The returned - /// `Vec` is still O(N) — required for Lean buffer C ABI. Prefer this over `read_to_end` - /// when the source is unbounded / adversarial so intermediate growth stays on disk until - /// the final materialize (with optional DoS cap via [`Read::take`]). - #[cfg(feature = "backend-lean")] - pub fn spool_then_materialize( - mut input: R, - max_len: Option, - ) -> Result, CarbonadoError> { - let mut spool = Self::new()?; - match max_len { - Some(cap) => { - let mut limited = input.by_ref().take(cap.saturating_add(1)); - io::copy(&mut limited, &mut spool).map_err(CarbonadoError::StdIoError)?; - let len = spool.content_len()?; - if len > cap { - return Err(CarbonadoError::InternalStateError(format!( - "spool materialize exceeds max_len {cap}" - ))); - } - } - None => { - io::copy(&mut input, &mut spool).map_err(CarbonadoError::StdIoError)?; - } - } - spool.rewind()?; - let mut out = Vec::new(); - io::copy(&mut spool, &mut out).map_err(CarbonadoError::StdIoError)?; - Ok(out) - } } impl Read for SeekableSpool { diff --git a/src/utils.rs b/src/utils.rs index 5b0fbb5..402962d 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, RwLock}, }; -use bao::{encode::Encoder, Hash}; +use bao::{Hash, encode::Encoder}; use log::trace; use crate::{ diff --git a/tests/bao_keyed_contract.rs b/tests/bao_keyed_contract.rs index 9151844..e40f976 100644 --- a/tests/bao_keyed_contract.rs +++ b/tests/bao_keyed_contract.rs @@ -14,12 +14,11 @@ use std::io::Cursor; use anyhow::Result; use bao_tree::{ - blake3, + BaoTree, ChunkNum, ChunkRanges, blake3, io::{ outboard::PostOrderMemOutboard, sync::{decode_ranges, keyed_encode_ranges_validated, keyed_valid_ranges}, }, - BaoTree, ChunkNum, ChunkRanges, }; use carbonado::{ carbonado_verification_key, diff --git a/tests/codec.rs b/tests/codec.rs index 17c7bd8..39b9e17 100644 --- a/tests/codec.rs +++ b/tests/codec.rs @@ -1,5 +1,5 @@ use std::{ - fs::{read, OpenOptions}, + fs::{OpenOptions, read}, io::Write, path::PathBuf, }; @@ -11,7 +11,7 @@ use carbonado::{ constants::Format, decode, encode, error::CarbonadoError, extract_slice, file::Header, scrub, structs::Encoded, verify_slice, }; -use common::corruption::{scattered_stream_knockout, InboardShardLayout}; +use common::corruption::{InboardShardLayout, scattered_stream_knockout}; use log::{debug, info}; use rand::{Rng, RngCore}; use wasm_bindgen_test::{wasm_bindgen_test, wasm_bindgen_test_configure}; diff --git a/tests/common/inboard_parity.rs b/tests/common/inboard_parity.rs index 502fca6..7008994 100644 --- a/tests/common/inboard_parity.rs +++ b/tests/common/inboard_parity.rs @@ -5,7 +5,7 @@ use std::io::{Cursor, Read, Seek, SeekFrom}; use bao::Hash; use carbonado::constants::Format; use carbonado::file::Header; -use carbonado::stream::encode::{stream_encode_inboard_body, PreprocessStats}; +use carbonado::stream::encode::{PreprocessStats, stream_encode_inboard_body}; use carbonado::stream::{stream_decode_buffer, stream_encode_buffer, stream_preprocess}; use carbonado::structs::EncodeInfo; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a9021b0..ec42dc1 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -9,6 +9,7 @@ pub mod corruption; pub mod format_matrix; pub mod header_layout; pub mod inboard_parity; +pub mod zstd_frame; use std::fs; use std::path::Path; diff --git a/tests/common/zstd_frame.rs b/tests/common/zstd_frame.rs new file mode 100644 index 0000000..51c5ee3 --- /dev/null +++ b/tests/common/zstd_frame.rs @@ -0,0 +1,159 @@ +//! RFC 8878 / `ref/zstd/doc/zstd_compression_format.md` frame-header parser. +//! +//! Mirrors Lean `Carbonado.Compress.parseZstdFrameHeader` so Rust tests can +//! assert the same parameter bits the spec names. + +use carbonado::constants::ZSTD_MAGIC; + +/// Frame-header parse errors (RFC reserved-bit + framing). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ZstdFrameError { + TruncatedHeader, + BadMagic, + ReservedBitSet, +} + +/// Parsed zstd `Frame_Header` (magic included in `header_len`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedZstdFrameHeader { + pub descriptor: u8, + pub content_size_flag: u8, + pub single_segment: bool, + pub unused_bit: bool, + pub reserved_bit: bool, + pub content_checksum: bool, + pub dictionary_id_flag: u8, + pub window_descriptor: Option, + pub window_log: Option, + pub window_size: Option, + pub dictionary_id: Option, + pub content_size: Option, + pub header_len: usize, +} + +fn did_field_size(flag: u8) -> usize { + match flag { + 0 => 0, + 1 => 1, + 2 => 2, + 3 => 4, + _ => 0, + } +} + +fn fcs_field_size(fcs_flag: u8, single_segment: bool) -> usize { + match (fcs_flag, single_segment) { + (1, _) => 2, + (2, _) => 4, + (3, _) => 8, + (0, true) => 1, + (0, false) => 0, + _ => 0, + } +} + +fn read_le_u16(bytes: &[u8], off: usize) -> u16 { + u16::from_le_bytes([bytes[off], bytes[off + 1]]) +} + +fn read_le_u32(bytes: &[u8], off: usize) -> u32 { + u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]) +} + +fn read_le_u64(bytes: &[u8], off: usize) -> u64 { + u64::from_le_bytes([ + bytes[off], + bytes[off + 1], + bytes[off + 2], + bytes[off + 3], + bytes[off + 4], + bytes[off + 5], + bytes[off + 6], + bytes[off + 7], + ]) +} + +fn window_log_from_descriptor(wd: u8) -> u32 { + 10 + u32::from(wd >> 3) +} + +fn window_size_from_descriptor(wd: u8) -> u64 { + let exponent = u32::from(wd >> 3); + let mantissa = u64::from(wd & 7); + let window_log = 10 + exponent; + let window_base = 1u64 << window_log; + window_base + (window_base / 8) * mantissa +} + +/// Parse magic + `Frame_Header`. Rejects RFC reserved bit. +pub fn parse_zstd_frame_header(bytes: &[u8]) -> Result { + if bytes.len() < 5 { + return Err(ZstdFrameError::TruncatedHeader); + } + if bytes[0..4] != ZSTD_MAGIC { + return Err(ZstdFrameError::BadMagic); + } + let descriptor = bytes[4]; + let content_size_flag = descriptor >> 6; + let single_segment = (descriptor & 0x20) != 0; + let unused_bit = (descriptor & 0x10) != 0; + let reserved_bit = (descriptor & 0x08) != 0; + let content_checksum = (descriptor & 0x04) != 0; + let dictionary_id_flag = descriptor & 0x03; + if reserved_bit { + return Err(ZstdFrameError::ReservedBitSet); + } + let need_win = if single_segment { 0 } else { 1 }; + let did_sz = did_field_size(dictionary_id_flag); + let fcs_sz = fcs_field_size(content_size_flag, single_segment); + let header_len = 5 + need_win + did_sz + fcs_sz; + if bytes.len() < header_len { + return Err(ZstdFrameError::TruncatedHeader); + } + let window_descriptor = if single_segment { None } else { Some(bytes[5]) }; + let (window_log, window_size) = if let Some(wd) = window_descriptor { + ( + Some(window_log_from_descriptor(wd)), + Some(window_size_from_descriptor(wd)), + ) + } else { + (None, None) + }; + let did_off = 5 + need_win; + let dictionary_id = match did_sz { + 0 => None, + 1 => Some(u32::from(bytes[did_off])), + 2 => Some(u32::from(read_le_u16(bytes, did_off))), + 4 => Some(read_le_u32(bytes, did_off)), + _ => None, + }; + let fcs_off = did_off + did_sz; + let content_size = match fcs_sz { + 0 => None, + 1 => Some(u64::from(bytes[fcs_off])), + 2 => Some(u64::from(read_le_u16(bytes, fcs_off)) + 256), + 4 => Some(u64::from(read_le_u32(bytes, fcs_off))), + 8 => Some(read_le_u64(bytes, fcs_off)), + _ => None, + }; + let window_size = if single_segment { + content_size + } else { + window_size + }; + Ok(ParsedZstdFrameHeader { + descriptor, + content_size_flag, + single_segment, + unused_bit, + reserved_bit, + content_checksum, + dictionary_id_flag, + window_descriptor, + window_log, + window_size, + dictionary_id, + content_size, + header_len, + }) +} diff --git a/tests/deprecation_aliases.rs b/tests/deprecation_aliases.rs index 6eadd8e..1cc2a6a 100644 --- a/tests/deprecation_aliases.rs +++ b/tests/deprecation_aliases.rs @@ -2,8 +2,9 @@ #![allow(deprecated)] use carbonado::{ - pack_index, PackEntry, PackIndex, PackSegmentRef, MAX_PACK_ENTRIES, PACK_INDEX_FORMAT_LEVEL, - PACK_INDEX_FORMAT_LEVEL_ENCRYPTED, PACK_INDEX_FORMAT_LEVEL_PUBLIC, PACK_INDEX_VERSION, + MAX_PACK_ENTRIES, PACK_INDEX_FORMAT_LEVEL, PACK_INDEX_FORMAT_LEVEL_ENCRYPTED, + PACK_INDEX_FORMAT_LEVEL_PUBLIC, PACK_INDEX_VERSION, PackEntry, PackIndex, PackSegmentRef, + pack_index, }; #[test] diff --git a/tests/determinism_roundtrip.rs b/tests/determinism_roundtrip.rs index fb397a4..546ccc7 100644 --- a/tests/determinism_roundtrip.rs +++ b/tests/determinism_roundtrip.rs @@ -28,18 +28,16 @@ //! ## Directory (**W2b**) //! //! Same-engine catalog+segment codecode/decodec under pinned options. Cross-engine -//! encode residual is **live rust root vs live lean root** (hard-asserted pins), -//! not the committed decode seed. `tests/fixtures/phase3_g9_directory` is -//! **decode-only SSOT** (catalog root may lag live re-encode). -//! -//! Runs under default `backend-rust` and lean freeze features (auto-include). +//! encode residual is **live rust root vs historical Lean AOT catalog pin**. +//! `tests/fixtures/phase3_g9_directory` matches live rust encode after catalog +//! bundle append follows sorted `rel_path` (not `read_dir` order). use std::fs; use std::path::{Path, PathBuf}; use carbonado::{ - constants::Format, decode, decode_outboard, encode_with_nonce, file, - stream_encode_outboard_buffer, structs::Encoded, OutboardEncoded, + OutboardEncoded, constants::Format, decode, decode_outboard, encode_with_nonce, file, + stream_encode_outboard_buffer, structs::Encoded, }; /// Same master as G9 / Phase 2. @@ -72,33 +70,20 @@ const OUTBOARD_COMPRESS: &[u8] = &[6, 7, 14, 15]; /// Live `backend-rust` catalog Bao root for [`dir_files`] + zero master + default options. /// -/// **Not** the committed `phase3_g9_directory` catalog (that seed is decode-only SSOT; -/// catalog packaging can lag while segment mains stay stable). +/// Matches [`PHASE3_SEED_DIR_CATALOG_ROOT`]: encode sorts by `rel_path` before appending +/// verification outboard / FEC (`a.txt` then `sub/b.bin`). const LIVE_RUST_DIR_CATALOG_ROOT: &str = - "0b119f121a003dd4136f340cdcb8de9dc91d8e15d6f402df485e9ffd123cea4e"; + "16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f"; -/// Live `backend-lean` catalog Bao root for the same tree/options as [`LIVE_RUST_DIR_CATALOG_ROOT`]. +/// Historical Lean AOT catalog Bao root for the same tree/options as [`LIVE_RUST_DIR_CATALOG_ROOT`]. const LIVE_LEAN_DIR_CATALOG_ROOT: &str = - "f67b6f49b9d2f3d8ac8b3906e9771dfaa6e2101fe8501b48d24700c3eb64d189"; + "d468ea7a9e8afc13f3c4a533c0d9614ecc6d032dae2255013bcf433c592e07b5"; -/// Committed phase3 G9 directory catalog root — **decode seed only**, not live re-encode golden. +/// Committed phase3 G9 directory catalog root. Live rust encode of [`dir_files`] matches this +/// seed once the catalog bundle is appended in sorted `rel_path` order. const PHASE3_SEED_DIR_CATALOG_ROOT: &str = "16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f"; -#[cfg(feature = "backend-lean")] -fn require_lean_lib() { - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - panic!( - "CARBONADO_LEAN_LIB unset. Build and export first:\n \ - nix build .#libcarbonado -o result-libcarbonado\n \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ - export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ - export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ - # or: just test-lean-ci" - ); - } -} - fn is_encrypted(format: u8) -> bool { Format::from(format).contains(Format::Encryption) } @@ -112,11 +97,7 @@ fn nonce_for(format: u8) -> Option<[u8; 16]> { } fn active_engine() -> &'static str { - if cfg!(feature = "backend-lean") { - "lean" - } else { - "rust" - } + "rust" } // --------------------------------------------------------------------------- @@ -367,8 +348,6 @@ fn decodec_outboard(format: u8, pt: &[u8]) { #[test] fn codecode_body_no_compress_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in BODY_NO_COMPRESS { codecode_body(format, PLAINTEXT); } @@ -376,8 +355,6 @@ fn codecode_body_no_compress_matrix() { #[test] fn decodec_body_no_compress_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in BODY_NO_COMPRESS { decodec_body(format, PLAINTEXT); } @@ -385,8 +362,6 @@ fn decodec_body_no_compress_matrix() { #[test] fn codecode_headered_no_compress_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in HEADERED_NO_COMPRESS { codecode_headered(format, PLAINTEXT); } @@ -394,8 +369,6 @@ fn codecode_headered_no_compress_matrix() { #[test] fn decodec_headered_no_compress_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in HEADERED_NO_COMPRESS { decodec_headered(format, PLAINTEXT); } @@ -403,8 +376,6 @@ fn decodec_headered_no_compress_matrix() { #[test] fn codecode_outboard_no_compress_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in OUTBOARD_NO_COMPRESS { codecode_outboard(format, PLAINTEXT); } @@ -412,8 +383,6 @@ fn codecode_outboard_no_compress_matrix() { #[test] fn decodec_outboard_no_compress_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in OUTBOARD_NO_COMPRESS { decodec_outboard(format, PLAINTEXT); } @@ -425,8 +394,6 @@ fn decodec_outboard_no_compress_matrix() { #[test] fn codecode_body_compress_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in BODY_COMPRESS { codecode_body(format, PLAINTEXT); } @@ -434,8 +401,6 @@ fn codecode_body_compress_same_engine() { #[test] fn decodec_body_compress_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in BODY_COMPRESS { decodec_body(format, PLAINTEXT); } @@ -443,8 +408,6 @@ fn decodec_body_compress_same_engine() { #[test] fn codecode_headered_compress_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in HEADERED_COMPRESS { codecode_headered(format, PLAINTEXT); } @@ -452,8 +415,6 @@ fn codecode_headered_compress_same_engine() { #[test] fn decodec_headered_compress_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in HEADERED_COMPRESS { decodec_headered(format, PLAINTEXT); } @@ -461,8 +422,6 @@ fn decodec_headered_compress_same_engine() { #[test] fn codecode_outboard_compress_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in OUTBOARD_COMPRESS { codecode_outboard(format, PLAINTEXT); } @@ -470,8 +429,6 @@ fn codecode_outboard_compress_same_engine() { #[test] fn decodec_outboard_compress_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); for &format in OUTBOARD_COMPRESS { decodec_outboard(format, PLAINTEXT); } @@ -533,8 +490,6 @@ fn compress_cross_engine_encode_not_bit_identical_documented() { /// bit-matches the golden wire under the same pins. #[test] fn decodec_body_from_g9_fixture_no_compress() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); let engine = active_engine(); let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -648,8 +603,6 @@ fn dir_files() -> [(&'static str, &'static [u8]); 2] { #[test] fn codecode_directory_public_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); let src = tempdir("dir_src"); write_tree(&src, &dir_files()); @@ -697,8 +650,6 @@ fn codecode_directory_public_same_engine() { #[test] fn decodec_directory_public_same_engine() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); let src = tempdir("dir_src_ded"); write_tree(&src, &dir_files()); @@ -751,28 +702,26 @@ fn decodec_directory_public_same_engine() { /// W2b residual canary: **live-vs-live** catalog roots under identical pins. /// /// Compares pinned live rust encode root vs pinned live lean encode root for -/// [`dir_files`] + zero master + default options. Does **not** use -/// `phase3_g9_directory` catalog as a re-encode golden (that seed is decode-only; -/// its catalog root lags live rust while segment mains may still match). +/// [`dir_files`] + zero master + default options. Live rust matches the committed +/// `phase3_g9_directory` catalog (sorted `rel_path` bundle append). Cross-engine +/// residual is rust vs lean catalog bytes (zstd / catalog packaging), not readdir order. /// /// Hard asserts: /// - active engine live root matches its pin (`LIVE_RUST_*` / `LIVE_LEAN_*`) /// - `LIVE_RUST_DIR_CATALOG_ROOT != LIVE_LEAN_DIR_CATALOG_ROOT` (cross-engine residual) -/// - live rust root ≠ phase3 seed catalog (documents seed packaging drift) +/// - live rust root equals the phase3 seed catalog /// - seed catalog file still present (decode SSOT) #[test] fn directory_cross_engine_live_roots_residual() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); - // Pin table integrity: residual is live-vs-live, not seed-vs-live. + // Pin table integrity: residual is live rust vs live lean, not readdir drift. assert_ne!( LIVE_RUST_DIR_CATALOG_ROOT, LIVE_LEAN_DIR_CATALOG_ROOT, "W2b residual pin table: live rust and live lean catalog roots must differ" ); - assert_ne!( + assert_eq!( LIVE_RUST_DIR_CATALOG_ROOT, PHASE3_SEED_DIR_CATALOG_ROOT, - "phase3_g9_directory catalog is decode-only SSOT — not equal to live rust re-encode" + "live rust directory encode must match the phase3_g9_directory catalog seed" ); let fixture = @@ -791,40 +740,19 @@ fn directory_cross_engine_live_roots_residual() { .unwrap_or_else(|e| panic!("[{}] dir encode for residual: {e}", active_engine())); let live = hex32(&arch.catalog_bao_root); - #[cfg(feature = "backend-rust")] - { - assert_eq!( - live, LIVE_RUST_DIR_CATALOG_ROOT, - "live rust directory catalog root drifted from W2b pin — update \ - LIVE_RUST_DIR_CATALOG_ROOT (and re-check lean residual) if intentional" - ); - assert_ne!( - live, PHASE3_SEED_DIR_CATALOG_ROOT, - "live rust catalog must not silently equal stale seed (decode-only SSOT)" - ); - assert_ne!( - live, LIVE_LEAN_DIR_CATALOG_ROOT, - "W2b residual: live rust catalog must still differ from live lean pin" - ); - } - - #[cfg(feature = "backend-lean")] - { - assert_eq!( - live, LIVE_LEAN_DIR_CATALOG_ROOT, - "live lean directory catalog root drifted from W2b pin — update \ - LIVE_LEAN_DIR_CATALOG_ROOT (and re-check rust residual) if intentional" - ); - assert_ne!( - live, LIVE_RUST_DIR_CATALOG_ROOT, - "W2b residual evidence: live lean catalog must still differ from live rust pin \ - (if equal, residual may have closed — investigate zstd/catalog packaging)" - ); - assert_ne!( - live, PHASE3_SEED_DIR_CATALOG_ROOT, - "live lean catalog must not equal decode-only seed root" - ); - } + assert_eq!( + live, LIVE_RUST_DIR_CATALOG_ROOT, + "live rust directory catalog root drifted from pin — update \ + LIVE_RUST_DIR_CATALOG_ROOT if intentional" + ); + assert_eq!( + live, PHASE3_SEED_DIR_CATALOG_ROOT, + "live rust catalog must match the phase3_g9_directory seed (sorted rel_path bundle)" + ); + assert_ne!( + live, LIVE_LEAN_DIR_CATALOG_ROOT, + "historical Lean AOT catalog pin must still differ from live rust encode" + ); let _ = fs::remove_dir_all(&src); let _ = fs::remove_dir_all(&enc); diff --git a/tests/directory_archive.rs b/tests/directory_archive.rs index d014fa0..cfdd372 100644 --- a/tests/directory_archive.rs +++ b/tests/directory_archive.rs @@ -3,11 +3,11 @@ mod common; #[cfg(feature = "ots")] -use carbonado::ots::{verify_stamp, OtsPolicy}; +use carbonado::ots::{OtsPolicy, verify_stamp}; use carbonado::{ adamantine::{ - decode_adamantine, encode_adamantine, ADAMANTINE_CARBONADO_FMT_ENCRYPTED, - ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_MAGIC, + ADAMANTINE_CARBONADO_FMT_ENCRYPTED, ADAMANTINE_CARBONADO_FMT_PUBLIC, + ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_MAGIC, decode_adamantine, encode_adamantine, }, adamantine_payload::{ build_adamantine_payload, fec_slice_from_bundle, split_adamantine_payload, @@ -15,18 +15,18 @@ use carbonado::{ }, decode_outboard, directory::format_policy::{ - SegmentFormatPolicy, SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, + SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, SegmentFormatPolicy, }, encode_outboard, error::CarbonadoError, file::{ - decode, decode_directory, encode_directory, encode_directory_with_options, - DirectoryEncodeOptions, DIRECTORY_ARCHIVE_FORMAT, DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, - DIRECTORY_TEST_SEGMENT_BUDGET, + DIRECTORY_ARCHIVE_FORMAT, DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, + DIRECTORY_TEST_SEGMENT_BUDGET, DirectoryEncodeOptions, decode, decode_directory, + encode_directory, encode_directory_with_options, }, filepack_manifest::{ - FilepackEntry, FilepackManifest, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, - FILEPACK_MANIFEST_VERSION, MAX_SEGMENT_MAIN_LEN, + FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, FilepackEntry, + FilepackManifest, MAX_SEGMENT_MAIN_LEN, }, scrub_outboard, }; @@ -681,6 +681,68 @@ fn decode_rejects_tampered_catalog_body_returns_verification_failed() { ); } +/// Same tree, opposite creation order and a reversed `read_dir` walk, must produce +/// one catalog Bao root. Bundle FEC/outboard append order follows sorted `rel_path`. +#[test] +#[cfg(debug_assertions)] +fn directory_encode_independent_of_readdir_order() { + use carbonado::file::directory_encode_test_hooks::with_reverse_readdir; + + let src_ab = tempdir("order_src_ab"); + fs::write(src_ab.join("a.txt"), b"phase3 g9 hello").expect("a.txt first"); + fs::create_dir_all(src_ab.join("sub")).expect("sub"); + fs::write(src_ab.join("sub/b.bin"), b"nested data").expect("b.bin second"); + + let src_ba = tempdir("order_src_ba"); + fs::create_dir_all(src_ba.join("sub")).expect("sub first"); + fs::write(src_ba.join("sub/b.bin"), b"nested data").expect("b.bin first"); + fs::write(src_ba.join("a.txt"), b"phase3 g9 hello").expect("a.txt second"); + + let enc_ab = tempdir("order_enc_ab"); + let arch_ab = encode_directory(&ZERO_KEY, &src_ab, &enc_ab).expect("encode creation a-then-b"); + + let enc_ba = tempdir("order_enc_ba"); + let arch_ba = encode_directory(&ZERO_KEY, &src_ba, &enc_ba).expect("encode creation b-then-a"); + + let enc_rev = tempdir("order_enc_rev"); + let arch_rev = with_reverse_readdir(|| { + encode_directory(&ZERO_KEY, &src_ab, &enc_rev).expect("encode reversed read_dir") + }); + + assert_eq!( + arch_ab.catalog_bao_root, arch_ba.catalog_bao_root, + "catalog Bao root must not depend on file creation order" + ); + assert_eq!( + arch_ab.catalog_bao_root, arch_rev.catalog_bao_root, + "catalog Bao root must not depend on read_dir listing order" + ); + + let catalog_path = + adam_catalog_path(&enc_ab, &arch_ab.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); + let (manifest, _) = load_catalog_manifest_and_parts(&catalog_path, &arch_ab.catalog_bao_root); + assert_eq!( + manifest + .entries + .iter() + .map(|e| e.rel_path.as_str()) + .collect::>(), + vec!["a.txt", "sub/b.bin"] + ); + let a_off = manifest.entries[0].segments[0].fec_parity_offset; + let b_off = manifest.entries[1].segments[0].fec_parity_offset; + assert!( + a_off < b_off, + "FEC blobs must be appended in sorted rel_path order, got a.txt offset {a_off} sub/b.bin {b_off}" + ); + + let _ = fs::remove_dir_all(&src_ab); + let _ = fs::remove_dir_all(&src_ba); + let _ = fs::remove_dir_all(&enc_ab); + let _ = fs::remove_dir_all(&enc_ba); + let _ = fs::remove_dir_all(&enc_rev); +} + #[test] fn encode_rollback_removes_segments_on_symlink_error() { let src = tempdir("symlink_src"); @@ -1388,7 +1450,7 @@ fn decode_rejects_headered_segment_main_layout() { /// `scrub_outboard` recovers corrupt bare mains (≤4 shard taints) using bundle parity slices. #[test] fn directory_segment_corruption_bao_bundle_extract_scrub_roundtrip() { - use common::corruption::{scattered_outboard_main_knockout, OutboardShardLayout}; + use common::corruption::{OutboardShardLayout, scattered_outboard_main_knockout}; use rand::thread_rng; let src = tempdir("fec_scrub_src"); @@ -1549,7 +1611,7 @@ fn directory_segment_corruption_bao_bundle_extract_scrub_roundtrip() { #[test] fn directory_fec_scrub_matrix_c12_c13_c14_c15() { - use common::corruption::{scattered_outboard_main_knockout, OutboardShardLayout}; + use common::corruption::{OutboardShardLayout, scattered_outboard_main_knockout}; use rand::thread_rng; for (label, policy, key, encrypted) in [ diff --git a/tests/fec_chaos.rs b/tests/fec_chaos.rs index 8435cfa..ffca373 100644 --- a/tests/fec_chaos.rs +++ b/tests/fec_chaos.rs @@ -11,8 +11,8 @@ use carbonado::{ structs::Encoded, }; use common::corruption::{ - scattered_outboard_main_knockout, scattered_stream_knockout, InboardShardLayout, - OutboardShardLayout, + InboardShardLayout, OutboardShardLayout, scattered_outboard_main_knockout, + scattered_stream_knockout, }; use common::format_matrix::{format_label, verification_fec_levels}; use proptest::prelude::*; diff --git a/tests/fec_scrub_matrix.rs b/tests/fec_scrub_matrix.rs index 95eb5cb..7259548 100644 --- a/tests/fec_scrub_matrix.rs +++ b/tests/fec_scrub_matrix.rs @@ -7,7 +7,7 @@ use carbonado::{ decode, decode_outboard, encode, encode_outboard, error::CarbonadoError, scrub, scrub_outboard, structs::Encoded, }; -use common::corruption::{flip_byte, InboardShardLayout}; +use common::corruption::{InboardShardLayout, flip_byte}; use common::format_matrix::{format_label, public_fec_levels, verification_fec_levels}; use rand::Rng; diff --git a/tests/filepack_interop.rs b/tests/filepack_interop.rs index 5af072d..764609b 100644 --- a/tests/filepack_interop.rs +++ b/tests/filepack_interop.rs @@ -7,15 +7,14 @@ use carbonado::{ adamantine::decode_adamantine, adamantine_payload::split_adamantine_payload, error::CarbonadoError, - file::{decode, encode_directory, DirectoryArchive, DIRECTORY_ARCHIVE_FORMAT}, + file::{DIRECTORY_ARCHIVE_FORMAT, DirectoryArchive, decode, encode_directory}, filepack::{self, parse_filepack_cbor}, filepack_manifest::{ - FilepackEntry, FilepackManifest, FilepackSegmentMap, SegmentRef, - FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, MAX_REL_PATH_LEN, + FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, FilepackEntry, + FilepackManifest, FilepackSegmentMap, MAX_REL_PATH_LEN, SegmentRef, }, }; use ciborium::value::Value as CborValue; -#[cfg(feature = "backend-rust")] use serde_json::Value as JsonValue; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; @@ -327,7 +326,6 @@ fn parse_rejects_oversized_rel_path_at_flatten() { ); } -#[cfg(feature = "backend-rust")] fn golden_fixture_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/directory_interop_golden.json") } @@ -398,10 +396,6 @@ fn adamantine_decimal_segment_naming_contract() { } /// Pins rust-engine directory roots / rkyv SHA-256 for `tests/samples`. -/// Under `backend-lean`, segment crypto differs until full G9 encode bit-match; -/// dual-suite directory green is exercised by `lean_backend_phase3` + functional -/// `directory_archive` tests (not rust-root checksums). -#[cfg(feature = "backend-rust")] #[test] fn golden_directory_interop_checksums_and_manifest_wire() { let fixture_text = fs::read_to_string(golden_fixture_path()).expect("read golden fixture"); diff --git a/tests/fixtures/g9/README.md b/tests/fixtures/g9/README.md index 546f7e3..75c4a5e 100644 --- a/tests/fixtures/g9/README.md +++ b/tests/fixtures/g9/README.md @@ -1,6 +1,6 @@ # G9 cross-backend fixtures (Milestone R8) -Committed wire goldens for **Rust ↔ Lean** encode/decode parity on **no-compress** formats. +Committed wire goldens. Rust still **decodes** historical Lean AOT bytes. There is no live Cargo Lean encoder. ## Pins @@ -20,7 +20,7 @@ Committed wire goldens for **Rust ↔ Lean** encode/decode parity on **no-compre ```text g9/ rust/ # encoded under default backend-rust - lean/ # encoded under backend-lean + fixed NONCE when encrypted + lean/ # historical Lean AOT encode + fixed NONCE when encrypted ``` ### Body (`body_c{fmt}.bin` + `.meta.json`) @@ -53,38 +53,29 @@ geometry yields an empty post-order outboard for some tiny payloads. ## Matrix scope (R8 DoD) -- **In scope:** body/headered/outboard public + encrypted fixed-nonce, **both directions**. -- **Continuous re-encode bit-match (CI under lean):** body c0/c1/c4/c5/c8/c9/c12/c13; - headered c4/c5/c12/c13; outboard c4/c5/c12/c13 (no compress). Live lean re-encode vs - rust golden. -- **Committed fixture identity:** rust/ and lean/ trees are regenerated together under the - same pins; c14 outboard mains may differ (Compression residual). +- **In scope now:** Rust decode of committed `lean/` goldens + rust self-roundtrip. +- **Committed fixture identity:** `lean/` is frozen historical AOT output; `just g9-gen-fixtures` + regenerates `rust/` only. c14 outboard mains may differ (Compression residual). - **Residuals (W2 settled):** cross-engine Compression encode bit-match is **permanent** - (W2a — zstd frames differ; decode interop only). Cross-engine directory encode bit-match + (W2a — zstd frames differ; decode interop only). Frame **parameters** (magic, checksum + off, no dict, rust `0x00`+windowLog 25 vs lean `0x20`+FCS) are specified in + `Carbonado.Compress` and checked by `tests/zstd_frame_params.rs`. Cross-engine directory encode bit-match is **permanent** (W2b; `phase3_g9_directory` decode seed remains SSOT). Same-engine codecode/decodec shipped in `tests/determinism_roundtrip.rs` (W2d). ## Regeneration ```bash -# Both engines (recommended) just g9-gen-fixtures - -# Or manually: +# or: G9_WRITE_FIXTURES=1 cargo test --test g9_cross_backend write_fixtures -- --ignored --nocapture - -eval "$(just _lean-env)" -G9_WRITE_FIXTURES=1 cargo test --no-default-features --features "backend-lean,pqc,ots" \ - --test g9_cross_backend write_fixtures -- --ignored --nocapture ``` -Do **not** hand-edit binaries; regenerate and commit both `rust/` and `lean/` trees together. +Do **not** hand-edit binaries. `lean/` goldens are historical; do not invent a Cargo Lean encoder to regenerate them. ## Tests -| Command | Direction | +| Command | What it does | |---------|-----------| -| `cargo test --test g9_cross_backend` | lean→rust + self RT + zero-nonce contract (no libcarbonado) | -| lean features + `CARBONADO_LEAN_LIB` | rust→lean + continuous re-encode bit-match + self RT | -| `just test-g9` | both directions | -| `just test-lean-ci` | full dual suite (includes this file after R7 freeze) | +| `cargo test --test g9_cross_backend` | Rust decode of `lean/` goldens + rust self-roundtrip + zero-nonce contract | +| `just test-g9` | same | diff --git a/tests/fixtures/rkyv/README.md b/tests/fixtures/rkyv/README.md index 8efcfd6..7085f56 100644 --- a/tests/fixtures/rkyv/README.md +++ b/tests/fixtures/rkyv/README.md @@ -25,6 +25,5 @@ cargo run --example dump_rkyv_r9 --features backend-rust **W3 acceptance:** Lean `encodeRkyvManifest` must bit-match these fixtures (encode twice → same bytes). AOT demo greps: `rkyv FilepackManifestWire encode/decode goldens ok`. -**Dual-suite honesty:** directory encode/decode under `backend-lean` still uses **Rust rkyv -composition** as product SSOT (segment/catalog *crypto* via Lean C ABI). Pure Lean path is -wire-compatible when claimed; dual-suite does **not** require pure Lean encode. +Rust directory encode uses this rkyv wire. Pure Lean `encodeRkyvManifest` must bit-match +these fixtures. There is no Cargo Lean directory encoder. diff --git a/tests/format.rs b/tests/format.rs index 699ace0..aeeb59a 100644 --- a/tests/format.rs +++ b/tests/format.rs @@ -314,24 +314,24 @@ fn outboard_and_keyed_c_number() -> Result<()> { assert!(matches!(err_z, CarbonadoError::MissingFecParity)); // tampered sidecar (flip byte in a real ob if present) -> verification error (strict) - if let Some(mut good_ob) = o4.verification_outboard.clone() { - if !good_ob.is_empty() { - good_ob[0] ^= 0xff; - let err_verify = decode_outboard( - &PUBLIC_MASTER, - o4.hash.as_bytes(), - &o4.main, - Some(good_ob.as_slice()), - None, - o4.info.padding_len, - 4, - ) - .unwrap_err(); - assert!(matches!( - err_verify, - CarbonadoError::OutboardVerificationFailed(_) - )); - } + if let Some(mut good_ob) = o4.verification_outboard.clone() + && !good_ob.is_empty() + { + good_ob[0] ^= 0xff; + let err_verify = decode_outboard( + &PUBLIC_MASTER, + o4.hash.as_bytes(), + &o4.main, + Some(good_ob.as_slice()), + None, + o4.info.padding_len, + 4, + ) + .unwrap_err(); + assert!(matches!( + err_verify, + CarbonadoError::OutboardVerificationFailed(_) + )); } Ok(()) diff --git a/tests/format_policy.rs b/tests/format_policy.rs index 96c32a7..1ff04da 100644 --- a/tests/format_policy.rs +++ b/tests/format_policy.rs @@ -2,12 +2,12 @@ use carbonado::{ directory::{ + SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, format_policy::{ - is_likely_incompressible, resolve_catalog_format, SegmentFormatPolicy, SEGMENT_FORMAT_ENCRYPTED_RAW, SEGMENT_FORMAT_PUBLIC_COMPRESSED, - SEGMENT_FORMAT_PUBLIC_RAW, + SEGMENT_FORMAT_PUBLIC_RAW, SegmentFormatPolicy, is_likely_incompressible, + resolve_catalog_format, }, - SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, }, error::CarbonadoError, filepack_manifest::{ diff --git a/tests/g9_cross_backend.rs b/tests/g9_cross_backend.rs index bca0c08..6fa68b3 100644 --- a/tests/g9_cross_backend.rs +++ b/tests/g9_cross_backend.rs @@ -1,31 +1,21 @@ -//! Milestone R8 / G9: full cross-backend encode/decode matrix (no-compress formats). +//! G9: Rust decode of committed Lean AOT goldens, plus rust self-roundtrip. //! -//! Both directions: -//! - **rust→lean** (`#[cfg(feature = "backend-lean")]`): decode committed `tests/fixtures/g9/rust/*` -//! - **lean→rust** (`#[cfg(feature = "backend-rust")]`): decode committed `tests/fixtures/g9/lean/*` +//! There is no Cargo Lean engine. Committed `tests/fixtures/g9/lean/*` bytes stay as +//! historical Lean-AOT goldens that the Rust library must still decode. //! -//! Fixtures are no-compress formats only (c0/c1/c4/c5/c8/c9/c12/c13 + selected headered/outboard) -//! so re-encode bit-match is meaningful. Compression (Zstd) is an intentional residual. -//! -//! Regenerate (writes under `tests/fixtures/g9/{rust,lean}/` for the active backend): +//! Regenerate rust goldens: //! ```bash -//! # Rust goldens //! G9_WRITE_FIXTURES=1 cargo test --test g9_cross_backend write_fixtures -- --ignored --nocapture -//! # Lean goldens (needs libcarbonado) -//! eval "$(just _lean-env)" -//! G9_WRITE_FIXTURES=1 cargo test --no-default-features --features "backend-lean,pqc,ots" \ -//! --test g9_cross_backend write_fixtures -- --ignored --nocapture -//! # or: just g9-gen-fixtures //! ``` //! -//! Pins: [`MASTER`], [`NONCE`], [`PLAINTEXT`] — same MASTER/NONCE as `lean_backend_phase2`. +//! Pins: [`MASTER`], [`NONCE`], [`PLAINTEXT`]. use std::fs; use std::path::{Path, PathBuf}; use carbonado::{ - constants::Format, decode, decode_outboard, encode_with_nonce, file, - stream_encode_outboard_buffer, structs::Encoded, OutboardEncoded, + OutboardEncoded, constants::Format, decode, decode_outboard, encode_with_nonce, file, + stream_encode_outboard_buffer, structs::Encoded, }; use serde::{Deserialize, Serialize}; @@ -77,20 +67,6 @@ fn require_write_env() { } } -#[cfg(feature = "backend-lean")] -fn require_lean_lib() { - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - panic!( - "CARBONADO_LEAN_LIB unset. Build and export first:\n \ - nix build .#libcarbonado -o result-libcarbonado\n \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ - export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ - export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ - # or: just test-lean-ci / just g9-gen-fixtures" - ); - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] struct BodyMeta { layout: String, @@ -148,11 +124,7 @@ fn write_json(path: &Path, value: &T) { } fn active_engine() -> &'static str { - if cfg!(feature = "backend-lean") { - "lean" - } else { - "rust" - } + "rust" } fn is_encrypted(format: u8) -> bool { @@ -227,8 +199,6 @@ fn encode_outboard_fixture(format: u8) -> (OutboardEncoded, Option>, boo #[ignore = "set G9_WRITE_FIXTURES=1 to regenerate tests/fixtures/g9/{engine}/"] fn write_fixtures() { require_write_env(); - #[cfg(feature = "backend-lean")] - require_lean_lib(); let engine = active_engine(); let root = engine_dir(engine); @@ -556,11 +526,10 @@ fn decode_outboard_fixture(engine: &str, format: u8) { } // --------------------------------------------------------------------------- -// lean→rust: decode lean fixtures under default backend-rust +// Decode committed Lean AOT goldens with the Rust library // --------------------------------------------------------------------------- -#[cfg(feature = "backend-rust")] -mod lean_to_rust { +mod lean_goldens { use super::*; #[test] @@ -585,149 +554,12 @@ mod lean_to_rust { } } -// --------------------------------------------------------------------------- -// rust→lean: decode rust fixtures under backend-lean (+ optional re-encode bit-match) -// --------------------------------------------------------------------------- - -#[cfg(feature = "backend-lean")] -mod rust_to_lean { - use super::*; - use carbonado::structs::Encoded; - - #[test] - fn body_matrix() { - require_lean_lib(); - for &format in BODY_FORMATS { - decode_body_fixture("rust", format); - } - } - - #[test] - fn headered_matrix() { - require_lean_lib(); - for &format in HEADERED_FORMATS { - decode_headered_fixture("rust", format); - } - } - - #[test] - fn outboard_matrix() { - require_lean_lib(); - for &format in OUTBOARD_FORMATS { - decode_outboard_fixture("rust", format); - } - } - - /// Public no-compress body re-encode under lean must bit-match rust golden. - #[test] - fn public_body_reencode_bit_match() { - require_lean_lib(); - for &format in &[0u8, 4, 8, 12] { - let (rust_body, meta) = load_body("rust", format); - let Encoded(lean_body, lean_hash, _) = - encode_with_nonce(&MASTER, PLAINTEXT, format, None) - .unwrap_or_else(|e| panic!("lean re-encode c{format}: {e}")); - assert_eq!( - lean_body, rust_body, - "lean re-encode must bit-match rust body c{format}" - ); - assert_eq!( - to_hex(lean_hash.as_bytes()), - meta.hash_hex, - "lean re-encode hash c{format}" - ); - } - } - - /// Encrypted fixed-nonce body re-encode under lean must bit-match rust golden. - #[test] - fn encrypted_body_reencode_bit_match() { - require_lean_lib(); - for &format in &[1u8, 5, 9, 13] { - let (rust_body, meta) = load_body("rust", format); - let Encoded(lean_body, lean_hash, _) = - encode_with_nonce(&MASTER, PLAINTEXT, format, Some(NONCE)) - .unwrap_or_else(|e| panic!("lean re-encode enc c{format}: {e}")); - assert_eq!( - lean_body, rust_body, - "lean re-encode must bit-match rust encrypted body c{format}" - ); - assert_eq!( - to_hex(lean_hash.as_bytes()), - meta.hash_hex, - "lean re-encode enc hash c{format}" - ); - } - } - - /// Headered re-encode under lean must bit-match rust golden (no-compress formats). - #[test] - fn headered_reencode_bit_match() { - require_lean_lib(); - for &format in HEADERED_FORMATS { - let (rust_arch, meta) = load_headered("rust", format); - let nonce = if is_encrypted(format) { - Some(NONCE) - } else { - None - }; - let (lean_arch, _) = file::encode_with_nonce(&MASTER, PLAINTEXT, format, None, nonce) - .unwrap_or_else(|e| panic!("lean re-encode headered c{format}: {e}")); - assert_eq!( - lean_arch, rust_arch, - "lean re-encode must bit-match rust headered c{format}" - ); - let (hdr, _) = file::decode(&MASTER, &lean_arch).expect("decode lean headered"); - assert_eq!(to_hex(hdr.hash.as_bytes()), meta.hash_hex); - } - } - - /// Outboard re-encode under lean must bit-match rust golden (skip compressed c14). - #[test] - fn outboard_reencode_bit_match_no_compress() { - require_lean_lib(); - for &format in &[4u8, 5, 12, 13] { - let loaded = load_outboard("rust", format); - let (oenc, header, _) = encode_outboard_fixture(format); - assert_eq!( - oenc.main, loaded.main, - "lean re-encode main must bit-match rust outboard c{format}" - ); - assert_eq!( - oenc.verification_outboard.as_deref(), - loaded.verification_outboard.as_deref(), - "outboard c{format} verification outboard" - ); - assert_eq!( - oenc.fec_parity.as_deref(), - loaded.fec_parity.as_deref(), - "outboard c{format} fec parity" - ); - if is_encrypted(format) { - assert_eq!( - header.as_deref(), - loaded.header.as_deref(), - "outboard c{format} header.bin" - ); - } - assert_eq!( - to_hex(oenc.hash.as_bytes()), - loaded.meta.hash_hex, - "outboard c{format} hash" - ); - } - } -} - -/// `Some([0u8; 16])` must be honored literally on the active backend (no CSPRNG override). +/// `Some([0u8; 16])` must be honored literally (no CSPRNG override). /// /// Uses c1 (encryption-only body, embedded layout) and c5 headered so the zero nonce is -/// visible on the wire. Dual-backend identity is the same contract on both engines. +/// visible on the wire. #[test] fn explicit_zero_nonce_is_honored_headered_and_body() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); - let zero = [0u8; 16]; let pt = b"g9 zero nonce dual contract"; @@ -763,9 +595,6 @@ fn explicit_zero_nonce_is_honored_headered_and_body() { #[test] fn active_backend_self_roundtrip_matrix() { - #[cfg(feature = "backend-lean")] - require_lean_lib(); - for &format in BODY_FORMATS { let (body, hash, pad) = encode_body(format); let d = decode(&MASTER, &hash, &body, pad, format).expect("body decode"); diff --git a/tests/lean_backend_phase2.rs b/tests/lean_backend_phase2.rs deleted file mode 100644 index f7a6a22..0000000 --- a/tests/lean_backend_phase2.rs +++ /dev/null @@ -1,512 +0,0 @@ -//! Phase 2 allowlist for `backend-lean` (docs/TEST_CONTRACT.md). -//! -//! Covers outboard roundtrip, scrub happy/error paths, verify_slice, stream buffer -//! composition, and G9 cross-backend body/headered buffers (Rust encode → Lean decode -//! requires both engines available — under pure `backend-lean` we check Lean self -//! parity and document G9 via lean-encode / lean-decode identity against fixed vectors). -//! -//! ```bash -//! nix build .#libcarbonado -o result-libcarbonado -//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB -//! cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_phase2 -//! # or: just test-lean-phase2 -//! ``` -//! Only compiled under `backend-lean` (avoids breaking default/`backend-rust` clippy of all targets). - -#![cfg(feature = "backend-lean")] - -use carbonado::{ - decode, decode_outboard, encode, encode_outboard, error::CarbonadoError, extract_slice, file, - scrub, scrub_outboard, stream_decode_buffer, stream_encode_buffer, structs::Encoded, - verify_slice, OutboardEncoded, -}; - -const MASTER: [u8; 32] = [ - 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, - 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, -]; - -const NONCE: [u8; 16] = [ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, -]; - -fn require_lean_lib() { - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - panic!( - "CARBONADO_LEAN_LIB unset. Build and export first:\n \ - nix build .#libcarbonado -o result-libcarbonado\n \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ - export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ - export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ - # or: just test-lean-phase2" - ); - } -} - -#[test] -fn abi_version_is_one() { - require_lean_lib(); - assert_eq!(carbonado::backend::lean::abi_version(), 1); -} - -#[test] -fn outboard_roundtrip_public_c4_c12_c14() { - require_lean_lib(); - let plaintext = b"phase2 outboard public smoke"; - for format in [4u8, 12, 14] { - let oenc = encode_outboard(&MASTER, plaintext, format) - .unwrap_or_else(|e| panic!("encode_outboard c{format}: {e}")); - assert_eq!(oenc.info.input_len, plaintext.len() as u32); - if format & 0x4 != 0 { - assert!( - oenc.verification_outboard.is_some(), - "c{format}: Verification formats always yield Some(outboard) (may be empty single-leaf)" - ); - } - if format & 0x8 != 0 { - assert!( - oenc.fec_parity.is_some(), - "c{format}: Fec formats always yield Some(parity)" - ); - assert_eq!( - oenc.info.bytes_ecc, - oenc.fec_parity - .as_ref() - .map(|p| p.len() as u32) - .unwrap_or(0), - "c{format}: bytes_ecc must equal parity sidecar length" - ); - // plaintext is non-empty smoke payload → FEC parity sidecar must be non-empty. - assert!( - oenc.fec_parity.as_ref().is_some_and(|p| !p.is_empty()), - "c{format}: expected non-empty FEC parity for non-empty plaintext" - ); - } - let decoded = decode_outboard( - &MASTER, - oenc.hash.as_bytes(), - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - oenc.info.padding_len, - format, - ) - .unwrap_or_else(|e| panic!("decode_outboard c{format}: {e}")); - assert_eq!(decoded, plaintext, "c{format} outboard plaintext mismatch"); - } -} - -#[test] -fn outboard_encrypted_fixed_nonce_roundtrip_c5() { - require_lean_lib(); - let plaintext = b"encrypted outboard fixed nonce"; - // c5 = Encryption | Verification — low-level embedded-nonce layout - let oenc = - carbonado::backend::lean::encode_outboard(&MASTER, plaintext, 5, Some(&NONCE), false) - .expect("lean encode_outboard c5 embedded"); - // Embedded layout embeds 16-byte nonce in main. - assert!( - oenc.main.len() >= 16 + 64, - "embedded main must hold nonce+tag" - ); - let decoded = carbonado::backend::lean::decode_outboard( - &MASTER, - oenc.hash.as_bytes(), - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - oenc.info.padding_len, - 5, - None, - false, - ) - .expect("lean decode_outboard c5 embedded"); - assert_eq!(decoded, plaintext); -} - -#[test] -fn outboard_header_path_encrypted_matches_file_layout() { - require_lean_lib(); - let plaintext = b"header-path outboard encrypted"; - // c5 with header_path=true → bare main is [tag|ct] (matches file::encode_outboard). - let oenc = carbonado::backend::lean::encode_outboard(&MASTER, plaintext, 5, Some(&NONCE), true) - .expect("lean encode_outboard c5 header_path"); - // Header-path main starts with tag (64 B), not a random-looking nonce prefix alone. - assert!(oenc.main.len() >= 64, "header-path main has at least tag"); - // Embedded would be 16 longer for same pt (nonce prefix); header_path is shorter by 16. - let oenc_emb = - carbonado::backend::lean::encode_outboard(&MASTER, plaintext, 5, Some(&NONCE), false) - .expect("embedded"); - assert_eq!( - oenc_emb.main.len(), - oenc.main.len() + 16, - "header-path main omits 16-byte embedded nonce" - ); - let decoded = carbonado::backend::lean::decode_outboard( - &MASTER, - oenc.hash.as_bytes(), - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - oenc.info.padding_len, - 5, - Some(&NONCE), - true, - ) - .expect("lean decode_outboard c5 header_path"); - assert_eq!(decoded, plaintext); - - // High-level file::encode_outboard under lean uses header_path (Some(payload_nonce)). - let (hdr, fo) = - file::encode_outboard(&MASTER, plaintext, 5, None).expect("file encode_outboard"); - let hdr = hdr.expect("encrypted outboard returns Header"); - let hdr_bytes = hdr.try_to_vec().expect("hdr vec"); - let d2 = file::decode_outboard( - &MASTER, - fo.hash.as_bytes(), - Some(&hdr_bytes), - &fo.main, - fo.verification_outboard.as_deref(), - fo.fec_parity.as_deref(), - fo.info.padding_len, - 5, - ) - .expect("file decode_outboard"); - assert_eq!(d2, plaintext); -} - -#[test] -fn inboard_scrub_pristine_unnecessary() { - require_lean_lib(); - let plaintext = b"scrub pristine"; - // c12 = Verification | Fec (public) - let Encoded(body, hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); - let err = scrub(&body, hash.as_bytes(), &info, 12).expect_err("pristine scrub"); - assert!( - matches!(err, CarbonadoError::UnnecessaryScrub), - "expected UnnecessaryScrub, got {err:?}" - ); -} - -#[test] -fn inboard_scrub_requires_verification() { - require_lean_lib(); - let plaintext = b"no verification bit"; - let Encoded(body, hash, info) = encode(&MASTER, plaintext, 0).expect("encode c0"); - let err = scrub(&body, hash.as_bytes(), &info, 0).expect_err("scrub c0"); - assert!( - matches!(err, CarbonadoError::ScrubRequiresVerification), - "expected ScrubRequiresVerification, got {err:?}" - ); -} - -#[test] -fn inboard_scrub_recovers_bitflip_in_fec_body() { - require_lean_lib(); - let plaintext = b"scrub recover bitflip phase2"; - let Encoded(mut body, hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); - // Flip a byte deep in the inboard (past 8-byte length prefix) to taint ≤1 shard. - if body.len() > 64 { - let i = body.len() / 2; - body[i] ^= 0xff; - } else if !body.is_empty() { - let i = body.len() - 1; - body[i] ^= 0x01; - } - let recovered = - scrub(&body, hash.as_bytes(), &info, 12).unwrap_or_else(|e| panic!("scrub recover: {e}")); - assert_eq!( - recovered.len(), - encode(&MASTER, plaintext, 12).unwrap().0.len() - ); - // Decode recovered body - let decoded = decode(&MASTER, hash.as_bytes(), &recovered, info.padding_len, 12) - .expect("decode recovered"); - assert_eq!(decoded, plaintext); -} - -#[test] -fn outboard_scrub_pristine_unnecessary() { - require_lean_lib(); - let plaintext = b"outboard scrub pristine"; - let oenc = encode_outboard(&MASTER, plaintext, 12).expect("encode_outboard c12"); - let err = scrub_outboard( - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - &oenc.info, - 12, - oenc.hash.as_bytes(), - ) - .expect_err("pristine outboard scrub"); - assert!( - matches!(err, CarbonadoError::UnnecessaryScrub), - "expected UnnecessaryScrub, got {err:?}" - ); -} - -#[test] -fn outboard_scrub_recovers_main_damage() { - require_lean_lib(); - let plaintext = b"outboard scrub damage recover"; - let oenc = encode_outboard(&MASTER, plaintext, 12).expect("encode_outboard c12"); - let mut damaged = oenc.main.clone(); - if damaged.len() > 8 { - damaged[0] ^= 0xff; - damaged[1] ^= 0xaa; - } - let recovered = scrub_outboard( - &damaged, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - &oenc.info, - 12, - oenc.hash.as_bytes(), - ) - .unwrap_or_else(|e| panic!("scrub_outboard recover: {e}")); - let decoded = decode_outboard( - &MASTER, - oenc.hash.as_bytes(), - &recovered, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - oenc.info.padding_len, - 12, - ) - .expect("decode recovered bare"); - assert_eq!(decoded, plaintext); -} - -#[test] -fn verify_slice_c4_content() { - require_lean_lib(); - let plaintext = b"slice verify content match!!"; - let Encoded(body, hash, info) = encode(&MASTER, plaintext, 4).expect("encode c4"); - // For non-FEC, verifiable_slice_count is 0; use count=1 for first leaf. - let _ = info.verifiable_slice_count; - let got = verify_slice(&body, 0, 1, hash.as_bytes(), 4).expect("verify_slice"); - let n = got.len().min(plaintext.len()); - assert_eq!(&got[..n], &plaintext[..n]); - let extracted = extract_slice(&body, 0, hash.as_bytes(), 4).expect("extract_slice"); - assert_eq!(extracted, got); -} - -#[test] -fn stream_buffer_composes_over_lean_body() { - require_lean_lib(); - // Under backend-lean, stream_encode_buffer / stream_decode_buffer compose over Lean C ABI. - let plaintext = b"stream buffer compose"; - let (body, hash, info) = - stream_encode_buffer(&MASTER, plaintext, 4).expect("stream_encode_buffer"); - let decoded = stream_decode_buffer(&MASTER, hash.as_bytes(), &body, info.padding_len, 4) - .expect("stream_decode_buffer"); - assert_eq!(decoded, plaintext); - // Same engine as crate::encode - let Encoded(body2, hash2, _) = encode(&MASTER, plaintext, 4).expect("encode"); - assert_eq!(body, body2); - assert_eq!(hash, hash2); -} - -#[test] -fn g9_headered_public_roundtrip_matrix() { - require_lean_lib(); - // G9 start: lean encode → lean decode for headered public formats (same ABI). - // Full Rust↔Lean cross process needs both engines; buffer identity under lean is - // the Phase 2 G9 seed. Cross-process G9 continues as residual until CI freezes both. - let plaintext = b"g9 headered public"; - for level in [0u8, 4, 12, 14] { - let (archive, info) = file::encode(&MASTER, plaintext, level, None) - .unwrap_or_else(|e| panic!("headered encode c{level}: {e}")); - assert!(archive.len() >= file::Header::LEN); - assert_eq!(info.input_len, plaintext.len() as u32); - let (header, decoded) = file::decode(&MASTER, &archive) - .unwrap_or_else(|e| panic!("headered decode c{level}: {e}")); - assert_eq!(header.format.bits(), level); - assert_eq!(decoded, plaintext, "c{level}"); - } -} - -#[test] -fn g9_body_public_formats_deterministic() { - require_lean_lib(); - let plaintext = b"g9 body deterministic"; - for format in [0u8, 4, 12, 14] { - let Encoded(b1, h1, i1) = encode(&MASTER, plaintext, format).expect("e1"); - let Encoded(b2, h2, i2) = encode(&MASTER, plaintext, format).expect("e2"); - assert_eq!(b1, b2, "c{format} body deterministic"); - assert_eq!(h1, h2); - assert_eq!(i1.padding_len, i2.padding_len); - assert_eq!(i1.chunk_len, i2.chunk_len); - let d = decode(&MASTER, h1.as_bytes(), &b1, i1.padding_len, format).expect("decode"); - assert_eq!(d, plaintext); - } -} - -#[test] -fn g9_fixed_nonce_encrypted_body() { - require_lean_lib(); - // Encrypted body with fixed nonce via lean helper (public encode uses random). - let plaintext = b"g9 enc fixed nonce"; - let Encoded(body, hash, info) = - carbonado::backend::lean::encode(&MASTER, plaintext, 5, Some(&NONCE)).expect("enc c5"); - let decoded = - carbonado::backend::lean::decode(&MASTER, hash.as_bytes(), &body, info.padding_len, 5) - .expect("dec c5"); - assert_eq!(decoded, plaintext); - // Second encode with same nonce matches (deterministic). - let Encoded(body2, hash2, _) = - carbonado::backend::lean::encode(&MASTER, plaintext, 5, Some(&NONCE)).expect("enc2"); - assert_eq!(body, body2); - assert_eq!(hash, hash2); -} - -#[test] -fn encode_info_fec_fields_populated() { - require_lean_lib(); - let plaintext = b"encode info meta"; - let Encoded(body, _hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); - assert_eq!(info.bytes_verifiable as usize, body.len()); - assert!(info.chunk_len > 0, "chunk_len must be set for FEC"); - assert!(info.bytes_ecc > 0, "bytes_ecc must be set for FEC"); - assert!( - info.verifiable_slice_count > 0, - "verifiable_slice_count must be set for FEC+V" - ); -} - -#[test] -fn outboard_missing_verification_maps() { - require_lean_lib(); - let plaintext = b"missing ob"; - let OutboardEncoded { - main, - verification_outboard: _, - fec_parity, - hash, - info, - } = encode_outboard(&MASTER, plaintext, 12).expect("encode"); - let err = scrub_outboard( - &main, - None, - fec_parity.as_deref(), - &info, - 12, - hash.as_bytes(), - ) - .expect_err("missing outboard"); - assert!( - matches!(err, CarbonadoError::MissingVerificationOutboard), - "expected MissingVerificationOutboard, got {err:?}" - ); -} - -#[test] -fn inboard_scrub_invalid_scrubbed_hash_excess_damage() { - require_lean_lib(); - let plaintext = b"scrub fail excess damage"; - let Encoded(mut body, hash, info) = encode(&MASTER, plaintext, 12).expect("encode c12"); - // Zero most of the body past the length prefix so >4 shards are wiped. - if body.len() > 8 { - for b in &mut body[8..] { - *b = 0; - } - } - let err = scrub(&body, hash.as_bytes(), &info, 12).expect_err("irrecoverable"); - assert!( - matches!(err, CarbonadoError::InvalidScrubbedHash), - "expected InvalidScrubbedHash, got {err:?}" - ); -} - -#[test] -fn outboard_scrub_requires_verification() { - require_lean_lib(); - // c0 has no Verification bit - let plaintext = b"outboard scrub no V"; - let oenc = encode_outboard(&MASTER, plaintext, 0).expect("encode c0"); - let err = scrub_outboard( - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - &oenc.info, - 0, - oenc.hash.as_bytes(), - ) - .expect_err("scrub without V"); - assert!( - matches!(err, CarbonadoError::ScrubRequiresVerification), - "expected ScrubRequiresVerification, got {err:?}" - ); -} - -#[test] -fn outboard_scrub_missing_fec_parity() { - require_lean_lib(); - let plaintext = b"outboard scrub missing parity"; - let oenc = encode_outboard(&MASTER, plaintext, 12).expect("encode c12"); - let mut damaged = oenc.main.clone(); - if !damaged.is_empty() { - damaged[0] ^= 0xff; - } - let err = scrub_outboard( - &damaged, - oenc.verification_outboard.as_deref(), - None, // missing parity after verify fail - &oenc.info, - 12, - oenc.hash.as_bytes(), - ) - .expect_err("missing parity"); - assert!( - matches!(err, CarbonadoError::MissingFecParity), - "expected MissingFecParity, got {err:?}" - ); -} - -fn from_hex(s: &str) -> Vec { - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("hex")) - .collect() -} - -/// G9: Rust-engine goldens (generated under default `backend-rust`) decoded by Lean AOT. -/// -/// Vectors from `encode` / `file::encode` with MASTER + public formats (deterministic). -#[test] -fn g9_rust_encode_lean_decode_body_c0_c4() { - require_lean_lib(); - // c0: body == plaintext (no verification) - let c0_body = from_hex("67392063726f73732d6261636b656e6420626f6479206330"); - let c0_hash = [0u8; 32]; - let pt0 = b"g9 cross-backend body c0"; - assert_eq!(c0_body, pt0); - let d0 = decode(&MASTER, &c0_hash, &c0_body, 0, 0).expect("lean decode rust c0"); - assert_eq!(d0, pt0); - - // c4: bao inboard over plaintext - let c4_body = from_hex("180000000000000067392063726f73732d6261636b656e6420626f6479206334"); - let c4_hash = from_hex("4174c6c3b5a0cf2d734a243b9cf3766afaf2c0e0e913fb09944bbcc9a8556c48"); - let pt4 = b"g9 cross-backend body c4"; - let d4 = decode(&MASTER, &c4_hash, &c4_body, 0, 4).expect("lean decode rust c4"); - assert_eq!(d4, pt4); - - // Lean re-encode of same input must match the Rust body (wire identity). - let Encoded(lean_body, lean_hash, _) = encode(&MASTER, pt4, 4).expect("lean encode c4"); - assert_eq!(lean_body, c4_body, "lean encode must bit-match rust body"); - assert_eq!(lean_hash.as_bytes(), c4_hash.as_slice()); -} - -#[test] -fn g9_rust_encode_lean_decode_headered_c4() { - require_lean_lib(); - let arch = from_hex( - "434152424f4e41444f32300a00000000000000000000000000000000fa7760aa360e9c232b9d7b544f8172f5dc64d99404fd77f94a7a934906034bdcf82b5662847ab76c4f45594cee2faeb45d97bc2ebed05e2e9df3936b4ff9c5f94174c6c3b5a0cf2d734a243b9cf3766afaf2c0e0e913fb09944bbcc9a8556c480000000000000000000000000000000000000000000000000000000000000000040000000020000000000000000000000000000000180000000000000067392063726f73732d6261636b656e6420626f6479206334", - ); - let pt = b"g9 cross-backend body c4"; - let (header, decoded) = file::decode(&MASTER, &arch).expect("lean decode rust headered c4"); - assert_eq!(header.format.bits(), 4); - assert_eq!(decoded, pt); -} diff --git a/tests/lean_backend_phase3.rs b/tests/lean_backend_phase3.rs deleted file mode 100644 index 07ba991..0000000 --- a/tests/lean_backend_phase3.rs +++ /dev/null @@ -1,447 +0,0 @@ -//! Phase 3 allowlist for `backend-lean` (docs/TEST_CONTRACT.md). -//! -//! Directory dual-backend via **composition**: -//! - Rust: FS + **rkyv** FilepackManifest v2 + Adamantine framing + path policy -//! - Lean C ABI: segment outboard encode/decode + catalog headered encode/decode -//! -//! OTS entry/catalog proof cases remain Phase 4. Full rust-root checksum goldens -//! (`filepack_interop::golden_directory_interop_*`) stay `backend-rust`-only. -//! -//! ```bash -//! nix build .#libcarbonado -o result-libcarbonado -//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB -//! cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_phase3 -//! # or: just test-lean-phase3 -//! ``` -//! Only compiled under `backend-lean` (avoids breaking default/`backend-rust` clippy of all targets). - -#![cfg(feature = "backend-lean")] - -use std::fs; -use std::path::{Path, PathBuf}; - -use carbonado::directory::format_policy::{ - is_likely_incompressible, SegmentFormatPolicy, SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, - SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, -}; -use carbonado::error::CarbonadoError; -use carbonado::file::{ - decode_directory, encode_directory, encode_directory_with_options, DirectoryEncodeOptions, - DIRECTORY_ARCHIVE_FORMAT, -}; -use carbonado::filepack_manifest::{ - FilepackEntry, FilepackManifest, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, - FILEPACK_MANIFEST_VERSION, MAX_REL_PATH_LEN, -}; -use carbonado::{ - build_adamantine_payload, decode_adamantine, encode_adamantine, split_adamantine_payload, - ADAMANTINE_CARBONADO_FMT_PUBLIC, -}; - -const ZERO_KEY: [u8; 32] = [0u8; 32]; - -const TEST_MASTER: [u8; 32] = [ - 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, - 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, -]; - -fn require_lean_lib() { - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - panic!( - "CARBONADO_LEAN_LIB unset. Build and export first:\n \ - nix build .#libcarbonado -o result-libcarbonado\n \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ - export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ - export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ - # or: just test-lean-phase3" - ); - } -} - -fn tempdir(name: &str) -> PathBuf { - let p = std::env::temp_dir().join(format!( - "carbonado-lean-p3-{}-{}-{}", - name, - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0) - )); - let _ = fs::remove_dir_all(&p); - fs::create_dir_all(&p).expect("tempdir"); - p -} - -fn hex32(bytes: &[u8; 32]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} - -fn adam_catalog_path(enc_dir: &Path, root: &[u8; 32], format: u8) -> PathBuf { - enc_dir.join(format!("{}.adam.c{format}", hex32(root))) -} - -fn write_tree(src: &Path, files: &[(&str, &[u8])]) { - for (rel, data) in files { - let path = src.join(rel); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).expect("mkdir"); - } - fs::write(&path, data).expect("write"); - } -} - -fn read_tree_file(dec: &Path, rel: &str) -> Vec { - fs::read(dec.join(rel)).unwrap_or_else(|e| panic!("read {rel}: {e}")) -} - -#[test] -fn abi_version_is_one() { - require_lean_lib(); - assert_eq!(carbonado::backend::lean::abi_version(), 1); -} - -#[test] -fn directory_public_roundtrip_under_lean() { - require_lean_lib(); - let src = tempdir("pub_src"); - write_tree( - &src, - &[ - ("hello.txt", b"phase3 public lean directory"), - ("nested/x.bin", b"\x00\x01\x02nested"), - ], - ); - let enc = tempdir("pub_enc"); - let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode_directory public"); - assert_eq!(archive.entry_count, 2); - - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); - assert!(catalog.is_file(), "missing catalog {}", catalog.display()); - // Catalog filename uses decimal c14 (not hex .c0e). - let name = catalog.file_name().unwrap().to_string_lossy(); - assert!( - name.ends_with(".adam.c14"), - "expected decimal .adam.c14, got {name}" - ); - - let dec = tempdir("pub_dec"); - decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode_directory public"); - assert_eq!( - read_tree_file(&dec, "hello.txt"), - b"phase3 public lean directory" - ); - assert_eq!(read_tree_file(&dec, "nested/x.bin"), b"\x00\x01\x02nested"); -} - -#[test] -fn directory_encrypted_roundtrip_under_lean() { - require_lean_lib(); - let src = tempdir("enc_src"); - write_tree(&src, &[("secret.txt", b"encrypted catalog+segments")]); - let enc = tempdir("enc_enc"); - let options = DirectoryEncodeOptions { - encrypted: true, - ..Default::default() - }; - let archive = encode_directory_with_options(&TEST_MASTER, &src, &enc, options) - .expect("encode_directory encrypted"); - assert_eq!(archive.entry_count, 1); - - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, 0x0F); - let name = catalog.file_name().unwrap().to_string_lossy(); - assert!( - name.ends_with(".adam.c15"), - "expected .adam.c15, got {name}" - ); - - let dec = tempdir("enc_dec"); - decode_directory(&TEST_MASTER, &catalog, &dec).expect("decode encrypted"); - assert_eq!( - read_tree_file(&dec, "secret.txt"), - b"encrypted catalog+segments" - ); -} - -#[test] -fn empty_directory_roundtrip_under_lean() { - require_lean_lib(); - let src = tempdir("empty_src"); - let enc = tempdir("empty_enc"); - let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode empty"); - assert_eq!(archive.entry_count, 0); - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); - let dec = tempdir("empty_dec"); - decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode empty"); -} - -#[test] -fn encode_rejects_zero_master_on_encrypted_strict() { - require_lean_lib(); - let src = tempdir("zmk_src"); - write_tree(&src, &[("a.txt", b"x")]); - let enc = tempdir("zmk_enc"); - let options = DirectoryEncodeOptions { - encrypted: true, - ..Default::default() - }; - let err = encode_directory_with_options(&ZERO_KEY, &src, &enc, options).unwrap_err(); - assert!( - matches!(err, CarbonadoError::ZeroMasterKeyNotAllowed), - "expected ZeroMasterKeyNotAllowed, got {err:?}" - ); -} - -#[test] -fn decode_rejects_nonzero_master_on_public_strict() { - require_lean_lib(); - let src = tempdir("nz_src"); - write_tree(&src, &[("a.txt", b"public")]); - let enc = tempdir("nz_enc"); - let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode"); - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); - let err = decode_directory(&TEST_MASTER, &catalog, &tempdir("nz_dec")).unwrap_err(); - assert!( - matches!(err, CarbonadoError::EncryptedDirectoryNotRequested), - "expected EncryptedDirectoryNotRequested, got {err:?}" - ); -} - -#[test] -fn decode_rejects_path_traversal_writes_no_files() { - require_lean_lib(); - // Unit SSOT: validate_rel_path / FilepackManifest::validate reject `..`. - let pe = FilepackManifest::validate_rel_path("../escape.txt").unwrap_err(); - assert!( - matches!(pe, CarbonadoError::InvalidFilepackManifest(_)), - "expected InvalidFilepackManifest from validate_rel_path, got {pe:?}" - ); - let _ = MAX_REL_PATH_LEN; - - // Full fail-closed: encode good tree → rewrite manifest rel_path → re-encode catalog → - // decode_directory must fail before any extract write. - let src = tempdir("mal_src"); - write_tree(&src, &[("one.txt", b"hello")]); - let enc = tempdir("mal_enc"); - let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode"); - let good_catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); - - let main_raw = fs::read(&good_catalog).expect("read catalog"); - let (_, body) = carbonado::file::decode(&ZERO_KEY, &main_raw).expect("headered decode"); - let (adam_payload, hdr) = decode_adamantine(&body).expect("adamantine"); - let (rkyv, bundle) = split_adamantine_payload(&adam_payload).expect("split"); - let good_index = - FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); - let entry = good_index.entries.first().expect("one entry"); - - let malicious = FilepackManifest { - version: FILEPACK_MANIFEST_VERSION, - format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, - catalog_bao_root: [0u8; 32], - catalog_ots_proof: None, - entries: vec![FilepackEntry { - rel_path: "../escape.txt".into(), - content_blake3: entry.content_blake3, - segment_format: entry.segment_format, - segments: entry.segments.clone(), - ots_proof: None, - }], - }; - // Structural validate also fails on the hand-built index (secondary assert). - let unit_err = malicious.validate().unwrap_err(); - assert!( - matches!(unit_err, CarbonadoError::InvalidFilepackManifest(_)), - "expected InvalidFilepackManifest from validate, got {unit_err:?}" - ); - - let mal_rkyv = malicious.to_bytes().expect("malicious rkyv"); - let mal_payload = build_adamantine_payload(&mal_rkyv, &bundle).expect("build payload"); - let mal_adam = encode_adamantine(&mal_payload, ADAMANTINE_CARBONADO_FMT_PUBLIC, hdr.flags); - let (mal_encoded, _) = - carbonado::file::encode(&ZERO_KEY, &mal_adam, DIRECTORY_ARCHIVE_FORMAT, None) - .expect("encode malicious catalog"); - let mal_header = - carbonado::file::Header::try_from(&mal_encoded[..carbonado::file::Header::LEN]) - .expect("header"); - let mal_root = *mal_header.hash.as_bytes(); - let mal_catalog = adam_catalog_path(&enc, &mal_root, DIRECTORY_ARCHIVE_FORMAT); - fs::write(&mal_catalog, &mal_encoded).expect("write malicious catalog"); - - let dec = tempdir("mal_dec"); - let err = decode_directory(&ZERO_KEY, &mal_catalog, &dec).unwrap_err(); - assert!( - matches!( - err, - CarbonadoError::InvalidFilepackManifest(ref msg) if msg.contains("..") - ), - "expected InvalidFilepackManifest containing '..', got {err:?}" - ); - assert!( - fs::read_dir(&dec) - .map(|mut d| d.next()) - .expect("read_dir") - .is_none(), - "decode_directory must not write files on path traversal" - ); -} - -#[test] -fn format_policy_pure_logic_under_lean() { - require_lean_lib(); - // Pure Rust policy (no crypto) — must stay available under backend-lean builds. - assert!(!is_likely_incompressible( - b"hello world text that compresses well" - )); - assert!(is_likely_incompressible(&[ - 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10 - ])); - - let text = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - let fmt = SegmentFormatPolicy::Auto - .resolve_segment_format(false, text) - .expect("auto public text"); - assert_eq!( - fmt, SEGMENT_FORMAT_PUBLIC_COMPRESSED, - "compressible public → c14" - ); - - let jpeg = [0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]; - let fmt = SegmentFormatPolicy::Auto - .resolve_segment_format(false, &jpeg) - .expect("auto public jpeg"); - assert_eq!( - fmt, SEGMENT_FORMAT_PUBLIC_RAW, - "incompressible public → c12" - ); - - let fmt = SegmentFormatPolicy::Auto - .resolve_segment_format(true, text) - .expect("auto encrypted text"); - assert_eq!( - fmt, SEGMENT_FORMAT_ENCRYPTED_COMPRESSED, - "compressible encrypted → c15" - ); - - let err = SegmentFormatPolicy::ForceC12 - .resolve_segment_format(true, text) - .unwrap_err(); - assert!( - matches!(err, CarbonadoError::SegmentFormatMismatch(_)), - "ForceC12 on encrypted catalog must fail, got {err:?}" - ); -} - -#[test] -fn catalog_rkyv_wire_roundtrip_under_lean_encode() { - require_lean_lib(); - // Dual-suite claim: under backend-lean, directory catalogs still carry **rkyv** - // FilepackManifest v2 (not CFP2) inside Adamantine payload. - let src = tempdir("rkyv_src"); - write_tree(&src, &[("only.txt", b"rkyv normative wire")]); - let enc = tempdir("rkyv_enc"); - let archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode"); - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); - - // Peel catalog: headered decode → adamantine → rkyv body. - let main_raw = fs::read(&catalog).expect("read catalog"); - let (header, body) = carbonado::file::decode(&ZERO_KEY, &main_raw).expect("headered decode"); - assert_eq!(header.format.bits(), DIRECTORY_ARCHIVE_FORMAT); - assert_eq!(header.hash.as_bytes(), &archive.catalog_bao_root); - - let (adam_payload, adam_hdr) = carbonado::decode_adamantine(&body).expect("adamantine"); - assert_eq!(adam_hdr.carbonado_fmt, 0x0E); - let (rkyv_payload, _bundle) = - carbonado::split_adamantine_payload(&adam_payload).expect("split"); - // CFP2 magic would be b"CFP2"; rkyv has no that prefix at byte 0 typically, but - // definitive check is successful FilepackManifest deserialize + version 2. - assert!( - !rkyv_payload.starts_with(b"CFP2"), - "dual-suite catalog must not be Lean-only CFP2 wire" - ); - let manifest = FilepackManifest::from_bytes_with_root(&rkyv_payload, archive.catalog_bao_root) - .expect("rkyv FilepackManifest v2"); - assert_eq!(manifest.version, FILEPACK_MANIFEST_VERSION); - assert_eq!(manifest.format_level, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC); - assert_eq!(manifest.entries.len(), 1); - assert_eq!(manifest.entries[0].rel_path, "only.txt"); -} - -/// G9 directory seed: rust-encoded fixture (tests/fixtures/phase3_g9_directory) → lean decode. -#[test] -fn g9_rust_encode_lean_decode_directory_fixture() { - require_lean_lib(); - let fixture = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/phase3_g9_directory"); - assert!( - fixture.is_dir(), - "missing G9 fixture dir {}", - fixture.display() - ); - - let catalog = - fixture.join("16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14"); - assert!( - catalog.is_file(), - "missing G9 catalog {}", - catalog.display() - ); - - // Fixture segments must sit next to the catalog (decode looks in parent dir). - let dec = tempdir("g9_dec"); - // Copy entire fixture archive next to a writable extract root so decode can - // resolve segment mains relative to the catalog path without mutating fixtures. - let work = tempdir("g9_work"); - for entry in fs::read_dir(&fixture).expect("read fixture") { - let entry = entry.expect("entry"); - let path = entry.path(); - if path.is_file() - && path - .file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n != "README.txt") - { - fs::copy(&path, work.join(path.file_name().unwrap())).expect("copy artifact"); - } - } - let work_catalog = work.join(catalog.file_name().unwrap()); - decode_directory(&ZERO_KEY, &work_catalog, &dec).expect("lean decode of rust G9 fixture"); - assert_eq!(read_tree_file(&dec, "a.txt"), b"phase3 g9 hello"); - assert_eq!(read_tree_file(&dec, "sub/b.bin"), b"nested data"); -} - -#[test] -fn multi_segment_sharding_under_lean() { - require_lean_lib(); - let src = tempdir("shard_src"); - // 5 bytes with budget 2 → 3 segments. - write_tree(&src, &[("shard.bin", b"abcde")]); - let enc = tempdir("shard_enc"); - let options = DirectoryEncodeOptions { - segment_plaintext_budget: 2, - ..Default::default() - }; - let archive = encode_directory_with_options(&ZERO_KEY, &src, &enc, options).expect("encode"); - assert_eq!(archive.entry_count, 1); - - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, DIRECTORY_ARCHIVE_FORMAT); - let mains: Vec<_> = fs::read_dir(&enc) - .expect("read_dir") - .filter_map(|e| e.ok()) - .filter(|e| { - let n = e.file_name().to_string_lossy().into_owned(); - n.contains(".c") && !n.contains(".adam.") - }) - .collect(); - assert_eq!( - mains.len(), - 3, - "expected exactly 3 segment mains for budget=2 on 5-byte file, got {}", - mains.len() - ); - - let dec = tempdir("shard_dec"); - decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode sharded"); - assert_eq!(read_tree_file(&dec, "shard.bin"), b"abcde"); -} diff --git a/tests/lean_backend_phase4.rs b/tests/lean_backend_phase4.rs deleted file mode 100644 index 98f4dc0..0000000 --- a/tests/lean_backend_phase4.rs +++ /dev/null @@ -1,676 +0,0 @@ -//! Phase 4 allowlist for `backend-lean` (docs/TEST_CONTRACT.md, docs/GAPS.md). -//! -//! Dual-backend composition (G10 strategy A): -//! - **Container crypto** (outboard / headered / directory segments+catalog): Lean C ABI -//! - **SLH-DSA** (`crypto::slh_*`, SLH1 sidecar, header `slh_public_key`): Rust `bitcoinpqc` -//! under both backends until pure Lean/libbitcoinpqc FFI lands (G10 residual) -//! - **OTS** offline CBOTS stubs: pure Rust (`ots` feature) — same path under lean -//! - **CLI dual path (honest):** -//! - Directory library + subprocess → Lean composition (dual-engine) -//! - Buffer APIs (`file::encode` / `encode_outboard`) → Lean -//! - Single-file CLI streaming (`encode_stream` / `stream_*_outboard`) remains pure Rust -//! under lean builds; lean-linked binary subprocess for single-file is link/smoke only -//! -//! ```bash -//! nix build .#libcarbonado -o result-libcarbonado -//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB -//! cargo test --no-default-features --features "backend-lean,pqc,ots,cli" --test lean_backend_phase4 -//! # or: just test-lean-phase4 -//! ``` -//! Only compiled under `backend-lean` + `pqc` (avoids breaking default/`backend-rust` clippy of all targets). - -#![cfg(all(feature = "backend-lean", feature = "pqc"))] - -use std::fs; -use std::path::{Path, PathBuf}; - -use carbonado::constants::Format; -use carbonado::crypto::{ - read_slh_sidecar, slh_dsa_generate_keypair, slh_dsa_sign, slh_dsa_verify, write_slh_sidecar, - Algorithm, PublicKey, Signature, SLH1_MAGIC, SLH1_SIDECAR_LEN, SLH1_SIGNATURE_LEN, -}; -use carbonado::error::CarbonadoError; -use carbonado::file::{ - self, decode_directory, encode_directory, encode_directory_with_options, encode_stream, - DirectoryEncodeOptions, Header, DIRECTORY_ARCHIVE_FORMAT, -}; -use carbonado::{ - build_adamantine_payload, decode_adamantine, encode_adamantine, split_adamantine_payload, - ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, -}; -use getrandom::getrandom; -use rand::RngCore; - -#[cfg(feature = "ots")] -use carbonado::filepack_manifest::FilepackManifest; -#[cfg(feature = "ots")] -use carbonado::ots::{verify_stamp, OtsPolicy}; - -const ZERO_KEY: [u8; 32] = [0u8; 32]; - -const TEST_MASTER: [u8; 32] = [ - 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, - 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, -]; - -/// Header wire offset of `slh_public_key` (AGENTS § Header layout; 12+16+64+32 = 124). -mod offsets { - pub const SLH_PK: usize = 124; -} - -fn require_lean_lib() { - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - panic!( - "CARBONADO_LEAN_LIB unset. Build and export first:\n \ - nix build .#libcarbonado -o result-libcarbonado\n \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ - export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ - export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ - # or: just test-lean-phase4" - ); - } -} - -fn tempdir(name: &str) -> PathBuf { - let p = std::env::temp_dir().join(format!( - "carbonado-lean-p4-{}-{}-{}", - name, - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0) - )); - let _ = fs::remove_dir_all(&p); - fs::create_dir_all(&p).expect("tempdir"); - p -} - -fn hex32(bytes: &[u8; 32]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} - -fn adam_catalog_path(enc_dir: &Path, root: &[u8; 32], format: u8) -> PathBuf { - enc_dir.join(format!("{}.adam.c{format}", hex32(root))) -} - -fn random_master() -> [u8; 32] { - let mut k = [0u8; 32]; - rand::thread_rng().fill_bytes(&mut k); - k -} - -fn slh_entropy() -> [u8; 128] { - let mut e = [0u8; 128]; - getrandom(&mut e).expect("entropy"); - e -} - -fn slh_public_key_bytes(pk: &PublicKey) -> [u8; 32] { - let mut out = [0u8; 32]; - out.copy_from_slice(&pk.bytes[..32]); - out -} - -#[test] -fn abi_version_is_one() { - require_lean_lib(); - assert_eq!(carbonado::backend::lean::abi_version(), 1); -} - -#[test] -fn backend_name_is_lean() { - require_lean_lib(); - assert_eq!(carbonado::backend::lean::NAME, "lean"); -} - -/// G10 dual-suite: Lean outboard encode/decode + Rust bitcoinpqc SLH sidecar over Bao root. -#[test] -fn slh_outboard_sidecar_binds_header_public_key_under_lean() { - require_lean_lib(); - let key = random_master(); - let input = b"Phase4 SLH under lean: Lean container + Rust SLH-DSA-SHA2-128s"; - - let (hdr_opt, oenc) = file::encode_outboard(&key, input, 14, None).expect("encode_outboard"); - let base_hdr = hdr_opt.expect("header for outboard high-level path"); - let bao_root = base_hdr.hash.as_bytes(); - - let keypair = slh_dsa_generate_keypair(&slh_entropy()).expect("slh keygen"); - let slh_pk = slh_public_key_bytes(&keypair.public_key); - let signature = slh_dsa_sign(&keypair.secret_key, bao_root).expect("slh sign"); - - let signed_hdr = Header::new( - &key, - base_hdr.payload_nonce, - bao_root, - slh_pk, - Format::from(14), - base_hdr.chunk_index, - base_hdr.encoded_len, - base_hdr.padding_len, - base_hdr.metadata, - ) - .expect("Header::new with slh_pk"); - assert_eq!(signed_hdr.slh_public_key, slh_pk); - - let hdr_bytes = signed_hdr.try_to_vec().expect("header wire"); - let rec = file::decode_outboard( - &key, - signed_hdr.hash.as_bytes(), - Some(&hdr_bytes), - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - oenc.info.padding_len, - 14, - ) - .expect("decode_outboard lean"); - assert_eq!(rec, input); - - let sidecar_path = tempdir("slh_sc").join(format!("{}.slh", signed_hdr.file_name())); - write_slh_sidecar(&sidecar_path, &signature.bytes).expect("write slh"); - let sig_bytes = read_slh_sidecar(&sidecar_path).expect("read slh"); - assert_eq!(sig_bytes.len(), SLH1_SIGNATURE_LEN); - let on_disk = fs::read(&sidecar_path).expect("raw sidecar"); - assert_eq!(on_disk.len(), SLH1_SIDECAR_LEN); - assert_eq!(&on_disk[..4], SLH1_MAGIC); - - let sig = Signature { - algorithm: Algorithm::SLH_DSA_SHA2_128S, - bytes: sig_bytes, - }; - assert!( - slh_dsa_verify(&keypair.public_key, bao_root, &sig).expect("verify"), - "signature must verify over Bao root" - ); - - let hdr_pk = PublicKey { - algorithm: Algorithm::SLH_DSA_SHA2_128S, - bytes: signed_hdr.slh_public_key.to_vec(), - }; - assert!( - slh_dsa_verify(&hdr_pk, bao_root, &sig).expect("verify via header pk"), - "header slh_public_key must verify sidecar" - ); - - // Fail-closed: wrong root - let mut bad_root = *bao_root; - bad_root[0] ^= 0x01; - assert!( - !slh_dsa_verify(&hdr_pk, &bad_root, &sig).expect("verify bad root"), - "wrong Bao root must not verify" - ); - - // Fail-closed: wrong public key - let other = slh_dsa_generate_keypair(&slh_entropy()).expect("other keygen"); - let wrong_pk = PublicKey { - algorithm: Algorithm::SLH_DSA_SHA2_128S, - bytes: slh_public_key_bytes(&other.public_key).to_vec(), - }; - assert!( - !slh_dsa_verify(&wrong_pk, bao_root, &sig).expect("verify wrong pk"), - "wrong header pk must not verify" - ); - - // Fail-closed: tampered slh_public_key fails header_mac (lean headered path) - let mut bad_hdr_bytes = hdr_bytes.clone(); - bad_hdr_bytes[offsets::SLH_PK] ^= 0x01; - let err_pk = file::decode_outboard( - &key, - signed_hdr.hash.as_bytes(), - Some(&bad_hdr_bytes), - &oenc.main, - oenc.verification_outboard.as_deref(), - oenc.fec_parity.as_deref(), - oenc.info.padding_len, - 14, - ) - .unwrap_err(); - assert!( - matches!(err_pk, CarbonadoError::AuthenticationFailed), - "tampered slh_public_key must fail header_mac, got {err_pk:?}" - ); -} - -#[test] -fn slh_sidecar_bad_magic_and_length_fail_closed() { - require_lean_lib(); - let dir = tempdir("slh_wire"); - - // Bad magic, correct length - let bad_magic = dir.join("bad_magic.slh"); - let mut wire = vec![0u8; SLH1_SIDECAR_LEN]; - wire[..4].copy_from_slice(b"XXXX"); - fs::write(&bad_magic, &wire).expect("write"); - let err = read_slh_sidecar(&bad_magic).unwrap_err(); - assert!( - matches!(err, CarbonadoError::InvalidMagicNumber(_)), - "bad SLH1 magic must be InvalidMagicNumber, got {err:?}" - ); - - // Truncated (good magic prefix) - let short = dir.join("short.slh"); - fs::write(&short, b"SLH1").expect("write"); - let err2 = read_slh_sidecar(&short).unwrap_err(); - assert!( - matches!(err2, CarbonadoError::OutboardVerificationFailed(_)), - "short sidecar must be OutboardVerificationFailed, got {err2:?}" - ); - - // Wrong signature length on write. - // Taxonomy freeze (P4): short sig / short sidecar map to `OutboardVerificationFailed` - // (pre-existing; not a dedicated SlhWire error). Update these asserts if refined later. - let err3 = write_slh_sidecar(dir.join("short_sig.slh"), b"short").unwrap_err(); - assert!( - matches!(err3, CarbonadoError::OutboardVerificationFailed(_)), - "short signature write must fail, got {err3:?}" - ); -} - -#[cfg(feature = "ots")] -#[test] -fn directory_ots_entry_and_catalog_under_lean() { - require_lean_lib(); - let src = tempdir("ots_src"); - fs::write(src.join("one.txt"), b"phase4 ots lean payload").expect("write"); - - let enc_dir = tempdir("ots_enc"); - let dec_dir = tempdir("ots_dec"); - let options = DirectoryEncodeOptions { - ots_policy: Some(OtsPolicy { - stamp_entries: true, - stamp_catalog: true, - }), - ..DirectoryEncodeOptions::default() - }; - let archive = - encode_directory_with_options(&ZERO_KEY, &src, &enc_dir, options).expect("encode"); - let catalog_path = adam_catalog_path( - &enc_dir, - &archive.catalog_bao_root, - DIRECTORY_ARCHIVE_FORMAT, - ); - - let catalog_bytes = fs::read(&catalog_path).expect("read catalog"); - assert!( - catalog_bytes.windows(4).any(|w| w == b"COTS"), - "catalog must contain COTS trailer when stamp_catalog is set" - ); - - let (_, body) = carbonado::file::decode(&ZERO_KEY, &catalog_bytes).expect("headered decode"); - let (adam_payload, hdr) = decode_adamantine(&body).expect("adam"); - assert_ne!( - hdr.flags & ADAMANTINE_FLAG_REQUIRE_OTS, - 0, - "REQUIRE_OTS must be set when stamp_entries" - ); - let (rkyv, _) = split_adamantine_payload(&adam_payload).expect("split"); - let index = - FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); - let proof = index.entries[0].ots_proof.as_ref().expect("entry ots"); - let primary_root = index.entries[0].segments[0].segment_bao_root; - assert!( - verify_stamp(proof, &primary_root) - .expect("verify entry") - .valid, - "entry OTS must verify primary segment Bao root" - ); - let catalog_ots = catalog_ots_proof_from_cots_trailer(&catalog_bytes).expect("catalog ots"); - assert!( - verify_stamp(&catalog_ots, &archive.catalog_bao_root) - .expect("verify catalog") - .valid, - "catalog OTS must verify catalog Bao root" - ); - - decode_directory(&ZERO_KEY, &catalog_path, &dec_dir).expect("decode_directory"); - assert_eq!( - fs::read(dec_dir.join("one.txt")).expect("read"), - b"phase4 ots lean payload" - ); -} - -#[cfg(feature = "ots")] -#[test] -fn directory_ots_tampered_entry_fails_under_lean() { - require_lean_lib(); - let src = tempdir("ots_tamper_src"); - fs::write(src.join("one.txt"), b"tamper me lean").expect("write"); - let enc_dir = tempdir("ots_tamper_enc"); - let dec_dir = tempdir("ots_tamper_dec"); - let options = DirectoryEncodeOptions { - ots_policy: Some(OtsPolicy { - stamp_entries: true, - stamp_catalog: false, - }), - ..DirectoryEncodeOptions::default() - }; - let archive = - encode_directory_with_options(&ZERO_KEY, &src, &enc_dir, options).expect("encode"); - let catalog_path = adam_catalog_path( - &enc_dir, - &archive.catalog_bao_root, - DIRECTORY_ARCHIVE_FORMAT, - ); - - let (_, body) = carbonado::file::decode(&ZERO_KEY, &fs::read(&catalog_path).expect("read")) - .expect("decode"); - let (adam_payload, hdr) = decode_adamantine(&body).expect("adam"); - let (rkyv, bundle) = split_adamantine_payload(&adam_payload).expect("split"); - let mut index = - FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); - let proof = index.entries[0].ots_proof.as_mut().expect("proof"); - if let Some(byte) = proof.first_mut() { - *byte ^= 0xFF; - } - let tampered_rkyv = index.to_bytes().expect("to_bytes"); - let tampered_payload = build_adamantine_payload(&tampered_rkyv, &bundle).expect("payload"); - let tampered_adam = encode_adamantine( - &tampered_payload, - ADAMANTINE_CARBONADO_FMT_PUBLIC, - hdr.flags, - ); - let (tampered_encoded, _) = - carbonado::file::encode(&ZERO_KEY, &tampered_adam, DIRECTORY_ARCHIVE_FORMAT, None) - .expect("re-encode"); - let tampered_header = Header::try_from(&tampered_encoded[..Header::LEN]).expect("header"); - let tampered_root = *tampered_header.hash.as_bytes(); - let tampered_catalog = adam_catalog_path(&enc_dir, &tampered_root, DIRECTORY_ARCHIVE_FORMAT); - fs::write(&tampered_catalog, &tampered_encoded).expect("write tampered"); - - let err = decode_directory(&ZERO_KEY, &tampered_catalog, &dec_dir).unwrap_err(); - assert!( - matches!(err, CarbonadoError::OtsVerificationFailed), - "tampered entry OTS must be OtsVerificationFailed, got {err:?}" - ); -} - -#[cfg(feature = "ots")] -#[test] -fn directory_ots_missing_when_required_fails_under_lean() { - require_lean_lib(); - let src = tempdir("ots_req_src"); - fs::write(src.join("one.txt"), b"x").expect("write"); - let enc_dir = tempdir("ots_req_enc"); - let archive = encode_directory(&ZERO_KEY, &src, &enc_dir).expect("encode"); - let catalog_path = adam_catalog_path( - &enc_dir, - &archive.catalog_bao_root, - DIRECTORY_ARCHIVE_FORMAT, - ); - let (_, body) = carbonado::file::decode(&ZERO_KEY, &fs::read(&catalog_path).expect("read")) - .expect("decode"); - let (adam_payload, _) = decode_adamantine(&body).expect("adam"); - let (rkyv, bundle) = split_adamantine_payload(&adam_payload).expect("split"); - let mut index = - FilepackManifest::from_bytes_with_root(&rkyv, archive.catalog_bao_root).expect("index"); - index.entries[0].ots_proof = None; - let payload = build_adamantine_payload(&index.to_bytes().expect("bytes"), &bundle).expect("p"); - let adam = encode_adamantine( - &payload, - ADAMANTINE_CARBONADO_FMT_PUBLIC, - ADAMANTINE_FLAG_REQUIRE_OTS, - ); - let (encoded, _) = - carbonado::file::encode(&ZERO_KEY, &adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); - let header = Header::try_from(&encoded[..Header::LEN]).expect("hdr"); - let root = *header.hash.as_bytes(); - let bad_catalog = adam_catalog_path(&enc_dir, &root, DIRECTORY_ARCHIVE_FORMAT); - fs::write(&bad_catalog, &encoded).expect("write"); - let err = decode_directory(&ZERO_KEY, &bad_catalog, &tempdir("ots_req_dec")).unwrap_err(); - assert!( - matches!( - err, - CarbonadoError::OtsProofRequired(ref rel) if rel == "one.txt" - ), - "expected OtsProofRequired(one.txt), got {err:?}" - ); -} - -/// CLI-shaped library paths under lean — engines called out per half. -#[test] -fn cli_library_encode_decode_paths_under_lean() { - require_lean_lib(); - - // --- Cross-engine (mirrors CLI single-file inboard wire assembly) --- - // Encode: pure-Rust `encode_stream` (same as bin/carbonado; no stream→Lean dispatch). - // Decode: Lean headered `file::decode` (G9-style rust-encode → lean-decode). - let input = b"phase4 cli library single-file lean"; - let mut body_bytes = Vec::new(); - let (enc_hdr, _info) = encode_stream(&ZERO_KEY, &mut &input[..], 14, None, &mut body_bytes) - .expect("encode_stream"); - assert_eq!(enc_hdr.hash.as_bytes().len(), 32); - assert!(!body_bytes.is_empty()); - let mut archive = enc_hdr.try_to_vec().expect("header wire"); - archive.extend_from_slice(&body_bytes); - - let (hdr, body) = carbonado::file::decode(&ZERO_KEY, &archive).expect("lean headered decode"); - assert_eq!(hdr.hash, enc_hdr.hash); - assert_eq!(body, input); - - // --- Same-engine dual path (buffer API the CLI does *not* use for streaming) --- - // `file::encode` under backend-lean → Lean `carbonado_encode_headered`. - let (lean_archive, _) = - carbonado::file::encode(&ZERO_KEY, input, 14, None).expect("lean file::encode"); - let (lean_hdr, lean_body) = - carbonado::file::decode(&ZERO_KEY, &lean_archive).expect("lean file::decode"); - assert_eq!(lean_hdr.format.bits(), 14); - assert_eq!(lean_body, input); - - // --- Directory path (CLI `encode

` / `decode .adam.c14`) — dual-engine --- - let src = tempdir("cli_lib_src"); - fs::write(src.join("hi.txt"), b"cli dir lean").expect("write"); - let enc = tempdir("cli_lib_enc"); - let dir_archive = encode_directory(&ZERO_KEY, &src, &enc).expect("encode_directory"); - let catalog = adam_catalog_path( - &enc, - &dir_archive.catalog_bao_root, - DIRECTORY_ARCHIVE_FORMAT, - ); - let dec = tempdir("cli_lib_dec"); - decode_directory(&ZERO_KEY, &catalog, &dec).expect("decode_directory"); - assert_eq!(fs::read(dec.join("hi.txt")).expect("read"), b"cli dir lean"); -} - -/// Encrypted directory under lean (CLI `--encrypted --master`). -#[test] -fn cli_library_encrypted_directory_under_lean() { - require_lean_lib(); - let src = tempdir("cli_enc_src"); - fs::write(src.join("secret.txt"), b"encrypted cli lean").expect("write"); - let enc = tempdir("cli_enc_enc"); - let options = DirectoryEncodeOptions { - encrypted: true, - ..Default::default() - }; - let archive = encode_directory_with_options(&TEST_MASTER, &src, &enc, options).expect("enc"); - let catalog = adam_catalog_path(&enc, &archive.catalog_bao_root, 0x0F); - let dec = tempdir("cli_enc_dec"); - decode_directory(&TEST_MASTER, &catalog, &dec).expect("dec"); - assert_eq!( - fs::read(dec.join("secret.txt")).expect("read"), - b"encrypted cli lean" - ); -} - -/// Lean-linked binary: single-file `--outboard` is **link/smoke only** (pure Rust streaming). -/// Does **not** exercise Lean C ABI — see `cli_subprocess_directory_roundtrip_under_lean`. -#[cfg(feature = "cli")] -#[test] -fn cli_subprocess_single_file_link_smoke_under_lean() { - require_lean_lib(); - let bin = PathBuf::from(env!("CARGO_BIN_EXE_carbonado")); - assert!( - bin.is_file(), - "carbonado binary missing at {} — build with features backend-lean,cli", - bin.display() - ); - - let work = tempdir("cli_sub_sf"); - let input = work.join("input.txt"); - let outdir = work.join("enc"); - let recovered = work.join("recovered.bin"); - fs::create_dir_all(&outdir).expect("outdir"); - fs::write(&input, b"phase4 subprocess single-file link smoke").expect("write"); - - let enc = std::process::Command::new(&bin) - .args([ - "encode", - input.to_str().unwrap(), - "--format", - "14", - "--outboard", - "--output", - outdir.to_str().unwrap(), - ]) - .env( - "LD_LIBRARY_PATH", - std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), - ) - .output() - .expect("spawn encode"); - assert!( - enc.status.success(), - "encode failed: status={:?} stderr={}", - enc.status, - String::from_utf8_lossy(&enc.stderr) - ); - - // One bare main archive (not .out/.par) - let archive = fs::read_dir(&outdir) - .expect("read") - .filter_map(|e| e.ok()) - .map(|e| e.path()) - .find(|p| { - p.is_file() - && p.file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| !n.ends_with(".out") && !n.ends_with(".par")) - }) - .expect("archive main missing"); - - let dec = std::process::Command::new(&bin) - .args([ - "decode", - archive.to_str().unwrap(), - "--output", - recovered.to_str().unwrap(), - ]) - .env( - "LD_LIBRARY_PATH", - std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), - ) - .output() - .expect("spawn decode"); - assert!( - dec.status.success(), - "decode failed: status={:?} stderr={}", - dec.status, - String::from_utf8_lossy(&dec.stderr) - ); - assert_eq!( - fs::read(&recovered).expect("read recovered"), - b"phase4 subprocess single-file link smoke" - ); -} - -/// Dual-engine CLI subprocess: directory encode/decode hits Lean segment/catalog crypto. -#[cfg(feature = "cli")] -#[test] -fn cli_subprocess_directory_roundtrip_under_lean() { - require_lean_lib(); - let bin = PathBuf::from(env!("CARGO_BIN_EXE_carbonado")); - assert!( - bin.is_file(), - "carbonado binary missing at {} — build with features backend-lean,cli", - bin.display() - ); - - let work = tempdir("cli_sub_dir"); - let src = work.join("src"); - let outdir = work.join("enc"); - let recovered = work.join("recovered"); - fs::create_dir_all(&src).expect("src"); - fs::create_dir_all(&outdir).expect("outdir"); - fs::write(src.join("hi.txt"), b"phase4 subprocess directory lean").expect("write"); - - let enc = std::process::Command::new(&bin) - .args([ - "encode", - src.to_str().unwrap(), - "--output", - outdir.to_str().unwrap(), - ]) - .env( - "LD_LIBRARY_PATH", - std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), - ) - .output() - .expect("spawn directory encode"); - assert!( - enc.status.success(), - "directory encode failed: status={:?} stderr={}", - enc.status, - String::from_utf8_lossy(&enc.stderr) - ); - - let catalog = fs::read_dir(&outdir) - .expect("read enc") - .filter_map(|e| e.ok()) - .map(|e| e.path()) - .find(|p| { - p.is_file() - && p.file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.ends_with(".adam.c14")) - }) - .expect("catalog .adam.c14 missing"); - - let dec = std::process::Command::new(&bin) - .args([ - "decode", - catalog.to_str().unwrap(), - "--output", - recovered.to_str().unwrap(), - ]) - .env( - "LD_LIBRARY_PATH", - std::env::var_os("LD_LIBRARY_PATH").unwrap_or_default(), - ) - .output() - .expect("spawn directory decode"); - assert!( - dec.status.success(), - "directory decode failed: status={:?} stderr={}", - dec.status, - String::from_utf8_lossy(&dec.stderr) - ); - assert_eq!( - fs::read(recovered.join("hi.txt")).expect("read recovered"), - b"phase4 subprocess directory lean" - ); -} - -#[cfg(feature = "ots")] -fn catalog_ots_proof_from_cots_trailer(bytes: &[u8]) -> Option> { - if bytes.len() < Header::LEN + 8 { - return None; - } - let max_scan = carbonado::filepack_manifest::MAX_OTS_PROOF_LEN + 8; - let scan_start = bytes.len().saturating_sub(max_scan).max(Header::LEN); - for i in (scan_start..=bytes.len().saturating_sub(8)).rev() { - if bytes.get(i..i + 4)? != b"COTS" { - continue; - } - let ots_len = u32::from_le_bytes(bytes[i + 4..i + 8].try_into().ok()?) as usize; - if ots_len > carbonado::filepack_manifest::MAX_OTS_PROOF_LEN { - return None; - } - if i + 8 + ots_len == bytes.len() { - return Some(bytes[i + 8..].to_vec()); - } - } - None -} diff --git a/tests/lean_backend_smoke.rs b/tests/lean_backend_smoke.rs deleted file mode 100644 index 0d30dc7..0000000 --- a/tests/lean_backend_smoke.rs +++ /dev/null @@ -1,277 +0,0 @@ -//! Phase 1 allowlist smoke for `backend-lean` (docs/TEST_CONTRACT.md). -//! -//! ```bash -//! nix build .#libcarbonado -o result-libcarbonado -//! export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib -//! export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include -//! export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB -//! cargo test --no-default-features --features "backend-lean,pqc,ots" --test lean_backend_smoke -//! # or: just test-lean-smoke -//! ``` -//! -//! Primary allowlist: public (even) formats c0, c4, c12. Fixed master keys. -//! R2 also smokes encrypted headered + non-zero SLH with a fixed nonce (no RNG). -//! Only compiled under `backend-lean` (avoids breaking default/`backend-rust` clippy of all targets). - -#![cfg(feature = "backend-lean")] - -mod common; - -use carbonado::{ - carbonado_verification_key, constants::Format, decode, encode, error::CarbonadoError, file, - file::Header, structs::Encoded, -}; -use common::header_layout::offsets; - -/// Fixed 32-byte master (public formats may use zeros; we use a non-zero pattern). -const MASTER: [u8; 32] = [ - 0x0c, 0xa1, 0xb0, 0xda, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, - 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, -]; - -fn require_lean_lib() { - if std::env::var_os("CARBONADO_LEAN_LIB").is_none() { - panic!( - "CARBONADO_LEAN_LIB unset. Build and export first:\n \ - nix build .#libcarbonado -o result-libcarbonado\n \ - export CARBONADO_LEAN_LIB=$PWD/result-libcarbonado/lib\n \ - export CARBONADO_LEAN_INCLUDE=$PWD/result-libcarbonado/include\n \ - export LD_LIBRARY_PATH=$CARBONADO_LEAN_LIB\n \ - # or: just test-lean-smoke" - ); - } -} - -#[test] -fn abi_version_is_one() { - require_lean_lib(); - let v = carbonado::backend::lean::abi_version(); - assert_eq!(v, 1, "CARBONADO_ABI_VERSION"); -} - -#[test] -fn verification_key_lean_abi_matches_blake3_formula() { - require_lean_lib(); - // Public API is pure blake3; parity is via the Lean C ABI helper. - for format in [0u8, 4, 6, 12, 14, 15] { - let lean = carbonado::backend::lean::verification_key(format) - .unwrap_or_else(|e| panic!("lean verification_key c{format}: {e}")); - let rust_formula = carbonado_verification_key(format); - assert_eq!( - lean, rust_formula, - "Lean AOT verification key mismatch for format {format}" - ); - assert_eq!( - rust_formula, - blake3::derive_key("carbonado-v2/verification", &[format]) - ); - } -} - -#[test] -fn headered_roundtrip_public_formats() { - require_lean_lib(); - let plaintext = b"carbonado phase1 lean headered smoke"; - // c0 raw, c4 bao, c12 bao+fec (padding lives in Header) - for level in [0u8, 4, 12] { - let (archive, info) = file::encode(&MASTER, plaintext, level, None) - .unwrap_or_else(|e| panic!("encode c{level}: {e}")); - assert!( - archive.len() >= file::Header::LEN, - "c{level}: archive shorter than header" - ); - assert_eq!(info.input_len, plaintext.len() as u32); - let (header, decoded) = - file::decode(&MASTER, &archive).unwrap_or_else(|e| panic!("decode c{level}: {e}")); - assert_eq!(header.format.bits(), level); - assert_eq!(decoded, plaintext, "c{level} plaintext mismatch"); - } -} - -#[test] -fn low_level_roundtrip_c0_c4() { - require_lean_lib(); - let plaintext = b"low-level body smoke c0/c4"; - for format in [0u8, 4] { - let Encoded(body, hash, info) = encode(&MASTER, plaintext, format) - .unwrap_or_else(|e| panic!("encode body c{format}: {e}")); - assert_eq!(info.bytes_verifiable as usize, body.len()); - let decoded = decode(&MASTER, hash.as_bytes(), &body, info.padding_len, format) - .unwrap_or_else(|e| panic!("decode body c{format}: {e}")); - assert_eq!(decoded, plaintext, "c{format} body plaintext mismatch"); - } -} - -#[test] -fn low_level_encode_short_master_invalid_key_length() { - require_lean_lib(); - let short = [0u8; 16]; - let err = match encode(&short, b"x", 0) { - Ok(_) => panic!("short master low-level encode must fail"), - Err(e) => e, - }; - assert!( - matches!(err, CarbonadoError::InvalidKeyLength), - "expected InvalidKeyLength (not InternalStateError from packEncodeErr bug), got {err:?}" - ); -} - -#[test] -fn low_level_decode_wrong_hash_fails() { - require_lean_lib(); - let plaintext = b"low-level wrong hash"; - let Encoded(body, hash, info) = encode(&MASTER, plaintext, 4).expect("encode"); - let mut bad_hash = *hash.as_bytes(); - bad_hash[0] ^= 0xff; - let err = decode(&MASTER, &bad_hash, &body, info.padding_len, 4).expect_err("bad hash"); - // R4: Bao root/auth mismatch maps to AuthenticationFailed (same as pure Rust - // map_decode_error Parent/LeafHashMismatch). Truncation alone stays BaoResponseTruncated. - assert!( - matches!(err, CarbonadoError::AuthenticationFailed), - "expected AuthenticationFailed, got {err:?}" - ); -} - -#[test] -fn headered_bad_magic_fails() { - require_lean_lib(); - let plaintext = b"tamper magic"; - let (mut archive, _) = file::encode(&MASTER, plaintext, 4, None).expect("encode"); - // Corrupt MAGICNO first byte (CARBONADO20\n) - archive[0] ^= 0xff; - let err = file::decode(&MASTER, &archive).expect_err("bad magic must fail"); - assert!( - matches!(err, CarbonadoError::InvalidMagicNumber(_)), - "expected InvalidMagicNumber, got {err:?}" - ); -} - -#[test] -fn headered_auth_fail_on_header_mac_tamper() { - require_lean_lib(); - let plaintext = b"tamper header mac"; - let (mut archive, _) = file::encode(&MASTER, plaintext, 4, None).expect("encode"); - // header_mac sits at offset 28 (12 magic + 16 nonce); flip one byte - let mac_off = 28; - archive[mac_off] ^= 0x01; - let err = file::decode(&MASTER, &archive).expect_err("tampered MAC must fail"); - assert!( - matches!(err, CarbonadoError::AuthenticationFailed), - "expected AuthenticationFailed, got {err:?}" - ); -} - -#[test] -fn invalid_key_length_rejected_headered() { - require_lean_lib(); - let short = [0u8; 16]; - let err = file::encode(&short, b"x", 0, None).expect_err("short master"); - assert!( - matches!(err, CarbonadoError::InvalidKeyLength), - "expected InvalidKeyLength, got {err:?}" - ); -} - -#[test] -fn headered_metadata_roundtrip_and_mac_binding() { - require_lean_lib(); - let meta = *b"metameta"; - let (arch, _) = file::encode(&MASTER, b"meta payload", 0, Some(meta)).expect("encode meta"); - let hdr = Header::try_from(&arch[..Header::LEN]).expect("header"); - assert_eq!(hdr.metadata, Some(meta)); - let (_hdr2, pt) = file::decode(&MASTER, &arch).expect("decode"); - assert_eq!(pt, b"meta payload"); - - // Tamper metadata byte → header_mac fail. - let mut tampered = arch.clone(); - tampered[offsets::METADATA] ^= 0xff; - let err = file::decode(&MASTER, &tampered).expect_err("tampered meta"); - assert!( - matches!(err, CarbonadoError::AuthenticationFailed), - "expected AuthenticationFailed, got {err:?}" - ); -} - -/// R2: non-zero SLH pk via `lean::encode_headered` (file::encode leaves SLH zeroed by design). -#[test] -fn headered_slh_pk_roundtrip_and_mac_binding() { - require_lean_lib(); - let slh = [0xABu8; 32]; - let meta = *b"slh-meta"; - let (arch, info) = carbonado::backend::lean::encode_headered( - &MASTER, - b"slh payload", - 0, - None, - Some(&slh), - Some(&meta), - ) - .expect("encode with slh+meta"); - assert_eq!(info.bytes_compressed, 0); - assert_eq!(info.bytes_encrypted, 0); - let hdr = Header::try_from(&arch[..Header::LEN]).expect("header"); - assert_eq!(hdr.slh_public_key, slh); - assert_eq!(hdr.metadata, Some(meta)); - let (hdr2, pt) = file::decode(&MASTER, &arch).expect("decode"); - assert_eq!(pt, b"slh payload"); - assert_eq!(hdr2.slh_public_key, slh); - assert_eq!(hdr2.metadata, Some(meta)); - - // Tamper SLH pk byte → header_mac fail. - let mut tampered = arch.clone(); - tampered[offsets::SLH_PUBLIC_KEY] ^= 0xff; - let err = file::decode(&MASTER, &tampered).expect_err("tampered slh"); - assert!( - matches!(err, CarbonadoError::AuthenticationFailed), - "expected AuthenticationFailed, got {err:?}" - ); -} - -/// R2: non-zero SLH + metadata on encrypted headered path (fixed nonce; no RNG). -#[test] -fn headered_encrypted_slh_pk_and_metadata_roundtrip() { - require_lean_lib(); - let slh = [0xCDu8; 32]; - let meta = *b"enc-meta"; - let nonce = [0x11u8; 16]; - let format = Format::Encryption.bits(); // c1 - let (arch, info) = carbonado::backend::lean::encode_headered( - &MASTER, - b"enc slh payload", - format, - Some(&nonce), - Some(&slh), - Some(&meta), - ) - .expect("encrypted encode with slh+meta"); - assert_eq!(info.bytes_compressed, 0); - assert!(info.bytes_encrypted > 0); - let hdr = Header::try_from(&arch[..Header::LEN]).expect("header"); - assert_eq!(hdr.slh_public_key, slh); - assert_eq!(hdr.metadata, Some(meta)); - assert_eq!(hdr.payload_nonce, nonce); - let (hdr2, pt) = file::decode(&MASTER, &arch).expect("decode"); - assert_eq!(pt, b"enc slh payload"); - assert_eq!(hdr2.slh_public_key, slh); - assert_eq!(hdr2.metadata, Some(meta)); - - let mut tampered = arch.clone(); - tampered[offsets::SLH_PUBLIC_KEY] ^= 0xff; - let err = file::decode(&MASTER, &tampered).expect_err("tampered slh on encrypted"); - assert!( - matches!(err, CarbonadoError::AuthenticationFailed), - "expected AuthenticationFailed, got {err:?}" - ); -} - -#[test] -fn format_bits_even_are_public() { - // Sanity: Phase 1 allowlist uses even formats only. - for f in [0u8, 4, 12] { - let fmt = Format::from(f); - assert!( - !fmt.contains(Format::Encryption), - "format {f} should be public (even)" - ); - } -} diff --git a/tests/parallel_determinism.rs b/tests/parallel_determinism.rs index 5d84002..1e04f49 100644 --- a/tests/parallel_determinism.rs +++ b/tests/parallel_determinism.rs @@ -16,16 +16,16 @@ use carbonado::constants::{FEC_K, FEC_M}; use carbonado::error::CarbonadoError; use carbonado::stream::encode::stream_encode_inboard_body; use carbonado::stream::fec::{ - encode_inboard_buffer, encode_outboard_parity_buffer, write_outboard_parity, FecInboardEncoder, - FecStripe, + FecInboardEncoder, FecStripe, encode_inboard_buffer, encode_outboard_parity_buffer, + write_outboard_parity, }; use carbonado::stream::parallel::{ - encode_rs_parity_serial, encode_rs_parity_with_config, rs_parity_parallelism_active, - ParallelConfig, + ParallelConfig, encode_rs_parity_serial, encode_rs_parity_with_config, + rs_parity_parallelism_active, }; use carbonado::stream::{stream_decode_buffer, stream_encode_buffer}; use carbonado::{decode, encode, scrub, structs::Encoded}; -use common::corruption::{flip_byte, InboardShardLayout}; +use common::corruption::{InboardShardLayout, flip_byte}; use reed_solomon_erasure::galois_8::ReedSolomon; use common::inboard_parity::{assert_inboard_body_roundtrip, preprocess_and_body}; diff --git a/tests/serial_fec_path.rs b/tests/serial_fec_path.rs index c3656a5..784b1a1 100644 --- a/tests/serial_fec_path.rs +++ b/tests/serial_fec_path.rs @@ -9,7 +9,7 @@ use std::io::Cursor; use carbonado::constants::FEC_M; -use carbonado::stream::fec::{encode_inboard_buffer, FecInboardEncoder}; +use carbonado::stream::fec::{FecInboardEncoder, encode_inboard_buffer}; fn patterned(len: usize) -> Vec { (0..len).map(|i| (i % 251) as u8).collect() diff --git a/tests/shard_fec_scrub.rs b/tests/shard_fec_scrub.rs index d648a3e..9968250 100644 --- a/tests/shard_fec_scrub.rs +++ b/tests/shard_fec_scrub.rs @@ -8,7 +8,7 @@ use anyhow::Result; use carbonado::{ error::CarbonadoError, scrub, - stream::{decode_shards_stream, encode_shard_stream, ShardEncodeResult, ShardSource}, + stream::{ShardEncodeResult, ShardSource, decode_shards_stream, encode_shard_stream}, }; use common::corruption::InboardShardLayout; diff --git a/tests/sharding.rs b/tests/sharding.rs index 050fa2c..a94e33f 100644 --- a/tests/sharding.rs +++ b/tests/sharding.rs @@ -8,8 +8,8 @@ use carbonado::{ constants::MAGICNO, error::CarbonadoError, stream::{ - decode_shards_stream, encode_shard_stream, ShardEncodeResult, ShardSource, - DEFAULT_SEGMENT_PLAINTEXT_BUDGET, + DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, decode_shards_stream, + encode_shard_stream, }, }; use rand::RngCore; diff --git a/tests/slh_outboard.rs b/tests/slh_outboard.rs index 3bba0ee..e31b4a8 100644 --- a/tests/slh_outboard.rs +++ b/tests/slh_outboard.rs @@ -1,9 +1,7 @@ //! Phase 1B: SLH-DSA sidecar E2E (requires `pqc` feature). //! -//! Default features already enable `pqc`. Also in the lean freeze allowlist: -//! `just test-lean-ci` / `cargo test --no-default-features --features "backend-lean,pqc,ots,cli" --test slh_outboard`. -//! Never `cargo test --all-features` (enables both backends → `compile_error!`). -//! Builds without `pqc` skip this crate (`#![cfg(feature = "pqc")]`). +//! Default features already enable `pqc`. Builds without `pqc` skip this crate +//! (`#![cfg(feature = "pqc")]`). #![cfg(feature = "pqc")] @@ -12,9 +10,9 @@ use std::fs; use carbonado::{ constants::Format, crypto::{ + Algorithm, PublicKey, SLH1_MAGIC, SLH1_SIDECAR_LEN, SLH1_SIGNATURE_LEN, Signature, read_slh_sidecar, slh_dsa_generate_keypair, slh_dsa_sign, slh_dsa_verify, - write_slh_sidecar, Algorithm, PublicKey, Signature, SLH1_MAGIC, SLH1_SIDECAR_LEN, - SLH1_SIGNATURE_LEN, + write_slh_sidecar, }, error::CarbonadoError, file::{self, Header}, diff --git a/tests/streaming.rs b/tests/streaming.rs index 16144fb..694fcb4 100644 --- a/tests/streaming.rs +++ b/tests/streaming.rs @@ -336,9 +336,7 @@ fn file_stream_format_sweep() { /// W1b: public **non-Compression** outboard stream (c4 Bao, c12 Bao+FEC) multi-MiB /// codecode/decodec. /// -/// Under `backend-lean` this is the **S4 O(chunk/stripe) composition E2** path (not pure Lean -/// buffer; not Compression — bulk zstd under lean is O(logical)). Under `backend-rust` it is -/// the same S4 pipeline. Wire must match buffer path; public re-encode is deterministic. +/// S4 O(chunk/stripe) pipeline. Wire must match the buffer path; public re-encode is deterministic. /// /// **Peak RAM:** architectural O(chunk/stripe) claim (SeekableSpool / stripe FEC / leaf Bao); /// not RSS-instrumented here (optional W4 measurement residual). @@ -427,7 +425,7 @@ fn stream_outboard_public_e2_codecode_decodec_c4_c12() { .expect("decode2"); assert_eq!(out2, pt, "c{format} decodec plaintext"); - // Match buffer path (Lean dual under backend-lean for buffer APIs) + // Match buffer path let buf = stream_encode_outboard_buffer(&MASTER, &pt, format, None).expect("buf encode"); assert_eq!(buf.hash, hash1, "c{format} stream vs buffer hash"); assert_eq!(buf.main, main1_bytes, "c{format} stream vs buffer main"); diff --git a/tests/streaming_async.rs b/tests/streaming_async.rs index 924193e..f31acb1 100644 --- a/tests/streaming_async.rs +++ b/tests/streaming_async.rs @@ -212,9 +212,7 @@ async fn stream_decode_async_truncated_bounded_body_staging_errors_c4_c8() { } /// Verification c12 truncated body: async always fails at spool staging -/// (`truncated encoded body`). Sync taxonomy is engine-dependent (R10): -/// - `backend-rust`: incremental Bao → `BaoResponseTruncated` (divergence from async). -/// - `backend-lean`: R5 E1 spool `read_exact` → `UnexpectedEof` (both paths fail before Bao). +/// (`truncated encoded body`). Sync incremental Bao → `BaoResponseTruncated`. #[tokio::test] async fn stream_decode_async_truncated_bounded_verification_diverges_from_sync_c12() { let input: Vec = (0..8192).map(|i| (i % 251) as u8).collect(); @@ -233,18 +231,9 @@ async fn stream_decode_async_truncated_bounded_verification_diverges_from_sync_c &mut sync_out, ) .expect_err("sync truncated bounded c12"); - #[cfg(feature = "backend-rust")] assert!( matches!(err_sync, CarbonadoError::BaoResponseTruncated(_)), - "sync rust must yield BaoResponseTruncated, got {err_sync:?}" - ); - #[cfg(feature = "backend-lean")] - assert!( - matches!( - err_sync, - CarbonadoError::StdIoError(ref e) if e.kind() == ErrorKind::UnexpectedEof - ), - "sync lean E1 must yield UnexpectedEof on short body, got {err_sync:?}" + "sync must yield BaoResponseTruncated, got {err_sync:?}" ); assert!(sync_out.is_empty()); diff --git a/tests/streaming_limits.rs b/tests/streaming_limits.rs index ca42a67..980d014 100644 --- a/tests/streaming_limits.rs +++ b/tests/streaming_limits.rs @@ -11,13 +11,13 @@ use std::io::Cursor; use carbonado::constants::FEC_M; use carbonado::decode as low_level_decode; use carbonado::error::CarbonadoError; -use carbonado::file::{decode, decode_stream, encode, encode_stream, Header}; +use carbonado::file::{Header, decode, decode_stream, encode, encode_stream}; use carbonado::stream::crypto_stream::{ stream_decrypt, stream_decrypt_seek, stream_decrypt_with_nonce, stream_decrypt_with_nonce_seek, }; use carbonado::stream::encode::stream_encode_outboard; -use carbonado::stream::encode::{stream_encode_inboard_body, PreprocessStats}; -use carbonado::stream::fec::{encode_inboard_buffer, FecInboardEncoder}; +use carbonado::stream::encode::{PreprocessStats, stream_encode_inboard_body}; +use carbonado::stream::fec::{FecInboardEncoder, encode_inboard_buffer}; use carbonado::stream::{ stream_decode, stream_decode_buffer, stream_decode_outboard, stream_decode_outboard_buffer, stream_encode_buffer, @@ -26,7 +26,7 @@ use carbonado::{encode_outboard, scrub, scrub_outboard, verify_inboard_keyed_ora use rand::RngCore; use common::inboard_parity::{ - assert_bounded_inboard_body_roundtrip, assert_inboard_body_roundtrip, BoundedReadSeek, + BoundedReadSeek, assert_bounded_inboard_body_roundtrip, assert_inboard_body_roundtrip, }; const MASTER: [u8; 32] = [0x42; 32]; diff --git a/tests/udp_fec_sim.rs b/tests/udp_fec_sim.rs index b42ffcf..8f48b91 100644 --- a/tests/udp_fec_sim.rs +++ b/tests/udp_fec_sim.rs @@ -27,7 +27,7 @@ use carbonado::{ scrub, structs::Encoded, }; -use common::corruption::{erase_shards, InboardShardLayout, OutboardShardLayout}; +use common::corruption::{InboardShardLayout, OutboardShardLayout, erase_shards}; /// Chaos-injection datagram: `shard_index` + payload at `InboardShardLayout` coordinates. #[derive(Clone, Debug)] @@ -323,8 +323,8 @@ fn directory_bundle_parity_outboard_scrub_recovery() -> Result<()> { directory::SegmentFormatPolicy, encode_outboard, file::{ - decode, decode_directory, encode_directory_with_options, DirectoryEncodeOptions, - DIRECTORY_ARCHIVE_FORMAT, + DIRECTORY_ARCHIVE_FORMAT, DirectoryEncodeOptions, decode, decode_directory, + encode_directory_with_options, }, filepack_manifest::FilepackManifest, scrub_outboard, diff --git a/tests/zstd_frame_params.rs b/tests/zstd_frame_params.rs new file mode 100644 index 0000000..44a6975 --- /dev/null +++ b/tests/zstd_frame_params.rs @@ -0,0 +1,204 @@ +//! Zstd frame-header parameters named by Lean `Carbonado.Compress`. +//! +//! These tests **read the frame** (magic + `Frame_Header_Descriptor` bits, +//! optional window descriptor, optional content size). They do **not** claim +//! cross-engine compressed-block identity (W2a permanent residual). +//! +//! Fail-closed if G9 `outboard_c14` fixtures are missing. + +mod common; + +use std::fs; +use std::path::{Path, PathBuf}; + +use carbonado::constants::{ + ZSTD_CONTENT_CHECKSUM, ZSTD_DICTIONARY_ID_FLAG, ZSTD_LEVEL, ZSTD_LEVEL20_WINDOW_LOG_LARGE, + ZSTD_MAGIC, +}; +use carbonado::stream::compress::compress_buffer; + +use common::zstd_frame::{ParsedZstdFrameHeader, ZstdFrameError, parse_zstd_frame_header}; + +/// Lean AOT `ZSTD_compress` level-20 golden for `hello` (`Carbonado.Main`). +const LEAN_HELLO_LEVEL20: &[u8] = &[ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x05, 0x29, 0x00, 0x00, 0x68, 0x65, 0x6c, 0x6c, 0x6f, +]; + +/// Lean AOT `ZSTD_compress` level-20 golden for empty input. +const LEAN_EMPTY_LEVEL20: &[u8] = &[0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x00, 0x01, 0x00, 0x00]; + +fn g9_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/g9") +} + +fn require_file(path: &Path) -> Vec { + assert!( + path.is_file(), + "missing committed zstd frame fixture {}", + path.display() + ); + fs::read(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +fn assert_product_shared_flags(h: &ParsedZstdFrameHeader) { + assert!(!h.unused_bit, "Unused_bit must be 0 (Lean zstdUnusedBit)"); + assert!( + !h.reserved_bit, + "Reserved_bit must be 0 (Lean zstdReservedBit)" + ); + assert_eq!( + h.content_checksum, ZSTD_CONTENT_CHECKSUM, + "Content_Checksum_flag (Lean zstdContentChecksum)" + ); + assert_eq!( + h.dictionary_id_flag, ZSTD_DICTIONARY_ID_FLAG, + "Dictionary_ID_flag (Lean zstdDictionaryIdFlag)" + ); + assert_eq!(h.dictionary_id, None); +} + +#[test] +fn rust_constants_match_lean_spec() { + const { + assert!(ZSTD_LEVEL == 20); + assert!(matches!(ZSTD_MAGIC, [0x28, 0xb5, 0x2f, 0xfd])); + assert!(!ZSTD_CONTENT_CHECKSUM); + assert!(ZSTD_DICTIONARY_ID_FLAG == 0); + assert!(ZSTD_LEVEL20_WINDOW_LOG_LARGE == 25); + } +} + +#[test] +fn parse_rejects_truncated_bad_magic_reserved() { + assert_eq!( + parse_zstd_frame_header(&[]).unwrap_err(), + ZstdFrameError::TruncatedHeader + ); + assert_eq!( + parse_zstd_frame_header(&[0x28, 0xb5, 0x2f, 0xfd]).unwrap_err(), + ZstdFrameError::TruncatedHeader + ); + assert_eq!( + parse_zstd_frame_header(&[0x00, 0x01, 0x02, 0x03, 0x20]).unwrap_err(), + ZstdFrameError::BadMagic + ); + assert_eq!( + parse_zstd_frame_header(&[0x28, 0xb5, 0x2f, 0xfd, 0x08]).unwrap_err(), + ZstdFrameError::ReservedBitSet + ); +} + +#[test] +fn bulk_level20_hello_matches_lean_aot_golden() { + let frame = zstd::bulk::Compressor::new(ZSTD_LEVEL) + .expect("compressor") + .compress(b"hello") + .expect("compress hello"); + assert_eq!( + frame.as_slice(), + LEAN_HELLO_LEVEL20, + "zstd-sys 1.5.7 ZSTD_compress must match Lean AOT hello golden" + ); + let h = parse_zstd_frame_header(&frame).expect("parse hello"); + assert_eq!(&frame[..4], &ZSTD_MAGIC); + assert_product_shared_flags(&h); + assert!(h.single_segment, "Lean productBufferSmallFrameOk"); + assert_eq!(h.content_size_flag, 0); + assert_eq!(h.content_size, Some(5)); + assert_eq!(h.window_descriptor, None); + assert_eq!(h.descriptor, 0x20); +} + +#[test] +fn bulk_level20_empty_matches_lean_aot_golden() { + let frame = zstd::bulk::Compressor::new(ZSTD_LEVEL) + .expect("compressor") + .compress(b"") + .expect("compress empty"); + assert_eq!( + frame.as_slice(), + LEAN_EMPTY_LEVEL20, + "zstd-sys 1.5.7 ZSTD_compress must match Lean AOT empty golden" + ); + let h = parse_zstd_frame_header(&frame).expect("parse empty"); + assert_product_shared_flags(&h); + assert!(h.single_segment); + assert_eq!(h.content_size, Some(0)); + assert_eq!(h.descriptor, 0x20); +} + +#[test] +fn product_compress_buffer_frame_params() { + let frame = compress_buffer(b"hello").expect("compress_buffer hello"); + assert_eq!(&frame[..4], &ZSTD_MAGIC); + let h = parse_zstd_frame_header(&frame).expect("parse product hello"); + assert_product_shared_flags(&h); + assert!( + !h.single_segment, + "copy_encode leaves Single_Segment clear (differs from Lean AOT one-shot frames)" + ); + assert_eq!(h.content_size, None); + assert_eq!(h.window_log, Some(ZSTD_LEVEL20_WINDOW_LOG_LARGE)); + assert_eq!(h.window_descriptor, Some(0x78)); + assert_ne!( + frame.as_slice(), + LEAN_HELLO_LEVEL20, + "streaming rust frame must still differ from Lean AOT hello golden" + ); +} + +#[test] +fn g9_outboard_c14_fixtures_match_lean_named_params() { + let rust_main = g9_root().join("rust/outboard_c14/main.bin"); + let lean_main = g9_root().join("lean/outboard_c14/main.bin"); + let rust = require_file(&rust_main); + let lean = require_file(&lean_main); + assert!( + rust.len() >= 6, + "rust c14 main too short for a zstd frame header" + ); + assert!( + lean.len() >= 6, + "lean c14 main too short for a zstd frame header" + ); + + let rh = parse_zstd_frame_header(&rust).expect("parse rust c14"); + let lh = parse_zstd_frame_header(&lean).expect("parse lean c14"); + assert_eq!(&rust[..4], &ZSTD_MAGIC); + assert_eq!(&lean[..4], &ZSTD_MAGIC); + assert_product_shared_flags(&rh); + assert_product_shared_flags(&lh); + + assert_eq!(lh.descriptor, 0x20, "Lean AOT G9 c14 descriptor"); + assert!(lh.single_segment); + assert_eq!(lh.content_size, Some(26), "g9_matrix_v1 plaintext len"); + assert_eq!(lh.window_descriptor, None); + + assert_eq!(rh.descriptor, 0x00, "Rust streaming G9 c14 descriptor"); + assert!(!rh.single_segment); + assert_eq!(rh.content_size, None); + assert_eq!(rh.window_log, Some(ZSTD_LEVEL20_WINDOW_LOG_LARGE)); + assert_eq!(rh.window_descriptor, Some(0x78)); + assert_eq!(rh.window_size, Some(1u64 << 25)); + + assert_eq!( + rh.header_len, lh.header_len, + "both G9 c14 headers are 6 bytes (desc+window vs desc+FCS)" + ); + assert_eq!( + &rust[rh.header_len..], + &lean[lh.header_len..], + "G9 c14 compressed blocks match; residual is the frame header only" + ); +} + +#[test] +fn stream_copy_encode_unknown_size_window_log_25() { + let mut frame = Vec::new(); + zstd::stream::copy_encode(b"hello" as &[u8], &mut frame, ZSTD_LEVEL).expect("copy_encode"); + let h = parse_zstd_frame_header(&frame).expect("parse copy_encode"); + assert_eq!(&frame[..4], &ZSTD_MAGIC); + assert_product_shared_flags(&h); + assert!(!h.single_segment); + assert_eq!(h.window_log, Some(ZSTD_LEVEL20_WINDOW_LOG_LARGE)); +} From 2a2c8f7bbb0b7e9117e3e75a4362641ca53bd316 Mon Sep 17 00:00:00 2001 From: Hunter Beast Date: Thu, 27 Aug 2026 15:17:34 -0600 Subject: [PATCH 5/6] carbonado 0.7 --- .cargo/config.toml.example | 6 +++--- AGENTS.md | 6 +++--- CHANGELOG.md | 2 +- Cargo.toml | 10 ++++++---- README.md | 4 ++-- docs/PARITY.md | 2 +- justfile | 8 ++++---- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.cargo/config.toml.example b/.cargo/config.toml.example index fdd4b89..4abcff3 100644 --- a/.cargo/config.toml.example +++ b/.cargo/config.toml.example @@ -1,12 +1,12 @@ # Optional local development overrides (copy to `.cargo/config.toml`). # -# Faster iteration: use a sibling checkout of n0-computer/bao-tree (PR 78 merge -# SHA, see just setup-bao-tree) instead of fetching from git on every clean build. +# Faster iteration: use a sibling checkout of n0-computer/bao-tree 0.16.1 +# (see just setup-bao-tree) instead of crates.io on every clean build. # # just setup-bao-tree # cp .cargo/config.toml.example .cargo/config.toml -[patch."https://github.com/n0-computer/bao-tree.git"] +[patch.crates-io] bao-tree = { path = "../bao-tree" } # `bitcoinpqc` 0.4 mirror lag: the repo ships `.cargo/config.toml` with a temporary diff --git a/AGENTS.md b/AGENTS.md index c4072c3..3ebb00c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,7 +169,7 @@ The overarching principle is a **clean cryptographic break** (see §1). v1 ECIES | Passphrase KDF | Argon2id wrapper inside library | Removed; caller responsibility (Argon2id recommended outside) | Keeps container security contract simple. Master key is 32/64B high-entropy material. | | Magic number | CARBONADO01 or similar (ECIES) | CARBONADO20\n (stable v2); 02 was dev transitional | Signals official stabilized 2.0 format. Old magic → clear external migration error. | | Version | Pre-0.7 (ECIES) | 2.0.0 (post-FEC + docs stabilization) | Marks end of fluid dev period. API now stable for semver. | -| Dependencies | ecies + secp + ... | aes+ctr+hmac+sha2 + reed-solomon-erasure + n0-computer/bao-tree (keyed git pin) + bitcoinpqc (optional pqc) | Clean break removal of ECIES-only crates. Hardware-accel friendly. | +| Dependencies | ecies + secp + ... | aes+ctr+hmac+sha2 + reed-solomon-erasure + n0-computer/bao-tree 0.16.1 (keyed) + bitcoinpqc (optional pqc) | Clean break removal of ECIES-only crates. Hardware-accel friendly. | | Optional hybrid layer | (the only encryption was the ECIES hybrid) | Pure symmetric is default. Added *optional* inner secp256k1-ECDH + ChaCha20-Poly1305 AEAD wrapped by outer AES-CTR + HMAC-EtM (via new hybrid_* and ecc_aead_* APIs) | "Maximal paranoia" defense-in-depth: different cipher families, different key-gen (ECDH+derive vs pure HMAC labels), HMAC + AEAD. See dedicated rationale below. Pure sym path and Encrypted bit semantics unchanged for normal use. secp here is *not* for the main container (no pubkeys in headers etc.). | #### Detailed Decision Rationales @@ -673,7 +673,7 @@ Current registered labels (must be kept in sync with code — full table in **Su - Suggestion: Use a keyed variant of the Bao tree (keyed on the format bitmask byte, or a small header prefix) so that the root hash cryptographically commits to which processing pipeline was used. - This would be extremely useful for data markets (see §9), because different format combinations (especially encrypted vs public) would produce distinguishable roots even for related data. - **Endianness for key material**: All integer fields in Carbonado (and in the Bao format itself) are little-endian. If a keyed Bao implementation derives a 32-byte key from header fields, those fields should be serialized in LE order for consistency. A minimal implementation that only keys on the single-byte `format` bitmask has no endianness issues at all. - - (Implemented) Original `bao` 0.13 lacked BlockSize and public keyed. Now using n0-computer/bao-tree (PR 78 merge, git rev pin) with BlockSize(2) for 4KB + keyed_hash on format byte (root commits to pipeline). See constants::BAO_BLOCK_SIZE and stream::bao. Not published on crates.io yet. + - (Implemented) Original `bao` 0.13 lacked BlockSize and public keyed. Now using n0-computer/bao-tree 0.16.1 (crates.io; PR 78 keyed) with BlockSize(2) for 4KB + keyed_hash on format byte (root commits to pipeline). See constants::BAO_BLOCK_SIZE and stream::bao. Because there are 16 possible format combinations, the same logical input can produce up to 16 different Bao hashes. In this sense the naming is **multi-dimensional**: - When the `Encrypted` bit is set (symmetric encryption), the hash primarily names an *encrypted+protected container*. @@ -954,7 +954,7 @@ This tension is acknowledged but not resolved in the current design. Carbonado i Remaining open (documented; active work called out): - **Pipeline memory residual (hard-break track):** fused encode/decode is O(chunk) spool + O(stripe) FEC encode; non-FEC verification decode is O(chunk) via `SeekWriteAt`; FEC verification decode retains O(FEC body) shard buffers (`FecInboardWriteAt`); outboard verify uses `PostOrderOutboard` + `ReadAt` (O(hash pair) per node); `stream_decode_async` fully spools encoded body to disk. Distinct from Bao **slice** verification (already O(slice) memory). See [doc/STREAMING_PARALLELISM.md](doc/STREAMING_PARALLELISM.md). - **WASM:** `cargo clippy --target wasm32-unknown-unknown --no-default-features --features "backend-rust"` is green (CI `lint-wasm`). **wasm32 + `pqc` probe (2026-07-08):** pointing global `CC_wasm32-unknown-unknown` at `libbitcoinpqc-bindings/wasm/clang-wasm32.sh` breaks **`zstd-sys`** (it tries to assemble `huf_decompress_amd64.S` with the wasm clang). Residual is build-env / dep CC scoping — not Carbonado crypto logic. Keep CI wasm lint **no-pqc** until bitcoinpqc (or zstd) wasm build is isolated. -- Bao crate: n0-computer/bao-tree git pin (PR 78 merge `dbc952e32cbda8ffd14c106b770e72987b01618e`), 4 KiB groups, `default-features = false` (no tokio/fs on wasm). Not a crates.io version yet. +- Bao crate: n0-computer/bao-tree **0.16.1** (crates.io; PR 78 keyed APIs), 4 KiB groups, `default-features = false` (no tokio/fs on wasm). - reed-solomon-erasure: upstream "looking for maintainers"; periodic re-eval (no runtime issues). - (Perf: inboard `verify_slice` is O(slice) memory but O(N) encoded-byte I/O; outboard slice verify is O(slice) time+memory; scrub pre-check uses `verify_inboard_keyed` with O(1) retained decode memory (S5).) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96e3833..6491b15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ First crates.io release of the v2 format (`CARBONADO20\n`). Last published crate - **Crate version 0.7.0** (crates.io next after 0.6.0). Format magic remains `CARBONADO20\n`. - **Rust edition 2024 / rustc 1.98:** package `edition = "2024"`, `rust-version = "1.98.0"`, `rust-toolchain.toml` channel `1.98.0`. CI uses the same toolchain pin. Flake adds `oxalica/rust-overlay` for 1.98 in the dev shell and a `rustc-1_98` check (separate nixpkgs overlay so Lean AOT does not rebuild on Rust pin changes). -- **bao-tree upstream pin:** Cargo git dep is n0-computer/bao-tree at PR 78 merge `dbc952e32cbda8ffd14c106b770e72987b01618e` (keyed 4 KiB groups). Not a crates.io version. Replaces the `keyed-bao` branch pin and the earlier Surmount `76-keyed-bao` fork docs. +- **bao-tree 0.16.1** from crates.io (keyed 4 KiB groups, PR 78). Replaces the git `rev` pin at the PR 78 merge. - **M1 pipeline memory (hard break):** non-FEC verification decode (c6) uses `SeekWriteAt` over the post-preprocess spool (O(chunk) RAM; no full logical `Vec`). FEC verification uses `FecInboardWriteAt::finish_into` (stream logical bytes without a second full logical buffer; shard buffers remain O(FEC body) under segment-wide RS geometry). See `doc/STREAMING_PARALLELISM.md`. - **M2 outboard verify memory:** `stream_verification_outboard_verify` uses `PostOrderOutboard` + `ReadAt` (on-demand hash pairs) instead of copying the full sidecar into `PostOrderMemOutboard`. Streaming outboard decode keeps the sidecar on a disk spool. - **S5 scrub verify oracle:** `scrub` pre-check uses `verify_inboard_keyed` (`DiscardWriteAt` sink) instead of buffer `verification()` full-body staging; `scrub_outboard` pre-check uses `stream_verification_outboard_verify` with `io::sink()`. Memory tiers in `doc/STREAMING_PARALLELISM.md`. diff --git a/Cargo.toml b/Cargo.toml index 9e8f8d1..40ca36f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,22 +11,24 @@ repository = "https://github.com/bitmask-stack/carbonado.git" readme = "README.md" keywords = ["archive", "encryption", "erasure-coding", "bao", "backup"] categories = ["command-line-utilities", "cryptography"] -include = ["src/**/*", "LICENSE", "README.md", "doc/man/*.1", "doc/man/README.md", ".cargo/config.toml.example"] +# Slim crates.io package: library + CLI sources, license, readme, man pages. +# Do not ship tests, examples, benches, Lean, nix, or ref/ oracles. +# `/LICENSE` and `/README.md` are crate-root only (`README.md` would match ref/**). +include = ["src/**/*", "/LICENSE", "/README.md", "doc/man/*.1", "doc/man/README.md"] [dependencies] # bao kept for Hash re-export (blake3::Hash), pub re-export in lib, and legacy error type bridging. # All Bao logic uses n0-computer/bao-tree keyed 4KB groups + format-keyed roots. bao = "0.13" -# Keyed Bao: n0-computer/bao-tree PR 78 merged to main (not on crates.io yet). -# Pin the merge commit, not a branch and not a crates.io version. +# Keyed Bao: n0-computer/bao-tree 0.16.1 (PR 78 keyed APIs, crates.io 2026-08-26). # Default 4KB groups (BlockSize::from_chunk_log(2)) via BAO_BLOCK_SIZE. Keyed mode # makes root = keyed_hash(key_from_format, data) so the Bao hash commits to the # exact format pipeline (multi-dimensional naming). See AGENTS.md and # constants::BAO_BLOCK_SIZE. # default-features = false: crate defaults pull in tokio/fs and break some # cross/wasm targets. We only need sync keyed + validate. -bao_tree = { package = "bao-tree", git = "https://github.com/n0-computer/bao-tree.git", rev = "dbc952e32cbda8ffd14c106b770e72987b01618e", default-features = false, features = ["validate"] } +bao_tree = { package = "bao-tree", version = "0.16.1", default-features = false, features = ["validate"] } futures-lite = { version = "2", optional = true, default-features = false, features = ["std"] } tokio = { version = "1", features = ["rt"], optional = true } bitmask-enum = "2.1.0" diff --git a/README.md b/README.md index 3e0370a..c86b7ed 100644 --- a/README.md +++ b/README.md @@ -345,10 +345,10 @@ Code, dependencies, and programs can be vendored and preserved wherever they are ## Development -Requires [just](https://github.com/casey/just), [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`), and **Rust 1.98** (edition 2024; see `rust-toolchain.toml`). Cargo fetches keyed `bao-tree` from n0-computer at the PR 78 merge SHA. An optional sibling checkout at `../bao-tree` speeds clean builds (`just setup-bao-tree`): +Requires [just](https://github.com/casey/just), [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`), and **Rust 1.98** (edition 2024; see `rust-toolchain.toml`). Cargo uses `bao-tree` 0.16.1 from crates.io. An optional sibling checkout at `../bao-tree` speeds clean builds (`just setup-bao-tree`): ```bash -just setup-bao-tree # optional; pins n0-computer/bao-tree at the merge SHA +just setup-bao-tree # optional; sibling checkout of bao-tree 0.16.1 just # list recipes just all # everything (fmt, lint, tests, release build, source grep) ``` diff --git a/docs/PARITY.md b/docs/PARITY.md index 9b04544..9ab72be 100644 --- a/docs/PARITY.md +++ b/docs/PARITY.md @@ -21,7 +21,7 @@ Pin the exact **third-party** trees the Rust product used (Bao, RS, crypto crate | ref path | Source | Pin (commit / tag) | |----------|--------|--------------------| -| `ref/bao-tree` | `https://github.com/SurmountSystems/bao-tree.git` | Oracle snapshot **`02916e784bb0afe0fd5a73c291c8c5335865e166`** (keyed work before upstream squash). **Product** cargo dep is n0-computer/bao-tree **`dbc952e32cbda8ffd14c106b770e72987b01618e`** (PR 78 merge; git rev, not crates.io). | +| `ref/bao-tree` | `https://github.com/SurmountSystems/bao-tree.git` | Oracle snapshot **`02916e784bb0afe0fd5a73c291c8c5335865e166`** (keyed work before upstream squash). **Product** cargo dep is crates.io **bao-tree 0.16.1**. | | `ref/reed-solomon-erasure` | `https://github.com/darrenldl/reed-solomon-erasure.git` | tag **`v5.0.3`** → **`9f974918f8c598eee351406c36fa0295f4bb4d69`** | | `ref/rustcrypto-block-ciphers` | `https://github.com/RustCrypto/block-ciphers.git` | tag **`aes-v0.8.4`** → **`f2dbee516b4d0cf4cb4f3045d09e35b5fd80087b`** | | `ref/rustcrypto-macs` | `https://github.com/RustCrypto/MACs.git` | tag **`hmac-v0.12.1`** → **`46797e3b44973a30edb9d7f3a3ebb41810061d90`** | diff --git a/justfile b/justfile index 16bef48..8bd8f39 100644 --- a/justfile +++ b/justfile @@ -17,8 +17,8 @@ system := env_var_or_default("CI_SYSTEM", `case "$(uname -s)-$(uname -m)" in Lin default: @just --list -# Clone n0-computer/bao-tree at the PR 78 merge SHA (optional sibling path patch). -bao_tree_rev := "dbc952e32cbda8ffd14c106b770e72987b01618e" +# Clone n0-computer/bao-tree 0.16.1 (optional sibling path patch). +bao_tree_rev := "0.16.1" setup-bao-tree: #!/usr/bin/env bash @@ -30,7 +30,7 @@ setup-bao-tree: git clone https://github.com/n0-computer/bao-tree.git ../bao-tree fi git -C ../bao-tree fetch --all --tags - git -C ../bao-tree checkout "$PIN" + git -C ../bao-tree checkout "v${PIN}" 2>/dev/null || git -C ../bao-tree checkout "$PIN" rg -q 'create_keyed|keyed_outboard_post_order' ../bao-tree/src echo "bao-tree OK (n0-computer $PIN)" @@ -44,7 +44,7 @@ require-bao-tree: exit 1 fi if ! rg -q 'create_keyed|keyed_outboard_post_order' ../bao-tree/src 2>/dev/null; then - echo "Wrong bao-tree at ../bao-tree — need n0-computer PR 78 merge ($PIN)" + echo "Wrong bao-tree at ../bao-tree — need n0-computer bao-tree $PIN (keyed APIs)" exit 1 fi From b8bab6181bdf846de78016de9d9ed3d7ba132b9a Mon Sep 17 00:00:00 2001 From: Hunter Beast Date: Thu, 27 Aug 2026 22:01:41 -0600 Subject: [PATCH 6/6] carbonado 0.7.1 --- .cargo/config.toml | 14 +- .gitignore | 2 +- AGENTS.md | 1 + CHANGELOG.md | 9 + Carbonado/Compress.lean | 5 +- CarbonadoTest/Compress.lean | 2 +- Cargo.lock | 2448 +++++++++++++++++ Cargo.toml | 8 +- RESIDUAL.md | 42 + benches/crypto_bench.rs | 1 + benches/fec_stripe_bench.rs | 74 + benches/parallel_bench.rs | 5 +- doc/STREAMING_PARALLELISM.md | 2 +- doc/TEST_STRATEGY.md | 2 +- doc/man/carbonado-encode.1 | 8 +- doc/man/carbonado.1 | 5 +- docs/LIMITS.md | 2 +- docs/TEST_CONTRACT.md | 3 +- examples/basic_roundtrip.rs | 6 +- examples/dir_archival.rs | 3 +- examples/dump_rkyv_r9.rs | 167 +- examples/slh_dsa_sidecar.rs | 11 +- justfile | 22 +- nix/cargo-quality.nix | 15 + src/adamantine.rs | 33 +- src/adamantine_payload.rs | 9 + src/bin/carbonado/main.rs | 247 +- src/cli_app.rs | 24 +- src/constants.rs | 17 +- src/decoding.rs | 414 ++- src/encoding.rs | 51 +- src/error.rs | 16 +- src/file.rs | 437 ++- src/filepack_manifest.rs | 175 +- src/lib.rs | 27 +- src/paths.rs | 168 +- src/stream/bao.rs | 10 +- src/stream/compress.rs | 225 +- src/stream/decode.rs | 105 +- src/stream/encode.rs | 98 +- src/stream/fec.rs | 776 +++--- src/stream/mod.rs | 23 +- src/stream/shard.rs | 26 + src/stream/slice.rs | 173 +- tests/adam_zstd.rs | 334 +++ tests/adversarial_proptest.rs | 5 +- tests/apocalypse.rs | 2 - tests/bao_keyed_contract.rs | 6 +- tests/bin_cli.rs | 50 +- tests/bin_heuristics.rs | 102 +- tests/bin_smoke.rs | 4 + tests/codec.rs | 23 +- tests/common/cli.rs | 14 +- tests/common/corruption.rs | 146 +- tests/common/inboard_parity.rs | 4 +- tests/common/mod.rs | 107 + tests/common/zstd_frame.rs | 3 +- tests/determinism_roundtrip.rs | 76 +- tests/directory_archive.rs | 96 +- tests/fec_chaos.rs | 262 +- tests/fec_scrub_matrix.rs | 4 +- tests/filepack_interop.rs | 20 +- tests/fixtures/directory_interop_golden.json | 34 +- tests/fixtures/phase3_g9_directory/README.txt | 26 +- ...76868f091eec0d614a129c506125c114.adam.c14} | Bin 33401 -> 33401 bytes tests/fixtures/rkyv/README.md | 6 +- tests/fixtures/rkyv/empty_manifest.bin | Bin 13 -> 13 bytes tests/fixtures/rkyv/multi_entry_ots.bin | Bin 275 -> 291 bytes tests/fixtures/rkyv/ots_first_only.bin | Bin 251 -> 267 bytes tests/fixtures/rkyv/path_inline_8.bin | Bin 131 -> 139 bytes tests/fixtures/rkyv/path_ool_9.bin | Bin 140 -> 148 bytes tests/fixtures/rkyv/rkyv_cfp2_prefix.bin | Bin 131 -> 139 bytes tests/fixtures/rkyv/single_entry.bin | Bin 131 -> 139 bytes tests/fixtures/rkyv/two_segments.bin | Bin 191 -> 207 bytes tests/format.rs | 30 +- tests/format_amplification.rs | 7 +- tests/g9_cross_backend.rs | 31 +- tests/header_tamper.rs | 12 +- tests/parallel_determinism.rs | 86 +- tests/rkyv_golden_lock.rs | 137 +- tests/seekable_slices.rs | 7 +- tests/serial_fec_path.rs | 13 +- tests/shard_fec_scrub.rs | 12 +- tests/sharding.rs | 2 +- tests/slh_outboard.rs | 5 +- tests/streaming.rs | 38 +- tests/streaming_async.rs | 2 +- tests/streaming_limits.rs | 34 +- tests/udp_fec_sim.rs | 88 +- tests/zstd_frame_params.rs | 12 +- 90 files changed, 6434 insertions(+), 1317 deletions(-) create mode 100644 Cargo.lock create mode 100644 RESIDUAL.md create mode 100644 benches/fec_stripe_bench.rs create mode 100644 tests/adam_zstd.rs rename tests/fixtures/phase3_g9_directory/{16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14 => f14bfeb50f1d3072e5510d7eab42fea176868f091eec0d614a129c506125c114.adam.c14} (90%) diff --git a/.cargo/config.toml b/.cargo/config.toml index 29e1896..0e54896 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,10 @@ -# TEMPORARY — remove on or after 2026-07-18. +# This directory's `.cargo/config.toml` is closer than `~/.cargo`, so it +# overrides menhera-cooldown for this repo only. Do not edit ~/.cargo. +# +# Nix crane vendors crates.io and would lose if this replace-with stayed in +# the sandbox. `nix/cargo-quality.nix` strips the two tables below on unpack. +# +# TEMPORARY bitcoinpqc patch — remove on or after 2026-07-18. # # `bitcoinpqc` 0.4.0 (published 2026-07-08) may not be on mirrored registries yet. # This patch satisfies `version = "0.4"` until mirrors sync (target lift: 2026-07-18). @@ -10,6 +16,12 @@ # # Environments that already see `bitcoinpqc` 0.4 on the default index may delete early. +[registries.crates-io-official] +index = "sparse+https://index.crates.io/" + +[source.crates-io] +replace-with = "crates-io-official" + [patch.crates-io] bitcoinpqc = { git = "https://github.com/cryptoquick/libbitcoinpqc-bindings.git", rev = "7936b56" } # Monorepo local iteration (optional): `bitcoinpqc = { path = "../libbitcoinpqc-bindings" }` \ No newline at end of file diff --git a/.gitignore b/.gitignore index 330ccf2..810d126 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /target -/Cargo.lock + *.7z # CI (and optional local) in-workspace bao-tree checkout. Local `just setup-bao-tree` diff --git a/AGENTS.md b/AGENTS.md index 3ebb00c..caf637f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -833,6 +833,7 @@ Offset Size Field **Directory archive layout (fixed v1.0):** - **Catalog:** inboard headered `{catalog_root}.adam.c14` or `.adam.c15` (`CARBONADO20\n` + body); no `.out`/`.par` +- **Single-file (0.7.1):** inboard is one `{hash}.adam.cXX` (Header + body + optional Adamantine after `encoded_len`). Outboard is `{hash}.cXX` + `{hash}.adam.cXX` sidecar starting with `ADAMANTINE10\n`. Same-stem pair is not a directory catalog. Zstd level is encoder input (no silent default 20). Dict lives in the Adamantine bundle, not a `.dict` sibling. - **Segments:** bare mains `{seg_root}.c12`/`.c14`/`.c13`/`.c15` only; verification outboard + FEC parity centralized in Adam payload bundle - **No** directory `.out`, `.par`, or `.ots` sidecar files - **Scrub:** directory segments are FEC-capable (c12–c15). Slice verification + FEC parity from the centralized bundle; `scrub_outboard` recovers corrupt bare mains within the RS 4/8 budget (≤4 shard taints). `MissingFecParity` when `Format::Fec` is set, `main_len > 0`, and `fec_parity_len` is zero (zero-byte mains use empty FEC slice at decode). diff --git a/CHANGELOG.md b/CHANGELOG.md index 6491b15..423d5b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to the Carbonado crate and `carbonado` CLI are documented here. +## [0.7.1] — 2026-08-27 + +### Changed + +- **Zstd level is encoder input.** Compression encode fails with `MissingZstdLevel` unless an explicit level is supplied. There is no silent library default of 20. Tests and the CLI may pass `20` explicitly (`--zstd-level`). +- **Single-file Adamantine sidecar.** Outboard writes exactly `{hash}.cXX` + `{hash}.adam.cXX` (`ADAMANTINE10\n`). Inboard writes exactly one `{hash}.adam.cXX` (Header + body + Adamantine after `encoded_len`). No `.par` or `.dict` siblings. Same-stem `.cXX` + `.adam.cXX` is single-file outboard, not a directory catalog. +- **FilepackManifest v3.** One bundle blob with keyed Bao outboard, RS parity (empty if no FEC), and RFC 8878 dictionary (empty if none). `SegmentRef` adds `dict_offset` / `dict_len`. Until 1.0 there is no dual-read of old `.out`/`.par` companions. +- **Optional `--zstd-dict`.** Dictionary bytes are stored in the Adamantine dict section. Frame `Dictionary_ID` must match. Decode with a named ID and empty dict section fails (`MissingZstdDictionary`). + ## [0.7.0] — 2026-08-24 First crates.io release of the v2 format (`CARBONADO20\n`). Last published crate was **0.6.0** (v1/ECIES). In-tree `2.0.0` / `2.1.0` numbers were never published. diff --git a/Carbonado/Compress.lean b/Carbonado/Compress.lean index aba542a..9589135 100644 --- a/Carbonado/Compress.lean +++ b/Carbonado/Compress.lean @@ -28,7 +28,8 @@ inductive ZstdError where | invalidInput deriving DecidableEq, Repr -/-- Normative compression level (AGENTS: zstd-20). -/ +/-- Encoder compression level is an input, not a hidden library default. + The AOT demo and G9 goldens use 20; tests may pass 20 explicitly. -/ def zstdLevel : UInt32 := 20 /-- DoS cap on decompressed output (Rust `MAX_SEGMENT_MAIN_LEN` = 256 MiB). -/ @@ -300,7 +301,7 @@ def decodeStatusPayload (raw : ByteArray) : Except ZstdError ByteArray := else .error (ofStatus code) -/-- Compress at normative level 20 (AOT: real zstd). -/ +/-- Compress at explicit level 20 (AOT demo / G9 goldens). Level is encoder input. -/ def compressLevel20 (input : ByteArray) : Except ZstdError ByteArray := decodeStatusPayload (compressRaw input zstdLevel) diff --git a/CarbonadoTest/Compress.lean b/CarbonadoTest/Compress.lean index a396a75..1797e1b 100644 --- a/CarbonadoTest/Compress.lean +++ b/CarbonadoTest/Compress.lean @@ -92,7 +92,7 @@ theorem decompress_bit_clear : | .error _ => false) = true := by native_decide -/-- Level constant is 20. -/ +/-- AOT demo / G9 still name level 20; it is not a silent product default. -/ theorem level_20 : zstdLevel = 20 := zstdLevel_eq_20 theorem magic_literal : zstdMagic = [0x28, 0xb5, 0x2f, 0xfd] := zstdMagic_eq_literal diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..9b637f3 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2448 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bao" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9125f93587b894c196d58b0a752ce2213552d0d483c98ee41530e38202a511" +dependencies = [ + "arrayvec", + "blake3", +] + +[[package]] +name = "bao-tree" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149a2a6017771141e2cd5d0c55b3892d8ff1958df5c318b2e496bf3544b426ed" +dependencies = [ + "blake3", + "bytes", + "futures-lite", + "genawaiter", + "iroh-io", + "positioned-io", + "range-collections", + "self_cell", + "smallvec", +] + +[[package]] +name = "binary-merge" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bip39" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90dbd31c98227229239363921e60fcf5e558e43ec69094d46fc4996f08d1d5bc" +dependencies = [ + "bitcoin_hashes", + "rand 0.8.7", + "rand_core 0.6.4", + "serde", + "unicode-normalization", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitcoin-consensus-encoding" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6712f9c6fd6785b3b270884e57c441c403dc5d7e19ca45368c97c7a1de3000ec" +dependencies = [ + "bitcoin-internals", + "hex-conservative 1.2.0", + "serde", +] + +[[package]] +name = "bitcoin-internals" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d573f4cf32996a8dce612e4348cece65a241f1882ed594047c9ba348e8869fa5" + +[[package]] +name = "bitcoin-io" +version = "0.1.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5de036369d1ac59d3c1819ebc4d850f89466f5401c571a285b6ed564a4cb78" +dependencies = [ + "bitcoin-consensus-encoding", +] + +[[package]] +name = "bitcoin_hashes" +version = "0.14.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca4c7abb40c8817d77403c880988cfd484f23ab2365726afb2f798363e2c4a2" +dependencies = [ + "bitcoin-io", + "hex-conservative 0.2.2", +] + +[[package]] +name = "bitcoinpqc" +version = "0.4.0" +source = "git+https://github.com/cryptoquick/libbitcoinpqc-bindings.git?rev=7936b56#7936b56f15e86b6764947c9298215ecfe38b712b" +dependencies = [ + "bindgen", + "bitmask-enum", + "cc", + "cmake", + "getrandom 0.3.4", + "hex", + "libc", + "secp256k1 0.31.1", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitmask-enum" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6cbbb8f56245b5a479b30a62cdc86d26e2f35c2b9f594bc4671654b03851380" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "blake3" +version = "1.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecheck" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26333eeac754f0ad8a6bcd0eb0ac012156302e4e16b852b72ee399aea4f12c29" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46d07918caa9eeaaf06b7873925c53a61daac173539b4f7715090745e44e4e69" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "carbonado" +version = "0.7.1" +dependencies = [ + "aes", + "anyhow", + "bao", + "bao-tree", + "bip39", + "bitcoinpqc", + "bitmask-enum", + "blake3", + "chacha20poly1305", + "ciborium", + "clap", + "clap_mangen", + "criterion", + "ctr", + "directories", + "futures-lite", + "getrandom 0.2.17", + "hmac", + "infer", + "log", + "positioned-io", + "pretty_env_logger", + "proptest", + "rand 0.8.7", + "reed-solomon-erasure", + "rkyv", + "secp256k1 0.29.1", + "serde", + "serde_json", + "sha2", + "thiserror", + "tiny_http", + "tokio", + "wasm-bindgen-test", + "zstd", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "clap_mangen" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e30ffc187e2e3aeafcd1c6e2aa416e29739454c0ccaa419226d5ecd181f2d78" +dependencies = [ + "clap", + "roff", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "futures-core", + "genawaiter-macro", + "genawaiter-proc-macro", + "proc-macro-hack", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + +[[package]] +name = "genawaiter-proc-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" +dependencies = [ + "proc-macro-error", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-conservative" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "hex-conservative" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35431185f361ccf3ffc58254628af5f1f5d5f28531da2e02e5d6c82bbc282a10" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "infer" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +dependencies = [ + "cfb", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "inplace-vec-builder" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf64c2edc8226891a71f127587a2861b132d2b942310843814d5001d99a1d307" +dependencies = [ + "smallvec", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "iroh-io" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a5feb781017b983ff1b155cd1faf8174da2acafd807aa482876da2d7e6577a" +dependencies = [ + "bytes", + "futures-lite", + "pin-project", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minicov" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" +dependencies = [ + "cc", + "walkdir", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "positioned-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ec4b80060f033312b99b6874025d9503d2af87aef2dd4c516e253fbfcdada7" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty_env_logger" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" +dependencies = [ + "env_logger", + "log", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro-error" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "syn-mid", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.13.1", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "ptr_meta" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743da816b98c921cdbe8628ef7381b76f25ecf4da599fc80aca90eae7ef70cc0" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8d9ca532f185d5d4db7a7c9d51420b452168ea1c2b913953281bd6fe1fcbd0" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rancor" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b534442d0fcdb55d66f373d9cac6d33b6293a2335bc2136dbd06ce0e87d2572" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "range-collections" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "861706ea9c4aded7584c5cd1d241cec2ea7f5f50999f236c22b65409a1f1a0d0" +dependencies = [ + "binary-merge", + "inplace-vec-builder", + "ref-cast", + "smallvec", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "reed-solomon-erasure" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fe31452b684b8b33f65f8730c8b8812c3f5a0bb8a096934717edb1ac488641" +dependencies = [ + "libm", + "parking_lot", + "smallvec", + "spin", +] + +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rend" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "663ba70707f96e871406fe10d68128412e619b06d1d47cb91c3a4c6501176240" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "rkyv" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c25ef604ac7dd839d44d64648952ea23c97866f124ff671b0ed2cf3ad9bb06e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "roff" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "secp256k1" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" +dependencies = [ + "rand 0.8.7", + "secp256k1-sys 0.10.1", +] + +[[package]] +name = "secp256k1" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" +dependencies = [ + "bitcoin_hashes", + "rand 0.9.5", + "secp256k1-sys 0.11.0", +] + +[[package]] +name = "secp256k1-sys" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" +dependencies = [ + "cc", +] + +[[package]] +name = "secp256k1-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" +dependencies = [ + "cc", +] + +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea305d57546cc8cd04feb14b62ec84bf17f50e3f7b12560d7bfa9265f39d9ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "895a2607575412a4eda1df892084a375ea10dfeadc4d7d2ab87b854e4ddc7ba1" +dependencies = [ + "async-trait", + "cast", + "js-sys", + "libm", + "minicov", + "nu-ansi-term", + "num-traits", + "oorandom", + "serde", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", + "wasm-bindgen-test-shared", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288cb0ebe215033bf949ae1fd046726daa4c32a157f24b9dc6ac387a52aa759" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "wasm-bindgen-test-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ff1c1b360982e93b6d8ea9c04836f71dba0817a16f91e229cf3a51bdd9d987" + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 40ca36f..87ae2a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,9 +2,9 @@ name = "carbonado" edition = "2024" rust-version = "1.98.0" -version = "0.7.0" +version = "0.7.1" license = "MIT" -description = "An apocalypse-resistant data storage format for the truly paranoid. Fully symmetric AES-256-CTR + HMAC-SHA512 (EtM) container. Callers supply high-entropy master keys; passphrase KDF (e.g. Argon2id) is caller responsibility. SLH-DSA signatures are provided only as sidecars (via `bitcoinpqc`). Clean cryptographic break from ECIES — v1 encrypted archives require external migration." +description = "Apocalypse-resistant archival format for consensus-critical data. One portable file: AES-256-CTR + HMAC-SHA512, keyed Bao, Reed-Solomon 4/8, optional zstd, SLH-DSA sidecars." documentation = "https://docs.rs/carbonado" homepage = "https://github.com/bitmask-stack/carbonado" repository = "https://github.com/bitmask-stack/carbonado.git" @@ -123,6 +123,10 @@ name = "parallel_bench" harness = false required-features = ["parallel"] +[[bench]] +name = "fec_stripe_bench" +harness = false + # Primary CLI: `cargo install carbonado` or `cargo install --path . --bin carbonado` [[bin]] name = "carbonado" diff --git a/RESIDUAL.md b/RESIDUAL.md new file mode 100644 index 0000000..5cf08a7 --- /dev/null +++ b/RESIDUAL.md @@ -0,0 +1,42 @@ +# Carbonado remaining work (2026-08-27 closer) + +Compression encode no longer has a silent library zstd level of 20. Callers that set the Compression bit must pass an explicit level. Tests and the CLI may pass 20 as an explicit choice. + +## What landed this closer + +- Product encode wrappers take explicit zstd: `file::encode_with_zstd`, `file::encode_stream_with_zstd`, `file::encode_outboard_with_zstd`, `encode_shard_stream_with_zstd`, `decoding::decode_outboard_with_dict`. +- Tests that need compressed success paths call those APIs (or `tests/common` helpers) with `ZstdEncode::level(20)`. The 3-arg `encode` / `encode_outboard` / `stream_encode_buffer` still fail with `MissingZstdLevel` when Compression is set. That is the contract. +- FilepackManifest v3 rkyv goldens were regenerated (`tests/fixtures/rkyv/*.bin` and the hex constants in `tests/rkyv_golden_lock.rs`). Directory interop JSON and the phase3 G9 directory catalog seed were updated for v3 wire (dict offset fields). +- Directory decode loads a segment dictionary from the Adamantine bundle when `dict_len > 0`. Catalog Carbonado compression does not use the file-segment dictionary (the catalog is inboard and has no dict of its own). +- Single-file inboard `{hash}.adam.c0e` trailers can carry a dict without outboard FEC blobs. Decode reads that trailer without applying directory FEC-geometry validation. +- Named tests: `cargo test --test fec_chaos` (17 passed), `cargo test --test adam_zstd` (11 passed), and `cargo test --test udp_fec_sim` (6 passed). +- UDP chaos datagrams are concatenated 4 KiB stripe leaves per RS symbol (`inboard_symbol_payload`), matching `erase_shards`. Five-drop c12 is irrecoverable. c14 may still recover because zstd padding leaves are already zeros. The 50% leaf budget is unchanged. + +## What remains + +- **Lean FilepackManifest wire is still v2.** `Carbonado/Filepack.lean` `SegmentRef` has no `dict_offset` / `dict_len`. Rust goldens are v3. Do not copy the new hex strings into Lean until the Lean encoder grows those fields. Updating Lean hex without that change would be a lie. +- **`bao-tree` 0.16.1 is not on the menhera-cooldown index yet.** Product `Cargo.toml` still asks for 0.16.1. This host ran tests with a path patch to the already-cached crates.io 0.16.1 crate. Do not fetch crates.io to skip the cooldown. +- **`streaming_async` is empty under default features** (`cfg(feature = "async")`). `serial_fec_path` is empty under default `parallel`. +- Directory catalogs for small compressed files can still store `verification_outboard_len = 0` while FEC parity is present. Scrub and decode still work. The linear bao-outboard slot for those files is empty. + +## Highest-value next work + +1. Add `dict_offset` / `dict_len` to Lean `SegmentRef` and regen Lean hex goldens without a sorry. Rust FilepackManifest is v3; Lean still describes v2. That is the spec/proof bottleneck. +2. Update Lean FEC to 4 KiB stripe geometry so proofs match the Rust engine (still segment-wide columns in Lean). +3. When menhera-cooldown lists `bao-tree` 0.16.1, drop the local path patch. +4. Re-run full default `cargo test` after the UDP stripe-helper fix (named `udp_fec_sim` is green; the full suite was last run with that test skipped). + +## Commands that actually passed + +Path patch used on this host (cached 0.16.1, not a crates.io fetch): + +```text +cargo test --config 'patch.crates-io.bao-tree.path=""' --test fec_chaos +cargo test --config 'patch.crates-io.bao-tree.path=""' --test adam_zstd +cargo test --config 'patch.crates-io.bao-tree.path=""' --doc +cargo test --config 'patch.crates-io.bao-tree.path=""' --test udp_fec_sim +``` + +`udp_fec_sim`: 6 passed, 0 failed. Default features only. Never `--all-features`. Full default `cargo test` was not re-run after this stripe-helper fix. + +`cargo test --offline` failed to resolve `bao-tree = 0.16.1` against menhera-cooldown (candidate 0.16.0 only). diff --git a/benches/crypto_bench.rs b/benches/crypto_bench.rs index c1cf32f..f6250d0 100644 --- a/benches/crypto_bench.rs +++ b/benches/crypto_bench.rs @@ -139,6 +139,7 @@ fn bench_encode_directory(c: &mut Criterion) { black_box(&master_key), black_box(&input), black_box(&out_base), + black_box(&carbonado::ZstdEncode::level(20)), ) .unwrap(); black_box(archive.entry_count); diff --git a/benches/fec_stripe_bench.rs b/benches/fec_stripe_bench.rs new file mode 100644 index 0000000..49bf67e --- /dev/null +++ b/benches/fec_stripe_bench.rs @@ -0,0 +1,74 @@ +//! c12 (Bao + FEC, no compress) encode / decode / scrub at 1 MiB and 16 MiB. +//! +//! Run: +//! RUSTFLAGS="-C target-cpu=native" cargo bench --bench fec_stripe_bench +//! +//! First land is measurement, not a CI cap. Sample on this host with +//! `RUSTFLAGS="-C target-cpu=native"` (10 samples, 3 s): +//! 1 MiB encode ~7.2 ms (~139 MiB/s), decode ~1.13 ms (~882 MiB/s), scrub ~8.4 ms (~118 MiB/s) +//! 16 MiB encode ~112 ms (~143 MiB/s), decode ~28 ms (~572 MiB/s), scrub ~132 ms (~121 MiB/s) + +use carbonado::{decode, encode, scrub, structs::Encoded}; +use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main}; + +const C12: u8 = 12; +const ZERO_MASTER: [u8; 32] = [0u8; 32]; + +fn patterned(len: usize) -> Vec { + (0..len).map(|i| (i % 251) as u8).collect() +} + +fn bench_c12_stripe(c: &mut Criterion) { + let mut group = c.benchmark_group("fec_stripe_c12"); + for &size in &[1024 * 1024usize, 16 * 1024 * 1024] { + let data = patterned(size); + group.throughput(Throughput::Bytes(size as u64)); + + group.bench_function(format!("encode_{}mib", size / (1024 * 1024)), |b| { + b.iter(|| { + let Encoded(_body, _hash, _info) = + encode(black_box(&ZERO_MASTER), black_box(&data), C12).unwrap(); + }) + }); + + let Encoded(body, hash, info) = encode(&ZERO_MASTER, &data, C12).unwrap(); + let hash_bytes = hash.as_bytes().to_vec(); + + group.bench_function(format!("decode_{}mib", size / (1024 * 1024)), |b| { + b.iter(|| { + let _ = decode( + black_box(&ZERO_MASTER), + black_box(&hash_bytes), + black_box(&body), + black_box(info.padding_len), + C12, + ) + .unwrap(); + }) + }); + + let mut nicked = body.clone(); + // One 4 KiB leaf (stripe 0, symbol 0) so scrub has work without a 16 MiB copy each iter. + if let Ok(ranges) = carbonado::stream::inboard_leaf_data_ranges(&nicked) + && let Some(range) = ranges.first() + { + nicked[range.clone()].fill(0xEE); + } + + group.bench_function(format!("scrub_{}mib", size / (1024 * 1024)), |b| { + b.iter(|| { + let _ = scrub( + black_box(&nicked), + black_box(&hash_bytes), + black_box(&info), + C12, + ) + .unwrap(); + }) + }); + } + group.finish(); +} + +criterion_group!(benches, bench_c12_stripe); +criterion_main!(benches); diff --git a/benches/parallel_bench.rs b/benches/parallel_bench.rs index 79cfef9..a399e34 100644 --- a/benches/parallel_bench.rs +++ b/benches/parallel_bench.rs @@ -27,8 +27,9 @@ fn pre_parity_shards(logical_len: usize) -> (ReedSolomon, Vec>, usize) { let rs = ReedSolomon::new(4, 4).expect("rs"); let mut enc = FecInboardEncoder::new(logical_len).expect("new"); let input = patterned(logical_len); - enc.feed(Cursor::new(&input)).expect("feed"); - let stripe = enc.finish().expect("finish").expect("stripe"); + let mut stripes = enc.feed(Cursor::new(&input)).expect("feed"); + stripes.extend(enc.finish().expect("finish")); + let stripe = stripes.into_iter().next().expect("stripe"); let chunk_len = stripe.chunk_len as usize; let mut shards = stripe.shards; for s in shards.iter_mut().skip(4) { diff --git a/doc/STREAMING_PARALLELISM.md b/doc/STREAMING_PARALLELISM.md index 3c66b54..e917aac 100644 --- a/doc/STREAMING_PARALLELISM.md +++ b/doc/STREAMING_PARALLELISM.md @@ -142,7 +142,7 @@ Receiver: - Bao ordering independent of datagram arrival order ``` -50% packet loss ≈ 50% shard loss at the **chaos-injection coordinates** (`InboardShardLayout` / `erase_shards`) — within RS 4/8 if losses are spread (not concentrated on >4 shards). True inboard wire is Bao-wrapped; `tests/udp_fec_sim.rs` documents the approximate model explicitly. `fec_chaos.rs` models distributed knockout at the same coordinates. +50% packet loss ≈ 50% RS **symbol** loss (`erase_shards` / `inboard_symbol_payload`: every 4 KiB leaf with that symbol, not a tall column). That stays within RS 4/8 if losses stay at ≤4 of 8 symbols per stripe. True inboard wire is Bao-wrapped; `tests/udp_fec_sim.rs` is the datagram model. `fec_chaos.rs` knocks out the same leaves. ## JBOD / RAID replacement diff --git a/doc/TEST_STRATEGY.md b/doc/TEST_STRATEGY.md index 7bd53fa..e957d16 100644 --- a/doc/TEST_STRATEGY.md +++ b/doc/TEST_STRATEGY.md @@ -122,7 +122,7 @@ Carbonado uses **reed-solomon-erasure 4/8**: any **4 of 8** shards reconstruct t 1. ~~Directory segment corruption + centralized bundle extract + FEC scrub~~ **Done** — `tests/directory_archive.rs::{directory_segment_corruption_bao_bundle_extract_scrub_roundtrip,directory_fec_scrub_matrix_c12_c13_c14_c15,directory_multi_segment_fec_bundle_indices}` (c12–c15 segments: verification + FEC parity indexed in Adamantine bundle; `scrub_outboard` recovers corrupt bare mains within ≤4 shard taints; c15 encrypted five-shard knockout documented as `InvalidScrubbedHash` negative) 2. ~~Cross-tool interop fixtures (manifest + segment naming)~~ **Done** — `tests/fixtures/directory_interop_golden.json` + `tests/filepack_interop.rs::{adamantine_decimal_segment_naming_contract,golden_directory_interop_checksums_and_manifest_wire}` -3. ~~UDP shard mapping contract test (chaos-injection datagram ↔ shard slot at `InboardShardLayout` coordinates)~~ **Done** — `tests/udp_fec_sim.rs` (datagram drop = `erase_shards` at approximate coordinates; not normative Bao-wrapped wire; ≤4-drop scrub recovery; c12 five-drop irrecoverable) +3. ~~UDP shard mapping contract test (chaos-injection datagram ↔ RS symbol leaves)~~ **Done** — `tests/udp_fec_sim.rs` (datagram = concatenated 4 KiB stripe leaves per symbol; drop = `erase_shards`; not normative Bao-wrapped wire; ≤4-drop scrub recovery; five-drop irrecoverable at c12; c14 may recover on zero padding leaves) ### P4 — External normative diff --git a/doc/man/carbonado-encode.1 b/doc/man/carbonado-encode.1 index 38332b8..8be91b1 100644 --- a/doc/man/carbonado-encode.1 +++ b/doc/man/carbonado-encode.1 @@ -13,7 +13,13 @@ Encode a file or directory into a Carbonado archive Format level 0–15 (default 14 = public verifiable; odd values = encrypted) .TP \fB\-\-outboard\fR -Single\-file only: bare main + `.out`/`.par` sidecars (default single\-file is inboard) +Single\-file only: `{hash}.cXX` + `{hash}.adam.cXX` sidecar (default single\-file is inboard) +.TP +\fB\-\-zstd\-level\fR \fI\fR +Zstd compression level (required when the Compression bit is set) +.TP +\fB\-\-zstd\-dict\fR \fI\fR +Optional RFC 8878 zstd dictionary file (bytes stored in the Adamantine dict section) .TP \fB\-\-encrypted\fR Directory only: encrypted catalog c15 and segment formats c13/c15 (auto\-creates BIP39 seed if needed) diff --git a/doc/man/carbonado.1 b/doc/man/carbonado.1 index 6157147..250a685 100644 --- a/doc/man/carbonado.1 +++ b/doc/man/carbonado.1 @@ -12,8 +12,9 @@ KEY MATERIAL: First encrypted encode auto\-generates a BIP39 mnemonic (24 words), saved in plaintext at the path from `carbonado key path` (override: CARBONADO_MNEMONIC_PATH). Later encode/decode reuse it unless `\-\-master` is given. Decode never auto\-generates. .PP ARTIFACTS: - Single\-file default: inboard headered `{hash}.c{fmt:02x}` (format 14 → `.c0e`). - `\-\-outboard`: bare main + optional `.out`/`.par` sidecars (single\-file only). + Single\-file inboard: one `{hash}.adam.c{fmt:02x}` (format 14 → `.adam.c0e`). + `\-\-outboard`: `{hash}.c{fmt:02x}` + `{hash}.adam.c{fmt:02x}` (ADAMANTINE10 sidecar). + `\-\-zstd-level` is required when Compression is set. Directory: inboard Adamantine 1.0 catalog `.adam.c14` (or `.adam.c15` with `\-\-encrypted`) and heterogeneous bare segment mains (c12/c14 or c13/c15). Output defaults to `{input}\-archive/`. .PP See `carbonado \-\-help` for per\-command options. diff --git a/docs/LIMITS.md b/docs/LIMITS.md index f09de73..bf06f8b 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -12,7 +12,7 @@ There is no `carbonado-sys`, no Cargo `backend-lean`, and no product C ABI. Do not claim G8 C-ABI parity. -- AOT CLI (`packages.carbonado` / `nix run`) runs **Programs A–G**: constants, EtM, FEC, keyed Bao, full pipeline (c0–c15), Header wire, scrub, stream bounds, multi-segment shards, **zstd-20 compression (linked)**, **SLH1 sidecar wire + bind-to-root model**, **Adamantine 1.0 directories**, **encode/decode/slh CLI**. +- AOT CLI (`packages.carbonado` / `nix run`) runs **Programs A–G**: constants, EtM, FEC, keyed Bao, full pipeline (c0–c15), Header wire, scrub, stream bounds, multi-segment shards, **zstd compression (level is encoder input; AOT demo uses 20)**, **SLH1 sidecar wire + bind-to-root model**, **Adamantine 1.0 directories**, **encode/decode/slh CLI**. - Rust tree (`src/`, `tests/`, …) **stays** first-class. **G1/W5a closed:** permanent policy — **no** `ref/carbonado-rust` product pin. Not a license to delete `src/` or `tests/`. - Lean theorem/test tree is **`CarbonadoTest/`** (not `Tests/`) so it does not collide with Rust `tests/` on case-insensitive filesystems (Darwin APFS). - Dependency direction is **CarbonadoTest → Carbonado** only. diff --git a/docs/TEST_CONTRACT.md b/docs/TEST_CONTRACT.md index d5f1b95..b244970 100644 --- a/docs/TEST_CONTRACT.md +++ b/docs/TEST_CONTRACT.md @@ -56,7 +56,8 @@ Helpers under `tests/common/` are not separate contract files; they support the | **parallel** | `serial_fec_path.rs` | Serial FEC encoder vs buffer path | | **g9_goldens** | `g9_cross_backend.rs` | Rust decode of committed Lean AOT goldens + rust self-roundtrip (`just test-g9`) | | **determinism** | `determinism_roundtrip.rs` | codecode (EDE) + decodec (DED); same-engine compress + directory | -| **zstd** | `zstd_frame_params.rs` | Frame flags vs Lean AOT goldens (honest descriptor residual) | +| **zstd** | `zstd_frame_params.rs` | Frame flags vs Lean AOT goldens (honest descriptor residual); explicit level, no library default | +| **zstd** | `adam_zstd.rs` | Required zstd level; inboard/outboard Adamantine file counts; dict ID in bundle; layout detect | | **rkyv** | `rkyv_golden_lock.rs` | Directory catalog rkyv goldens | Removed 2026-08-24: `lean_backend_smoke.rs`, `lean_backend_phase2.rs`, `lean_backend_phase3.rs`, `lean_backend_phase4.rs` (they existed only for Cargo `backend-lean` via C). diff --git a/examples/basic_roundtrip.rs b/examples/basic_roundtrip.rs index b5fe824..7e829c4 100644 --- a/examples/basic_roundtrip.rs +++ b/examples/basic_roundtrip.rs @@ -8,7 +8,9 @@ //! - Never reuse a master key across unrelated datasets without rotation. //! - See AGENTS.md §2 for full invariants, nonce rules, and recommendations. -use carbonado::{constants, decode, decode_outboard, encode, encode_outboard}; +use carbonado::{ + ZstdEncode, constants, decode, decode_outboard, encode_outboard, encode_with_zstd, +}; use getrandom::getrandom; fn main() -> Result<(), Box> { @@ -28,7 +30,7 @@ fn main() -> Result<(), Box> { // Using the low-level encode/decode API here for the demo. // Most production code will prefer the high-level carbonado::file API. - let encoded = encode(&master_key, plaintext, level)?; + let encoded = encode_with_zstd(&master_key, plaintext, level, None, &ZstdEncode::level(20))?; println!("Encoded size: {} bytes", encoded.0.len()); println!( diff --git a/examples/dir_archival.rs b/examples/dir_archival.rs index 64b6a49..d4e715b 100644 --- a/examples/dir_archival.rs +++ b/examples/dir_archival.rs @@ -36,7 +36,8 @@ fn main() -> Result<(), Box> { match cmd.as_str() { "encode" => { - let archive = encode_directory(&master, &input, &output)?; + let archive = + encode_directory(&master, &input, &output, &carbonado::ZstdEncode::level(20))?; let root_hex: String = archive .catalog_bao_root .iter() diff --git a/examples/dump_rkyv_r9.rs b/examples/dump_rkyv_r9.rs index 643a7af..3239ec9 100644 --- a/examples/dump_rkyv_r9.rs +++ b/examples/dump_rkyv_r9.rs @@ -1,44 +1,181 @@ -//! R9 golden dump helper (maintainer); not part of product CLI. +//! FilepackManifest v3 golden dump helper (maintainer); not part of product CLI. +//! +//! Writes `tests/fixtures/rkyv/*.bin` and prints hex for `tests/rkyv_golden_lock.rs`. + use carbonado::filepack_manifest::*; +use std::path::Path; fn hex(b: &[u8]) -> String { b.iter().map(|x| format!("{x:02x}")).collect() } -fn main() { - let seg = |root_fill: u8, main_len: u64| SegmentRef { +fn seg(root_fill: u8, main_len: u64, chunk: u32, vo: u32) -> SegmentRef { + SegmentRef { segment_bao_root: [root_fill; 32], - chunk_index: 0, + chunk_index: chunk, main_len, - verification_outboard_offset: 0, + verification_outboard_offset: vo, verification_outboard_len: 64, - fec_parity_offset: 64, + fec_parity_offset: vo + 64, fec_parity_len: 128, + dict_offset: 0, + dict_len: 0, + } +} + +fn write_named(dir: &Path, name: &str, bytes: &[u8]) { + let path = dir.join(name); + std::fs::write(&path, bytes).unwrap_or_else(|e| panic!("write {name}: {e}")); + println!("{name} LEN={}", bytes.len()); + println!("{name} HEX={}", hex(bytes)); +} + +fn main() { + let dir = Path::new("tests/fixtures/rkyv"); + std::fs::create_dir_all(dir).expect("fixtures dir"); + + let empty = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![], }; + write_named(dir, "empty_manifest.bin", &empty.to_bytes().expect("empty")); + let e1 = FilepackEntry { rel_path: "a.txt".into(), content_blake3: [0x22; 32], segment_format: 0x0E, - segments: vec![seg(0x11, 100)], + segments: vec![seg(0x11, 100, 0, 0)], ots_proof: None, }; + let single = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0x33; 32], + catalog_ots_proof: None, + entries: vec![e1.clone()], + }; + write_named(dir, "single_entry.bin", &single.to_bytes().expect("single")); + let e2 = FilepackEntry { - rel_path: "b/longer-path-name.txt".into(), // >8 bytes → out-of-line string + rel_path: "b/longer-path-name.txt".into(), content_blake3: [0x33; 32], segment_format: 0x0E, - segments: vec![seg(0x44, 200)], + segments: vec![seg(0x44, 200, 0, 0)], ots_proof: Some(vec![0xAB, 0xCD, 0xEF, 0x01]), }; - let m = FilepackManifest { + let multi = FilepackManifest { version: FILEPACK_MANIFEST_VERSION, format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, catalog_bao_root: [0x55; 32], catalog_ots_proof: None, entries: vec![e1, e2], }; - let b = m.to_bytes().expect("to_bytes"); - println!("MULTI_LEN={}", b.len()); - println!("MULTI_HEX={}", hex(&b)); - std::fs::write("tests/fixtures/rkyv/multi_entry_ots.bin", &b).expect("write"); - println!("wrote tests/fixtures/rkyv/multi_entry_ots.bin"); + write_named( + dir, + "multi_entry_ots.bin", + &multi.to_bytes().expect("multi"), + ); + + let path8 = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "12345678".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: None, + }], + }; + write_named(dir, "path_inline_8.bin", &path8.to_bytes().expect("path8")); + + let path9 = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "123456789".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: None, + }], + }; + write_named(dir, "path_ool_9.bin", &path9.to_bytes().expect("path9")); + + let two = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0), seg(0x12, 50, 1, 192)], + ots_proof: None, + }], + }; + write_named(dir, "two_segments.bin", &two.to_bytes().expect("two")); + + let ots_first = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![ + FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: Some(vec![0xDE, 0xAD]), + }, + FilepackEntry { + rel_path: "b.txt".into(), + content_blake3: [0x33; 32], + segment_format: 0x0E, + segments: vec![seg(0x44, 200, 0, 0)], + ots_proof: None, + }, + ], + }; + write_named( + dir, + "ots_first_only.bin", + &ots_first.to_bytes().expect("ots_first"), + ); + + let mut cfp2_root = [0x11u8; 32]; + cfp2_root[0..4].copy_from_slice(b"CFP2"); + let cfp2 = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![SegmentRef { + segment_bao_root: cfp2_root, + chunk_index: 0, + main_len: 100, + verification_outboard_offset: 0, + verification_outboard_len: 64, + fec_parity_offset: 64, + fec_parity_len: 128, + dict_offset: 0, + dict_len: 0, + }], + ots_proof: None, + }], + }; + write_named(dir, "rkyv_cfp2_prefix.bin", &cfp2.to_bytes().expect("cfp2")); } diff --git a/examples/slh_dsa_sidecar.rs b/examples/slh_dsa_sidecar.rs index fbe796b..4b8e442 100644 --- a/examples/slh_dsa_sidecar.rs +++ b/examples/slh_dsa_sidecar.rs @@ -7,10 +7,11 @@ //! //! See AGENTS.md §2.3 for the exact sidecar format and security model. +use carbonado::ZstdEncode; use carbonado::crypto::{ read_slh_sidecar, slh_dsa_generate_keypair, slh_dsa_sign, slh_dsa_verify, write_slh_sidecar, }; -use carbonado::file::{Header, encode}; +use carbonado::file::{Header, encode_with_zstd}; use getrandom::getrandom; fn main() -> Result<(), Box> { @@ -23,7 +24,13 @@ fn main() -> Result<(), Box> { let important_data = b"This could be a manifest, a checkpoint, or a critical archive."; - let (encoded, _info) = encode(&master_key, important_data, 15, None)?; + let (encoded, _info) = encode_with_zstd( + &master_key, + important_data, + 15, + None, + &ZstdEncode::level(20), + )?; // The high-level encode includes a Header. Parse it to get the authoritative Bao hash // that represents this archive (this is the value we sign for a sidecar). diff --git a/justfile b/justfile index 8bd8f39..87d566d 100644 --- a/justfile +++ b/justfile @@ -353,24 +353,12 @@ nix_retry +cmd: n=$((n + 1)) done -# Sequential host-Nix flake checks (fmt, clippy, nextest, then Lean proofs). -# Does not force the remote builder; this laptop may rustc. +# This laptop: fmt, clippy, nextest. No Nix (no store copy, no builder). +# Lean flake checks: `just test-lean-ci` or `just check-remote`. check-local: - #!/usr/bin/env bash - set -euo pipefail - sys="{{ system }}" - echo "==> just check-local: fmt" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.fmt" - echo "==> just check-local: clippy (backend-rust + async,async-tokio,man-gen)" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.clippy-rust" - echo "==> just check-local: nextest (backend-rust)" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.nextest-rust" - echo "==> just check-local: Lean gates" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.no-sorry" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.tooling-purity" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.carbonado" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.demo" - nix build --impure -L --print-out-paths "path:.#checks.${sys}.rustc-1_98" + cargo fmt --all -- --check + cargo clippy --all-targets --features "async,async-tokio,man-gen" -- -D warnings + cargo nextest run --features async,async-tokio,man-gen # Sequential force-remote gate. rustc requires surmount-remote. Quote # .#attr; unquoted # is a bash comment. diff --git a/nix/cargo-quality.nix b/nix/cargo-quality.nix index c23fc96..4b98ef7 100644 --- a/nix/cargo-quality.nix +++ b/nix/cargo-quality.nix @@ -94,6 +94,21 @@ CARGO_BUILD_JOBS = "32"; CARGO_PROFILE = "dev"; hardeningDisable = ["all"]; + # Host `.cargo/config.toml` points crates-io at index.crates.io so this + # laptop is not stuck on menhera-cooldown. Crane already vendors crates.io; + # that replace-with would override the vendor directory in the sandbox. + postPatch = '' + if [ -f .cargo/config.toml ]; then + awk ' + /^\[source\.crates-io\]/ { skip=1; next } + /^\[registries\.crates-io-official\]/ { skip=1; next } + /^\[/ { skip=0 } + skip { next } + { print } + ' .cargo/config.toml > .cargo/config.toml.vendor + mv .cargo/config.toml.vendor .cargo/config.toml + fi + ''; # Presence of ZSTD_SYS_USE_PKG_CONFIG (even =0) makes zstd-sys probe # nixpkgs libzstd. Unset on the deps layer and the test layer. preConfigure = '' diff --git a/src/adamantine.rs b/src/adamantine.rs index 01a9683..47a866b 100644 --- a/src/adamantine.rs +++ b/src/adamantine.rs @@ -10,7 +10,7 @@ //! ```text //! Offset Size Field //! 0 13 magic ADAMANTINE10\n (version 1.0 in magic) -//! 13 1 carbonado_fmt 0x0E | 0x0F (catalog only) +//! 13 1 carbonado_fmt catalog c14/c15 or single-file format 0–15 //! 14 1 flags u8 (bit0 REQUIRE_OTS = per-entry proofs required at decode; bits 1–7 reserved, must be 0) //! 15 4 payload_len u32 LE //! 19 N payload rkyv + Bao bundle (see adamantine_payload) @@ -132,6 +132,31 @@ pub fn decode_adamantine(bytes: &[u8]) -> Result<(Vec, AdamantineHeader), Ca )) } +/// Parse an Adamantine envelope at the start of `bytes`, allowing trailing bytes (COTS). +pub fn decode_adamantine_prefix( + bytes: &[u8], +) -> Result<(Vec, AdamantineHeader, usize), CarbonadoError> { + if bytes.len() < ADAMANTINE_HEADER_LEN { + return Err(CarbonadoError::InvalidAdamantineHeader); + } + let payload_len = u32::from_le_bytes( + bytes[15..19] + .try_into() + .map_err(|_| CarbonadoError::InvalidAdamantineHeader)?, + ); + let payload_end = ADAMANTINE_HEADER_LEN + .checked_add(payload_len as usize) + .ok_or(CarbonadoError::InvalidAdamantineHeader)?; + if bytes.len() < payload_end { + return Err(CarbonadoError::InvalidAdamantinePayloadLength { + expected: payload_len, + available: bytes.len().saturating_sub(ADAMANTINE_HEADER_LEN), + }); + } + let (payload, hdr) = decode_adamantine(&bytes[..payload_end])?; + Ok((payload, hdr, payload_end)) +} + /// Parse `ADAMANTINE{digit}{digit}\n` or `ADAMANTINE{digit}\n` version from unsupported magic. fn parse_unsupported_magic_version(magic: &[u8]) -> Option<(u8, u8)> { if magic.len() < 12 { @@ -155,7 +180,7 @@ fn parse_unsupported_magic_version(magic: &[u8]) -> Option<(u8, u8)> { } fn validate_carbonado_fmt(fmt: u8) -> Result<(), CarbonadoError> { - if fmt != ADAMANTINE_CARBONADO_FMT_PUBLIC && fmt != ADAMANTINE_CARBONADO_FMT_ENCRYPTED { + if fmt > 15 { return Err(CarbonadoError::InvalidAdamantineCarbonadoFormat(fmt)); } Ok(()) @@ -280,11 +305,11 @@ mod tests { #[test] fn reject_invalid_carbonado_fmt() { let mut bytes = encode_adamantine(b"x", ADAMANTINE_CARBONADO_FMT_PUBLIC, 0); - bytes[13] = 6; + bytes[13] = 16; let err = decode_adamantine(&bytes).unwrap_err(); assert!(matches!( err, - CarbonadoError::InvalidAdamantineCarbonadoFormat(6) + CarbonadoError::InvalidAdamantineCarbonadoFormat(16) )); } diff --git a/src/adamantine_payload.rs b/src/adamantine_payload.rs index be515a5..5c6b3ec 100644 --- a/src/adamantine_payload.rs +++ b/src/adamantine_payload.rs @@ -151,6 +151,15 @@ pub fn fec_slice_from_bundle( bundle_slice_from_bundle(bundle, offset, len, "fec_parity") } +/// Extract one segment's RFC 8878 dictionary slice from the bundle. +pub fn dict_slice_from_bundle( + bundle: &[u8], + offset: u32, + len: u32, +) -> Result<&[u8], CarbonadoError> { + bundle_slice_from_bundle(bundle, offset, len, "dict") +} + fn bundle_slice_from_bundle<'a>( bundle: &'a [u8], offset: u32, diff --git a/src/bin/carbonado/main.rs b/src/bin/carbonado/main.rs index 2148591..1563045 100644 --- a/src/bin/carbonado/main.rs +++ b/src/bin/carbonado/main.rs @@ -19,19 +19,18 @@ use carbonado::cli_app::{Cli, Commands, KeyCommands}; use carbonado::constants::Format; use carbonado::file::{ - DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, DirectoryEncodeOptions, decode_directory, decode_stream, - encode_directory_with_options, encode_stream, + DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, DirectoryEncodeOptions, EncodeToDirOptions, + decode_directory, decode_stream, encode_directory_with_options, encode_to_dir, }; use carbonado::paths::{ - ArchiveLayout, detect_archive_layout, guess_format_from_filename, parse_bao_root_from_filename, - sidecar_sibling_path, + ArchiveLayout, adamantine_sidecar_for_main, companion_main_for_adam_sidecar, + detect_archive_layout, guess_format_from_filename, parse_bao_root_from_filename, }; -use carbonado::stream::decode::stream_decode_outboard; -use carbonado::stream::encode::stream_encode_outboard; -use carbonado::structs::OutboardEncoded; +use carbonado::stream::ZstdEncode; +use carbonado::stream::decode::stream_decode_outboard_with_dict; use clap::Parser; use std::fs::{self, File}; -use std::io::{Cursor, Read, Seek, Write}; +use std::io::{Cursor, Read, Seek}; use std::path::{Path, PathBuf}; fn validate_format(format: u8) -> Result<(), Box> { @@ -70,8 +69,6 @@ fn reject_bare_outboard_flags_on_headered( format: &Option, hash: &Option, padding: u32, - verification_outboard: &Option, - fec_parity: &Option, ) -> Result<(), Box> { if format.is_some() { return Err( @@ -85,15 +82,31 @@ fn reject_bare_outboard_flags_on_headered( if padding != 0 { return Err("--padding is for bare outboard decode only".into()); } - if verification_outboard.is_some() { - return Err("--verification-outboard is for bare outboard decode only".into()); - } - if fec_parity.is_some() { - return Err("--fec-parity is for bare outboard decode only".into()); - } Ok(()) } +fn zstd_from_cli( + format: u8, + zstd_level: Option, + zstd_dict: Option, +) -> Result> { + let fmt = Format::from(format); + let dict = match zstd_dict { + Some(p) => Some(fs::read(p)?), + None => None, + }; + if fmt.contains(Format::Compression) && zstd_level.is_none() { + return Err( + "this format uses compression; pass --zstd-level (level is encoder input, not a default)" + .into(), + ); + } + Ok(ZstdEncode { + level: zstd_level, + dict, + }) +} + fn parse_master_hex(hexs: &str) -> Result<[u8; 32], Box> { if hexs.len() != 64 { return Err("master must be 64 hex chars (32 bytes)".into()); @@ -217,6 +230,8 @@ fn run_cli() -> Result<(), Box> { format, outboard, encrypted, + zstd_level, + zstd_dict, master, output, } => { @@ -227,7 +242,19 @@ fn run_cli() -> Result<(), Box> { let policy = master_policy_for_format(format, true); let master_key = resolve_master_key(master, policy)?; reject_zero_encrypted_master(format, &master_key)?; - do_encode_file_streaming(&input, format, outboard, &master_key, &outdir)?; + let plaintext = fs::read(&input)?; + let zstd = zstd_from_cli(format, zstd_level, zstd_dict)?; + let written = encode_to_dir( + &master_key, + &plaintext, + format, + &outdir, + EncodeToDirOptions { outboard, zstd }, + )?; + println!("encoded: {}", written.main_path.display()); + if written.adam_path != written.main_path { + println!(" + adamantine: {}", written.adam_path.display()); + } } else if input.is_dir() { if outboard { return Err("--outboard is for single-file encode only".into()); @@ -242,8 +269,10 @@ fn run_cli() -> Result<(), Box> { let policy = master_policy_for_format(dir_fmt, true); let master_key = resolve_master_key(master, policy)?; reject_zero_encrypted_master(dir_fmt, &master_key)?; + let zstd = zstd_from_cli(dir_fmt, zstd_level, zstd_dict)?; let options = DirectoryEncodeOptions { encrypted, + zstd, ..DirectoryEncodeOptions::default() }; let archive = encode_directory_with_options(&master_key, &input, &outdir, options)?; @@ -268,8 +297,6 @@ fn run_cli() -> Result<(), Box> { hash, format, padding, - verification_outboard, - fec_parity, } => { let layout = detect_archive_layout(&input)?; @@ -287,13 +314,7 @@ fn run_cli() -> Result<(), Box> { println!("decoded directory to {}", out_base.display()); } ArchiveLayout::InboardHeadered { path } => { - reject_bare_outboard_flags_on_headered( - &format, - &hash, - padding, - &verification_outboard, - &fec_parity, - )?; + reject_bare_outboard_flags_on_headered(&format, &hash, padding)?; let out_base = output.unwrap_or_else(|| PathBuf::from("recovered.bin")); let mut header_bytes = [0u8; carbonado::file::Header::LEN]; let mut input_f = File::open(&path)?; @@ -322,8 +343,6 @@ fn run_cli() -> Result<(), Box> { hash, format, padding, - verification_outboard, - fec_parity, )?; println!("decoded to {}", out_base.display()); } @@ -333,99 +352,6 @@ fn run_cli() -> Result<(), Box> { } } -fn do_encode_file_streaming( - input: &Path, - format: u8, - outboard: bool, - master: &[u8; 32], - outdir: &Path, -) -> Result<(), Box> { - let mut in_f = File::open(input)?; - if outboard { - let oenc = stream_encode_outboard_cli(master, &mut in_f, format, outdir)?; - write_outboard_artifacts_from_oenc(&oenc, format, outdir)?; - } else { - let mut body_bytes = Vec::new(); - let (header, _info) = encode_stream(master, &mut in_f, format, None, &mut body_bytes)?; - let mut archive = header.try_to_vec()?; - archive.extend_from_slice(&body_bytes); - let hhex = hex_encode_slice(header.hash.as_bytes()); - let name = format!("{}.c{:02x}", hhex, format); - let p = outdir.join(&name); - File::create(&p)?.write_all(&archive)?; - println!("encoded: {}", p.display()); - } - Ok(()) -} - -fn stream_encode_outboard_cli( - master: &[u8; 32], - input: &mut File, - format: u8, - _outdir: &Path, -) -> Result> { - let fmt = Format::from(format); - let mut main_buf = Cursor::new(Vec::new()); - let mut bao_buf = Vec::new(); - let mut par_buf = Vec::new(); - - let bao_out = fmt.contains(Format::Verification).then_some(&mut bao_buf); - let par_out = fmt.contains(Format::Fec).then_some(&mut par_buf); - - let mut payload_nonce = [0u8; 16]; - let (hash, info) = stream_encode_outboard( - master, - input, - format, - &mut main_buf, - bao_out, - par_out, - &mut payload_nonce, - false, - )?; - - Ok(OutboardEncoded { - main: main_buf.into_inner(), - verification_outboard: if fmt.contains(Format::Verification) { - Some(bao_buf) - } else { - None - }, - fec_parity: if fmt.contains(Format::Fec) { - Some(par_buf) - } else { - None - }, - hash, - info, - }) -} - -fn write_outboard_artifacts_from_oenc( - res: &OutboardEncoded, - format: u8, - outdir: &Path, -) -> Result<(), Box> { - let h = res.hash; - let hhex = hex_encode_slice(h.as_bytes()); - let main_name = format!("{}.c{:02x}", hhex, format); - let main_p = outdir.join(&main_name); - File::create(&main_p)?.write_all(&res.main)?; - println!("outboard bare: {}", main_p.display()); - if let Some(ob) = &res.verification_outboard { - let op = outdir.join(format!("{}.c{:02x}.out", hhex, format)); - File::create(&op)?.write_all(ob)?; - println!(" + bao outboard: {}", op.display()); - } - if let Some(par) = &res.fec_parity { - let pp = outdir.join(format!("{}.c{:02x}.par", hhex, format)); - File::create(&pp)?.write_all(par)?; - println!(" + fec parity: {}", pp.display()); - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] fn do_decode_outboard_streaming( input: &Path, master: &[u8; 32], @@ -433,22 +359,70 @@ fn do_decode_outboard_streaming( hash: Option, format: Option, padding: u32, - verification_outboard: Option, - fec_parity: Option, ) -> Result<(), Box> { - let mut main_f = File::open(input)?; - let ob_path = verification_outboard.unwrap_or_else(|| sidecar_sibling_path(input, "out")); - let par_path = fec_parity.unwrap_or_else(|| sidecar_sibling_path(input, "par")); - - let bao_ob = if ob_path.exists() { - Some(fs::read(&ob_path)?) + let main_path = if companion_main_for_adam_sidecar(input).is_some_and(|p| p.is_file()) + && input + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.contains(".adam.")) + { + companion_main_for_adam_sidecar(input).expect("companion main") } else { - None + input.to_path_buf() }; - let fec_p = if par_path.exists() { - Some(fs::read(&par_path)?) + let mut main_f = File::open(&main_path)?; + let adam_path = adamantine_sidecar_for_main(&main_path) + .filter(|p| p.is_file()) + .or_else(|| { + if input + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.contains(".adam.")) + { + Some(input.to_path_buf()) + } else { + None + } + }); + + let (bao_ob, fec_p, dict) = if let Some(adam) = adam_path { + let bytes = fs::read(&adam)?; + let (payload, _hdr, _n) = carbonado::adamantine::decode_adamantine_prefix(&bytes)?; + let (rkyv, bundle) = carbonado::split_adamantine_payload(&payload)?; + let manifest = carbonado::FilepackManifest::from_bytes(&rkyv)?; + let seg = manifest + .entries + .first() + .and_then(|e| e.segments.first()) + .ok_or("Adamantine sidecar has no segment")?; + let bao = carbonado::verification_slice_from_bundle( + &bundle, + seg.verification_outboard_offset, + seg.verification_outboard_len, + )? + .to_vec(); + let fec = if seg.fec_parity_len > 0 { + Some( + carbonado::fec_slice_from_bundle( + &bundle, + seg.fec_parity_offset, + seg.fec_parity_len, + )? + .to_vec(), + ) + } else { + None + }; + let dict = if seg.dict_len > 0 { + Some( + carbonado::dict_slice_from_bundle(&bundle, seg.dict_offset, seg.dict_len)?.to_vec(), + ) + } else { + None + }; + (Some(bao), fec, dict) } else { - None + (None, None, None) }; let fmt = match format { @@ -456,7 +430,7 @@ fn do_decode_outboard_streaming( validate_format(f)?; f } - None => guess_format_from_filename(input).ok_or( + None => guess_format_from_filename(&main_path).ok_or( "could not guess Carbonado format level from filename; provide --format (0-15)", )?, }; @@ -464,7 +438,7 @@ fn do_decode_outboard_streaming( let bao_hash = if let Some(hs) = &hash { parse_hash_hex(hs)? } else { - parse_bao_root_from_filename(input) + parse_bao_root_from_filename(&main_path) .ok_or("could not parse valid 64-hex bao root from bare filename; provide --hash")? }; @@ -478,7 +452,7 @@ fn do_decode_outboard_streaming( }; let mut out_f = File::create(out_base)?; - stream_decode_outboard( + stream_decode_outboard_with_dict( master, &bao_hash, &mut main_f, @@ -488,6 +462,7 @@ fn do_decode_outboard_streaming( fmt, None, &mut out_f, + dict.as_deref(), )?; Ok(()) } diff --git a/src/cli_app.rs b/src/cli_app.rs index e1a445f..8b40aac 100644 --- a/src/cli_app.rs +++ b/src/cli_app.rs @@ -20,11 +20,13 @@ use clap::{Parser, Subcommand}; plaintext at the path from `carbonado key path` (override: CARBONADO_MNEMONIC_PATH). \ Later encode/decode reuse it unless `--master` is given. Decode never auto-generates.\n\n\ ARTIFACTS:\n \ - Single-file default: inboard headered `{hash}.c{fmt:02x}` (format 14 → `.c0e`).\n \ - `--outboard`: bare main + optional `.out`/`.par` sidecars (single-file only).\n \ + Single-file inboard: one `{hash}.adam.c{fmt:02x}` (format 14 → `.adam.c0e`).\n \ + `--outboard`: `{hash}.c{fmt:02x}` bare + `{hash}.adam.c{fmt:02x}` sidecar \ + (starts with ADAMANTINE10\\n). No `.par` / `.dict` siblings.\n \ Directory: inboard Adamantine 1.0 catalog `.adam.c14` (or `.adam.c15` with \ `--encrypted`) and heterogeneous bare segment mains (c12/c14 or c13/c15). Output \ - defaults to `{input}-archive/`.\n\n\ + defaults to `{input}-archive/`.\n \ + `--zstd-level` is required when the Compression bit is set (including default format 14).\n\n\ See `carbonado --help` for per-command options.", after_help = "EXAMPLES:\n \ carbonado encode secret.bin --format 15\n \ @@ -56,12 +58,18 @@ pub enum Commands { /// Format level 0–15 (default 14 = public verifiable; odd values = encrypted) #[arg(short, long, default_value_t = 14, value_name = "LEVEL")] format: u8, - /// Single-file only: bare main + `.out`/`.par` sidecars (default single-file is inboard). + /// Single-file only: `{hash}.cXX` + `{hash}.adam.cXX` sidecar (default single-file is inboard). #[arg(long)] outboard: bool, /// Directory only: encrypted catalog c15 and segment formats c13/c15 (auto-creates BIP39 seed if needed) #[arg(long)] encrypted: bool, + /// Zstd compression level (required when the Compression bit is set) + #[arg(long, value_name = "LEVEL")] + zstd_level: Option, + /// Optional RFC 8878 zstd dictionary file (bytes stored in the Adamantine dict section) + #[arg(long, value_name = "PATH")] + zstd_dict: Option, /// 32-byte master key as 64 hex chars (overrides stored BIP39 seed) #[arg(long, value_name = "HEX")] master: Option, @@ -85,15 +93,9 @@ pub enum Commands { /// Bare outboard only: format level 0–15 when not encoded in filename (rejected on headered inboard) #[arg(short, long, value_name = "LEVEL")] format: Option, - /// Bare outboard only: FEC padding in bytes [default: 0, auto when `.par` sidecar present] + /// Bare outboard only: FEC padding in bytes [default: 0, auto from Adamantine sidecar] #[arg(long, default_value = "0", value_name = "BYTES")] padding: u32, - /// Bare outboard only: path to verification `.out` sidecar [default: sibling of input] - #[arg(long, alias = "bao-outboard", value_name = "PATH")] - verification_outboard: Option, - /// Bare outboard only: path to FEC `.par` sidecar [default: sibling of input] - #[arg(long, value_name = "PATH")] - fec_parity: Option, }, } diff --git a/src/constants.rs b/src/constants.rs index dfdbfb5..6bfabc2 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -18,8 +18,14 @@ pub const FEC_K: usize = 4; /// FEC total shards (m) pub const FEC_M: usize = 8; -/// Normative zstd compression level (AGENTS / Lean `Carbonado.Compress.zstdLevel`). -pub const ZSTD_LEVEL: i32 = 20; +/// Logical bytes in one RS stripe: four 4 KiB data leaves (`FEC_K * SLICE_LEN`). +pub const FEC_STRIPE_LOGICAL_LEN: u32 = SLICE_LEN * FEC_K as u32; +/// Inboard bytes in one RS stripe: eight 4 KiB leaves (4 data + 4 parity). +pub const FEC_STRIPE_INBOARD_LEN: u32 = SLICE_LEN * FEC_M as u32; + +/// Level-20 `windowLog` reference only. Encoder level is caller input, not a silent default. +/// Tests and the Lean AOT demo may pass `20` explicitly. +pub const ZSTD_LEVEL20: i32 = 20; /// Zstandard frame magic, little-endian `0xFD2FB528` /// (Lean `zstdMagic`; `ref/zstd/doc/zstd_compression_format.md`). @@ -28,7 +34,10 @@ pub const ZSTD_MAGIC: [u8; 4] = [0x28, 0xb5, 0x2f, 0xfd]; /// Product frames do not set `Content_Checksum_flag` (Lean `zstdContentChecksum`). pub const ZSTD_CONTENT_CHECKSUM: bool = false; -/// Product frames do not emit a dictionary ID (Lean `zstdDictionaryIdFlag`). +/// RFC 8878 zstd dictionary magic (`MAGIC_DICTIONARY` / `0xEC30A437` little-endian). +pub const ZSTD_DICTIONARY_MAGIC: [u8; 4] = [0x37, 0xa4, 0x30, 0xec]; + +/// No-dictionary `Dictionary_ID_flag` (Lean `zstdDictionaryIdFlag` when no dict is supplied). pub const ZSTD_DICTIONARY_ID_FLAG: u8 = 0; /// Level-20 `windowLog` from `ref/zstd` `ZSTD_defaultCParameters[0][20]` @@ -62,7 +71,7 @@ pub const ZSTD_LEVEL20_WINDOW_LOG_LARGE: u32 = 25; /// | Bit name in enum | Meaning when set | /// |-------|-------| /// | Encryption | Apply symmetric encryption (AES-256-CTR + HMAC-SHA512 EtM) | -/// | Compression | Apply Zstd compression at level 20 | +/// | Compression | Apply Zstd compression (level is encoder input) | /// | Verification | Add streaming verifiability (keyed Bao, 4 KiB leaves) | /// | Fec | Add forward error correction (reed-solomon-erasure 4/8) | /// diff --git a/src/decoding.rs b/src/decoding.rs index 0fb2d68..38f381a 100644 --- a/src/decoding.rs +++ b/src/decoding.rs @@ -6,68 +6,15 @@ pub use crate::stream::compress::decompress_buffer as decompress; pub use crate::stream::decode::{stream_decode_buffer, stream_decode_outboard_buffer}; use crate::{ - constants::{FEC_K, FEC_M}, + constants::{FEC_K, FEC_M, FEC_STRIPE_INBOARD_LEN, Format, SLICE_LEN}, + encoding, error::CarbonadoError, + stream::fec::{concat_data_leaves, encode_stripes, reconstruct_stripe}, + stream::{classify_inboard_leaves, verify_slice_inboard_seekable, verify_slice_outboard}, structs::EncodeInfo, -}; - -use reed_solomon_erasure::ReedSolomon; -use reed_solomon_erasure::galois_8::Field; - -use crate::{ - constants::{Format, SLICE_LEN}, - encoding, - stream::{extract_slice_inboard_for_scrub, verify_slice_inboard_seekable}, utils::decode_bao_hash, }; -use log::{debug, info, warn}; - -fn fec_chunks(chunked_bytes: &[(usize, &[u8])], padding: u32) -> Result, CarbonadoError> { - let data_shards = FEC_K; - let parity_shards = FEC_M - FEC_K; - let total_shards = FEC_M; - - let shard_size = if let Some((_, first)) = chunked_bytes.iter().find(|(_, c)| !c.is_empty()) { - first.len() - } else if !chunked_bytes.is_empty() { - chunked_bytes[0].1.len() - } else { - return Err(CarbonadoError::UnevenFecChunks); - }; - - let mut shards: Vec>> = vec![None; total_shards]; - for &(idx, data) in chunked_bytes { - if idx < total_shards && !data.is_empty() { - shards[idx] = Some(data.to_vec()); - } - } - for d in shards.iter().flatten() { - if d.len() != shard_size { - return Err(CarbonadoError::UnevenFecChunks); - } - } - - let rs = ReedSolomon::::new(data_shards, parity_shards)?; - rs.reconstruct(&mut shards)?; - - let mut decoded = vec![]; - for sh in shards.iter().take(data_shards) { - if let Some(s) = sh { - decoded.extend_from_slice(s); - } else { - decoded.resize(decoded.len() + shard_size, 0); - } - } - - if padding as usize > decoded.len() { - return Err(CarbonadoError::ScrubbedLengthMismatch( - decoded.len(), - padding as usize, - )); - } - decoded.truncate(decoded.len() - padding as usize); - Ok(decoded) -} +use log::warn; pub fn verification_with_outboard( bare: &[u8], @@ -97,64 +44,7 @@ pub fn fec_with_parity( padding: u32, ) -> Result, CarbonadoError> { trace!("forward error correcting from bare + parity sidecar (reed-solomon outboard)"); - if input.is_empty() && parity.is_empty() { - return Ok(vec![]); - } - let parity_shards = FEC_M - FEC_K; - if !parity.len().is_multiple_of(parity_shards) { - return Err(CarbonadoError::UnevenFecChunks); - } - let shard_len = parity.len() / parity_shards; - let padded_total = shard_len * FEC_K; - let pad = padding as usize; - if pad > padded_total { - return Err(CarbonadoError::ScrubbedLengthMismatch(padded_total, pad)); - } - // Logical length from parity stripe geometry + encode-time padding (not truncated main len). - let logical_len = padded_total - pad; - - // Stripe geometry comes from the parity sidecar (encode-time chunk_len), not truncated main len. - let mut padded = vec![0u8; padded_total]; - let copy = input.len().min(logical_len); - padded[..copy].copy_from_slice(&input[..copy]); - - let mut shards: Vec>> = vec![None; FEC_M]; - for (i, sh) in shards.iter_mut().enumerate().take(FEC_K) { - let start = i * shard_len; - let end = start + shard_len; - if end <= copy { - *sh = Some(padded[start..end].to_vec()); - } else { - // Truncated or missing data column — erasure; RS reconstructs from parity. - *sh = None; - } - } - for j in 0..parity_shards { - let start = j * shard_len; - let end = start + shard_len; - shards[FEC_K + j] = Some(parity[start..end].to_vec()); - } - - let rs = ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?; - rs.reconstruct(&mut shards)?; - - let mut decoded = vec![]; - for sh in shards.iter().take(FEC_K) { - if let Some(s) = sh { - decoded.extend_from_slice(s); - } else { - decoded.resize(decoded.len() + shard_len, 0); - } - } - - if decoded.len() < logical_len { - return Err(CarbonadoError::ScrubbedLengthMismatch( - decoded.len(), - logical_len, - )); - } - decoded.truncate(logical_len); - Ok(decoded) + crate::stream::fec::decode_outboard_stripes(input, parity, padding) } pub fn fec(input: &[u8], padding: u32) -> Result, CarbonadoError> { @@ -162,14 +52,30 @@ pub fn fec(input: &[u8], padding: u32) -> Result, CarbonadoError> { if input.is_empty() { return Ok(vec![]); } - let input_len = input.len(); - #[allow(clippy::manual_is_multiple_of)] - if input_len % FEC_M != 0 { + const STRIPE: usize = FEC_STRIPE_INBOARD_LEN as usize; + const LEAF: usize = SLICE_LEN as usize; + let (stripes, remainder) = input.as_chunks::(); + if !remainder.is_empty() { return Err(CarbonadoError::UnevenFecChunks); } - let chunk_len = input_len / FEC_M; - let chunks: Vec<(usize, &[u8])> = input.chunks_exact(chunk_len).enumerate().collect(); - fec_chunks(&chunks, padding) + let mut logical = Vec::new(); + for stripe in stripes { + let (leaves, leaf_rem) = stripe.as_chunks::(); + debug_assert!(leaf_rem.is_empty()); + let mut shards: Vec>> = leaves.iter().map(|c| Some(c.to_vec())).collect(); + let rebuilt = reconstruct_stripe(&mut shards)?; + for s in rebuilt.iter().take(FEC_K) { + logical.extend_from_slice(s); + } + } + if padding as usize > logical.len() { + return Err(CarbonadoError::ScrubbedLengthMismatch( + logical.len(), + padding as usize, + )); + } + logical.truncate(logical.len() - padding as usize); + Ok(logical) } pub fn verification(input: &[u8], hash: &[u8], format: u8) -> Result, CarbonadoError> { @@ -227,6 +133,34 @@ pub fn decode_outboard( ) } +/// Outboard decode with an optional RFC 8878 dictionary from the Adamantine bundle. +/// +/// When the compressed frame names a Dictionary_ID, `dict` must be the matching trained +/// dictionary bytes. Missing dict is [`CarbonadoError::MissingZstdDictionary`]. +#[allow(clippy::too_many_arguments)] +pub fn decode_outboard_with_dict( + master_key: &[u8], + hash: &[u8], + main: &[u8], + verification_outboard: Option<&[u8]>, + fec_parity: Option<&[u8]>, + padding: u32, + format: u8, + dict: Option<&[u8]>, +) -> Result, CarbonadoError> { + crate::stream::decode::stream_decode_outboard_buffer_with_dict( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + None, + dict, + ) +} + pub fn extract_slice( encoded: &[u8], index: u32, @@ -247,11 +181,13 @@ pub fn verify_slice( verify_slice_inboard_seekable(input, index, count, hash, format) } -/// Recover a damaged inboard Bao+FEC archive via RS subset search and re-encode oracle. +/// Recover a damaged inboard Bao+FEC archive per 16 KiB stripe. +/// +/// Bao-verify each 4 KiB leaf. Failed leaves are erasures in that stripe. +/// Reconstruct when the stripe has at least 4 good leaves; five bad leaves in +/// one stripe yields [`CarbonadoError::InvalidScrubbedHash`]. Re-Bao of the +/// reconstructed body must match `hash`. /// -/// **Scrub entry:** all [`verify_inboard_keyed`] failures (`AuthenticationFailed`, -/// `InvalidHeaderLength`, `BaoResponseTruncated`, `StdIoError`, etc.) route into combinatorial -/// FEC recovery — the API does not distinguish tamper from truncation before attempting recovery. /// Pristine archives return [`CarbonadoError::UnnecessaryScrub`]. pub fn scrub( input: &[u8], @@ -264,64 +200,49 @@ pub fn scrub( return Err(CarbonadoError::ScrubRequiresVerification); } let hash = decode_bao_hash(hash)?; - let chunk_size = encode_info.chunk_len; let padding = encode_info.padding_len; - let slices_per_chunk = chunk_size / SLICE_LEN; - // S5: slice-bounded keyed Bao verify oracle — no O(decoded) body staging on scrub entry. match crate::stream::bao::verify_inboard_keyed(input, hash.as_bytes(), format) { Ok(()) => Err(CarbonadoError::UnnecessaryScrub), Err(e) => { warn!("Data failed to verify with error: {e}. Scrubbing..."); - let mut chunks: Vec<(usize, Vec)> = vec![]; - - for i in 0..FEC_M { - let slice_index = (i as u32) * slices_per_chunk; - match extract_slice_inboard_for_scrub(input, slice_index, slices_per_chunk) { - Ok(chunk) if chunk.len() == chunk_size as usize => chunks.push((i, chunk)), - Ok(_) => debug!("Chunk {i} wrong length after seekable slice extract"), - Err(e) => { - debug!("At least one chunk was bad, at chunk index {i}. Error was: {e}.") - } - } + if !fmt.contains(Format::Fec) { + return Err(CarbonadoError::InvalidScrubbedHash); } - - info!( - "{} candidate chunks extracted, of {FEC_K} needed.", - chunks.len() - ); - - let mut recovered: Option> = None; - let n = chunks.len(); - for mask in 0..(1usize << n) { - if mask.count_ones() < FEC_K as u32 { - continue; - } - let mut sel: Vec<(usize, &[u8])> = vec![]; - for (j, c) in chunks.iter().enumerate().take(n) { - if (mask & (1 << j)) != 0 { - sel.push((c.0, &c.1)); - } + let leaves = classify_inboard_leaves(input, hash.as_bytes(), format) + .map_err(|_| CarbonadoError::InvalidScrubbedHash)?; + if leaves.is_empty() || !leaves.len().is_multiple_of(FEC_M) { + return Err(CarbonadoError::InvalidScrubbedHash); + } + let mut logical = Vec::new(); + for stripe in leaves.chunks(FEC_M) { + let good = stripe.iter().filter(|l| l.is_some()).count(); + if good < FEC_K { + return Err(CarbonadoError::InvalidScrubbedHash); } - if let Ok(cand_inner) = fec_chunks(&sel, padding) { - let (scrubbed, sp, _) = encoding::encode_inboard_buffer(&cand_inner)?; - if sp != padding { - continue; - } - if let Ok((verif, got_h)) = - encoding::verification_inboard_buffer(&scrubbed, format) - && got_h == hash - && verif.len() == input.len() - { - recovered = Some(verif); - break; - } + let mut shards: Vec>> = stripe.to_vec(); + let rebuilt = reconstruct_stripe(&mut shards) + .map_err(|_| CarbonadoError::InvalidScrubbedHash)?; + for s in rebuilt.iter().take(FEC_K) { + logical.extend_from_slice(s); } } - - match recovered { - Some(v) => Ok(v), - None => Err(CarbonadoError::InvalidScrubbedHash), + if padding as usize > logical.len() { + return Err(CarbonadoError::ScrubbedLengthMismatch( + logical.len(), + padding as usize, + )); + } + logical.truncate(logical.len() - padding as usize); + let (scrubbed, sp, _) = encoding::encode_inboard_buffer(&logical)?; + if sp != padding { + return Err(CarbonadoError::ScrubbedPaddingMismatch); + } + let (verif, got_h) = encoding::verification_inboard_buffer(&scrubbed, format)?; + if got_h == hash && verif.len() == input.len() { + Ok(verif) + } else { + Err(CarbonadoError::InvalidScrubbedHash) } } } @@ -340,92 +261,91 @@ pub fn scrub_outboard( return Err(CarbonadoError::ScrubRequiresVerification); } - let good = if let Some(ob) = verification_outboard { - crate::stream::bao::stream_verification_outboard_verify( - bare, - bare.len() as u64, - ob, - hash, - format, - ) - .is_ok() - } else { + let Some(ob) = verification_outboard else { return Err(CarbonadoError::MissingVerificationOutboard); }; + let good = crate::stream::bao::stream_verification_outboard_verify( + bare, + bare.len() as u64, + ob, + hash, + format, + ) + .is_ok(); + if good { return Err(CarbonadoError::UnnecessaryScrub); } - let padding = encode_info.padding_len; - let recovered_bare = if fmt.contains(Format::Fec) { - let Some(ob) = verification_outboard else { - return Err(CarbonadoError::MissingVerificationOutboard); - }; - let Some(par) = fec_parity else { - return Err(CarbonadoError::MissingFecParity); - }; + if !fmt.contains(Format::Fec) { + return Err(CarbonadoError::InvalidScrubbedHash); + } + let Some(par) = fec_parity else { + return Err(CarbonadoError::MissingFecParity); + }; - // Encode-time geometry (parity sidecar + EncodeInfo), not calc_padding_len(bare.len()). - let shard_len = encode_info.chunk_len as usize; - if shard_len == 0 { - return Err(CarbonadoError::UnevenFecChunks); - } - let parity_shards = FEC_M - FEC_K; - if !par.len().is_multiple_of(shard_len) || par.len() / shard_len != parity_shards { - return Err(CarbonadoError::UnevenFecChunks); + let padding = encode_info.padding_len; + const LEAF: usize = SLICE_LEN as usize; + const PARITY_STRIPE: usize = (FEC_M - FEC_K) * LEAF; + let (parity_stripes, remainder) = par.as_chunks::(); + if !remainder.is_empty() { + return Err(CarbonadoError::UnevenFecChunks); + } + let n_stripes = parity_stripes.len(); + let padded_total = n_stripes * FEC_K * LEAF; + let pad = padding as usize; + if pad > padded_total { + return Err(CarbonadoError::ScrubbedLengthMismatch(padded_total, pad)); + } + let logical_len = padded_total - pad; + let data_len = bare.len() as u64; + + let mut logical = Vec::with_capacity(padded_total); + for (stripe_idx, parity_stripe) in parity_stripes.iter().enumerate() { + let mut shards: Vec>> = vec![None; FEC_M]; + for (symbol, shard) in shards.iter_mut().take(FEC_K).enumerate() { + let leaf_index = (stripe_idx * FEC_K + symbol) as u32; + match verify_slice_outboard(bare, ob, data_len, leaf_index, 1, hash, format) { + Ok(bytes) if bytes.len() == LEAF => *shard = Some(bytes), + _ => {} + } } - let padded_total = shard_len * FEC_K; - let pad = padding as usize; - if pad > padded_total { - return Err(CarbonadoError::ScrubbedLengthMismatch(padded_total, pad)); + let (parity_leaves, leaf_rem) = parity_stripe.as_chunks::(); + debug_assert!(leaf_rem.is_empty()); + for (shard, chunk) in shards[FEC_K..].iter_mut().zip(parity_leaves) { + *shard = Some(chunk.to_vec()); } - let logical_len = padded_total - pad; - let copy = bare.len().min(logical_len); - - let mut padded = vec![0u8; padded_total]; - padded[..copy].copy_from_slice(&bare[..copy]); - - let mut chunks: Vec<(usize, Vec)> = vec![]; - for i in 0..FEC_K { - let start = i * shard_len; - let end = start + shard_len; - if end <= copy { - chunks.push((i, padded[start..end].to_vec())); - } + let good = shards.iter().filter(|s| s.is_some()).count(); + if good < FEC_K { + return Err(CarbonadoError::InvalidScrubbedHash); } - for j in 0..parity_shards { - let start = j * shard_len; - chunks.push((FEC_K + j, par[start..start + shard_len].to_vec())); + let rebuilt = reconstruct_stripe(&mut shards)?; + for s in rebuilt.iter().take(FEC_K) { + logical.extend_from_slice(s); } + } + logical.truncate(logical_len); - let n = chunks.len(); - let mut recovered: Option> = None; - for mask in 0..(1usize << n) { - if mask.count_ones() < FEC_K as u32 { - continue; - } - let mut sel: Vec<(usize, &[u8])> = vec![]; - for (j, c) in chunks.iter().enumerate().take(n) { - if (mask & (1 << j)) != 0 { - sel.push((c.0, &c.1)); - } - } - if let Ok(cand_inner) = fec_chunks(&sel, padding) - && verification_with_outboard(&cand_inner, ob, hash, format).is_ok() - { - recovered = Some(cand_inner); - break; - } + if verification_with_outboard(&logical, ob, hash, format).is_ok() { + Ok(logical) + } else { + // Reconstruct may have included padding zeros; re-encode data leaves and + // compare against the Bao root of the original (unpadded) main. + let (stripes, sp, _) = encode_stripes(&logical)?; + if sp != padding { + return Err(CarbonadoError::InvalidScrubbedHash); } - - match recovered { - Some(v) => v, - None => return Err(CarbonadoError::InvalidScrubbedHash), + let data = concat_data_leaves(&stripes); + let recovered = if data.len() >= logical_len { + data[..logical_len].to_vec() + } else { + data + }; + if verification_with_outboard(&recovered, ob, hash, format).is_ok() { + Ok(recovered) + } else { + Err(CarbonadoError::InvalidScrubbedHash) } - } else { - return Err(CarbonadoError::InvalidScrubbedHash); - }; - - Ok(recovered_bare) + } } diff --git a/src/encoding.rs b/src/encoding.rs index 31e987f..b4f4ae5 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -2,15 +2,28 @@ use log::trace; use crate::{error::CarbonadoError, structs::Encoded}; +use crate::stream::ZstdEncode; use crate::stream::encode::stream_encode_buffer_with_nonce; use crate::stream::encode::stream_encode_outboard_buffer; /// Encode data into Carbonado format (delegates to the streaming pipeline). /// /// Encrypted formats use a CSPRNG nonce (embedded layout). For deterministic encrypted -/// bodies (G9 fixtures), use [`encode_with_nonce`]. +/// bodies (G9 fixtures), use [`encode_with_nonce`]. Compression requires +/// [`encode_with_zstd`] with an explicit level. pub fn encode(master_key: &[u8], input: &[u8], format: u8) -> Result { - encode_with_nonce(master_key, input, format, None) + encode_with_zstd(master_key, input, format, None, &ZstdEncode::default()) +} + +/// Encode with explicit zstd parameters (required when the Compression bit is set). +pub fn encode_with_zstd( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, +) -> Result { + encode_with_nonce_and_zstd(master_key, input, format, explicit_nonce, zstd) } /// Low-level body encode with optional fixed nonce for encrypted formats. @@ -31,20 +44,48 @@ pub fn encode_with_nonce( input: &[u8], format: u8, explicit_nonce: Option<[u8; 16]>, +) -> Result { + encode_with_nonce_and_zstd( + master_key, + input, + format, + explicit_nonce, + &ZstdEncode::default(), + ) +} + +fn encode_with_nonce_and_zstd( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result { let (verifiable, hash, info) = - stream_encode_buffer_with_nonce(master_key, input, format, explicit_nonce)?; + stream_encode_buffer_with_nonce(master_key, input, format, explicit_nonce, zstd)?; Ok(Encoded(verifiable, hash, info)) } -/// Outboard variant for public and encrypted formats. +/// Outboard variant for public and encrypted formats. Compression requires +/// [`encode_outboard_with_zstd`]. pub fn encode_outboard( master_key: &[u8], input: &[u8], format: u8, +) -> Result { + encode_outboard_with_zstd(master_key, input, format, None, &ZstdEncode::default()) +} + +/// Outboard encode with explicit zstd parameters. +pub fn encode_outboard_with_zstd( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result { trace!("encode_outboard format=0x{format:02x}"); - stream_encode_outboard_buffer(master_key, input, format, None) + stream_encode_outboard_buffer(master_key, input, format, explicit_nonce, zstd) } // Scrub recovery re-exports diff --git a/src/error.rs b/src/error.rs index 9c58e71..6fab82b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -18,6 +18,16 @@ pub enum CarbonadoError { #[error("zstd encode/decode failed: {0}")] ZstdError(String), + /// Compression bit is set but no zstd level was supplied (level is encoder input, not a library default). + #[error("zstd compression requires an explicit level")] + MissingZstdLevel, + + /// Zstd frame names a Dictionary_ID but the Adamantine dict section is empty or absent. + #[error( + "zstd frame Dictionary_ID {dictionary_id} requires dictionary bytes in the Adamantine bundle" + )] + MissingZstdDictionary { dictionary_id: u32 }, + // The old EciesError variant was removed as part of the clean break to the v2 symmetric model. // All encryption-related errors now go through the new symmetric primitives (see crypto.rs). /// bao decode error @@ -166,10 +176,8 @@ pub enum CarbonadoError { #[error("Invalid Adamantine flags: {0}")] InvalidAdamantineFlags(u8), - /// Adamantine carbonado_fmt byte is not a valid directory catalog format (c14/c15) - #[error( - "Invalid Adamantine carbonado format: expected 0x0E (c14) or 0x0F (c15), got 0x{0:02x}" - )] + /// Adamantine carbonado_fmt byte is not a valid format (0–15) + #[error("Invalid Adamantine carbonado format: expected 0–15, got 0x{0:02x}")] InvalidAdamantineCarbonadoFormat(u8), /// Adamantine header `carbonado_fmt` disagrees with the format parsed from the `.adam.c{N}` filename diff --git a/src/file.rs b/src/file.rs index ff4c6af..f57ea98 100644 --- a/src/file.rs +++ b/src/file.rs @@ -9,16 +9,15 @@ use bao::Hash; // nom imports removed — legacy parse_bytes / old header parsing was deleted as part of the v2 replacement. // (secp256k1 imports removed - clean break, legacy Header parsing deleted) -use crate::stream::decode::stream_decrypt_header_path; use crate::{ adamantine::{ ADAMANTINE_CARBONADO_FMT_ENCRYPTED, ADAMANTINE_CARBONADO_FMT_PUBLIC, - ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_HEADER_LEN, AdamantineHeader, decode_adamantine, - encode_adamantine, + ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_HEADER_LEN, ADAMANTINE_MAGIC, AdamantineHeader, + decode_adamantine, decode_adamantine_prefix, encode_adamantine, }, adamantine_payload::{ MAX_ADAMANTINE_PAYLOAD_LEN, MAX_BAO_BUNDLE_LEN, build_adamantine_payload, - split_adamantine_payload, verification_slice_from_bundle, + dict_slice_from_bundle, split_adamantine_payload, verification_slice_from_bundle, }, constants::{Format, MAGICNO}, decoding, @@ -31,7 +30,7 @@ use crate::{ SegmentRef, }, paths::parse_bao_root_from_filename, - stream::{DEFAULT_SEGMENT_PLAINTEXT_BUDGET, encode::stream_encode_outboard}, + stream::{DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ZstdEncode, encode::stream_encode_outboard}, structs::{EncodeInfo, OutboardEncoded}, utils::{calc_padding_len, decode_bao_hash, encode_bao_hash}, }; @@ -57,6 +56,8 @@ pub struct DirectoryEncodeOptions { pub segment_format_policy: SegmentFormatPolicy, /// Max logical plaintext bytes per segment before sharding a file. pub segment_plaintext_budget: u64, + /// Zstd parameters (level required when any segment or the catalog uses Compression). + pub zstd: ZstdEncode, /// Optional OpenTimestamps stamping policy (requires `ots` feature). #[cfg(feature = "ots")] pub ots_policy: Option, @@ -68,12 +69,63 @@ impl Default for DirectoryEncodeOptions { encrypted: false, segment_format_policy: SegmentFormatPolicy::default(), segment_plaintext_budget: DEFAULT_SEGMENT_PLAINTEXT_BUDGET, + zstd: ZstdEncode::default(), #[cfg(feature = "ots")] ots_policy: None, } } } +/// Options for [`encode_to_dir`]. +#[derive(Clone, Debug, Default)] +pub struct EncodeToDirOptions { + /// When true, write `{hash}.cXX` + `{hash}.adam.cXX`. When false, one `{hash}.adam.cXX`. + pub outboard: bool, + /// Zstd level (required when Compression is set) and optional dictionary. + pub zstd: ZstdEncode, +} + +/// Paths written by [`encode_to_dir`]. +#[derive(Clone, Debug)] +pub struct EncodedToDir { + /// Keyed Bao root used in filenames. + pub hash: [u8; 32], + /// Format level 0–15. + pub format: u8, + /// Bare main (`{hash}.cXX`) or the single inboard `{hash}.adam.cXX`. + pub main_path: PathBuf, + /// Adamantine sidecar or the same path as `main_path` for inboard. + pub adam_path: PathBuf, + dict: Vec, +} + +impl EncodedToDir { + /// Filename of the main artifact. + pub fn main_name(&self) -> String { + self.main_path + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() + } + + /// Filename of the Adamantine sidecar (inboard: same as [`Self::main_name`]). + pub fn adam_name(&self) -> String { + self.adam_path + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() + } + + /// Dictionary bytes stored in the Adamantine bundle, if any. + pub fn dict_bytes(&self) -> Option<&[u8]> { + if self.dict.is_empty() { + None + } else { + Some(self.dict.as_slice()) + } + } +} + impl DirectoryEncodeOptions { /// Resolve the catalog format level (c14 public or c15 encrypted). pub fn resolved_catalog_format(&self) -> u8 { @@ -353,16 +405,27 @@ pub fn decode_stream( &mut post_preprocess, )?; post_preprocess.rewind()?; + let mut trailer = Vec::new(); + input + .read_to_end(&mut trailer) + .map_err(CarbonadoError::StdIoError)?; + let dict = dict_from_inboard_trailer(&trailer)?; + let out_len = if fmt.contains(Format::Encryption) { - stream_decrypt_header_path( + crate::stream::decode::stream_decrypt_header_path_with_dict( master_key, header.payload_nonce, &mut post_preprocess, fmt.bits(), output, + dict.as_deref(), )? } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(post_preprocess, output)? + crate::stream::compress::stream_decompress_with_dict( + post_preprocess, + output, + dict.as_deref(), + )? } else { std::io::copy(&mut post_preprocess, output).map_err(CarbonadoError::StdIoError)? }; @@ -404,17 +467,24 @@ pub fn decode(master_key: &[u8], encoded: &[u8]) -> Result<(Header, Vec), Ca &mut post_preprocess, )?; post_preprocess.rewind()?; + let trailer = &body[body_len..]; + let dict = dict_from_inboard_trailer(trailer)?; let mut decompressed = Vec::new(); if fmt.contains(Format::Encryption) { - crate::stream::stream_decrypt_header_path( + crate::stream::decode::stream_decrypt_header_path_with_dict( master_key, header.payload_nonce, &mut post_preprocess, fmt.bits(), &mut decompressed, + dict.as_deref(), )?; } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(post_preprocess, &mut decompressed)?; + crate::stream::compress::stream_decompress_with_dict( + post_preprocess, + &mut decompressed, + dict.as_deref(), + )?; } else { std::io::copy(&mut post_preprocess, &mut decompressed) .map_err(CarbonadoError::StdIoError)?; @@ -439,7 +509,18 @@ pub fn encode( level: u8, metadata: Option<[u8; 8]>, ) -> Result<(Vec, EncodeInfo), CarbonadoError> { - encode_with_nonce(master_key, input, level, metadata, None) + encode_with_zstd(master_key, input, level, metadata, &ZstdEncode::default()) +} + +/// Headered inboard encode with explicit zstd parameters (required when Compression is set). +pub fn encode_with_zstd( + master_key: &[u8], + input: &[u8], + level: u8, + metadata: Option<[u8; 8]>, + zstd: &ZstdEncode, +) -> Result<(Vec, EncodeInfo), CarbonadoError> { + encode_with_nonce_and_zstd(master_key, input, level, metadata, None, zstd) } /// Headered inboard encode with optional fixed `payload_nonce` for encrypted formats. @@ -460,10 +541,36 @@ pub fn encode_with_nonce( level: u8, metadata: Option<[u8; 8]>, explicit_nonce: Option<[u8; 16]>, +) -> Result<(Vec, EncodeInfo), CarbonadoError> { + encode_with_nonce_and_zstd( + master_key, + input, + level, + metadata, + explicit_nonce, + &ZstdEncode::default(), + ) +} + +/// Headered inboard encode with explicit zstd parameters. +pub fn encode_with_nonce_and_zstd( + master_key: &[u8], + input: &[u8], + level: u8, + metadata: Option<[u8; 8]>, + explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result<(Vec, EncodeInfo), CarbonadoError> { let mut out = Vec::new(); - let (header, info) = - encode_stream_with_nonce(master_key, input, level, metadata, &mut out, explicit_nonce)?; + let (header, info) = encode_stream_with_nonce_and_zstd( + master_key, + input, + level, + metadata, + &mut out, + explicit_nonce, + zstd, + )?; let mut body = header.try_to_vec()?; body.extend_from_slice(&out); Ok((body, info)) @@ -482,7 +589,26 @@ pub fn encode_stream( metadata: Option<[u8; 8]>, output: &mut W, ) -> Result<(Header, EncodeInfo), CarbonadoError> { - encode_stream_with_nonce(master_key, input, level, metadata, output, None) + encode_stream_with_zstd( + master_key, + input, + level, + metadata, + output, + &ZstdEncode::default(), + ) +} + +/// Like [`encode_stream`], with explicit zstd parameters (required when Compression is set). +pub fn encode_stream_with_zstd( + master_key: &[u8], + input: R, + level: u8, + metadata: Option<[u8; 8]>, + output: &mut W, + zstd: &ZstdEncode, +) -> Result<(Header, EncodeInfo), CarbonadoError> { + encode_stream_with_nonce_and_zstd(master_key, input, level, metadata, output, None, zstd) } /// Like [`encode_stream`], with optional fixed `payload_nonce` for encrypted formats. @@ -490,12 +616,33 @@ pub fn encode_stream( /// When `explicit_nonce` is `Some(n)`, both backends use `n` literally (including /// all-zero). See [`encode_with_nonce`] for safety notes (test/determinism only). pub fn encode_stream_with_nonce( + master_key: &[u8], + input: R, + level: u8, + metadata: Option<[u8; 8]>, + output: &mut W, + explicit_nonce: Option<[u8; 16]>, +) -> Result<(Header, EncodeInfo), CarbonadoError> { + encode_stream_with_nonce_and_zstd( + master_key, + input, + level, + metadata, + output, + explicit_nonce, + &ZstdEncode::default(), + ) +} + +/// Like [`encode_stream_with_nonce`], with explicit zstd parameters. +pub fn encode_stream_with_nonce_and_zstd( master_key: &[u8], mut input: R, level: u8, metadata: Option<[u8; 8]>, output: &mut W, explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result<(Header, EncodeInfo), CarbonadoError> { let format = Format::from(level); let mut payload_nonce = [0u8; 16]; @@ -507,6 +654,7 @@ pub fn encode_stream_with_nonce( &mut payload_nonce, true, explicit_nonce, + zstd, )?; let header = Header::new( @@ -558,6 +706,17 @@ pub fn encode_outboard( input: &[u8], level: u8, metadata: Option<[u8; 8]>, +) -> Result<(Option
, OutboardEncoded), CarbonadoError> { + encode_outboard_with_zstd(master_key, input, level, metadata, &ZstdEncode::default()) +} + +/// High-level outboard encode with explicit zstd parameters (required when Compression is set). +pub fn encode_outboard_with_zstd( + master_key: &[u8], + input: &[u8], + level: u8, + metadata: Option<[u8; 8]>, + zstd: &ZstdEncode, ) -> Result<(Option
, OutboardEncoded), CarbonadoError> { let format = Format::from(level); let mut payload_nonce = [0u8; 16]; @@ -572,6 +731,7 @@ pub fn encode_outboard( input, format.bits(), explicit_nonce, + zstd, )?; let hdr = Header::new( master_key, @@ -610,6 +770,7 @@ pub fn encode_outboard_stream( par_ref, &mut payload_nonce, true, + &ZstdEncode::default(), )?; let main_bytes = read_file_from_start(main_out)?; @@ -660,6 +821,177 @@ fn read_file_from_start(f: &mut File) -> Result, CarbonadoError> { Ok(buf) } +/// Write a single-file archive into `outdir`. +/// +/// Inboard: one `{hash}.adam.c{fmt:02x}` (Header + body + Adamantine after `encoded_len`). +/// Outboard: `{hash}.c{fmt:02x}` bare main + `{hash}.adam.c{fmt:02x}` sidecar starting with +/// `ADAMANTINE10\n`. No `.par` or `.dict` siblings. +pub fn encode_to_dir( + master_key: &[u8], + input: &[u8], + format: u8, + outdir: &Path, + options: EncodeToDirOptions, +) -> Result { + fs::create_dir_all(outdir).map_err(CarbonadoError::StdIoError)?; + let zstd = &options.zstd; + let content_blake3 = *blake3::hash(input).as_bytes(); + if options.outboard { + encode_to_dir_outboard(master_key, input, format, outdir, zstd, content_blake3) + } else { + encode_to_dir_inboard(master_key, input, format, outdir, zstd, content_blake3) + } +} + +fn encode_to_dir_outboard( + master_key: &[u8], + input: &[u8], + format: u8, + outdir: &Path, + zstd: &ZstdEncode, + content_blake3: [u8; 32], +) -> Result { + let oenc = encoding::encode_outboard_with_zstd(master_key, input, format, None, zstd)?; + let hash = *oenc.hash.as_bytes(); + let main_name = single_file_main_name(&hash, format); + let adam_name = single_file_adam_name(&hash, format); + let main_path = outdir.join(&main_name); + let adam_path = outdir.join(&adam_name); + write_file(&main_path, &oenc.main)?; + let bao = oenc.verification_outboard.as_deref().unwrap_or(&[]); + let parity = oenc.fec_parity.as_deref().unwrap_or(&[]); + let dict = zstd.dict.as_deref().unwrap_or(&[]); + let adam = build_single_file_adamantine( + format, + hash, + oenc.main.len() as u64, + content_blake3, + bao, + parity, + dict, + )?; + write_file(&adam_path, &adam)?; + Ok(EncodedToDir { + hash, + format, + main_path, + adam_path, + dict: dict.to_vec(), + }) +} + +fn encode_to_dir_inboard( + master_key: &[u8], + input: &[u8], + format: u8, + outdir: &Path, + zstd: &ZstdEncode, + content_blake3: [u8; 32], +) -> Result { + let (encoded, info) = encode_with_nonce_and_zstd(master_key, input, format, None, None, zstd)?; + let header = Header::try_from(&encoded[..Header::LEN])?; + let hash = *header.hash.as_bytes(); + let body = &encoded[Header::LEN..]; + if body.len() < header.encoded_len as usize { + return Err(CarbonadoError::InvalidHeaderLength); + } + let pipeline = &body[..header.encoded_len as usize]; + let bao = []; + let parity = []; + let dict = zstd.dict.as_deref().unwrap_or(&[]); + let adam = build_single_file_adamantine( + format, + hash, + info.bytes_verifiable as u64, + content_blake3, + &bao, + &parity, + dict, + )?; + let mut on_disk = Vec::with_capacity(Header::LEN + pipeline.len() + adam.len()); + on_disk.extend_from_slice(&encoded[..Header::LEN]); + on_disk.extend_from_slice(pipeline); + on_disk.extend_from_slice(&adam); + let adam_name = single_file_adam_name(&hash, format); + let adam_path = outdir.join(&adam_name); + write_file(&adam_path, &on_disk)?; + Ok(EncodedToDir { + hash, + format, + main_path: adam_path.clone(), + adam_path, + dict: dict.to_vec(), + }) +} + +fn single_file_main_name(hash: &[u8; 32], format: u8) -> String { + format!("{}.c{:02x}", hex_encode(hash), format) +} + +fn single_file_adam_name(hash: &[u8; 32], format: u8) -> String { + format!("{}.adam.c{:02x}", hex_encode(hash), format) +} + +fn build_single_file_adamantine( + format: u8, + hash: [u8; 32], + main_len: u64, + content_blake3: [u8; 32], + bao: &[u8], + parity: &[u8], + dict: &[u8], +) -> Result, CarbonadoError> { + let mut bundle = Vec::new(); + let vo = 0u32; + let vl = bao.len() as u32; + bundle.extend_from_slice(bao); + let (fo, fl) = if parity.is_empty() { + (0u32, 0u32) + } else { + let off = bundle.len() as u32; + bundle.extend_from_slice(parity); + (off, parity.len() as u32) + }; + let (d_off, d_len) = if dict.is_empty() { + (0u32, 0u32) + } else { + let off = bundle.len() as u32; + bundle.extend_from_slice(dict); + (off, dict.len() as u32) + }; + let manifest = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: format, + catalog_bao_root: hash, + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "file".into(), + content_blake3, + segment_format: format, + segments: vec![SegmentRef { + segment_bao_root: hash, + chunk_index: 0, + main_len, + verification_outboard_offset: vo, + verification_outboard_len: vl, + fec_parity_offset: fo, + fec_parity_len: fl, + dict_offset: d_off, + dict_len: d_len, + }], + ots_proof: None, + }], + }; + // Inboard single-file: Bao and FEC are in the Carbonado body. The trailer may + // carry only a dict. Skip outboard FEC/bao geometry validation used for directory catalogs. + if !bao.is_empty() || !parity.is_empty() { + manifest.validate()?; + } + let rkyv = manifest.into_bytes()?; + let payload = build_adamantine_payload(&rkyv, &bundle)?; + Ok(encode_adamantine(&payload, format, 0)) +} + /// High-level outboard decode at the `file` layer (accepts bare main + sidecars + optional out-of-band header). /// /// `hash` is required (the keyed Bao root; from Header or filename); used for verification_with_outboard verify. @@ -778,8 +1110,17 @@ pub fn encode_directory( master_key: &[u8], dir: &Path, outdir: &Path, + zstd: &ZstdEncode, ) -> Result { - encode_directory_with_options(master_key, dir, outdir, DirectoryEncodeOptions::default()) + encode_directory_with_options( + master_key, + dir, + outdir, + DirectoryEncodeOptions { + zstd: zstd.clone(), + ..DirectoryEncodeOptions::default() + }, + ) } /// Encode a directory with explicit options (encryption, sharding budget, OTS policy). @@ -979,7 +1320,16 @@ pub fn decode_directory( } else { 0 }; - let part = decoding::decode_outboard( + let dict = if seg_ref.dict_len > 0 { + Some(dict_slice_from_bundle( + &bao_bundle, + seg_ref.dict_offset, + seg_ref.dict_len, + )?) + } else { + None + }; + let part = decoding::decode_outboard_with_dict( master_key, &segment_root, &seg_main, @@ -987,6 +1337,7 @@ pub fn decode_directory( fec_par, padding, entry.segment_format, + dict, )?; recovered.extend_from_slice(&part); } @@ -1227,6 +1578,7 @@ fn encode_file_segments( chunk_index, bao_bundle, written_segment_paths, + &options.zstd, )?; segments.push(seg_ref); } @@ -1234,6 +1586,7 @@ fn encode_file_segments( } /// Encode one bare segment main and append its Bao outboard blob to the bundle. +#[allow(clippy::too_many_arguments)] fn write_bare_segment( master_key: &[u8], data: &[u8], @@ -1242,8 +1595,9 @@ fn write_bare_segment( chunk_index: usize, bao_bundle: &mut BaoBundleBuilder, written_segment_paths: &mut Vec, + zstd: &ZstdEncode, ) -> Result { - let oenc = encoding::encode_outboard(master_key, data, segment_format)?; + let oenc = encoding::encode_outboard_with_zstd(master_key, data, segment_format, None, zstd)?; let root = *oenc.hash.as_bytes(); let main_len = oenc.main.len() as u64; if main_len > MAX_SEGMENT_MAIN_LEN { @@ -1271,6 +1625,12 @@ fn write_bare_segment( } else { (0, 0) }; + let (dict_offset, dict_len) = if let Some(dict) = zstd.dict.as_deref().filter(|d| !d.is_empty()) + { + bao_bundle.append(dict)? + } else { + (0, 0) + }; Ok(SegmentRef { segment_bao_root: root, chunk_index: chunk_index as u32, @@ -1279,6 +1639,8 @@ fn write_bare_segment( verification_outboard_len: ver_len, fec_parity_offset: fec_offset, fec_parity_len: fec_len, + dict_offset, + dict_len, }) } @@ -1312,8 +1674,21 @@ fn write_catalog_artifact( }; let bundle = bao_bundle.as_slice(); - let encoded = - encode_inboard_catalog_bytes(master_key, entries, catalog_format, adam_fmt, flags, bundle)?; + // Catalog Carbonado is inboard c14/c15. Do not bind the file-segment zstd + // dictionary into the catalog frame: decode_directory has no dict for it. + let catalog_zstd = ZstdEncode { + level: options.zstd.level, + dict: None, + }; + let encoded = encode_inboard_catalog_bytes( + master_key, + entries, + catalog_format, + adam_fmt, + flags, + bundle, + &catalog_zstd, + )?; #[cfg(debug_assertions)] if directory_encode_test_hooks::take_catalog_write_failure() { return Err(CarbonadoError::StdIoError(std::io::Error::other( @@ -1353,6 +1728,7 @@ fn encode_inboard_catalog_bytes( adam_fmt: u8, flags: u8, bao_bundle: &[u8], + zstd: &ZstdEncode, ) -> Result, CarbonadoError> { let index = FilepackManifest { version: FILEPACK_MANIFEST_VERSION, @@ -1365,7 +1741,8 @@ fn encode_inboard_catalog_bytes( let rkyv = index.into_bytes()?; let adam_payload = build_adamantine_payload(&rkyv, bao_bundle)?; let adamantine = encode_adamantine(&adam_payload, adam_fmt, flags); - let (encoded, _info) = encode(master_key, &adamantine, catalog_format, None)?; + let (encoded, _info) = + encode_with_nonce_and_zstd(master_key, &adamantine, catalog_format, None, None, zstd)?; Ok(encoded) } @@ -1546,6 +1923,25 @@ fn is_inboard_wire(bytes: &[u8]) -> bool { bytes.len() > Header::LEN && &bytes[0..12] == MAGICNO } +fn dict_from_inboard_trailer(trailer: &[u8]) -> Result>, CarbonadoError> { + if trailer.len() < ADAMANTINE_MAGIC.len() + || &trailer[..ADAMANTINE_MAGIC.len()] != ADAMANTINE_MAGIC + { + return Ok(None); + } + let (payload, _hdr, _consumed) = decode_adamantine_prefix(trailer)?; + let (rkyv, bundle) = split_adamantine_payload(&payload)?; + let manifest = FilepackManifest::from_bytes_unvalidated(&rkyv, [0u8; 32])?; + let Some(seg) = manifest.entries.first().and_then(|e| e.segments.first()) else { + return Ok(None); + }; + if seg.dict_len == 0 { + return Ok(None); + } + let dict = dict_slice_from_bundle(&bundle, seg.dict_offset, seg.dict_len)?; + Ok(Some(dict.to_vec())) +} + fn hash_to_root(hash_bytes: &[u8]) -> [u8; 32] { let mut root = [0u8; 32]; root.copy_from_slice(hash_bytes); @@ -1863,11 +2259,12 @@ mod directory_decode_path_tests { ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, ); - let (encoded, _) = encode( + let (encoded, _) = encode_with_nonce_and_zstd( &[0u8; 32], &adam, FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, None, + &ZstdEncode::level(20), ) .unwrap(); let header = Header::try_from(&encoded[..Header::LEN]).unwrap(); diff --git a/src/filepack_manifest.rs b/src/filepack_manifest.rs index f96727d..86d5990 100644 --- a/src/filepack_manifest.rs +++ b/src/filepack_manifest.rs @@ -35,8 +35,8 @@ use crate::error::CarbonadoError; use crate::filepack::{self, FilepackCborEntry, Packed}; use crate::utils::calc_padding_len; -/// FilepackManifest wire schema version (v2). -pub const FILEPACK_MANIFEST_VERSION: u32 = 2; +/// FilepackManifest wire schema version (v3: bao + parity + dict offsets in one bundle blob). +pub const FILEPACK_MANIFEST_VERSION: u32 = 3; /// Public directory archive format level (c14 = 0x0E). pub const FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC: u8 = 0x0E; @@ -86,6 +86,10 @@ pub struct SegmentRef { pub fec_parity_offset: u32, /// Length of this segment's FEC parity blob in the Adamantine payload bundle (0 when absent). pub fec_parity_len: u32, + /// Byte offset of this segment's RFC 8878 zstd dictionary in the Adamantine payload bundle (0 when absent). + pub dict_offset: u32, + /// Length of this segment's RFC 8878 zstd dictionary in the Adamantine payload bundle (0 when absent). + pub dict_len: u32, } /// A single file entry in a directory catalog. @@ -134,9 +138,10 @@ struct FilepackManifestWire { /// use carbonado::file::{encode_directory, DirectoryArchive}; /// use carbonado::filepack::{pack_directory, Packed}; /// use carbonado::filepack_manifest::{FilepackManifest, FilepackSegmentMap}; +/// use carbonado::ZstdEncode; /// /// # fn example(master: &[u8; 32], src: &std::path::Path, enc: &std::path::Path) -> Result<(), carbonado::error::CarbonadoError> { -/// let archive: DirectoryArchive = encode_directory(master, src, enc)?; +/// let archive: DirectoryArchive = encode_directory(master, src, enc, &ZstdEncode::level(20))?; /// let packed: Packed = pack_directory(src)?; /// // Load rkyv manifest from catalog (see tests/filepack_interop.rs), then: /// # let encoded_entries: Vec = vec![]; @@ -258,6 +263,25 @@ impl FilepackManifest { Ok(index) } + /// Deserialize without outboard FEC/bao geometry checks. + /// + /// Used for single-file inboard Adamantine trailers, where Bao and FEC live in the + /// Carbonado body and the trailer may carry only a zstd dictionary. + pub(crate) fn from_bytes_unvalidated( + bytes: &[u8], + catalog_bao_root: [u8; 32], + ) -> Result { + if bytes.len() > MAX_RKYV_PAYLOAD_LEN { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "rkyv payload exceeds {MAX_RKYV_PAYLOAD_LEN} bytes" + ))); + } + Self::check_archived_wire_limits(bytes)?; + let wire: FilepackManifestWire = rkyv::from_bytes::(bytes) + .map_err(|e| CarbonadoError::InvalidFilepackManifest(e.to_string()))?; + Ok(Self::from_wire(catalog_bao_root, wire)) + } + /// Pre-deserialize limits on archived layout (entry count, string/proof sizes). fn check_archived_wire_limits(bytes: &[u8]) -> Result<(), CarbonadoError> { let archived = rkyv::access::(bytes) @@ -268,6 +292,8 @@ impl FilepackManifest { ))); } let catalog_encrypted = archived.format_level & 1 != 0; + let directory_catalog = archived.format_level == FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC + || archived.format_level == FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED; for entry in archived.entries.iter() { if entry.rel_path.len() > MAX_REL_PATH_LEN { return Err(CarbonadoError::InvalidFilepackManifest(format!( @@ -286,14 +312,15 @@ impl FilepackManifest { "ots_proof exceeds {MAX_OTS_PROOF_LEN} bytes" ))); } - validate_segment_format_for_catalog(entry.segment_format, catalog_encrypted).map_err( - |e| match e { - CarbonadoError::SegmentFormatMismatch(msg) => { - CarbonadoError::InvalidFilepackManifest(msg) - } - other => other, - }, - )?; + if directory_catalog { + validate_segment_format_for_catalog(entry.segment_format, catalog_encrypted) + .map_err(|e| match e { + CarbonadoError::SegmentFormatMismatch(msg) => { + CarbonadoError::InvalidFilepackManifest(msg) + } + other => other, + })?; + } } Ok(()) } @@ -381,15 +408,15 @@ impl FilepackManifest { self.version ))); } - if self.format_level != FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC - && self.format_level != FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED - { + if self.format_level > 15 { return Err(CarbonadoError::InvalidFilepackManifest(format!( - "catalog format_level must be c14 or c15, got 0x{:02x}", + "format_level must be 0–15, got 0x{:02x}", self.format_level ))); } let catalog_encrypted = self.format_level & 1 != 0; + let directory_catalog = self.format_level == FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC + || self.format_level == FILEPACK_MANIFEST_FORMAT_LEVEL_ENCRYPTED; if let Some(proof) = &self.catalog_ots_proof && proof.len() > MAX_OTS_PROOF_LEN { @@ -405,14 +432,20 @@ impl FilepackManifest { let mut prev: Option<&str> = None; for entry in &self.entries { Self::validate_rel_path(&entry.rel_path)?; - validate_segment_format_for_catalog(entry.segment_format, catalog_encrypted).map_err( - |e| match e { - CarbonadoError::SegmentFormatMismatch(msg) => { - CarbonadoError::InvalidFilepackManifest(msg) - } - other => other, - }, - )?; + if directory_catalog { + validate_segment_format_for_catalog(entry.segment_format, catalog_encrypted) + .map_err(|e| match e { + CarbonadoError::SegmentFormatMismatch(msg) => { + CarbonadoError::InvalidFilepackManifest(msg) + } + other => other, + })?; + } else if entry.segment_format != self.format_level { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "single-file segment_format 0x{:02x} must match format_level 0x{:02x}", + entry.segment_format, self.format_level + ))); + } Self::validate_segments(&entry.segments)?; let seg_fmt = Format::from(entry.segment_format); for seg in &entry.segments { @@ -498,6 +531,23 @@ impl FilepackManifest { format!("{} chunk {} fec_parity", entry.rel_path, seg.chunk_index), )); } + + let dict_end = seg.dict_offset.checked_add(seg.dict_len).ok_or_else(|| { + CarbonadoError::InvalidFilepackManifest("dict offset overflow".into()) + })?; + if dict_end as usize > bundle_len { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "dict range for {} chunk {} exceeds bundle length {bundle_len}", + entry.rel_path, seg.chunk_index + ))); + } + if seg.dict_len > 0 { + ranges.push(( + seg.dict_offset, + dict_end, + format!("{} chunk {} dict", entry.rel_path, seg.chunk_index), + )); + } } } @@ -691,6 +741,30 @@ fn validate_segment_bundle_semantics( } } + if seg.dict_len > 0 { + let expected_dict_off = if seg.fec_parity_len > 0 { + seg.fec_parity_offset + .checked_add(seg.fec_parity_len) + .ok_or_else(|| { + CarbonadoError::InvalidFilepackManifest("fec_parity offset overflow".into()) + })? + } else { + seg.verification_outboard_offset + .checked_add(seg.verification_outboard_len) + .ok_or_else(|| { + CarbonadoError::InvalidFilepackManifest( + "verification_outboard offset overflow".into(), + ) + })? + }; + if seg.dict_offset != expected_dict_off { + return Err(CarbonadoError::InvalidFilepackManifest(format!( + "dict_offset for {rel_path} chunk {} must follow bao/parity contiguously", + seg.chunk_index + ))); + } + } + Ok(()) } @@ -709,6 +783,8 @@ mod tests { verification_outboard_len: ver_len, fec_parity_offset: ver_len, fec_parity_len: fec_len, + dict_offset: 0, + dict_len: 0, } } @@ -784,6 +860,8 @@ mod tests { verification_outboard_len: 0, fec_parity_offset: 0, fec_parity_len: 0, + dict_offset: 0, + dict_len: 0, }); let err = manifest.validate().unwrap_err(); assert!( @@ -828,20 +906,23 @@ mod tests { manifest.format_level = 16; let err = manifest.validate().unwrap_err(); assert!( - matches!(err, CarbonadoError::InvalidFilepackManifest(ref msg) if msg.contains("c14 or c15")), + matches!(err, CarbonadoError::InvalidFilepackManifest(ref msg) if msg.contains("0–15")), "got {err:?}" ); } #[test] - fn rejects_non_catalog_format_level() { + fn single_file_sidecar_allows_non_catalog_format_level() { let mut manifest = sample_manifest(); manifest.format_level = 6; - let err = manifest.validate().unwrap_err(); - assert!( - matches!(err, CarbonadoError::InvalidFilepackManifest(ref msg) if msg.contains("c14 or c15")), - "got {err:?}" - ); + manifest.entries[0].segment_format = 6; + for seg in &mut manifest.entries[0].segments { + seg.fec_parity_offset = 0; + seg.fec_parity_len = 0; + } + manifest + .validate() + .expect("one-entry sidecar may use any c0–c15 format_level"); } #[test] @@ -863,6 +944,8 @@ mod tests { verification_outboard_len: 8, fec_parity_offset: 0, fec_parity_len: 0, + dict_offset: 0, + dict_len: 0, }], ots_proof: None, }], @@ -895,6 +978,8 @@ mod tests { verification_outboard_len: 64, fec_parity_offset: 64, fec_parity_len: fec_len, + dict_offset: 0, + dict_len: 0, }, SegmentRef { segment_bao_root: [4u8; 32], @@ -904,6 +989,8 @@ mod tests { verification_outboard_len: 64, fec_parity_offset: 96, fec_parity_len: fec_len, + dict_offset: 0, + dict_len: 0, }, ], ots_proof: None, @@ -937,6 +1024,8 @@ mod tests { verification_outboard_len: ver_len, fec_parity_offset: fec_off, fec_parity_len: fec_len, + dict_offset: 0, + dict_len: 0, }); } let manifest = FilepackManifest { @@ -988,6 +1077,8 @@ mod tests { verification_outboard_len: 8, fec_parity_offset: 8, fec_parity_len: 16, + dict_offset: 0, + dict_len: 0, }], ots_proof: None, }], @@ -1013,11 +1104,17 @@ mod tests { fn decode_outboard_rejects_missing_verification_outboard() { use crate::decoding::decode_outboard; use crate::directory::format_policy::SEGMENT_FORMAT_PUBLIC_COMPRESSED; - use crate::encoding::encode_outboard; + use crate::encoding::encode_outboard_with_zstd; let payload = b"payload"; - let oenc = encode_outboard(&[0u8; 32], payload, SEGMENT_FORMAT_PUBLIC_COMPRESSED) - .expect("encode_outboard"); + let oenc = encode_outboard_with_zstd( + &[0u8; 32], + payload, + SEGMENT_FORMAT_PUBLIC_COMPRESSED, + None, + &crate::stream::ZstdEncode::level(20), + ) + .expect("encode_outboard"); let err = decode_outboard( &[0u8; 32], oenc.hash.as_bytes(), @@ -1038,11 +1135,17 @@ mod tests { fn encode_outboard_empty_verification_outboard_still_roundtrips() { use crate::decoding::decode_outboard; use crate::directory::format_policy::SEGMENT_FORMAT_PUBLIC_COMPRESSED; - use crate::encoding::encode_outboard; + use crate::encoding::encode_outboard_with_zstd; let payload = b"payload"; - let oenc = encode_outboard(&[0u8; 32], payload, SEGMENT_FORMAT_PUBLIC_COMPRESSED) - .expect("encode_outboard"); + let oenc = encode_outboard_with_zstd( + &[0u8; 32], + payload, + SEGMENT_FORMAT_PUBLIC_COMPRESSED, + None, + &crate::stream::ZstdEncode::level(20), + ) + .expect("encode_outboard"); let ver = oenc.verification_outboard.as_deref().expect("Some"); assert_eq!(ver.len(), 0, "small payloads may have zero-length outboard"); let decoded = decode_outboard( diff --git a/src/lib.rs b/src/lib.rs index 4828d95..83272aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ //! //! It combines a fully symmetric, hardware-accelerated cryptographic stack //! (AES-256-CTR + full HMAC-SHA512 EtM) with Bao streaming verifiability, -//! FEC (reed-solomon-erasure 4/8) forward error correction, optional Zstd (level 20) compression, and +//! FEC (reed-solomon-erasure 4/8) forward error correction, optional Zstd compression (level is encoder input), and //! SLH-DSA post-quantum signatures delivered exclusively as **sidecars**. //! //! ## Security Model & Production Guidance @@ -30,14 +30,20 @@ //! Using the low-level API (recommended for documentation examples): //! //! ```rust -//! use carbonado::{encode, decode}; +//! use carbonado::{decode, encode_with_zstd}; //! use getrandom::getrandom; //! //! let mut master_key = [0u8; 32]; //! getrandom(&mut master_key).unwrap(); //! //! let data = b"important archival payload"; -//! let encoded = encode(&master_key, data, 15).unwrap(); +//! let encoded = encode_with_zstd( +//! &master_key, +//! data, +//! 15, +//! None, +//! &carbonado::ZstdEncode::level(20), +//! ).unwrap(); //! //! let recovered = decode( //! &master_key, @@ -112,9 +118,10 @@ pub mod paths; pub mod stream; pub use encoding::encode; -pub use encoding::encode_with_nonce; - pub use encoding::encode_outboard; +pub use encoding::encode_outboard_with_zstd; +pub use encoding::encode_with_nonce; +pub use encoding::encode_with_zstd; pub use decoding::decode; @@ -135,9 +142,10 @@ pub use paths::{ArchiveLayout, detect_archive_layout}; #[cfg(feature = "async")] pub use stream::stream_decode_async; pub use stream::{ - DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, decode_shards_stream, - encode_shard_stream, stream_decode, stream_decode_buffer, stream_decode_outboard, - stream_decode_outboard_buffer, stream_encode_buffer, stream_encode_buffer_with_nonce, + DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, ZstdEncode, + decode_shards_stream, encode_shard_stream, encode_shard_stream_with_zstd, stream_decode, + stream_decode_buffer, stream_decode_outboard, stream_decode_outboard_buffer, + stream_decode_outboard_buffer_with_dict, stream_encode_buffer, stream_encode_buffer_with_nonce, stream_encode_outboard_buffer, verify_slice_inboard_seekable, verify_slice_outboard, }; @@ -157,7 +165,8 @@ pub use adamantine::{ }; pub use adamantine_payload::{ MAX_ADAMANTINE_PAYLOAD_LEN, MAX_BAO_BUNDLE_LEN, build_adamantine_payload, - fec_slice_from_bundle, split_adamantine_payload, verification_slice_from_bundle, + dict_slice_from_bundle, fec_slice_from_bundle, split_adamantine_payload, + verification_slice_from_bundle, }; pub use directory::SegmentFormatPolicy; diff --git a/src/paths.rs b/src/paths.rs index 62bb51f..d9a6fc9 100644 --- a/src/paths.rs +++ b/src/paths.rs @@ -2,7 +2,7 @@ //! //! Used by the `carbonado` CLI and [`crate::file::decode_directory`]. Directory archives //! use inboard `.adam.c14`/`.adam.c15` catalogs and decimal segment suffixes `c12`–`c15`; -//! single-file outboard uses hex `c{fmt:02x}` plus optional `.out`/`.par` sidecars. +//! single-file outboard uses `{hash}.c{fmt:02x}` plus `{hash}.adam.c{fmt:02x}` (Adamantine sidecar). //! //! Decimal suffix parsing tries longest match first (`15` down to `0`) so e.g. `.c14` resolves //! to format 14, not format 1 via a `.c1` prefix. @@ -11,6 +11,7 @@ use std::fs; use std::io::Read; use std::path::{Path, PathBuf}; +use crate::adamantine::ADAMANTINE_MAGIC; use crate::{constants::MAGICNO, error::CarbonadoError}; /// Detected on-disk archive layout. @@ -24,22 +25,14 @@ pub enum ArchiveLayout { OutboardBare { main: PathBuf }, } -/// Detect archive layout per plan §2.6: -/// 1. Outboard adam first when `.out`/`.par` siblings exist -/// 2. Inboard fallback when input starts with `CARBONADO20\n` -/// 3. `MissingCatalog` when segment mains/sidecars exist but `{catalog}.adam.cXX` missing +/// Detect archive layout: +/// 1. Same-stem `{hash}.cXX` + `{hash}.adam.cXX` (sidecar starts with `ADAMANTINE10\n`) is +/// single-file outboard, not a directory catalog. +/// 2. Headered `{hash}.adam.cXX` starting with `CARBONADO20\n` is inboard (directory catalog +/// when decimal `.adam.c14`/`.adam.c15` without a same-stem bare `.cXX`). +/// 3. `MissingCatalog` when segment mains exist but no inboard catalog. pub fn detect_archive_layout(input: &Path) -> Result { if input.is_file() { - if is_adam_catalog(input) { - if !starts_with_magic(input)? { - return Err(CarbonadoError::DirectoryLayoutMismatch( - "directory catalog must be inboard headered .adam.c14 or .adam.c15".into(), - )); - } - return Ok(ArchiveLayout::InboardAdam { - catalog: input.to_path_buf(), - }); - } return detect_single_file(input); } @@ -54,20 +47,39 @@ pub fn detect_archive_layout(input: &Path) -> Result Result { - if starts_with_magic(path)? { - return Ok(ArchiveLayout::InboardHeadered { - path: path.to_path_buf(), - }); + if let Some(main) = companion_main_for_adam_sidecar(path) + && main.is_file() + && starts_with_adamantine(path)? + { + return Ok(ArchiveLayout::OutboardBare { main }); } - let out = sidecar_sibling_path(path, "out"); - let par = sidecar_sibling_path(path, "par"); - if out.exists() || par.exists() { + + if let Some(adam) = adamantine_sidecar_for_main(path) + && adam.is_file() + && starts_with_adamantine(&adam)? + { return Ok(ArchiveLayout::OutboardBare { main: path.to_path_buf(), }); } - // Bare main without sidecar siblings: still outboard-capable when CLI supplies - // explicit `--bao-outboard` / `--fec-parity` / `--hash` overrides. + + if starts_with_magic(path)? { + if is_adam_catalog(path) { + if let Some(main) = companion_main_for_adam_sidecar(path) + && main.is_file() + && !starts_with_magic(&main)? + { + return Ok(ArchiveLayout::OutboardBare { main }); + } + return Ok(ArchiveLayout::InboardAdam { + catalog: path.to_path_buf(), + }); + } + return Ok(ArchiveLayout::InboardHeadered { + path: path.to_path_buf(), + }); + } + if guess_format_from_filename(path).is_some() { return Ok(ArchiveLayout::OutboardBare { main: path.to_path_buf(), @@ -83,6 +95,10 @@ fn detect_directory_layout( dir: &Path, hint: Option<&Path>, ) -> Result { + if let Some(main) = find_single_file_outboard_pair(dir)? { + return Ok(ArchiveLayout::OutboardBare { main }); + } + let mut adam_catalogs: Vec = Vec::new(); let mut segment_mains: Vec = Vec::new(); let mut has_orphan_sidecars = false; @@ -243,6 +259,94 @@ fn starts_with_magic(path: &Path) -> Result { Ok(n >= MAGICNO.len() && &buf[..MAGICNO.len()] == MAGICNO) } +fn starts_with_adamantine(path: &Path) -> Result { + let mut f = fs::File::open(path).map_err(CarbonadoError::StdIoError)?; + let mut buf = [0u8; ADAMANTINE_MAGIC.len()]; + let n = f.read(&mut buf).map_err(CarbonadoError::StdIoError)?; + Ok(n >= ADAMANTINE_MAGIC.len() && &buf[..ADAMANTINE_MAGIC.len()] == ADAMANTINE_MAGIC) +} + +fn strip_hex_c_suffix(name: &str) -> Option<(&str, u8)> { + let (stem, ext) = name.rsplit_once('.')?; + if ext.len() == 3 + && ext.starts_with('c') + && ext[1..].chars().all(|c| c.is_ascii_hexdigit()) + && !stem.ends_with(".adam") + { + let fmt = u8::from_str_radix(&ext[1..], 16).ok()?; + return Some((stem, fmt)); + } + None +} + +fn strip_hex_adam_suffix(name: &str) -> Option<(&str, u8)> { + let (rest, ext) = name.rsplit_once('.')?; + if ext.len() == 3 && ext.starts_with('c') && ext[1..].chars().all(|c| c.is_ascii_hexdigit()) { + let stem = rest.strip_suffix(".adam")?; + let fmt = u8::from_str_radix(&ext[1..], 16).ok()?; + return Some((stem, fmt)); + } + None +} + +/// `{hash}.cXX` → `{hash}.adam.cXX` (hex `c{fmt:02x}` or decimal `c{n}`). +pub fn adamantine_sidecar_for_main(main: &Path) -> Option { + let name = main.file_name()?.to_str()?; + let parent = main.parent().unwrap_or_else(|| Path::new("")); + if let Some((stem, fmt)) = strip_hex_c_suffix(name) { + return Some(parent.join(format!("{stem}.adam.c{fmt:02x}"))); + } + if let Some((stem, fmt)) = strip_decimal_suffix(name) { + if name.contains(".adam.") { + return None; + } + return Some(parent.join(format!("{stem}.adam.c{fmt}"))); + } + None +} + +/// `{hash}.adam.cXX` → `{hash}.cXX`. +pub fn companion_main_for_adam_sidecar(adam: &Path) -> Option { + let name = adam.file_name()?.to_str()?; + let parent = adam.parent().unwrap_or_else(|| Path::new("")); + if let Some((stem, fmt)) = strip_hex_adam_suffix(name) { + return Some(parent.join(format!("{stem}.c{fmt:02x}"))); + } + if let Some((stem, fmt)) = strip_decimal_adam_suffix(name) { + return Some(parent.join(format!("{stem}.c{fmt}"))); + } + None +} + +fn find_single_file_outboard_pair(dir: &Path) -> Result, CarbonadoError> { + let mut pair: Option = None; + for entry in fs::read_dir(dir).map_err(CarbonadoError::StdIoError)? { + let path = entry.map_err(CarbonadoError::StdIoError)?.path(); + if !path.is_file() { + continue; + } + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if strip_hex_c_suffix(name).is_none() && strip_decimal_suffix(name).is_none() { + continue; + } + if name.contains(".adam.") { + continue; + } + if let Some(adam) = adamantine_sidecar_for_main(&path) + && adam.is_file() + && starts_with_adamantine(&adam)? + { + if pair.is_some() { + return Ok(None); + } + pair = Some(path); + } + } + Ok(pair) +} + /// Whether `path` names an Adamantine directory catalog (`.adam.c14` or `.adam.c15`). pub fn is_adam_catalog(path: &Path) -> bool { path.file_name() @@ -393,6 +497,22 @@ mod tests { ); } + #[test] + fn detect_single_file_outboard_pair_not_directory_catalog() { + let dir = tempdir("outboard_pair"); + let hash = "aa".repeat(32); + let main = dir.join(format!("{hash}.c0e")); + let adam = dir.join(format!("{hash}.adam.c0e")); + fs::write(&main, b"bare-main").expect("main"); + let mut sidecar = crate::adamantine::ADAMANTINE_MAGIC.to_vec(); + sidecar.extend_from_slice(&[0u8; 16]); + fs::write(&adam, &sidecar).expect("adam"); + let layout = detect_archive_layout(&main).expect("detect main"); + assert_eq!(layout, ArchiveLayout::OutboardBare { main: main.clone() }); + let from_dir = detect_archive_layout(&dir).expect("detect dir"); + assert_eq!(from_dir, ArchiveLayout::OutboardBare { main }); + } + #[test] fn detect_missing_catalog_strict() { let dir = tempdir("orphan"); diff --git a/src/stream/bao.rs b/src/stream/bao.rs index a9eea80..3f7b2e8 100644 --- a/src/stream/bao.rs +++ b/src/stream/bao.rs @@ -372,8 +372,14 @@ mod tests { for &format in &[6u8, 12, 14, 15] { for logical_len in [0usize, 1, 4095, 4096, 65_536] { let input: Vec = (0..logical_len).map(|i| (i % 251) as u8).collect(); - let (encoded, hash, _) = - stream_encode_buffer(&master, &input, format).expect("encode"); + let (encoded, hash, _) = crate::stream::encode::stream_encode_buffer_with_zstd( + &master, + &input, + format, + None, + &crate::stream::ZstdEncode::level(20), + ) + .expect("encode"); assert_oracle_parity(&encoded, hash.as_bytes(), format); } } diff --git a/src/stream/compress.rs b/src/stream/compress.rs index a2dc3ab..b102448 100644 --- a/src/stream/compress.rs +++ b/src/stream/compress.rs @@ -1,11 +1,102 @@ -//! Zstd level-20 streaming compression over [`Read`] / [`Write`]. +//! Zstd streaming compression over [`Read`] / [`Write`]. +//! +//! Compression **level is encoder input**. There is no silent library default. -use std::io::{Read, Write}; +use std::io::{BufReader, Read, Write}; use crate::{ - constants::ZSTD_LEVEL, error::CarbonadoError, filepack_manifest::MAX_SEGMENT_MAIN_LEN, + constants::ZSTD_DICTIONARY_MAGIC, error::CarbonadoError, + filepack_manifest::MAX_SEGMENT_MAIN_LEN, }; +/// Caller-supplied zstd parameters. `level` is required when the Compression bit is set. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ZstdEncode { + /// Compression level. `None` with the Compression bit set is [`CarbonadoError::MissingZstdLevel`]. + pub level: Option, + /// RFC 8878 dictionary bytes (optional). Stored in the Adamantine dict section when encoding files. + pub dict: Option>, +} + +impl ZstdEncode { + /// Explicit level, no dictionary. + pub fn level(level: i32) -> Self { + Self { + level: Some(level), + dict: None, + } + } + + /// Explicit level and dictionary. + pub fn with_dict(level: i32, dict: Vec) -> Self { + Self { + level: Some(level), + dict: Some(dict), + } + } +} + +/// Resolve level when the Compression bit is set. Missing level is an error even if `dict` is present. +pub fn require_zstd_level(zstd: &ZstdEncode) -> Result { + zstd.level.ok_or(CarbonadoError::MissingZstdLevel) +} + +/// RFC 8878 dictionary ID (little-endian u32 after magic), if `dict` is a trained zstd dict. +pub fn zstd_dictionary_id(dict: &[u8]) -> Option { + if dict.len() < 8 || dict[0..4] != ZSTD_DICTIONARY_MAGIC { + return None; + } + Some(u32::from_le_bytes(dict[4..8].try_into().ok()?)) +} + +/// Dictionary_ID from a zstd frame header, if present. +pub fn zstd_frame_dictionary_id(frame: &[u8]) -> Result, CarbonadoError> { + if frame.len() < 5 { + return Err(CarbonadoError::ZstdError( + "truncated zstd frame header".into(), + )); + } + if frame[0..4] != crate::constants::ZSTD_MAGIC { + return Ok(None); + } + let descriptor = frame[4]; + if (descriptor & 0x08) != 0 { + return Err(CarbonadoError::ZstdError("zstd reserved bit set".into())); + } + let dictionary_id_flag = descriptor & 0x03; + let single_segment = (descriptor & 0x20) != 0; + let need_win = if single_segment { 0 } else { 1 }; + let did_sz = match dictionary_id_flag { + 0 => 0, + 1 => 1, + 2 => 2, + 3 => 4, + _ => 0, + }; + let header_len = 5 + need_win + did_sz; + if frame.len() < header_len { + return Err(CarbonadoError::ZstdError( + "truncated zstd frame header".into(), + )); + } + let did_off = 5 + need_win; + let id = match did_sz { + 0 => None, + 1 => Some(u32::from(frame[did_off])), + 2 => Some(u32::from(u16::from_le_bytes([ + frame[did_off], + frame[did_off + 1], + ]))), + 4 => Some(u32::from_le_bytes( + frame[did_off..did_off + 4] + .try_into() + .map_err(|_| CarbonadoError::ZstdError("truncated dictionary id".into()))?, + )), + _ => None, + }; + Ok(id) +} + struct CountWriter { inner: W, count: u64, @@ -33,48 +124,142 @@ impl Write for CountWriter { } } -/// Stream-compress `input` into `output` at [`crate::constants::ZSTD_LEVEL`] (20). +/// Stream-compress `input` into `output` at the caller-supplied `level`. /// Returns compressed bytes written. -/// -/// Frame flags match Lean `Carbonado.Compress` where they can: checksum off, no dictionary. -/// Streaming `copy_encode` leaves content size unknown (descriptor differs from Lean AOT -/// one-shot `ZSTD_compress` frames). That is an honest frame-shape difference, not a bug. -pub fn stream_compress(mut input: R, output: W) -> Result { +pub fn stream_compress( + input: R, + output: W, + level: i32, +) -> Result { + stream_compress_with_dict(input, output, level, None) +} + +/// Stream-compress with an optional RFC 8878 dictionary. +pub fn stream_compress_with_dict( + mut input: R, + output: W, + level: i32, + dict: Option<&[u8]>, +) -> Result { let mut counter = CountWriter { inner: output, count: 0, max: None, }; - zstd::stream::copy_encode(&mut input, &mut counter, ZSTD_LEVEL) - .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + if let Some(dict) = dict.filter(|d| !d.is_empty()) { + let mut encoder = zstd::stream::Encoder::with_dictionary(&mut counter, level, dict) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + encoder + .include_checksum(false) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + std::io::copy(&mut input, &mut encoder).map_err(CarbonadoError::StdIoError)?; + encoder + .finish() + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + } else { + zstd::stream::copy_encode(&mut input, &mut counter, level) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + } Ok(counter.count) } /// Stream-decompress `input` into `output`. Returns decompressed bytes written. -pub fn stream_decompress( - mut input: R, +pub fn stream_decompress(input: R, output: W) -> Result { + stream_decompress_with_dict(input, output, None) +} + +/// Stream-decompress, using `dict` when the frame names a Dictionary_ID. +pub fn stream_decompress_with_dict( + input: R, output: W, + dict: Option<&[u8]>, ) -> Result { let mut counter = CountWriter { inner: output, count: 0, max: Some(MAX_SEGMENT_MAIN_LEN), }; - zstd::stream::copy_decode(&mut input, &mut counter) - .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + let mut prefixed = PrefixRead { + prefix: Vec::new(), + inner: input, + pos: 0, + }; + let mut hdr = [0u8; 16]; + let n = prefixed + .fill_prefix(&mut hdr) + .map_err(CarbonadoError::StdIoError)?; + let frame_id = zstd_frame_dictionary_id(&hdr[..n])?; + if let Some(id) = frame_id { + let dict_bytes = dict + .filter(|d| !d.is_empty()) + .ok_or(CarbonadoError::MissingZstdDictionary { dictionary_id: id })?; + let mut decoder = + zstd::stream::Decoder::with_dictionary(BufReader::new(&mut prefixed), dict_bytes) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + std::io::copy(&mut decoder, &mut counter).map_err(CarbonadoError::StdIoError)?; + } else { + zstd::stream::copy_decode(&mut prefixed, &mut counter) + .map_err(|e| CarbonadoError::ZstdError(e.to_string()))?; + } Ok(counter.count) } -/// Buffer convenience: compress `input` via the streaming helper. -pub fn compress_buffer(input: &[u8]) -> Result, CarbonadoError> { +/// Buffer convenience: compress `input` at `level` with no dictionary. +pub fn compress_buffer(input: &[u8], level: i32) -> Result, CarbonadoError> { + let mut out = Vec::new(); + stream_compress(input, &mut out, level)?; + Ok(out) +} + +/// Buffer convenience: compress `input` at `level` with an RFC 8878 dictionary. +pub fn compress_buffer_with_dict( + input: &[u8], + level: i32, + dict: &[u8], +) -> Result, CarbonadoError> { let mut out = Vec::new(); - stream_compress(input, &mut out)?; + stream_compress_with_dict(input, &mut out, level, Some(dict))?; Ok(out) } -/// Buffer convenience: decompress `input` via the streaming helper. +/// Buffer convenience: decompress `input`. pub fn decompress_buffer(input: &[u8]) -> Result, CarbonadoError> { + decompress_buffer_with_dict(input, None) +} + +/// Buffer convenience: decompress `input`, with optional dictionary. +pub fn decompress_buffer_with_dict( + input: &[u8], + dict: Option<&[u8]>, +) -> Result, CarbonadoError> { let mut out = Vec::new(); - stream_decompress(input, &mut out)?; + stream_decompress_with_dict(input, &mut out, dict)?; Ok(out) } + +/// Replay a short prefix then the rest of `inner` (for frame-header peek). +struct PrefixRead { + prefix: Vec, + inner: R, + pos: usize, +} + +impl PrefixRead { + fn fill_prefix(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.prefix.extend_from_slice(&buf[..n]); + Ok(n) + } +} + +impl Read for PrefixRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.pos < self.prefix.len() { + let n = (self.prefix.len() - self.pos).min(buf.len()); + buf[..n].copy_from_slice(&self.prefix[self.pos..self.pos + n]); + self.pos += n; + return Ok(n); + } + self.inner.read(buf) + } +} diff --git a/src/stream/decode.rs b/src/stream/decode.rs index 7a109c3..3347192 100644 --- a/src/stream/decode.rs +++ b/src/stream/decode.rs @@ -47,9 +47,35 @@ pub fn stream_decode_outboard_buffer( padding: u32, format: u8, explicit_nonce: Option<[u8; 16]>, +) -> Result, CarbonadoError> { + stream_decode_outboard_buffer_with_dict( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + explicit_nonce, + None, + ) +} + +/// Outboard buffer decode with an optional RFC 8878 dictionary from the Adamantine bundle. +#[allow(clippy::too_many_arguments)] +pub fn stream_decode_outboard_buffer_with_dict( + master_key: &[u8], + hash: &[u8], + main: &[u8], + verification_outboard: Option<&[u8]>, + fec_parity: Option<&[u8]>, + padding: u32, + format: u8, + explicit_nonce: Option<[u8; 16]>, + dict: Option<&[u8]>, ) -> Result, CarbonadoError> { let mut out = Vec::new(); - stream_decode_outboard( + stream_decode_outboard_with_dict( master_key, hash, Cursor::new(main), @@ -59,6 +85,7 @@ pub fn stream_decode_outboard_buffer( format, explicit_nonce, &mut out, + dict, )?; Ok(out) } @@ -287,12 +314,12 @@ fn stream_decode_post_preprocess_seek( let mut decrypted = SeekableSpool::new()?; stream_decrypt_seek(master_key, input, &mut decrypted, ct_len)?; decrypted.rewind()?; - crate::stream::compress::stream_decompress(decrypted, output) + crate::stream::compress::stream_decompress_with_dict(decrypted, output, None) } else { stream_decrypt_seek(master_key, input, output, ct_len) } } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(input, output) + crate::stream::compress::stream_decompress_with_dict(input, output, None) } else if let Some(len) = body_len { let mut limited = input.take(len); copy(&mut limited, output).map_err(CarbonadoError::StdIoError) @@ -315,6 +342,34 @@ pub fn stream_decode_outboard( format: u8, explicit_nonce: Option<[u8; 16]>, output: &mut W, +) -> Result { + stream_decode_outboard_with_dict( + master_key, + hash, + main, + verification_outboard, + fec_parity, + padding, + format, + explicit_nonce, + output, + None, + ) +} + +/// Outboard decode with an optional RFC 8878 dictionary from the Adamantine bundle. +#[allow(clippy::too_many_arguments)] +pub fn stream_decode_outboard_with_dict( + master_key: &[u8], + hash: &[u8], + main: M, + verification_outboard: Option, + fec_parity: Option

, + padding: u32, + format: u8, + explicit_nonce: Option<[u8; 16]>, + output: &mut W, + dict: Option<&[u8]>, ) -> Result { stream_decode_outboard_s4( master_key, @@ -326,6 +381,7 @@ pub fn stream_decode_outboard( format, explicit_nonce, output, + dict, ) } @@ -341,6 +397,7 @@ fn stream_decode_outboard_s4( format: u8, explicit_nonce: Option<[u8; 16]>, output: &mut W, + dict: Option<&[u8]>, ) -> Result { let fmt = Format::from(format); let mut after_bao_spool = SeekableSpool::new()?; @@ -401,7 +458,7 @@ fn stream_decode_outboard_s4( Some(ct_len), )?; decrypted.rewind()?; - crate::stream::compress::stream_decompress(decrypted, output) + crate::stream::compress::stream_decompress_with_dict(decrypted, output, dict) } else { stream_decrypt_with_nonce_seek( master_key, @@ -422,13 +479,13 @@ fn stream_decode_outboard_s4( Some(ct_len), )?; decrypted.rewind()?; - crate::stream::compress::stream_decompress(decrypted, output) + crate::stream::compress::stream_decompress_with_dict(decrypted, output, dict) } else { stream_decrypt_seek(master_key, &mut after_fec_spool, output, Some(ct_len)) } } } else if fmt.contains(Format::Compression) { - crate::stream::compress::stream_decompress(after_fec_spool, output) + crate::stream::compress::stream_decompress_with_dict(after_fec_spool, output, dict) } else { copy(&mut after_fec_spool, output).map_err(CarbonadoError::StdIoError) } @@ -461,7 +518,41 @@ pub fn stream_decrypt_header_path( Some(ct_len), )?; decrypted.rewind()?; - crate::stream::compress::stream_decompress(decrypted, output) + crate::stream::compress::stream_decompress_with_dict(decrypted, output, None) + } else { + stream_decrypt_with_nonce_seek(master_key, nonce, input, output, Some(ct_len)) + } +} + +/// Header-path decrypt then optional zstd decompress with dictionary. +pub fn stream_decrypt_header_path_with_dict( + master_key: &[u8], + nonce: [u8; 16], + mut input: R, + format: u8, + output: &mut W, + dict: Option<&[u8]>, +) -> Result { + let fmt = Format::from(format); + let ct_len = input + .seek(SeekFrom::End(0)) + .map_err(CarbonadoError::StdIoError)? + .saturating_sub(64); + input + .seek(SeekFrom::Start(0)) + .map_err(CarbonadoError::StdIoError)?; + + if fmt.contains(Format::Compression) { + let mut decrypted = SeekableSpool::new()?; + stream_decrypt_with_nonce_seek( + master_key, + nonce, + &mut input, + &mut decrypted, + Some(ct_len), + )?; + decrypted.rewind()?; + crate::stream::compress::stream_decompress_with_dict(decrypted, output, dict) } else { stream_decrypt_with_nonce_seek(master_key, nonce, input, output, Some(ct_len)) } diff --git a/src/stream/encode.rs b/src/stream/encode.rs index de1fd60..41290a9 100644 --- a/src/stream/encode.rs +++ b/src/stream/encode.rs @@ -5,14 +5,16 @@ use std::io::{Read, Seek, SeekFrom, Write}; use bao::Hash; use crate::{ - constants::{FEC_M, Format, SLICE_LEN}, + constants::{FEC_K, FEC_M, Format, SLICE_LEN}, error::CarbonadoError, stream::{ - compress::stream_compress, + compress::{ZstdEncode, require_zstd_level, stream_compress_with_dict}, crypto_stream::{ stream_encrypt, stream_encrypt_embedded_with_nonce, stream_encrypt_with_nonce_seek, }, - fec::{FecStripeReadAt, feed_inboard_fec_stripe, write_inboard_stripe}, + fec::{ + FecStripesReadAt, feed_inboard_fec_stripes, write_inboard_stripe, write_outboard_parity, + }, spool::SeekableSpool, }, structs::{EncodeInfo, OutboardEncoded}, @@ -21,7 +23,7 @@ use crate::{ use crate::stream::bao::stream_verification_outboard; use crate::stream::{ bao::{verification_inboard_buffer, verification_outboard_buffer}, - compress::compress_buffer, + compress::{compress_buffer, compress_buffer_with_dict}, crypto_stream::stream_encrypt_with_nonce, fec::{encode_inboard_buffer, encode_outboard_parity_buffer}, }; @@ -57,6 +59,7 @@ pub struct PreprocessStats { /// all-zero). When `None`, draw a CSPRNG nonce (production). /// Prefer CSPRNG for live archives; fixed nonces are for tests/determinism only — see /// AGENTS §2.1.4 (nonce uniqueness; reuse under the same master is catastrophic). +#[allow(clippy::too_many_arguments)] pub fn stream_preprocess( master_key: &[u8], format: Format, @@ -65,17 +68,19 @@ pub fn stream_preprocess( payload_nonce: &mut [u8; 16], header_path_encrypt: bool, fixed_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result { body_sink .seek(std::io::SeekFrom::Start(0)) .map_err(CarbonadoError::StdIoError)?; let mut input_len = 0u64; if format.contains(Format::Compression) { + let level = require_zstd_level(zstd)?; let mut counter = CountingReader { inner: &mut input, count: 0, }; - stream_compress(&mut counter, &mut *body_sink)?; + stream_compress_with_dict(&mut counter, &mut *body_sink, level, zstd.dict.as_deref())?; input_len = counter.count; body_sink.rewind().map_err(CarbonadoError::StdIoError)?; } else { @@ -149,6 +154,7 @@ pub fn stream_preprocess( /// [`stream_preprocess`] for [`SeekableSpool`] sinks — encrypt replace uses /// [`SeekableSpool::overwrite_from`] so file size matches ciphertext (no stale tail bytes). +#[allow(clippy::too_many_arguments)] pub(crate) fn stream_preprocess_spool( master_key: &[u8], format: Format, @@ -157,15 +163,17 @@ pub(crate) fn stream_preprocess_spool( payload_nonce: &mut [u8; 16], header_path_encrypt: bool, fixed_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result { body_sink.rewind()?; let mut input_len = 0u64; if format.contains(Format::Compression) { + let level = require_zstd_level(zstd)?; let mut counter = CountingReader { inner: &mut input, count: 0, }; - stream_compress(&mut counter, &mut *body_sink)?; + stream_compress_with_dict(&mut counter, &mut *body_sink, level, zstd.dict.as_deref())?; input_len = counter.count; body_sink.rewind()?; } else { @@ -300,7 +308,18 @@ pub fn stream_encode_buffer( input: &[u8], format: u8, ) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { - stream_encode_buffer_with_nonce(master_key, input, format, None) + stream_encode_buffer_with_zstd(master_key, input, format, None, &ZstdEncode::default()) +} + +/// Inboard buffer encode with explicit zstd parameters (required when Compression is set). +pub fn stream_encode_buffer_with_zstd( + master_key: &[u8], + input: &[u8], + format: u8, + explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, +) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { + stream_encode_buffer_with_nonce(master_key, input, format, explicit_nonce, zstd) } /// Inboard body encode with optional fixed nonce for encrypted formats. @@ -320,6 +339,7 @@ pub fn stream_encode_buffer_with_nonce( input: &[u8], format: u8, explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result<(Vec, Hash, EncodeInfo), CarbonadoError> { let fmt = Format::from(format); let input_len = input.len() as u32; @@ -328,7 +348,12 @@ pub fn stream_encode_buffer_with_nonce( let mut bytes_encrypted = 0u32; if fmt.contains(Format::Compression) { - body = compress_buffer(input)?; + let level = require_zstd_level(zstd)?; + body = if let Some(dict) = zstd.dict.as_deref().filter(|d| !d.is_empty()) { + compress_buffer_with_dict(input, level, dict)? + } else { + compress_buffer(input, level)? + }; bytes_compressed = body.len() as u32; } if fmt.contains(Format::Encryption) { @@ -411,13 +436,19 @@ pub fn stream_encode_outboard_buffer( input: &[u8], format: u8, explicit_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result { let fmt = Format::from(format); let input_len = input.len() as u32; let mut bytes_compressed = 0u32; let compressed = if fmt.contains(Format::Compression) { - let c = compress_buffer(input)?; + let level = require_zstd_level(zstd)?; + let c = if let Some(dict) = zstd.dict.as_deref().filter(|d| !d.is_empty()) { + compress_buffer_with_dict(input, level, dict)? + } else { + compress_buffer(input, level)? + }; bytes_compressed = c.len() as u32; c } else { @@ -453,8 +484,11 @@ pub fn stream_encode_outboard_buffer( let (post_fec_or_bare, padding_len, chunk_len, bytes_ecc, fec_parity, vslice, cslice) = if fmt.contains(Format::Fec) { let (pl, cl, parity) = encode_outboard_parity_buffer(&post_comp_or_enc)?; - let would_bytes = (FEC_M as u32) * cl; - let vs = would_bytes / SLICE_LEN; + let vs = if cl == 0 { + 0 + } else { + (parity.len() as u32 / ((FEC_M - FEC_K) as u32 * SLICE_LEN)) * FEC_M as u32 + }; if !vs.is_multiple_of(8) { return Err(CarbonadoError::InvalidVerifiableSliceCount(vs)); } @@ -519,6 +553,7 @@ pub fn stream_encode_outboard( parity_out: Option<&mut P>, payload_nonce: &mut [u8; 16], header_path_encrypt: bool, + zstd: &ZstdEncode, ) -> Result<(Hash, EncodeInfo), CarbonadoError> { stream_encode_outboard_s4( master_key, @@ -529,6 +564,7 @@ pub fn stream_encode_outboard( parity_out, payload_nonce, header_path_encrypt, + zstd, ) } @@ -543,6 +579,7 @@ fn stream_encode_outboard_s4( mut parity_out: Option<&mut P>, payload_nonce: &mut [u8; 16], header_path_encrypt: bool, + zstd: &ZstdEncode, ) -> Result<(Hash, EncodeInfo), CarbonadoError> { let fmt = Format::from(format); // Fail-closed: required sidecar writers when format bits demand them. @@ -562,6 +599,7 @@ fn stream_encode_outboard_s4( payload_nonce, header_path_encrypt, None, + zstd, )?; let bare_len = stats.bare_len; main_out.rewind().map_err(CarbonadoError::StdIoError)?; @@ -570,12 +608,14 @@ fn stream_encode_outboard_s4( if bare_len == 0 { (0, 0, 0, 0) } else { - let (stripe, pl, cl) = feed_inboard_fec_stripe(bare_len as usize, &mut *main_out)?; - // parity_out is Some after fail-closed check above. + let (stripes, pl, cl) = feed_inboard_fec_stripes(bare_len as usize, &mut *main_out)?; let par = parity_out .as_mut() .ok_or(CarbonadoError::MissingFecParity)?; - let par_len = crate::stream::fec::write_outboard_parity(&stripe, par)?; + let mut par_len = 0u64; + for stripe in &stripes { + par_len += write_outboard_parity(stripe, par)?; + } main_out.rewind().map_err(CarbonadoError::StdIoError)?; (pl, cl, par_len as u32, par_len as u32) } @@ -593,7 +633,11 @@ fn stream_encode_outboard_s4( }; let verifiable_slice_count = if fmt.contains(Format::Fec) { - ((FEC_M as u32) * chunk_len) / SLICE_LEN + if chunk_len == 0 { + 0 + } else { + (bytes_ecc / (FEC_M as u32 - FEC_K as u32).max(1)) * FEC_M as u32 / SLICE_LEN + } } else { 0 }; @@ -642,8 +686,8 @@ pub fn stream_encode_inboard_body_from_bytes( /// [`PreprocessStats::bare_len`] for accurate [`EncodeInfo`] bookkeeping. /// /// FEC (`Format::Fec`) feeds `data` incrementally via [`FecInboardEncoder`] — peak encode -/// memory is O(stripe), not O(bare_len). Verification reads the FEC stripe via -/// [`FecStripeReadAt`] without flattening to a staging `Vec` (S3). +/// memory is O(stripe), not O(bare_len). Verification reads stripes via +/// [`FecStripesReadAt`] without flattening to a staging `Vec` (S3). pub fn stream_encode_inboard_body( mut data: D, preprocess: PreprocessStats, @@ -665,14 +709,12 @@ pub fn stream_encode_inboard_body( (0, 0, 0, hash, bytes_verifiable) } else { data.rewind().map_err(CarbonadoError::StdIoError)?; - // S2: `Read::take(content_len)` + `feed_inboard_fec_stripe` — regression: - // `streaming_limits::stream_encode_inboard_body_fec_bounded_read_contract` - let (stripe, padding_len, chunk_len) = - feed_inboard_fec_stripe(content_len as usize, &mut data)?; - let bytes_ecc = (FEC_M as u32) * chunk_len; + let (stripes, padding_len, chunk_len) = + feed_inboard_fec_stripes(content_len as usize, &mut data)?; + let bytes_ecc = stripes.len() as u32 * FEC_M as u32 * chunk_len; if fmt.contains(Format::Verification) { - let stripe_view = FecStripeReadAt::new(&stripe); + let stripe_view = FecStripesReadAt::new(&stripes); let fec_len = stripe_view.len(); let (h, written) = crate::stream::bao::stream_verification_inboard( stripe_view, @@ -682,7 +724,10 @@ pub fn stream_encode_inboard_body( )?; (padding_len, chunk_len, bytes_ecc, h, written as u32) } else { - let written = write_inboard_stripe(&stripe, output)? as u32; + let mut written = 0u32; + for stripe in &stripes { + written += write_inboard_stripe(stripe, output)? as u32; + } ( padding_len, chunk_len, @@ -792,6 +837,7 @@ pub fn stream_encode_inboard( output: &mut W, payload_nonce: &mut [u8; 16], header_path_encrypt: bool, + zstd: &ZstdEncode, ) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { stream_encode_inboard_with_nonce( master_key, @@ -801,6 +847,7 @@ pub fn stream_encode_inboard( payload_nonce, header_path_encrypt, None, + zstd, ) } @@ -810,6 +857,7 @@ pub fn stream_encode_inboard( /// When `None`, a CSPRNG nonce is drawn. **Test/determinism only** for fixed nonces — /// nonce reuse under the same master is catastrophic (AGENTS §2.1.4). Prefer /// [`stream_encode_inboard`] for production. +#[allow(clippy::too_many_arguments)] pub fn stream_encode_inboard_with_nonce( master_key: &[u8], input: R, @@ -818,6 +866,7 @@ pub fn stream_encode_inboard_with_nonce( payload_nonce: &mut [u8; 16], header_path_encrypt: bool, fixed_nonce: Option<[u8; 16]>, + zstd: &ZstdEncode, ) -> Result<(Hash, EncodeInfo, PreprocessStats), CarbonadoError> { let fmt = Format::from(format); let mut spool = SeekableSpool::new()?; @@ -829,6 +878,7 @@ pub fn stream_encode_inboard_with_nonce( payload_nonce, header_path_encrypt, fixed_nonce, + zstd, )?; let (hash, info) = stream_encode_inboard_body(&mut spool, stats, format, output)?; Ok((hash, info, stats)) diff --git a/src/stream/fec.rs b/src/stream/fec.rs index d4d97cb..bdea548 100644 --- a/src/stream/fec.rs +++ b/src/stream/fec.rs @@ -1,4 +1,4 @@ -//! Reed-Solomon 4/8 FEC streaming with a 16 KiB (4×4 KiB slice) stripe accumulator. +//! Reed-Solomon 4/8 FEC as 16 KiB logical stripes of eight 4 KiB leaves. use std::io::{Read, Write}; @@ -6,25 +6,65 @@ use reed_solomon_erasure::ReedSolomon; use reed_solomon_erasure::galois_8::Field; use crate::{ - constants::{FEC_K, FEC_M, SLICE_LEN}, + constants::{FEC_K, FEC_M, FEC_STRIPE_INBOARD_LEN, FEC_STRIPE_LOGICAL_LEN, SLICE_LEN}, error::CarbonadoError, utils::calc_padding_len, }; -/// Result of one completed FEC stripe (8 shards × `chunk_len`). +/// Result of one completed FEC stripe (8 shards × 4 KiB). #[derive(Clone, Debug)] pub struct FecStripe { pub shards: Vec>, pub chunk_len: u32, } -/// Inboard FEC encoder: consumes logical bytes, emits one concatenated stripe. +impl FecStripe { + fn empty_shards() -> Vec> { + (0..FEC_M).map(|_| vec![0u8; SLICE_LEN as usize]).collect() + } +} + +/// Split one stripe into data leaves (4 × 4 KiB) and parity leaves (4 × 4 KiB). +/// +/// Outboard main is the concatenation of data leaves across stripes (padded +/// logical body). Parity leaves are the stream Adamantine stores in the bundle. +pub fn stripe_data_and_parity_leaves(stripe: &FecStripe) -> (&[Vec], &[Vec]) { + stripe.shards.split_at(FEC_K) +} + +/// Concatenate data leaves of every stripe (padded logical body, stripe order). +pub fn concat_data_leaves(stripes: &[FecStripe]) -> Vec { + let mut out = Vec::with_capacity(stripes.len() * FEC_STRIPE_LOGICAL_LEN as usize); + for stripe in stripes { + let (data, _) = stripe_data_and_parity_leaves(stripe); + for leaf in data { + out.extend_from_slice(leaf); + } + } + out +} + +/// Concatenate parity leaves of every stripe (Adamantine / `.par` stream). +pub fn concat_parity_leaves(stripes: &[FecStripe]) -> Vec { + let mut out = Vec::with_capacity( + stripes.len() * (FEC_STRIPE_INBOARD_LEN - FEC_STRIPE_LOGICAL_LEN) as usize, + ); + for stripe in stripes { + let (_, parity) = stripe_data_and_parity_leaves(stripe); + for leaf in parity { + out.extend_from_slice(leaf); + } + } + out +} + +/// Inboard FEC encoder: consumes logical bytes, emits one 32 KiB stripe per 16 KiB. pub struct FecInboardEncoder { + logical_len: usize, padded_len: usize, - chunk_len: usize, padding_total: u32, pos: usize, - shards: Vec>, + current: Vec, rs: ReedSolomon, finished: bool, } @@ -34,29 +74,24 @@ impl FecInboardEncoder { pub fn new(logical_len: usize) -> Result { if logical_len == 0 { return Ok(Self { + logical_len: 0, padded_len: 0, - chunk_len: 0, padding_total: 0, pos: 0, - shards: vec![], + current: Vec::new(), rs: ReedSolomon::new(FEC_K, FEC_M - FEC_K)?, finished: true, }); } - let (padding_total, chunk_len) = calc_padding_len(logical_len); + let (padding_total, _chunk_len) = calc_padding_len(logical_len); let padded_len = logical_len + padding_total as usize; - let rs = ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?; - let mut shards = Vec::with_capacity(FEC_M); - for _ in 0..FEC_M { - shards.push(vec![0u8; chunk_len as usize]); - } Ok(Self { + logical_len, padded_len, - chunk_len: chunk_len as usize, padding_total, pos: 0, - shards, - rs, + current: Vec::with_capacity(FEC_STRIPE_LOGICAL_LEN as usize), + rs: ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?, finished: false, }) } @@ -65,123 +100,135 @@ impl FecInboardEncoder { self.padding_total } + /// RS symbol size: one 4 KiB Bao leaf (not `padded_len / 4`). pub fn chunk_len(&self) -> u32 { - self.chunk_len as u32 + if self.padded_len == 0 { 0 } else { SLICE_LEN } } - /// Feed logical bytes from `input`. Caller must supply exactly `logical_len` bytes total - /// (via [`Read::take`] or equivalent) before [`Self::finish`]. Excess bytes error; - /// padding is zero-filled only in `finish`. - pub fn feed(&mut self, mut input: R) -> Result, CarbonadoError> { + /// Feed logical bytes from `input`. Completed 16 KiB stripes are returned. + /// Caller must supply exactly `logical_len` bytes total before [`Self::finish`]. + pub fn feed(&mut self, mut input: R) -> Result, CarbonadoError> { if self.finished { - return Ok(None); + return Ok(vec![]); } + let mut completed = Vec::new(); let mut buf = [0u8; SLICE_LEN as usize]; loop { let n = input.read(&mut buf).map_err(CarbonadoError::StdIoError)?; if n == 0 { break; } - self.feed_logical_bytes(&buf[..n])?; + self.feed_logical_bytes(&buf[..n], &mut completed)?; } - Ok(None) + Ok(completed) } /// Finalize when the caller has fed exactly `logical_len` bytes (padding added internally). - pub fn finish(&mut self) -> Result, CarbonadoError> { - if self.finished || self.padded_len == 0 { - return Ok(None); + pub fn finish(&mut self) -> Result, CarbonadoError> { + if self.finished { + return Ok(vec![]); } - let logical_len = self.logical_len(); - if self.pos < logical_len { + if self.pos < self.logical_len { return Err(CarbonadoError::StdIoError(std::io::Error::new( std::io::ErrorKind::UnexpectedEof, "FEC encoder: short read before finish", ))); } + let mut completed = Vec::new(); if self.pos < self.padded_len { let zeros = vec![0u8; self.padded_len - self.pos]; - self.feed_padding_bytes(&zeros)?; + self.feed_padding_bytes(&zeros, &mut completed)?; + } + if !self.current.is_empty() { + return Err(CarbonadoError::InternalStateError( + "FEC encoder: unfinished stripe after padding".to_string(), + )); } self.finished = true; - Ok(Some(self.take_stripe()?)) + Ok(completed) } - fn logical_len(&self) -> usize { - self.padded_len - self.padding_total as usize - } - - fn feed_logical_bytes(&mut self, data: &[u8]) -> Result<(), CarbonadoError> { - let logical_len = self.logical_len(); + fn feed_logical_bytes( + &mut self, + data: &[u8], + completed: &mut Vec, + ) -> Result<(), CarbonadoError> { let mut off = 0usize; while off < data.len() { - if self.pos >= logical_len { + if self.pos >= self.logical_len { return Err(CarbonadoError::StdIoError(std::io::Error::new( std::io::ErrorKind::InvalidData, "FEC encoder: input exceeds logical length", ))); } - let shard_idx = self.pos / self.chunk_len; - let shard_off = self.pos % self.chunk_len; - if shard_idx >= FEC_K { - break; - } - let room = self.chunk_len - shard_off; - let cap = logical_len - self.pos; + let cap = self.logical_len - self.pos; + let room = FEC_STRIPE_LOGICAL_LEN as usize - self.current.len(); let take = (data.len() - off).min(room).min(cap); - self.shards[shard_idx][shard_off..shard_off + take] - .copy_from_slice(&data[off..off + take]); + self.current.extend_from_slice(&data[off..off + take]); self.pos += take; off += take; + if self.current.len() == FEC_STRIPE_LOGICAL_LEN as usize { + completed.push(self.take_stripe()?); + } } Ok(()) } - fn feed_padding_bytes(&mut self, data: &[u8]) -> Result<(), CarbonadoError> { + fn feed_padding_bytes( + &mut self, + data: &[u8], + completed: &mut Vec, + ) -> Result<(), CarbonadoError> { let mut off = 0usize; while off < data.len() && self.pos < self.padded_len { - let shard_idx = self.pos / self.chunk_len; - let shard_off = self.pos % self.chunk_len; - if shard_idx >= FEC_K { - break; - } - let room = self.chunk_len - shard_off; + let room = FEC_STRIPE_LOGICAL_LEN as usize - self.current.len(); let take = (data.len() - off).min(room).min(self.padded_len - self.pos); - self.shards[shard_idx][shard_off..shard_off + take] - .copy_from_slice(&data[off..off + take]); + self.current.extend_from_slice(&data[off..off + take]); self.pos += take; off += take; + if self.current.len() == FEC_STRIPE_LOGICAL_LEN as usize { + completed.push(self.take_stripe()?); + } } Ok(()) } fn take_stripe(&mut self) -> Result { - #[cfg(feature = "parallel")] - { - crate::stream::parallel::encode_rs_parity(&self.rs, &mut self.shards, self.chunk_len)?; - } - #[cfg(not(feature = "parallel"))] - { - self.rs.encode(&mut self.shards)?; - } - for s in &self.shards { - if s.len() != self.chunk_len { - return Err(CarbonadoError::EncodeInvalidChunkLength( - self.chunk_len as u32, - s.len(), - )); - } + let mut shards = FecStripe::empty_shards(); + let leaf = SLICE_LEN as usize; + for (i, shard) in shards.iter_mut().enumerate().take(FEC_K) { + shard.copy_from_slice(&self.current[i * leaf..(i + 1) * leaf]); } + self.current.clear(); + encode_stripe_parity(&self.rs, &mut shards)?; Ok(FecStripe { - shards: std::mem::take(&mut self.shards), - chunk_len: self.chunk_len as u32, + shards, + chunk_len: SLICE_LEN, }) } } +fn encode_stripe_parity( + rs: &ReedSolomon, + shards: &mut [Vec], +) -> Result<(), CarbonadoError> { + #[cfg(feature = "parallel")] + { + crate::stream::parallel::encode_rs_parity(rs, shards, SLICE_LEN as usize)?; + } + #[cfg(not(feature = "parallel"))] + { + rs.encode(shards)?; + } + for s in shards.iter() { + if s.len() != SLICE_LEN as usize { + return Err(CarbonadoError::EncodeInvalidChunkLength(SLICE_LEN, s.len())); + } + } + Ok(()) +} + /// [`positioned_io::ReadAt`] view over concatenated inboard FEC stripe shards. -/// -/// Avoids flattening shard data into a staging `Vec` before keyed Bao inboard encode (S3). pub struct FecStripeReadAt<'a> { stripe: &'a FecStripe, len: u64, @@ -206,35 +253,88 @@ impl<'a> FecStripeReadAt<'a> { } impl positioned_io::ReadAt for FecStripeReadAt<'_> { + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { + read_at_shards(&self.stripe.shards, self.len, offset, buf) + } +} + +/// [`ReadAt`] over every stripe of an inboard FEC body, in stripe order. +pub struct FecStripesReadAt<'a> { + stripes: &'a [FecStripe], + len: u64, +} + +impl<'a> FecStripesReadAt<'a> { + pub fn new(stripes: &'a [FecStripe]) -> Self { + let len = stripes.len() as u64 * u64::from(FEC_STRIPE_INBOARD_LEN); + Self { stripes, len } + } + + pub fn len(&self) -> u64 { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl positioned_io::ReadAt for FecStripesReadAt<'_> { fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { if offset >= self.len || buf.is_empty() { return Ok(0); } + let stripe_len = u64::from(FEC_STRIPE_INBOARD_LEN); let mut written = 0usize; let mut pos = offset; - let mut cum = 0u64; - for shard in &self.stripe.shards { - let shard_len = shard.len() as u64; - let shard_end = cum + shard_len; - if pos >= shard_end { - cum = shard_end; - continue; - } - let start = (pos - cum) as usize; - let avail = shard.len() - start; - let to_copy = avail.min(buf.len() - written); - buf[written..written + to_copy].copy_from_slice(&shard[start..start + to_copy]); - written += to_copy; - pos += to_copy as u64; - cum = shard_end; - if written >= buf.len() { + while written < buf.len() && pos < self.len { + let stripe_idx = (pos / stripe_len) as usize; + let stripe_off = pos % stripe_len; + let view = FecStripeReadAt::new(&self.stripes[stripe_idx]); + let n = view.read_at(stripe_off, &mut buf[written..])?; + if n == 0 { break; } + written += n; + pos += n as u64; } Ok(written) } } +fn read_at_shards( + shards: &[Vec], + len: u64, + offset: u64, + buf: &mut [u8], +) -> std::io::Result { + if offset >= len || buf.is_empty() { + return Ok(0); + } + let mut written = 0usize; + let mut pos = offset; + let mut cum = 0u64; + for shard in shards { + let shard_len = shard.len() as u64; + let shard_end = cum + shard_len; + if pos >= shard_end { + cum = shard_end; + continue; + } + let start = (pos - cum) as usize; + let avail = shard.len() - start; + let to_copy = avail.min(buf.len() - written); + buf[written..written + to_copy].copy_from_slice(&shard[start..start + to_copy]); + written += to_copy; + pos += to_copy as u64; + cum = shard_end; + if written >= buf.len() { + break; + } + } + Ok(written) +} + /// Write all shards of a stripe to `output` (inboard layout). pub fn write_inboard_stripe( stripe: &FecStripe, @@ -248,7 +348,7 @@ pub fn write_inboard_stripe( Ok(n) } -/// Write parity shards only (outboard `.par` sidecar). +/// Write parity shards only (outboard `.par` sidecar / Adamantine bundle). pub fn write_outboard_parity( stripe: &FecStripe, output: &mut W, @@ -261,17 +361,62 @@ pub fn write_outboard_parity( Ok(n) } -/// [`WriteAt`] sink for keyed Bao inboard decode into one RS stripe (S4). -/// -/// Retains at most `FEC_M` shard buffers (`O(stripe)`); RS-reconstructs on [`Self::finish`]. +/// Write data leaves only (outboard main = data leaves in stripe order). +pub fn write_data_leaves( + stripe: &FecStripe, + output: &mut W, +) -> Result { + let mut n = 0u64; + for s in stripe.shards.iter().take(FEC_K) { + output.write_all(s).map_err(CarbonadoError::StdIoError)?; + n += s.len() as u64; + } + Ok(n) +} + +/// Reconstruct one stripe from up to 8 optional 4 KiB symbols (erasures are `None`). +pub fn reconstruct_stripe(shards: &mut [Option>]) -> Result>, CarbonadoError> { + if shards.len() != FEC_M { + return Err(CarbonadoError::UnevenFecChunks); + } + let good = shards.iter().filter(|s| s.is_some()).count(); + if good < FEC_K { + return Err(CarbonadoError::InvalidScrubbedHash); + } + let leaf = SLICE_LEN as usize; + for s in shards.iter().flatten() { + if s.len() != leaf { + return Err(CarbonadoError::UnevenFecChunks); + } + } + let rs = ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?; + rs.reconstruct(shards)?; + let mut out = Vec::with_capacity(FEC_M); + for s in shards.iter_mut() { + out.push(s.take().ok_or(CarbonadoError::UnevenFecChunks)?); + } + Ok(out) +} + +fn reconstruct_stripe_logical( + shards: &mut [Option>], + logical_out: &mut Vec, +) -> Result<(), CarbonadoError> { + let rebuilt = reconstruct_stripe(shards)?; + for leaf in rebuilt.iter().take(FEC_K) { + logical_out.extend_from_slice(leaf); + } + Ok(()) +} + +/// [`WriteAt`] sink for keyed Bao inboard decode of a multi-stripe FEC body. /// -/// `filled` tracks the maximum end offset written. Completion assumes `keyed_decode_ranges` with -/// `ChunkRanges::all()` populates `[0, content_len)` contiguously on success (bao-tree contract). +/// Retains the FEC body (`O(FEC body)` shard bytes) then RS-reconstructs per 16 KiB +/// stripe on [`Self::finish_into`]. pub struct FecInboardWriteAt { content_len: u64, padding: u32, - shard_len: usize, - shards: Vec>, + buf: Vec, filled: u64, finished: bool, } @@ -282,34 +427,25 @@ impl FecInboardWriteAt { return Ok(Self { content_len: 0, padding, - shard_len: 0, - shards: vec![], + buf: vec![], filled: 0, finished: false, }); } let len = content_len as usize; - if !len.is_multiple_of(FEC_M) { + if !len.is_multiple_of(FEC_STRIPE_INBOARD_LEN as usize) { return Err(CarbonadoError::UnevenFecChunks); } - let shard_len = len / FEC_M; - let mut shards = Vec::with_capacity(FEC_M); - for _ in 0..FEC_M { - shards.push(vec![0u8; shard_len]); - } Ok(Self { content_len, padding, - shard_len, - shards, + buf: vec![0u8; len], filled: 0, finished: false, }) } - /// RS-decode and stream logical bytes (padding stripped) into `output` without a full - /// intermediate logical `Vec`. Peak RAM remains O(FEC body) for the shard buffers - /// (one segment-wide stripe under current geometry). + /// RS-decode each stripe and stream logical bytes (padding stripped) into `output`. pub fn finish_into(mut self, output: &mut W) -> Result { if self.finished { return Err(CarbonadoError::InternalStateError( @@ -329,34 +465,14 @@ impl FecInboardWriteAt { ), ))); } - let mut shard_opts: Vec>> = self.shards.drain(..).map(Some).collect(); - let rs = ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?; - rs.reconstruct(&mut shard_opts)?; - let data_len = self.shard_len.saturating_mul(FEC_K); - if self.padding as usize > data_len { - return Err(CarbonadoError::ScrubbedLengthMismatch( - data_len, - self.padding as usize, - )); - } - let logical_len = data_len - self.padding as usize; - let mut remaining = logical_len; - let mut written = 0u64; - for s in shard_opts.iter().take(FEC_K).flatten() { - if remaining == 0 { - break; - } - let n = remaining.min(s.len()); - output - .write_all(&s[..n]) - .map_err(CarbonadoError::StdIoError)?; - remaining -= n; - written += n as u64; - } - Ok(written) + let decoded = decode_inboard_stripes(&self.buf, self.padding)?; + output + .write_all(&decoded) + .map_err(CarbonadoError::StdIoError)?; + Ok(decoded.len() as u64) } - /// RS-decode the accumulated stripe and return logical bytes (padding stripped). + /// RS-decode the accumulated body and return logical bytes (padding stripped). pub fn finish(self) -> Result, CarbonadoError> { let mut decoded = Vec::new(); self.finish_into(&mut decoded)?; @@ -380,20 +496,9 @@ impl positioned_io::WriteAt for FecInboardWriteAt { { return Err(write_past_content_len_error()); } - let mut written = 0usize; - let mut pos = offset; - while written < data.len() { - let shard_idx = (pos as usize) / self.shard_len; - let shard_off = (pos as usize) % self.shard_len; - let room = self.shard_len - shard_off; - let cap = (self.content_len - pos) as usize; - let take = (data.len() - written).min(room).min(cap); - self.shards[shard_idx][shard_off..shard_off + take] - .copy_from_slice(&data[written..written + take]); - self.filled = self.filled.max(pos + take as u64); - written += take; - pos += take as u64; - } + let rel = offset as usize; + self.buf[rel..rel + data.len()].copy_from_slice(data); + self.filled = self.filled.max(offset + data.len() as u64); Ok(data.len()) } @@ -411,9 +516,6 @@ impl positioned_io::WriteAt for FecInboardWriteAt { /// /// Production inboard non-FEC verification uses [`crate::stream::spool::SeekWriteAt`] (disk /// spool, O(chunk) RAM). This type remains for unit tests of WriteAt completeness contracts. -/// -/// `filled` tracks the maximum end offset written; relies on full-range Bao decode completion -/// (see [`FecInboardWriteAt`]). pub struct LogicalBufferWriteAt { content_len: u64, buf: Vec, @@ -468,38 +570,63 @@ impl positioned_io::WriteAt for LogicalBufferWriteAt { } } -/// Inboard FEC decode from a reader of concatenated shards. -pub fn stream_decode_inboard( - mut input: R, - padding: u32, - logical_shard_len: usize, - output: &mut W, -) -> Result { - if logical_shard_len == 0 { - return Ok(0); +fn decode_inboard_stripes(input: &[u8], padding: u32) -> Result, CarbonadoError> { + if input.is_empty() { + return Ok(vec![]); } - let shard_len = logical_shard_len; - let mut shards: Vec>> = vec![None; FEC_M]; - for shard in shards.iter_mut() { - let mut buf = vec![0u8; shard_len]; - input - .read_exact(&mut buf) - .map_err(CarbonadoError::StdIoError)?; - *shard = Some(buf); + const STRIPE: usize = FEC_STRIPE_INBOARD_LEN as usize; + const LEAF: usize = SLICE_LEN as usize; + let (stripes, remainder) = input.as_chunks::(); + if !remainder.is_empty() { + return Err(CarbonadoError::UnevenFecChunks); } - let rs = ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?; - rs.reconstruct(&mut shards)?; - let mut decoded = Vec::new(); - for s in shards.iter().take(FEC_K).flatten() { - decoded.extend_from_slice(s); + let mut logical = Vec::with_capacity(input.len() / 2); + for stripe in stripes { + let (leaves, leaf_rem) = stripe.as_chunks::(); + debug_assert!(leaf_rem.is_empty()); + let mut shards: Vec>> = leaves.iter().map(|c| Some(c.to_vec())).collect(); + reconstruct_stripe_logical(&mut shards, &mut logical)?; } - if padding as usize > decoded.len() { + if padding as usize > logical.len() { return Err(CarbonadoError::ScrubbedLengthMismatch( - decoded.len(), + logical.len(), padding as usize, )); } - decoded.truncate(decoded.len() - padding as usize); + logical.truncate(logical.len() - padding as usize); + Ok(logical) +} + +/// Inboard FEC decode from a reader of concatenated 32 KiB stripes. +pub fn stream_decode_inboard( + mut input: R, + padding: u32, + _logical_shard_len: usize, + output: &mut W, +) -> Result { + let stripe_len = FEC_STRIPE_INBOARD_LEN as usize; + let mut body = Vec::new(); + let mut buf = vec![0u8; stripe_len]; + loop { + let mut got = 0usize; + while got < stripe_len { + let n = input + .read(&mut buf[got..]) + .map_err(CarbonadoError::StdIoError)?; + if n == 0 { + break; + } + got += n; + } + if got == 0 { + break; + } + if got != stripe_len { + return Err(CarbonadoError::UnevenFecChunks); + } + body.extend_from_slice(&buf); + } + let decoded = decode_inboard_stripes(&body, padding)?; output .write_all(&decoded) .map_err(CarbonadoError::StdIoError)?; @@ -508,9 +635,8 @@ pub fn stream_decode_inboard( /// Outboard FEC decode: bare main reader + parity reader -> logical output. /// -/// **Degraded / truncated main:** prefer [`crate::decoding::fec_with_parity`] via -/// [`crate::stream::stream_decode_outboard_buffer`], which derives stripe geometry from -/// the parity sidecar (encode-time `chunk_len`) rather than `calc_padding_len(main_len)`. +/// Parity is the concatenated 4 KiB parity leaves (4 per stripe). Main is the +/// logical body (data leaves with padding stripped, or a prefix thereof). pub fn stream_decode_outboard( mut main: R, mut parity: R, @@ -521,22 +647,8 @@ pub fn stream_decode_outboard( if main_len == 0 && padding == 0 { return Ok(0); } - - // Read parity incrementally into a buffer (streamed via `copy`, not single `read_to_end`). let mut parity_buf = Vec::new(); std::io::copy(&mut parity, &mut parity_buf).map_err(CarbonadoError::StdIoError)?; - let parity_shards = FEC_M - FEC_K; - if !parity_buf.len().is_multiple_of(parity_shards) { - return Err(CarbonadoError::UnevenFecChunks); - } - let shard_len = parity_buf.len() / parity_shards; - let padded_total = shard_len * FEC_K; - let pad = padding as usize; - if pad > padded_total { - return Err(CarbonadoError::ScrubbedLengthMismatch(padded_total, pad)); - } - let logical_len = padded_total - pad; - let mut main_buf = Vec::new(); if main_len > 0 { let mut buf = [0u8; SLICE_LEN as usize]; @@ -551,29 +663,65 @@ pub fn stream_decode_outboard( read_main += take; } } - let copy = main_buf.len().min(logical_len); - - let mut padded = vec![0u8; padded_total]; - padded[..copy].copy_from_slice(&main_buf[..copy]); + let decoded = decode_outboard_stripes(&main_buf, &parity_buf, padding)?; + output + .write_all(&decoded) + .map_err(CarbonadoError::StdIoError)?; + Ok(decoded.len() as u64) +} - let mut shards: Vec>> = vec![None; FEC_M]; - for (i, shard) in shards.iter_mut().enumerate().take(FEC_K) { - let start = i * shard_len; - let end = start + shard_len; - if end <= copy { - *shard = Some(padded[start..end].to_vec()); - } +pub(crate) fn decode_outboard_stripes( + main: &[u8], + parity: &[u8], + padding: u32, +) -> Result, CarbonadoError> { + if main.is_empty() && parity.is_empty() { + return Ok(vec![]); } - for j in 0..parity_shards { - let start = j * shard_len; - shards[FEC_K + j] = Some(parity_buf[start..start + shard_len].to_vec()); + const LEAF: usize = SLICE_LEN as usize; + const PARITY_STRIPE: usize = (FEC_M - FEC_K) * LEAF; + const LOGICAL_STRIPE: usize = FEC_STRIPE_LOGICAL_LEN as usize; + let (parity_stripes, remainder) = parity.as_chunks::(); + if !remainder.is_empty() { + return Err(CarbonadoError::UnevenFecChunks); } - - let rs = ReedSolomon::::new(FEC_K, FEC_M - FEC_K)?; - rs.reconstruct(&mut shards)?; - let mut decoded = Vec::new(); - for s in shards.iter().take(FEC_K).flatten() { - decoded.extend_from_slice(s); + let n_stripes = parity_stripes.len(); + let padded_total = n_stripes * LOGICAL_STRIPE; + let pad = padding as usize; + if pad > padded_total { + return Err(CarbonadoError::ScrubbedLengthMismatch(padded_total, pad)); + } + let logical_len = padded_total - pad; + let mut padded = vec![0u8; padded_total]; + let copy = main.len().min(logical_len); + padded[..copy].copy_from_slice(&main[..copy]); + + let mut decoded = Vec::with_capacity(logical_len); + let (logical_stripes, logical_rem) = padded.as_chunks::(); + debug_assert!(logical_rem.is_empty()); + for (stripe_idx, (logical_stripe, parity_stripe)) in + logical_stripes.iter().zip(parity_stripes).enumerate() + { + let mut shards: Vec>> = vec![None; FEC_M]; + let data_off = stripe_idx * LOGICAL_STRIPE; + let (data_leaves, data_rem) = logical_stripe.as_chunks::(); + debug_assert!(data_rem.is_empty()); + for (i, (shard, chunk)) in shards.iter_mut().take(FEC_K).zip(data_leaves).enumerate() { + let start = data_off + i * LEAF; + let end = start + LEAF; + if end <= copy { + *shard = Some(chunk.to_vec()); + } else if start < copy { + // Partial last data leaf: treat as erasure. + *shard = None; + } + } + let (parity_leaves, par_rem) = parity_stripe.as_chunks::(); + debug_assert!(par_rem.is_empty()); + for (shard, chunk) in shards[FEC_K..].iter_mut().zip(parity_leaves) { + *shard = Some(chunk.to_vec()); + } + reconstruct_stripe_logical(&mut shards, &mut decoded)?; } if decoded.len() < logical_len { return Err(CarbonadoError::ScrubbedLengthMismatch( @@ -582,10 +730,7 @@ pub fn stream_decode_outboard( )); } decoded.truncate(logical_len); - output - .write_all(&decoded) - .map_err(CarbonadoError::StdIoError)?; - Ok(decoded.len() as u64) + Ok(decoded) } fn fec_short_read_error() -> CarbonadoError { @@ -595,59 +740,65 @@ fn fec_short_read_error() -> CarbonadoError { )) } -/// Feed exactly `logical_len` bytes from `input` and emit one inboard FEC stripe. -/// -/// Uses [`Read::take`] so callers cannot over-feed; returns an error on short read. -pub fn feed_inboard_fec_stripe( +/// Feed exactly `logical_len` bytes and emit every inboard FEC stripe. +pub fn feed_inboard_fec_stripes( logical_len: usize, input: &mut R, -) -> Result<(FecStripe, u32, u32), CarbonadoError> { +) -> Result<(Vec, u32, u32), CarbonadoError> { if logical_len == 0 { return Err(CarbonadoError::UnevenFecChunks); } let mut enc = FecInboardEncoder::new(logical_len)?; let mut limited = input.take(logical_len as u64); - enc.feed(&mut limited)?; + let mut stripes = enc.feed(&mut limited)?; if limited.limit() > 0 { return Err(fec_short_read_error()); } - let stripe = enc.finish()?.ok_or(CarbonadoError::UnevenFecChunks)?; - Ok((stripe, enc.padding_len(), enc.chunk_len())) + stripes.extend(enc.finish()?); + if stripes.is_empty() { + return Err(CarbonadoError::UnevenFecChunks); + } + Ok((stripes, enc.padding_len(), enc.chunk_len())) } -/// Buffer-path helper: encode entire logical blob in one stripe. -fn take_stripe(enc: &mut FecInboardEncoder, input: &[u8]) -> Result { - if let Some(stripe) = enc.feed(std::io::Cursor::new(input))? { - return Ok(stripe); +/// Back-compat alias: same as [`feed_inboard_fec_stripes`]. +pub fn feed_inboard_fec_stripe( + logical_len: usize, + input: &mut R, +) -> Result<(Vec, u32, u32), CarbonadoError> { + feed_inboard_fec_stripes(logical_len, input) +} + +/// Encode logical bytes into inboard stripes (4 KiB leaves, stripe order). +pub fn encode_stripes(input: &[u8]) -> Result<(Vec, u32, u32), CarbonadoError> { + if input.is_empty() { + return Ok((vec![], 0, 0)); } - enc.finish()?.ok_or(CarbonadoError::UnevenFecChunks) + let mut enc = FecInboardEncoder::new(input.len())?; + let mut stripes = enc.feed(std::io::Cursor::new(input))?; + stripes.extend(enc.finish()?); + Ok((stripes, enc.padding_len(), enc.chunk_len())) } pub fn encode_inboard_buffer(input: &[u8]) -> Result<(Vec, u32, u32), CarbonadoError> { if input.is_empty() { return Ok((vec![], 0, 0)); } - let mut enc = FecInboardEncoder::new(input.len())?; - let stripe = take_stripe(&mut enc, input)?; - let padding_len = enc.padding_len(); - let chunk_len = enc.chunk_len(); + let (stripes, padding_len, chunk_len) = encode_stripes(input)?; let mut out = Vec::new(); - write_inboard_stripe(&stripe, &mut out)?; + for stripe in &stripes { + write_inboard_stripe(stripe, &mut out)?; + } Ok((out, padding_len, chunk_len)) } -/// Buffer-path helper: parity shards only for outboard FEC. +/// Buffer-path helper: parity leaves only for outboard FEC / Adamantine. pub fn encode_outboard_parity_buffer(input: &[u8]) -> Result<(u32, u32, Vec), CarbonadoError> { if input.is_empty() { return Ok((0, 0, vec![])); } - let mut enc = FecInboardEncoder::new(input.len())?; - let stripe = take_stripe(&mut enc, input)?; - let padding_len = enc.padding_len(); - let chunk_len = enc.chunk_len(); - let mut parity = Vec::new(); - write_outboard_parity(&stripe, &mut parity)?; - Ok((padding_len, chunk_len, parity)) + let (stripes, padding_len, chunk_len) = encode_stripes(input)?; + Ok((padding_len, chunk_len, concat_parity_leaves(&stripes))) } #[cfg(test)] @@ -658,19 +809,16 @@ mod tests { use crate::decoding::fec; #[test] - fn fec_stripe_geometry_matches_calc_padding_len() { - for logical_len in [1usize, 4095, 4096, 4097, 16 * 1024 - 1, 16 * 1024] { + fn fec_stripe_geometry_is_4kib_leaves() { + for logical_len in [1usize, 4095, 4096, 4097, 16 * 1024 - 1, 16 * 1024, 32_768] { let input: Vec = (0..logical_len).map(|i| (i % 251) as u8).collect(); let (encoded, pl, cl) = encode_inboard_buffer(&input).expect("encode"); - let (exp_pl, exp_cl) = calc_padding_len(logical_len); + let (exp_pl, _exp_cl) = calc_padding_len(logical_len); assert_eq!(pl, exp_pl, "padding len for {logical_len}"); - assert_eq!(cl, exp_cl, "chunk len for {logical_len}"); - if logical_len == 0 { - assert!(encoded.is_empty()); - continue; - } - assert_eq!(encoded.len(), FEC_M * cl as usize); - assert_eq!(cl % SLICE_LEN, 0, "chunk_len must align to SLICE_LEN"); + assert_eq!(cl, SLICE_LEN, "chunk_len is 4 KiB for {logical_len}"); + let padded = logical_len + pl as usize; + let n_stripes = padded / FEC_STRIPE_LOGICAL_LEN as usize; + assert_eq!(encoded.len(), n_stripes * FEC_STRIPE_INBOARD_LEN as usize); } } @@ -688,17 +836,28 @@ mod tests { } } + #[test] + fn two_stripes_place_second_data_after_first_parity() { + let input: Vec = (0..32_768).map(|i| (i % 251) as u8).collect(); + let (encoded, _, cl) = encode_inboard_buffer(&input).expect("encode"); + assert_eq!(cl, SLICE_LEN); + assert_eq!(encoded.len(), 2 * FEC_STRIPE_INBOARD_LEN as usize); + assert_eq!(&encoded[0..4096], &input[0..4096]); + assert_ne!(&encoded[16_384..20_480], &input[16_384..20_480]); + assert_eq!(&encoded[32_768..36_864], &input[16_384..20_480]); + } + #[test] fn fec_stripe_read_at_matches_flattened_stripe() { use positioned_io::ReadAt; let input: Vec = (0..12_288).map(|i| (i % 251) as u8).collect(); - let mut enc = FecInboardEncoder::new(input.len()).expect("new"); - let stripe = take_stripe(&mut enc, &input).expect("stripe"); + let (stripes, _, _) = encode_stripes(&input).expect("stripes"); + assert_eq!(stripes.len(), 1); let mut flat = Vec::new(); - write_inboard_stripe(&stripe, &mut flat).expect("flatten"); + write_inboard_stripe(&stripes[0], &mut flat).expect("flatten"); - let view = FecStripeReadAt::new(&stripe); + let view = FecStripeReadAt::new(&stripes[0]); assert_eq!(view.len(), flat.len() as u64); let mut via_read_at = vec![0u8; flat.len()]; @@ -707,13 +866,20 @@ mod tests { .expect("read_at full stripe"); assert_eq!(n, flat.len()); assert_eq!(via_read_at, flat); + } - let mut tail = [0u8; 64]; - let n = view - .read_at(flat.len() as u64 - 32, &mut tail) - .expect("read_at tail"); - assert_eq!(n, 32); - assert_eq!(&tail[..32], &flat[flat.len() - 32..]); + #[test] + fn fec_stripes_read_at_matches_concatenated_body() { + use positioned_io::ReadAt; + + let input: Vec = (0..32_768).map(|i| (i % 251) as u8).collect(); + let (stripes, _, _) = encode_stripes(&input).expect("stripes"); + let (flat, _, _) = encode_inboard_buffer(&input).expect("flat"); + let view = FecStripesReadAt::new(&stripes); + let mut got = vec![0u8; flat.len()]; + let n = view.read_at(0, &mut got).expect("read_at"); + assert_eq!(n, flat.len()); + assert_eq!(got, flat); } #[test] @@ -736,7 +902,8 @@ mod tests { fn feed_inboard_fec_stripe_errors_on_short_read() { let input: Vec = (0..8192).map(|i| (i % 251) as u8).collect(); let short = &input[..4096]; - let err = feed_inboard_fec_stripe(input.len(), &mut Cursor::new(short)).expect_err("short"); + let err = + feed_inboard_fec_stripes(input.len(), &mut Cursor::new(short)).expect_err("short"); assert!( matches!( err, @@ -748,22 +915,27 @@ mod tests { #[test] fn fec_incremental_feed_matches_single_buffer_feed() { - let input: Vec = (0..12_288).map(|i| (i % 251) as u8).collect(); + let input: Vec = (0..32_768).map(|i| (i % 251) as u8).collect(); let (buf_encoded, _, _) = encode_inboard_buffer(&input).expect("buffer"); let mut enc = FecInboardEncoder::new(input.len()).expect("new"); let mut off = 0usize; + let mut stripes = Vec::new(); while off < input.len() { let step = 512.min(input.len() - off); - let _ = enc - .feed(Cursor::new(&input[off..off + step])) - .expect("feed"); + stripes.extend( + enc.feed(Cursor::new(&input[off..off + step])) + .expect("feed"), + ); off += step; } - let stripe = enc.finish().expect("finish").expect("final stripe"); + stripes.extend(enc.finish().expect("finish")); let mut incremental = Vec::new(); - write_inboard_stripe(&stripe, &mut incremental).expect("write"); + for stripe in &stripes { + write_inboard_stripe(stripe, &mut incremental).expect("write"); + } assert_eq!(incremental, buf_encoded); + assert_eq!(stripes.len(), 2); } #[test] @@ -772,10 +944,9 @@ mod tests { let input: Vec = (0..8192).map(|i| (i % 251) as u8).collect(); let (pl, chunk_len, parity) = encode_outboard_parity_buffer(&input).expect("parity"); - // Outboard bare main is the pre-FEC logical body; parity sidecar holds RS parity shards. let decoded = fec_with_parity(&input, &parity, pl).expect("fec outboard"); assert_eq!(decoded, input); - assert_eq!(chunk_len % SLICE_LEN, 0); + assert_eq!(chunk_len, SLICE_LEN); } #[test] @@ -828,49 +999,36 @@ mod tests { } #[test] - fn fec_with_parity_corrupt_parity_does_not_recover_original() { - use crate::decoding::fec_with_parity; - - let input: Vec = (0..16_384).map(|i| (i % 251) as u8).collect(); - let (pl, chunk_len, parity) = encode_outboard_parity_buffer(&input).expect("parity"); - let chunk = chunk_len as usize; - let mut bad_parity = parity.clone(); - for j in 0..3 { - bad_parity[j * chunk..(j + 1) * chunk].fill(0xFF); + fn stripe_data_and_parity_lens() { + let input: Vec = (0..32_768).map(|i| (i % 251) as u8).collect(); + let (stripes, _, _) = encode_stripes(&input).expect("stripes"); + assert_eq!(stripes.len(), 2); + for stripe in &stripes { + let (data, parity) = stripe_data_and_parity_leaves(stripe); + assert_eq!(data.len(), FEC_K); + assert_eq!(parity.len(), FEC_M - FEC_K); + assert!(data.iter().all(|l| l.len() == SLICE_LEN as usize)); + assert!(parity.iter().all(|l| l.len() == SLICE_LEN as usize)); } - // RS reconstruct treats present-but-corrupt shards as valid; output must differ. - let decoded = fec_with_parity(&[], &bad_parity, pl).expect("reconstruct returns Ok"); - assert_ne!(decoded, input); - } - - #[test] - fn fec_with_parity_empty_parity_with_nonempty_input_errors() { - use crate::decoding::fec_with_parity; - use crate::error::CarbonadoError; - - let input: Vec = (0..4096).map(|i| (i % 251) as u8).collect(); - let (pl, _, _) = encode_outboard_parity_buffer(&input).expect("parity"); - let err = fec_with_parity(&input, &[], pl).unwrap_err(); - assert!(matches!( - err, - CarbonadoError::UnevenFecChunks - | CarbonadoError::FecError(_) - | CarbonadoError::ScrubbedLengthMismatch(0, _) - )); + let data = concat_data_leaves(&stripes); + assert_eq!(data, input); + assert_eq!( + concat_parity_leaves(&stripes).len(), + 2 * (FEC_M - FEC_K) * SLICE_LEN as usize + ); } #[test] fn fec_inboard_write_at_roundtrip_matches_decoding_fec() { use positioned_io::WriteAt; - let input: Vec = (0..12_288).map(|i| (i % 251) as u8).collect(); + let input: Vec = (0..32_768).map(|i| (i % 251) as u8).collect(); let (encoded, pl, _) = encode_inboard_buffer(&input).expect("encode"); let content_len = encoded.len() as u64; let expected = fec(&encoded, pl).expect("buffer fec"); let mut sink = FecInboardWriteAt::new(content_len, pl).expect("new"); - // Out-of-order shard-sized writes (mirrors Bao leaf ordering). - let shard_len = encoded.len() / FEC_M; + let shard_len = SLICE_LEN as usize; for (i, shard) in encoded.chunks(shard_len).enumerate() { let off = (i * shard_len) as u64; sink.write_at(off, shard).expect("write_at shard"); @@ -886,7 +1044,7 @@ mod tests { let input: Vec = (0..4096).map(|i| (i % 251) as u8).collect(); let (encoded, pl, _) = encode_inboard_buffer(&input).expect("encode"); let content_len = encoded.len() as u64; - let shard_len = encoded.len() / FEC_M; + let shard_len = SLICE_LEN as usize; let mut sink = FecInboardWriteAt::new(content_len, pl).expect("new"); sink.write_at(0, &encoded[..shard_len]) diff --git a/src/stream/mod.rs b/src/stream/mod.rs index 0be486f..92866b6 100644 --- a/src/stream/mod.rs +++ b/src/stream/mod.rs @@ -22,22 +22,29 @@ pub mod shard; pub mod slice; pub(crate) mod spool; -#[cfg(feature = "backend-rust")] -pub(crate) use slice::extract_slice_inboard_for_scrub; -pub use slice::{slice_to_chunk_ranges, verify_slice_inboard_seekable, verify_slice_outboard}; +pub use fec::{ + concat_data_leaves, concat_parity_leaves, encode_stripes, stripe_data_and_parity_leaves, + write_data_leaves, write_outboard_parity, +}; +pub use slice::{ + classify_inboard_leaves, inboard_leaf_data_ranges, leaf_index_to_stripe_symbol, + slice_to_chunk_ranges, stripe_symbol_to_leaf_index, verify_slice_inboard_seekable, + verify_slice_outboard, +}; +pub use compress::ZstdEncode; pub use decode::{ stream_decode, stream_decode_buffer, stream_decode_outboard, stream_decode_outboard_buffer, - stream_decrypt_header_path, + stream_decode_outboard_buffer_with_dict, stream_decrypt_header_path, }; #[cfg(feature = "async")] pub use decode_async::stream_decode_async; pub use encode::{ - stream_encode_buffer, stream_encode_buffer_with_nonce, stream_encode_inboard, - stream_encode_inboard_body, stream_encode_inboard_with_nonce, stream_encode_outboard, - stream_encode_outboard_buffer, stream_preprocess, + stream_encode_buffer, stream_encode_buffer_with_nonce, stream_encode_buffer_with_zstd, + stream_encode_inboard, stream_encode_inboard_body, stream_encode_inboard_with_nonce, + stream_encode_outboard, stream_encode_outboard_buffer, stream_preprocess, }; pub use shard::{ DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, decode_shards_stream, - encode_shard_stream, + encode_shard_stream, encode_shard_stream_with_zstd, }; diff --git a/src/stream/shard.rs b/src/stream/shard.rs index 69216a9..2bdb270 100644 --- a/src/stream/shard.rs +++ b/src/stream/shard.rs @@ -6,6 +6,7 @@ use crate::{ constants::Format, error::CarbonadoError, file::{Header, decode_stream}, + stream::ZstdEncode, structs::EncodeInfo, }; @@ -41,6 +42,29 @@ fn has_more_buffered_input(reader: &mut R) -> Result( + master_key: &[u8], + input: R, + format: u8, + chunk_index: u32, + segment_plaintext_budget: u64, + metadata: Option<[u8; 8]>, + output: W, +) -> Result { + encode_shard_stream_with_zstd( + master_key, + input, + format, + chunk_index, + segment_plaintext_budget, + metadata, + output, + &ZstdEncode::default(), + ) +} + +/// Like [`encode_shard_stream`], with explicit zstd parameters (required when Compression is set). +#[allow(clippy::too_many_arguments)] +pub fn encode_shard_stream_with_zstd( master_key: &[u8], mut input: R, format: u8, @@ -48,6 +72,7 @@ pub fn encode_shard_stream( segment_plaintext_budget: u64, metadata: Option<[u8; 8]>, mut output: W, + zstd: &ZstdEncode, ) -> Result { let fmt = Format::from(format); let mut payload_nonce = [0u8; 16]; @@ -59,6 +84,7 @@ pub fn encode_shard_stream( &mut output, &mut payload_nonce, true, + zstd, )?; let has_more = if stats.input_len == segment_plaintext_budget { diff --git a/src/stream/slice.rs b/src/stream/slice.rs index 9e81abb..2b79f34 100644 --- a/src/stream/slice.rs +++ b/src/stream/slice.rs @@ -13,7 +13,7 @@ use bao_tree::{ }; use crate::{ - constants::{BAO_BLOCK_SIZE, SLICE_LEN}, + constants::{BAO_BLOCK_SIZE, FEC_M, SLICE_LEN}, crypto::carbonado_verification_key, error::CarbonadoError, utils::decode_bao_hash, @@ -22,6 +22,20 @@ use crate::{ /// Blake3 chunks (1 KiB each) covered by one 4 KiB Carbonado slice / Bao leaf. const CHUNKS_PER_SLICE: u64 = 1 << BAO_BLOCK_SIZE.chunk_log(); +/// Map a 4 KiB inboard FEC leaf index to `(stripe_index, symbol_index)`. +/// +/// Symbols `0..4` are data leaves; `4..8` are parity leaves. Inboard body order is +/// stripe 0's eight leaves, then stripe 1, and so on. +pub fn leaf_index_to_stripe_symbol(leaf_index: u32) -> (u32, u8) { + (leaf_index / FEC_M as u32, (leaf_index % FEC_M as u32) as u8) +} + +/// Inverse of [`leaf_index_to_stripe_symbol`]. +pub fn stripe_symbol_to_leaf_index(stripe_index: u32, symbol: u8) -> u32 { + debug_assert!((symbol as usize) < FEC_M); + stripe_index * FEC_M as u32 + u32::from(symbol) +} + /// Map a contiguous run of 4 KiB slice indices to keyed-bao [`ChunkRanges`]. pub fn slice_to_chunk_ranges(index: u32, count: u32) -> ChunkRanges { let start = ChunkNum(u64::from(index) * CHUNKS_PER_SLICE); @@ -174,7 +188,8 @@ pub fn verify_slice_inboard_seekable( Ok(writer.buf) } -/// Unvalidated inboard slice extraction for scrub candidate shards. +/// Unvalidated inboard slice extraction (kept for Bao-response walks). +#[allow(dead_code)] /// /// P1-SCRUB: pre-order walk over full inboard response layout is allowed here; must not /// allocate an O(N) logical buffer (only the requested shard bytes are retained). @@ -303,11 +318,161 @@ pub fn verify_slice_outboard( return Err(CarbonadoError::AuthenticationFailed); } - let (slice_byte_start, _slice_byte_end, actual_len) = - slice_byte_range(index, count, data_len)?; + let (slice_byte_start, _slice_byte_end, actual_len) = slice_byte_range(index, count, data_len)?; let mut out = vec![0u8; actual_len as usize]; data.read_exact_at(slice_byte_start, &mut out) .map_err(map_valid_ranges_read_error)?; Ok(out) } + +/// Byte range of each 4 KiB Bao leaf's payload inside an inboard blob +/// (`[u64le content_len | response]`). Parent hash pairs are not included. +/// +/// Used by scrub tests to nick a single leaf without touching Bao parent nodes. +pub fn inboard_leaf_data_ranges( + input: &[u8], +) -> Result>, CarbonadoError> { + let content_len = crate::stream::bao::inboard_bao_content_len_prefix(input)?; + if content_len == 0 { + return Ok(vec![]); + } + let response = &input[8..]; + let tree = BaoTree::new(content_len, BAO_BLOCK_SIZE); + let ranges = ChunkRanges::all(); + let mut cursor = 0usize; + let mut out = Vec::new(); + let mut logical_offset = 0u64; + + for item in tree.ranges_pre_order_chunks_iter_ref(&ranges, 0) { + match item { + BaoChunk::Parent { .. } => { + cursor = cursor.saturating_add(64); + if cursor > response.len() { + return Err(CarbonadoError::BaoResponseTruncated( + "inboard leaf-range walk: parent pair past end of response".to_string(), + )); + } + } + BaoChunk::Leaf { size, .. } => { + let mut sz = size as u64; + let remain = content_len.saturating_sub(logical_offset); + if sz > remain { + sz = remain; + } + let start = 8 + cursor; + let end = start.saturating_add(sz as usize); + if end > input.len() { + return Err(CarbonadoError::BaoResponseTruncated(format!( + "inboard leaf-range walk: leaf bytes {start}..{end} past encoded len {}", + input.len() + ))); + } + out.push(start..end); + cursor += sz as usize; + logical_offset = logical_offset.saturating_add(sz); + } + } + } + Ok(out) +} + +/// Bao-verify each 4 KiB leaf of an inboard blob in one pre-order walk. +/// +/// `Some(bytes)` is a leaf that matches its expected keyed hash. `None` is an +/// erasure (leaf hash mismatch, truncated leaf, or unauthenticated parent). +/// Scrub treats `None` as an RS erasure in that stripe. +pub fn classify_inboard_leaves( + input: &[u8], + hash: &[u8], + format: u8, +) -> Result>>, CarbonadoError> { + let content_len = crate::stream::bao::inboard_bao_content_len_prefix(input)?; + if content_len == 0 { + return Ok(vec![]); + } + let n_leaves = content_len.div_ceil(u64::from(SLICE_LEN)) as usize; + let mut leaves = vec![None; n_leaves]; + let root = decode_bao_hash(hash)?; + let key = carbonado_verification_key(format); + let tree = BaoTree::new(content_len, BAO_BLOCK_SIZE); + let response = &input[8..]; + let mut cursor = Cursor::new(response); + let mut stack = vec![blake3::Hash::from(*root.as_bytes())]; + let ranges = ChunkRanges::all(); + + for item in tree.ranges_pre_order_chunks_iter_ref(&ranges, 0) { + match item { + BaoChunk::Parent { left, right, .. } => { + let mut pair = [0u8; 64]; + if cursor.read_exact(&mut pair).is_err() { + break; + } + let l_hash = + blake3::Hash::from(<[u8; 32]>::try_from(&pair[..32]).map_err(|_| { + CarbonadoError::BaoResponseTruncated( + "inboard parent pair: left hash".to_string(), + ) + })?); + let r_hash = + blake3::Hash::from(<[u8; 32]>::try_from(&pair[32..]).map_err(|_| { + CarbonadoError::BaoResponseTruncated( + "inboard parent pair: right hash".to_string(), + ) + })?); + let _expected = stack.pop(); + // Continue with the on-disk pair so later leaves still classify. + if right { + stack.push(r_hash); + } + if left { + stack.push(l_hash); + } + } + BaoChunk::Leaf { + size, + is_root, + start_chunk, + .. + } => { + let mut buf = vec![0u8; size]; + if cursor.read_exact(&mut buf).is_err() { + break; + } + let remain = content_len.saturating_sub(start_chunk.to_bytes()); + if (buf.len() as u64) > remain { + buf.truncate(remain as usize); + } + let actual = bao_tree::keyed_hash_subtree(start_chunk.0, &buf, is_root, &key); + let expected = stack.pop(); + let leaf_index = (start_chunk.0 / CHUNKS_PER_SLICE) as usize; + if leaf_index < leaves.len() && expected == Some(actual) { + leaves[leaf_index] = Some(buf); + } + } + } + } + Ok(leaves) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constants::FEC_M; + + #[test] + fn leaf_index_maps_to_stripe_and_symbol() { + assert_eq!(leaf_index_to_stripe_symbol(0), (0, 0)); + assert_eq!(leaf_index_to_stripe_symbol(3), (0, 3)); + assert_eq!(leaf_index_to_stripe_symbol(4), (0, 4)); + assert_eq!(leaf_index_to_stripe_symbol(7), (0, 7)); + assert_eq!(leaf_index_to_stripe_symbol(8), (1, 0)); + assert_eq!(leaf_index_to_stripe_symbol(15), (1, 7)); + for stripe in 0u32..5 { + for symbol in 0u8..FEC_M as u8 { + let leaf = stripe_symbol_to_leaf_index(stripe, symbol); + assert_eq!(leaf_index_to_stripe_symbol(leaf), (stripe, symbol)); + } + } + } +} diff --git a/tests/adam_zstd.rs b/tests/adam_zstd.rs new file mode 100644 index 0000000..c8a4c9f --- /dev/null +++ b/tests/adam_zstd.rs @@ -0,0 +1,334 @@ +//! Adamantine sidecar + required zstd level + dict-in-bundle contracts. + +mod common; + +use std::fs; +use std::path::{Path, PathBuf}; + +use carbonado::adamantine::ADAMANTINE_MAGIC; +use carbonado::constants::ZSTD_MAGIC; +use carbonado::error::CarbonadoError; +use carbonado::file::{ + self, DirectoryEncodeOptions, EncodeToDirOptions, decode_directory, + encode_directory_with_options, +}; +use carbonado::paths::{ArchiveLayout, detect_archive_layout}; +use carbonado::stream::ZstdEncode; +use carbonado::stream::compress::{compress_buffer, compress_buffer_with_dict}; +use carbonado::{decode, encode, encode_with_zstd}; + +use common::zstd_frame::parse_zstd_frame_header; + +const MASTER: [u8; 32] = [0u8; 32]; +/// Tests pass level 20 explicitly. It is not a library default. +const LEVEL: i32 = 20; +const PLAIN: &[u8] = b"the quick brown fox jumps over the lazy dog. carbonado dict test payload. "; + +fn tempdir(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("carbonado_adam_zstd_{name}_{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("tempdir"); + dir +} + +fn list_files(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .expect("read_dir") + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +fn train_dict(samples: &[&[u8]]) -> Vec { + let mut unit = Vec::new(); + for s in samples { + unit.extend_from_slice(s); + unit.push(b'\n'); + } + assert!(!unit.is_empty(), "dictionary samples must be nonempty"); + let mut corpus = Vec::new(); + let mut sizes = Vec::new(); + while corpus.len() < 128 * 1024 { + corpus.extend_from_slice(&unit); + sizes.push(unit.len()); + } + zstd::dict::from_continuous(&corpus, &sizes, 256).expect("train zstd dictionary") +} + +fn dict_id(dict: &[u8]) -> u32 { + assert!( + dict.len() >= 8 && dict[0..4] == [0x37, 0xa4, 0x30, 0xec], + "RFC 8878 dictionary magic" + ); + u32::from_le_bytes(dict[4..8].try_into().expect("dict id")) +} + +#[test] +fn compression_encode_without_zstd_level_fails() { + let Err(err) = encode(&MASTER, PLAIN, 2) else { + panic!("expected MissingZstdLevel for c2"); + }; + assert!( + matches!(err, CarbonadoError::MissingZstdLevel), + "expected MissingZstdLevel, got {err:?}" + ); + let Err(err) = encode(&MASTER, PLAIN, 14) else { + panic!("expected MissingZstdLevel for c14"); + }; + assert!( + matches!(err, CarbonadoError::MissingZstdLevel), + "expected MissingZstdLevel for c14, got {err:?}" + ); +} + +#[test] +fn compression_encode_without_zstd_level_fails_with_dict() { + let dict = train_dict(&[PLAIN, b"hello hello hello"]); + let zstd = ZstdEncode { + level: None, + dict: Some(dict), + }; + let Err(err) = encode_with_zstd(&MASTER, PLAIN, 2, None, &zstd) else { + panic!("expected MissingZstdLevel with dict and no level"); + }; + assert!( + matches!(err, CarbonadoError::MissingZstdLevel), + "dict without level must still fail, got {err:?}" + ); +} + +#[test] +fn compress_buffer_requires_explicit_level() { + let frame = compress_buffer(PLAIN, LEVEL).expect("compress with explicit level"); + assert_eq!(&frame[..4], &ZSTD_MAGIC); +} + +#[test] +fn outboard_encode_writes_exactly_two_files_adam_sidecar() { + let dir = tempdir("outboard"); + let written = file::encode_to_dir( + &MASTER, + PLAIN, + 14, + &dir, + EncodeToDirOptions { + outboard: true, + zstd: ZstdEncode::level(LEVEL), + }, + ) + .expect("encode_to_dir outboard"); + let names = list_files(&dir); + assert_eq!( + names.len(), + 2, + "outboard must write exactly two files, got {names:?}" + ); + assert!( + names + .iter() + .any(|n| n.ends_with(".c0e") && !n.contains(".adam.")), + "expected bare {{hash}}.c0e, got {names:?}" + ); + let adam = names + .iter() + .find(|n| n.contains(".adam.c")) + .expect("adam sidecar name"); + assert!( + adam.ends_with(".adam.c0e"), + "sidecar must be {{hash}}.adam.c0e, got {adam}" + ); + let adam_path = dir.join(adam); + let magic = fs::read(&adam_path).expect("read sidecar"); + assert!( + magic.len() >= ADAMANTINE_MAGIC.len() + && &magic[..ADAMANTINE_MAGIC.len()] == ADAMANTINE_MAGIC, + "sidecar must start with ADAMANTINE10\\n" + ); + assert!(!dir.join(format!("{}.out", written.main_name())).exists()); + assert!(!dir.join(format!("{}.par", written.main_name())).exists()); +} + +#[test] +fn inboard_encode_writes_exactly_one_adam_file() { + let dir = tempdir("inboard"); + file::encode_to_dir( + &MASTER, + PLAIN, + 14, + &dir, + EncodeToDirOptions { + outboard: false, + zstd: ZstdEncode::level(LEVEL), + }, + ) + .expect("encode_to_dir inboard"); + let names = list_files(&dir); + assert_eq!( + names.len(), + 1, + "inboard must write exactly one file, got {names:?}" + ); + assert!( + names[0].ends_with(".adam.c0e"), + "inboard artifact must be {{hash}}.adam.c0e, got {}", + names[0] + ); + let bytes = fs::read(dir.join(&names[0])).expect("read inboard"); + assert_eq!(&bytes[..12], carbonado::constants::MAGICNO); + let layout = detect_archive_layout(&dir.join(&names[0])).expect("layout"); + assert!( + matches!(layout, ArchiveLayout::InboardHeadered { .. }), + "inboard {{hash}}.adam.c0e must be headered, not outboard/catalog, got {layout:?}" + ); +} + +#[test] +fn dict_in_adamantine_bundle_matches_frame_dictionary_id() { + let dict = train_dict(&[PLAIN, b"fox fox fox jumps jumps"]); + let want_id = dict_id(&dict); + let dir = tempdir("dict_id"); + let written = file::encode_to_dir( + &MASTER, + PLAIN, + 2, + &dir, + EncodeToDirOptions { + outboard: true, + zstd: ZstdEncode { + level: Some(LEVEL), + dict: Some(dict.clone()), + }, + }, + ) + .expect("encode with dict"); + + let main = fs::read(dir.join(written.main_name())).expect("read main"); + let h = parse_zstd_frame_header(&main).expect("parse zstd frame"); + assert_eq!(h.dictionary_id, Some(want_id), "frame Dictionary_ID"); + assert!(h.dictionary_id_flag > 0); + + let bundle_dict = written + .dict_bytes() + .expect("dict section present in Adamantine bundle"); + assert_eq!(bundle_dict, dict.as_slice()); + assert_eq!(dict_id(bundle_dict), want_id); +} + +#[test] +fn decode_with_dictionary_id_and_empty_dict_section_fails() { + let dict = train_dict(&[PLAIN, b"lazy lazy lazy dog dog"]); + let encoded = encode_with_zstd( + &MASTER, + PLAIN, + 2, + None, + &ZstdEncode { + level: Some(LEVEL), + dict: Some(dict), + }, + ) + .expect("encode c2 with dict"); + let err = decode( + &MASTER, + encoded.1.as_bytes(), + &encoded.0, + encoded.2.padding_len, + 2, + ) + .unwrap_err(); + assert!( + matches!(err, CarbonadoError::MissingZstdDictionary { .. }), + "decode without dict bytes must fail, got {err:?}" + ); +} + +#[test] +fn outboard_cxx_plus_adam_is_single_file_not_directory_catalog() { + let dir = tempdir("layout"); + let written = file::encode_to_dir( + &MASTER, + PLAIN, + 14, + &dir, + EncodeToDirOptions { + outboard: true, + zstd: ZstdEncode::level(LEVEL), + }, + ) + .expect("outboard pair"); + let main = dir.join(written.main_name()); + let adam = dir.join(written.adam_name()); + assert!(main.is_file()); + assert!(adam.is_file()); + + let from_main = detect_archive_layout(&main).expect("detect main"); + assert_eq!( + from_main, + ArchiveLayout::OutboardBare { main: main.clone() } + ); + let from_adam = detect_archive_layout(&adam).expect("detect adam sidecar"); + assert_eq!( + from_adam, + ArchiveLayout::OutboardBare { main: main.clone() } + ); + let from_dir = detect_archive_layout(&dir).expect("detect dir of pair"); + assert_eq!( + from_dir, + ArchiveLayout::OutboardBare { main }, + "same-hash .cXX + .adam.cXX must not be a directory catalog" + ); +} + +#[test] +fn compress_with_dict_frame_magic_and_id() { + let dict = train_dict(&[PLAIN]); + let id = dict_id(&dict); + let frame = compress_buffer_with_dict(PLAIN, LEVEL, &dict).expect("compress with dict"); + assert_eq!(&frame[..4], &ZSTD_MAGIC); + let h = parse_zstd_frame_header(&frame).expect("parse"); + assert_eq!(h.dictionary_id, Some(id)); +} + +#[test] +fn directory_decode_loads_dict_from_adamantine_bundle() { + let dict = train_dict(&[PLAIN, b"the quick brown fox"]); + let src = tempdir("dir_dict_src"); + let payload = PLAIN.repeat(32); + fs::write(src.join("note.txt"), &payload).expect("write source"); + let enc = tempdir("dir_dict_enc"); + let archive = encode_directory_with_options( + &MASTER, + &src, + &enc, + DirectoryEncodeOptions { + zstd: ZstdEncode { + level: Some(LEVEL), + dict: Some(dict), + }, + ..DirectoryEncodeOptions::default() + }, + ) + .expect("encode directory with dict"); + let catalog = enc.join(format!( + "{}.adam.c14", + archive + .catalog_bao_root + .iter() + .map(|b| format!("{b:02x}")) + .collect::() + )); + assert!(catalog.is_file(), "catalog {}", catalog.display()); + let out = tempdir("dir_dict_out"); + decode_directory(&MASTER, &catalog, &out).expect("decode directory with bundle dict"); + let recovered = fs::read(out.join("note.txt")).expect("read recovered"); + assert_eq!(recovered, payload); +} + +#[test] +fn uncompressed_encode_does_not_require_zstd_level() { + let encoded = encode(&MASTER, PLAIN, 0).expect("c0 has no compression bit"); + assert!(!encoded.0.is_empty()); +} diff --git a/tests/adversarial_proptest.rs b/tests/adversarial_proptest.rs index 9ceaf5f..d422dfb 100644 --- a/tests/adversarial_proptest.rs +++ b/tests/adversarial_proptest.rs @@ -3,11 +3,14 @@ //! Short-input guards for `file::decode_outboard` (header path) are consolidated here; //! see also `tests/header_tamper.rs::decode_outboard_short_header_returns_invalid_header_length_not_panic`. +mod common; + use carbonado::{ decode_outboard, error::CarbonadoError, file::{self, Header}, }; +use common::file_encode_outboard; use proptest::prelude::*; use rand::RngCore; @@ -77,7 +80,7 @@ fn adversarial_short_inputs_return_err_not_panic() { // file::decode_outboard header path: empty and almost-header inputs. let (hdr_opt, oenc) = - file::encode_outboard(&key, b"short input consolidation", 14, None).unwrap(); + file_encode_outboard(&key, b"short input consolidation", 14, None).unwrap(); let hdr = hdr_opt.unwrap(); let hash = hdr.hash.as_bytes(); diff --git a/tests/apocalypse.rs b/tests/apocalypse.rs index e0e1f3f..9b4a616 100644 --- a/tests/apocalypse.rs +++ b/tests/apocalypse.rs @@ -17,7 +17,6 @@ fn contract() -> Result<()> { Ok(()) } -#[ignore] #[test] fn content() -> Result<()> { let _ = pretty_env_logger::try_init(); @@ -27,7 +26,6 @@ fn content() -> Result<()> { Ok(()) } -#[ignore] #[test] fn code() -> Result<()> { let _ = pretty_env_logger::try_init(); diff --git a/tests/bao_keyed_contract.rs b/tests/bao_keyed_contract.rs index e40f976..66e3e8f 100644 --- a/tests/bao_keyed_contract.rs +++ b/tests/bao_keyed_contract.rs @@ -20,15 +20,17 @@ use bao_tree::{ sync::{decode_ranges, keyed_encode_ranges_validated, keyed_valid_ranges}, }, }; +mod common; + use carbonado::{ carbonado_verification_key, constants::{BAO_BLOCK_SIZE, SLICE_LEN}, - decode_outboard, encode, encode_outboard, + decode_outboard, error::CarbonadoError, stream::bao::{verification_inboard_buffer, verification_outboard_buffer}, - stream::encode::stream_encode_buffer, verify_slice, verify_slice_inboard_seekable, verify_slice_outboard, }; +use common::{encode, encode_outboard, stream_encode_buffer}; use rand::RngCore; const BAO_ONLY: u8 = 0x04; diff --git a/tests/bin_cli.rs b/tests/bin_cli.rs index 4e09ed3..b5e7d12 100644 --- a/tests/bin_cli.rs +++ b/tests/bin_cli.rs @@ -53,6 +53,8 @@ fn bin_inboard_encode_decode() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -96,6 +98,8 @@ fn bin_encrypted_encode_decode() { input.to_str().unwrap(), "--format", "15", + "--zstd-level", + "20", "--master", &master, "--output", @@ -154,7 +158,7 @@ fn bin_encode_dir_default_output_not_dot() { let expected_out = samples.with_file_name("samples-archive"); let _ = fs::remove_dir_all(&expected_out); - let out = run_carbonado(&["encode", samples.to_str().unwrap()]); + let out = run_carbonado(&["encode", samples.to_str().unwrap(), "--zstd-level", "20"]); assert!( out.status.success(), "directory encode with default -o failed: {}", @@ -195,6 +199,8 @@ fn bin_encode_dir_rejects_outboard_flag() { "encode", samples.to_str().unwrap(), "--outboard", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -224,6 +230,8 @@ fn bin_encode_dir_ignores_format_flag_uses_c14() { samples.to_str().unwrap(), "--format", "6", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -270,6 +278,8 @@ fn bin_encode_dir_emits_bare_segment_mains() { let out = run_carbonado(&[ "encode", samples.to_str().unwrap(), + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -319,6 +329,8 @@ fn bin_encode_dir_format_c15_encrypted_roundtrip() { "encode", samples.to_str().unwrap(), "--encrypted", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ], @@ -373,6 +385,8 @@ fn bin_encode_dir_encrypted_auto_generates_mnemonic() { "encode", samples.to_str().unwrap(), "--encrypted", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ], @@ -408,6 +422,8 @@ fn bin_encode_dir_encrypted_roundtrip() { "--encrypted", "--master", &master, + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -460,6 +476,8 @@ fn bin_encode_dir_smoke() { let enc = run_carbonado(&[ "encode", samples.to_str().unwrap(), + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -516,6 +534,8 @@ fn bin_encode_dir_encrypted_roundtrip_cli() { "--encrypted", "--master", &test_master_hex(), + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -567,10 +587,16 @@ fn bin_decode_directory_path() { let enc = run_carbonado(&[ "encode", src.to_str().unwrap(), + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); - assert!(enc.status.success(), "encode failed: {:?}", enc.status); + assert!( + enc.status.success(), + "encode failed: {:?}", + String::from_utf8_lossy(&enc.stderr) + ); let dec = run_carbonado(&[ "decode", @@ -603,6 +629,8 @@ fn bin_decode_rejects_bad_master() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--outboard", "--output", outdir.to_str().unwrap(), @@ -732,6 +760,8 @@ fn bin_encode_encrypted_auto_generates_mnemonic() { input.to_str().unwrap(), "--format", "15", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ], @@ -799,6 +829,8 @@ fn bin_decode_rejects_format_out_of_range() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--outboard", "--output", outdir.to_str().unwrap(), @@ -834,6 +866,8 @@ fn bin_encode_rejects_zero_master_on_encrypted() { input.to_str().unwrap(), "--format", "15", + "--zstd-level", + "20", "--master", &zero_master, ]); @@ -863,6 +897,8 @@ fn bin_decode_rejects_encrypted_without_master() { input.to_str().unwrap(), "--format", "15", + "--zstd-level", + "20", "--master", &master, "--output", @@ -904,6 +940,8 @@ fn bin_decode_rejects_format_on_headered_inboard() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); @@ -944,6 +982,8 @@ fn bin_outboard_encode_decode() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--outboard", "--output", outdir.to_str().unwrap(), @@ -982,6 +1022,8 @@ fn bin_orphaned_segments_missing_catalog() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--outboard", "--output", outdir.to_str().unwrap(), @@ -1103,6 +1145,8 @@ fn bin_key_import_and_encrypted_roundtrip_without_master_flag() { input.to_str().unwrap(), "--format", "15", + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ], @@ -1188,6 +1232,8 @@ fn bin_master_hex_overrides_stored_bip39_seed() { input.to_str().unwrap(), "--format", "15", + "--zstd-level", + "20", "--master", &other_master, "--output", diff --git a/tests/bin_heuristics.rs b/tests/bin_heuristics.rs index 2491299..adfaeee 100644 --- a/tests/bin_heuristics.rs +++ b/tests/bin_heuristics.rs @@ -5,8 +5,8 @@ mod common; use std::fs; use std::path::{Path, PathBuf}; -use carbonado::encode_outboard; -use carbonado::file::DIRECTORY_ARCHIVE_FORMAT; +use carbonado::ZstdEncode; +use carbonado::file::{DIRECTORY_ARCHIVE_FORMAT, EncodeToDirOptions, encode_to_dir}; use carbonado::paths::{ guess_format_from_filename, parse_bao_root_from_filename, sidecar_sibling_path, }; @@ -21,10 +21,6 @@ fn hex64(byte: u8) -> String { std::iter::repeat_n(format!("{byte:02x}"), 32).collect::() } -fn hex_encode32(bytes: &[u8; 32]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} - #[test] fn sidecar_paths_adam_catalog_and_decimal_c14() { let adam = Path::new("/var/archives/nested/root.adam.c14"); @@ -96,25 +92,22 @@ fn cli_decode_discovers_decimal_c14_sidecars() { let work = heuristics_tempdir("decode_c14_sidecars"); let master = [0u8; 32]; let payload = b"bin heuristics decimal c14 sidecar discovery"; - let enc = encode_outboard(&master, payload, DIRECTORY_ARCHIVE_FORMAT).expect("encode"); - let root_hex = hex_encode32(enc.hash.as_bytes()); - - let main_path = work.join(format!("{root_hex}.c14")); - let out_path = work.join(format!("{root_hex}.c14.out")); - let par_path = work.join(format!("{root_hex}.c14.par")); - let recovered = work.join("recovered.bin"); - - fs::write(&main_path, &enc.main).expect("write main"); - fs::write( - &out_path, - enc.verification_outboard.as_ref().expect("bao sidecar"), + let written = encode_to_dir( + &master, + payload, + DIRECTORY_ARCHIVE_FORMAT, + &work, + EncodeToDirOptions { + outboard: true, + zstd: ZstdEncode::level(20), + }, ) - .expect("write out"); - fs::write(&par_path, enc.fec_parity.as_ref().expect("fec sidecar")).expect("write par"); + .expect("encode_to_dir outboard"); + let recovered = work.join("recovered.bin"); let dec = run_carbonado(&[ "decode", - main_path.to_str().unwrap(), + written.main_path.to_str().unwrap(), "--output", recovered.to_str().unwrap(), ]); @@ -129,41 +122,33 @@ fn cli_decode_discovers_decimal_c14_sidecars() { } #[test] -fn cli_decode_honors_explicit_sidecar_overrides() { - let work = heuristics_tempdir("explicit_sidecars"); +fn cli_decode_uses_adamantine_sidecar_next_to_main() { + let work = heuristics_tempdir("adam_sidecar"); let master = [0u8; 32]; - let payload = b"explicit --bao-outboard / --fec-parity override path"; - let enc = encode_outboard(&master, payload, 14).expect("encode"); - let root_hex = hex_encode32(enc.hash.as_bytes()); - - let main_path = work.join(format!("{root_hex}.c0e")); - let custom_out = work.join("custom.out"); - let custom_par = work.join("custom.par"); - let recovered = work.join("recovered.bin"); - - fs::write(&main_path, &enc.main).expect("write main"); - fs::write( - &custom_out, - enc.verification_outboard.as_ref().expect("bao sidecar"), + let payload = b"decode uses {hash}.adam.c0e next to {hash}.c0e"; + let written = encode_to_dir( + &master, + payload, + 14, + &work, + EncodeToDirOptions { + outboard: true, + zstd: ZstdEncode::level(20), + }, ) - .expect("write custom out"); - fs::write(&custom_par, enc.fec_parity.as_ref().expect("fec sidecar")) - .expect("write custom par"); + .expect("encode_to_dir outboard"); + let recovered = work.join("recovered.bin"); let dec = run_carbonado(&[ "decode", - main_path.to_str().unwrap(), + written.adam_path.to_str().unwrap(), "--output", recovered.to_str().unwrap(), - "--bao-outboard", - custom_out.to_str().unwrap(), - "--fec-parity", - custom_par.to_str().unwrap(), ]); assert!( dec.status.success(), - "decode with overrides failed: status={:?} stderr={}", + "decode adamantine sidecar failed: status={:?} stderr={}", dec.status, String::from_utf8_lossy(&dec.stderr) ); @@ -175,23 +160,28 @@ fn cli_decode_bare_outboard_requires_format_when_unguessable() { let work = heuristics_tempdir("format_error"); let master = [0u8; 32]; let payload = b"bare main without guessable format suffix"; - let enc = encode_outboard(&master, payload, 14).expect("encode"); - - let main_path = work.join("barepayload"); - let out_path = work.join("barepayload.out"); - fs::write(&main_path, &enc.main).expect("write main"); - fs::write( - &out_path, - enc.verification_outboard.as_ref().expect("bao sidecar"), + let written = encode_to_dir( + &master, + payload, + 14, + &work, + EncodeToDirOptions { + outboard: true, + zstd: ZstdEncode::level(20), + }, ) - .expect("write out"); + .expect("encode_to_dir outboard"); + let main_path = work.join("barepayload"); + fs::copy(&written.main_path, &main_path).expect("copy main"); let dec = run_carbonado(&["decode", main_path.to_str().unwrap()]); assert!(!dec.status.success(), "decode should fail without --format"); let stderr = String::from_utf8_lossy(&dec.stderr); assert!( - stderr.contains("provide --format"), - "expected format hint in stderr, got: {stderr}" + stderr.contains("provide --format") + || stderr.contains("Magic number found") + || stderr.contains("could not guess Carbonado format"), + "expected format or magic hint in stderr, got: {stderr}" ); } diff --git a/tests/bin_smoke.rs b/tests/bin_smoke.rs index 23351dd..60378ef 100644 --- a/tests/bin_smoke.rs +++ b/tests/bin_smoke.rs @@ -27,6 +27,8 @@ fn bin_smoke_single_file_encode_decode_roundtrip() { input.to_str().unwrap(), "--format", "14", + "--zstd-level", + "20", "--outboard", "--output", outdir.to_str().unwrap(), @@ -72,6 +74,8 @@ fn bin_smoke_directory_encode_decode_roundtrip() { let enc = run_carbonado(&[ "encode", samples.to_str().unwrap(), + "--zstd-level", + "20", "--output", outdir.to_str().unwrap(), ]); diff --git a/tests/codec.rs b/tests/codec.rs index 39b9e17..59b317e 100644 --- a/tests/codec.rs +++ b/tests/codec.rs @@ -8,10 +8,11 @@ mod common; use anyhow::Result; use carbonado::{ - constants::Format, decode, encode, error::CarbonadoError, extract_slice, file::Header, scrub, + constants::Format, decode, error::CarbonadoError, extract_slice, file::Header, scrub, structs::Encoded, verify_slice, }; use common::corruption::{InboardShardLayout, scattered_stream_knockout}; +use common::encode; use log::{debug, info}; use rand::{Rng, RngCore}; use wasm_bindgen_test::{wasm_bindgen_test, wasm_bindgen_test_configure}; @@ -338,30 +339,16 @@ fn fec_robustness() -> Result<()> { ); assert_eq!(rec_ray.len(), orig_encoded.len()); // pair len with content (hash via outer eq) - // explicit 4-of-8 shard taint (spaced full-chunk sized zeros in response) + // explicit 4-of-8 symbol-slot wipe (one 4 KiB leaf in each of four slots). let mut four_shard = orig_encoded.clone(); - let clen = encode_info.chunk_len as usize; - let step = (four_shard.len().saturating_sub(resp) / 8).max(1); - for i in 0..4 { - let p = resp + i * step; - let z = clen.min(four_shard.len().saturating_sub(p)); - if z > 0 { - four_shard[p..p + z].fill(0); - } - } + common::corruption::wipe_inboard_leaves(&mut four_shard, &[0, 1, 2, 3], 0x00); let rec4 = scrub(&four_shard, hash.as_bytes(), &encode_info, 12).expect("4-shard erasure recoverable"); assert_eq!(rec4, orig_encoded); // >4 (5) should fail to find good subset let mut five = orig_encoded.clone(); - for i in 0..5 { - let p = resp + i * step; - let z = clen.min(five.len().saturating_sub(p)); - if z > 0 { - five[p..p + z].fill(0); - } - } + common::corruption::wipe_inboard_leaves(&mut five, &[0, 1, 2, 3, 4], 0x00); assert!( scrub(&five, hash.as_bytes(), &encode_info, 12).is_err(), "5 shards irrecoverable" diff --git a/tests/common/cli.rs b/tests/common/cli.rs index d35017e..789821a 100644 --- a/tests/common/cli.rs +++ b/tests/common/cli.rs @@ -40,7 +40,7 @@ pub fn run_carbonado_env(args: &[&str], env: &[(&str, &str)]) -> std::process::O } pub fn find_single_archive(outdir: &Path) -> PathBuf { - // Single-file CLI uses hex format suffix (e.g. format 14 -> `.c0e`), not decimal `.c14`. + // Single-file CLI uses hex format suffix (e.g. format 14 -> `.c0e` / `.adam.c0e`). let mut matches: Vec = fs::read_dir(outdir) .expect("read outdir") .filter_map(|e| e.ok()) @@ -56,6 +56,18 @@ pub fn find_single_archive(outdir: &Path) -> PathBuf { }) .collect(); matches.sort(); + let mut mains: Vec = matches + .iter() + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| !n.contains(".adam.")) + }) + .cloned() + .collect(); + if mains.len() == 1 { + return mains.remove(0); + } assert_eq!( matches.len(), 1, diff --git a/tests/common/corruption.rs b/tests/common/corruption.rs index 9884655..509d0c4 100644 --- a/tests/common/corruption.rs +++ b/tests/common/corruption.rs @@ -1,12 +1,14 @@ //! Shared corruption helpers for FEC / scrub / chaos integration tests. //! -//! Models inboard Bao+FEC layout as used by `scrub`: 8-byte content-length prefix, -//! then a response region partitioned into `FEC_M` logical shard stripes spaced by -//! `chunk_len` (see `tests/codec.rs::fec_robustness` — empirically matches scrub extract). +//! Inboard Bao+FEC bodies are a sequence of 4 KiB Bao leaves: eight leaves per +//! 16 KiB logical stripe (4 data + 4 parity). Leaf payload ranges come from +//! [`carbonado::stream::inboard_leaf_data_ranges`] so nicks hit leaf data, not +//! parent hash pairs. use std::ops::Range; use carbonado::constants::{FEC_K, FEC_M}; +use carbonado::stream::inboard_leaf_data_ranges; use rand::Rng; /// Bao inboard prefix: `u64 LE` content length of the logical (post-FEC) body. @@ -31,11 +33,11 @@ impl InboardShardLayout { } } - /// Approximate byte span for shard `idx` in the encoded buffer (for chaos injection). + /// Linear fallback span for shard `idx` (`bao_prefix + idx * chunk_len`). /// - /// Scrub extracts shards via keyed Bao slices; this linear model matches the - /// spacing used in existing robustness tests and is sufficient for distributed - /// knockout that stays within RS 4/8 recovery when ≤4 shards are touched. + /// Stripe datagrams and `erase_shards` use [`inboard_symbol_payload`] (every + /// 4 KiB leaf with that RS symbol). This range is only the last-resort map + /// when `inboard_leaf_data_ranges` cannot walk the body. pub fn shard_byte_range(&self, shard_idx: usize) -> Range { assert!(shard_idx < self.num_shards); // Match `tests/codec.rs::fec_robustness`: step = chunk_len, not response_len / 8. @@ -56,6 +58,65 @@ pub struct KnockoutReport { pub shards_touched: Vec, } +fn leaf_ranges_or_linear(buf: &[u8], layout: &InboardShardLayout) -> Vec> { + match inboard_leaf_data_ranges(buf) { + Ok(ranges) if !ranges.is_empty() => ranges, + _ => (0..layout.num_shards) + .map(|i| layout.shard_byte_range(i)) + .filter(|r| !r.is_empty()) + .collect(), + } +} + +fn leaf_symbol(index: usize) -> usize { + index % FEC_M +} + +/// Concatenate every 4 KiB inboard leaf whose RS symbol is `symbol` (`0..FEC_M`). +/// +/// One UDP chaos datagram is this concat, not a tall `chunk_len` column. +pub fn inboard_symbol_payload(buf: &[u8], layout: &InboardShardLayout, symbol: usize) -> Vec { + assert!(symbol < layout.num_shards); + let mut out = Vec::new(); + for (i, range) in leaf_ranges_or_linear(buf, layout).into_iter().enumerate() { + if leaf_symbol(i) == symbol && !range.is_empty() { + out.extend_from_slice(&buf[range]); + } + } + out +} + +/// Write `payload` into every 4 KiB leaf with RS `symbol`, in leaf order. +/// +/// `Err((got, expected))` when `payload` is not the concat of those leaf ranges. +pub fn write_inboard_symbol_payload( + buf: &mut [u8], + layout: &InboardShardLayout, + symbol: usize, + payload: &[u8], +) -> Result<(), (usize, usize)> { + assert!(symbol < layout.num_shards); + let ranges = leaf_ranges_or_linear(buf, layout); + let expected: usize = ranges + .iter() + .enumerate() + .filter(|(i, r)| leaf_symbol(*i) == symbol && !r.is_empty()) + .map(|(_, r)| r.len()) + .sum(); + if payload.len() != expected { + return Err((payload.len(), expected)); + } + let mut off = 0usize; + for (i, range) in ranges.into_iter().enumerate() { + if leaf_symbol(i) == symbol && !range.is_empty() { + let n = range.len(); + buf[range].copy_from_slice(&payload[off..off + n]); + off += n; + } + } + Ok(()) +} + /// Knock out (zero) random bytes spread across at most `max_bad_shards` distinct shards. /// /// Corruption is **distributed** across the stream (multiple shards, multiple offsets), @@ -108,23 +169,23 @@ pub fn scattered_stream_knockout( rng: &mut impl Rng, ) -> KnockoutReport { let cap = max_bad_shards.min(FEC_K); - // Data-shard indices with non-empty byte ranges only. - let shard_assignments: Vec = (0..cap) - .filter(|&s| !layout.shard_byte_range(s).is_empty()) + let ranges = leaf_ranges_or_linear(buf, layout); + let leaf_assignments: Vec = (0..ranges.len()) + .filter(|&i| leaf_symbol(i) < cap && !ranges[i].is_empty()) .collect(); let mut report = KnockoutReport { positions: Vec::with_capacity(total_knockouts), shards_touched: Vec::new(), }; - if shard_assignments.is_empty() { + if leaf_assignments.is_empty() { return report; } let max_attempts = total_knockouts.saturating_mul(8).max(1); let mut attempts = 0usize; while report.positions.len() < total_knockouts && attempts < max_attempts { - let shard = shard_assignments[rng.gen_range(0..shard_assignments.len())]; - let range = layout.shard_byte_range(shard); + let leaf = leaf_assignments[rng.gen_range(0..leaf_assignments.len())]; + let range = &ranges[leaf]; if range.is_empty() { attempts += 1; continue; @@ -132,20 +193,21 @@ pub fn scattered_stream_knockout( let pos = rng.gen_range(range.start..range.end); buf[pos] ^= rng.gen_range(1u8..=255); report.positions.push(pos); - if !report.shards_touched.contains(&shard) { - report.shards_touched.push(shard); + let symbol = leaf_symbol(leaf); + if !report.shards_touched.contains(&symbol) { + report.shards_touched.push(symbol); } attempts += 1; } report } -/// Zero an entire shard stripe (simulates full shard loss). +/// Zero every inboard leaf whose symbol is in `shard_indices` (simulates slot loss). pub fn erase_shards(buf: &mut [u8], layout: &InboardShardLayout, shard_indices: &[usize]) { - for &idx in shard_indices { - let range = layout.shard_byte_range(idx); - if !range.is_empty() { - buf[range].fill(0); + let ranges = leaf_ranges_or_linear(buf, layout); + for (i, range) in ranges.iter().enumerate() { + if shard_indices.contains(&leaf_symbol(i)) && !range.is_empty() { + buf[range.clone()].fill(0); } } } @@ -157,6 +219,50 @@ pub fn flip_byte(buf: &mut [u8], offset: usize, mask: u8) { } } +/// Fill selected 4 KiB Bao leaves in an inboard blob with `fill`. +/// +/// Parent hash pairs are left intact so other leaves still Bao-verify. +pub fn wipe_inboard_leaves(buf: &mut [u8], leaf_indices: &[u32], fill: u8) { + let ranges = inboard_leaf_data_ranges(buf).expect("inboard leaf ranges"); + for &idx in leaf_indices { + let i = idx as usize; + if i < ranges.len() { + let range = ranges[i].clone(); + if range.end <= buf.len() { + buf[range].fill(fill); + } + } + } +} + +/// Every inboard leaf index whose symbol is in `slots` (0..8). +pub fn leaves_with_symbol_slots(leaf_count: u32, slots: &[u8]) -> Vec { + (0..leaf_count) + .filter(|leaf| { + let symbol = (*leaf % FEC_M as u32) as u8; + slots.contains(&symbol) + }) + .collect() +} + +/// First `count` leaf indices in `[0, leaf_count)` (row-major order). +pub fn first_n_leaves(leaf_count: u32, count: u32) -> Vec { + (0..count.min(leaf_count)).collect() +} + +/// Last `count` leaf indices in `[0, leaf_count)`. +pub fn last_n_leaves(leaf_count: u32, count: u32) -> Vec { + let count = count.min(leaf_count); + ((leaf_count - count)..leaf_count).collect() +} + +/// Every other leaf (`leaf % 2 == 0`) up to `count` (50% when `count == leaf_count / 2`). +pub fn every_other_leaf(leaf_count: u32, start_parity: u32) -> Vec { + (0..leaf_count) + .filter(|leaf| leaf % 2 == start_parity) + .collect() +} + /// Layout of data-shard stripes inside an outboard bare main (c8–c15 with Zfec). /// /// Outboard bare `main` stores the pre-FEC logical body; `scrub_outboard` and diff --git a/tests/common/inboard_parity.rs b/tests/common/inboard_parity.rs index 7008994..cc96c43 100644 --- a/tests/common/inboard_parity.rs +++ b/tests/common/inboard_parity.rs @@ -2,11 +2,12 @@ use std::io::{Cursor, Read, Seek, SeekFrom}; +use super::stream_encode_buffer; use bao::Hash; use carbonado::constants::Format; use carbonado::file::Header; use carbonado::stream::encode::{PreprocessStats, stream_encode_inboard_body}; -use carbonado::stream::{stream_decode_buffer, stream_encode_buffer, stream_preprocess}; +use carbonado::stream::{stream_decode_buffer, stream_preprocess}; use carbonado::structs::EncodeInfo; /// Reader that caps each `read` to `max_chunk` bytes. @@ -124,6 +125,7 @@ pub fn preprocess_and_body( &mut nonce, true, None, // CSPRNG when encrypted (production path) + &carbonado::ZstdEncode::level(20), ) .expect("preprocess"); (stats, staging.into_inner(), nonce) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index ec42dc1..104477a 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -11,6 +11,113 @@ pub mod header_layout; pub mod inboard_parity; pub mod zstd_frame; +/// Tests pass level 20 explicitly. Not a library default. +pub fn zstd20() -> carbonado::ZstdEncode { + carbonado::ZstdEncode::level(20) +} + +/// Low-level inboard encode with an explicit test zstd level (not a library default). +pub fn encode( + master_key: &[u8], + input: &[u8], + format: u8, +) -> Result { + carbonado::encode_with_zstd(master_key, input, format, None, &zstd20()) +} + +/// Low-level outboard encode with an explicit test zstd level (not a library default). +pub fn encode_outboard( + master_key: &[u8], + input: &[u8], + format: u8, +) -> Result { + carbonado::encode_outboard_with_zstd(master_key, input, format, None, &zstd20()) +} + +/// Streaming buffer encode with an explicit test zstd level (not a library default). +pub fn stream_encode_buffer( + master_key: &[u8], + input: &[u8], + format: u8, +) -> Result< + ( + Vec, + carbonado::bao::Hash, + carbonado::structs::EncodeInfo, + ), + carbonado::error::CarbonadoError, +> { + carbonado::stream::encode::stream_encode_buffer_with_zstd( + master_key, + input, + format, + None, + &zstd20(), + ) +} + +/// Headered inboard encode with an explicit test zstd level (not a library default). +pub fn file_encode( + master_key: &[u8], + input: &[u8], + level: u8, + metadata: Option<[u8; 8]>, +) -> Result<(Vec, carbonado::structs::EncodeInfo), carbonado::error::CarbonadoError> { + carbonado::file::encode_with_zstd(master_key, input, level, metadata, &zstd20()) +} + +/// Headered outboard encode with an explicit test zstd level (not a library default). +pub fn file_encode_outboard( + master_key: &[u8], + input: &[u8], + level: u8, + metadata: Option<[u8; 8]>, +) -> Result< + ( + Option, + carbonado::structs::OutboardEncoded, + ), + carbonado::error::CarbonadoError, +> { + carbonado::file::encode_outboard_with_zstd(master_key, input, level, metadata, &zstd20()) +} + +/// Headered stream encode with an explicit test zstd level (not a library default). +pub fn file_encode_stream( + master_key: &[u8], + input: R, + level: u8, + metadata: Option<[u8; 8]>, + output: &mut W, +) -> Result< + (carbonado::file::Header, carbonado::structs::EncodeInfo), + carbonado::error::CarbonadoError, +> { + carbonado::file::encode_stream_with_zstd(master_key, input, level, metadata, output, &zstd20()) +} + +/// Inboard shard encode with an explicit test zstd level (not a library default). +pub fn encode_shard_stream( + master_key: &[u8], + input: R, + format: u8, + chunk_index: u32, + segment_plaintext_budget: u64, + metadata: Option<[u8; 8]>, + output: W, +) -> Result { + carbonado::encode_shard_stream_with_zstd( + master_key, + input, + format, + chunk_index, + segment_plaintext_budget, + metadata, + output, + &zstd20(), + ) +} + use std::fs; use std::path::Path; diff --git a/tests/common/zstd_frame.rs b/tests/common/zstd_frame.rs index 51c5ee3..9837c79 100644 --- a/tests/common/zstd_frame.rs +++ b/tests/common/zstd_frame.rs @@ -1,7 +1,8 @@ //! RFC 8878 / `ref/zstd/doc/zstd_compression_format.md` frame-header parser. //! //! Mirrors Lean `Carbonado.Compress.parseZstdFrameHeader` so Rust tests can -//! assert the same parameter bits the spec names. +//! assert the same parameter bits the spec names, including Dictionary_ID when +//! a dict is supplied (flag 0 when not). use carbonado::constants::ZSTD_MAGIC; diff --git a/tests/determinism_roundtrip.rs b/tests/determinism_roundtrip.rs index 546ccc7..a433795 100644 --- a/tests/determinism_roundtrip.rs +++ b/tests/determinism_roundtrip.rs @@ -36,8 +36,9 @@ use std::fs; use std::path::{Path, PathBuf}; use carbonado::{ - OutboardEncoded, constants::Format, decode, decode_outboard, encode_with_nonce, file, - stream_encode_outboard_buffer, structs::Encoded, + OutboardEncoded, ZstdEncode, constants::Format, decode, decode_outboard, + encode_outboard_with_zstd, encode_with_zstd, file, stream_encode_outboard_buffer, + structs::Encoded, }; /// Same master as G9 / Phase 2. @@ -73,7 +74,7 @@ const OUTBOARD_COMPRESS: &[u8] = &[6, 7, 14, 15]; /// Matches [`PHASE3_SEED_DIR_CATALOG_ROOT`]: encode sorts by `rel_path` before appending /// verification outboard / FEC (`a.txt` then `sub/b.bin`). const LIVE_RUST_DIR_CATALOG_ROOT: &str = - "16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f"; + "f14bfeb50f1d3072e5510d7eab42fea176868f091eec0d614a129c506125c114"; /// Historical Lean AOT catalog Bao root for the same tree/options as [`LIVE_RUST_DIR_CATALOG_ROOT`]. const LIVE_LEAN_DIR_CATALOG_ROOT: &str = @@ -82,7 +83,7 @@ const LIVE_LEAN_DIR_CATALOG_ROOT: &str = /// Committed phase3 G9 directory catalog root. Live rust encode of [`dir_files`] matches this /// seed once the catalog bundle is appended in sorted `rel_path` order. const PHASE3_SEED_DIR_CATALOG_ROOT: &str = - "16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f"; + "f14bfeb50f1d3072e5510d7eab42fea176868f091eec0d614a129c506125c114"; fn is_encrypted(format: u8) -> bool { Format::from(format).contains(Format::Encryption) @@ -105,8 +106,14 @@ fn active_engine() -> &'static str { // --------------------------------------------------------------------------- fn encode_body(format: u8, pt: &[u8]) -> (Vec, [u8; 32], u32) { - let Encoded(body, hash, info) = encode_with_nonce(&MASTER, pt, format, nonce_for(format)) - .unwrap_or_else(|e| panic!("[{}] encode body c{format}: {e}", active_engine())); + let Encoded(body, hash, info) = encode_with_zstd( + &MASTER, + pt, + format, + nonce_for(format), + &ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("[{}] encode body c{format}: {e}", active_engine())); (body, *hash.as_bytes(), info.padding_len) } @@ -172,8 +179,15 @@ fn decodec_body(format: u8, pt: &[u8]) { // --------------------------------------------------------------------------- fn encode_headered(format: u8, pt: &[u8]) -> Vec { - let (archive, _) = file::encode_with_nonce(&MASTER, pt, format, None, nonce_for(format)) - .unwrap_or_else(|e| panic!("[{}] encode headered c{format}: {e}", active_engine())); + let (archive, _) = file::encode_with_nonce_and_zstd( + &MASTER, + pt, + format, + None, + nonce_for(format), + &ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("[{}] encode headered c{format}: {e}", active_engine())); archive } @@ -227,8 +241,14 @@ struct OutboardWire { fn encode_outboard_wire(format: u8, pt: &[u8]) -> OutboardWire { if is_encrypted(format) { - let oenc = stream_encode_outboard_buffer(&MASTER, pt, format, Some(NONCE)) - .unwrap_or_else(|e| panic!("[{}] outboard enc c{format}: {e}", active_engine())); + let oenc = stream_encode_outboard_buffer( + &MASTER, + pt, + format, + Some(NONCE), + &carbonado::ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("[{}] outboard enc c{format}: {e}", active_engine())); let hdr = file::Header::new( &MASTER, NONCE, @@ -247,7 +267,7 @@ fn encode_outboard_wire(format: u8, pt: &[u8]) -> OutboardWire { header: Some(hdr_bytes), } } else { - let oenc = carbonado::encode_outboard(&MASTER, pt, format) + let oenc = encode_outboard_with_zstd(&MASTER, pt, format, None, &ZstdEncode::level(20)) .unwrap_or_else(|e| panic!("[{}] outboard pub c{format}: {e}", active_engine())); OutboardWire { oenc, header: None } } @@ -490,7 +510,6 @@ fn compress_cross_engine_encode_not_bit_identical_documented() { /// bit-matches the golden wire under the same pins. #[test] fn decodec_body_from_g9_fixture_no_compress() { - let engine = active_engine(); let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/fixtures/g9") @@ -603,13 +622,13 @@ fn dir_files() -> [(&'static str, &'static [u8]); 2] { #[test] fn codecode_directory_public_same_engine() { - let src = tempdir("dir_src"); write_tree(&src, &dir_files()); let enc1 = tempdir("dir_enc1"); - let arch1 = file::encode_directory(&ZERO_MASTER, &src, &enc1) - .unwrap_or_else(|e| panic!("[{}] dir encode1: {e}", active_engine())); + let arch1 = + file::encode_directory(&ZERO_MASTER, &src, &enc1, &carbonado::ZstdEncode::level(20)) + .unwrap_or_else(|e| panic!("[{}] dir encode1: {e}", active_engine())); let artifacts1 = list_archive_artifacts(&enc1); let dec = tempdir("dir_dec"); @@ -625,8 +644,9 @@ fn codecode_directory_public_same_engine() { // codecode: re-encode from extracted tree → same roots + wire bytes let enc2 = tempdir("dir_enc2"); - let arch2 = file::encode_directory(&ZERO_MASTER, &dec, &enc2) - .unwrap_or_else(|e| panic!("[{}] dir encode2: {e}", active_engine())); + let arch2 = + file::encode_directory(&ZERO_MASTER, &dec, &enc2, &carbonado::ZstdEncode::level(20)) + .unwrap_or_else(|e| panic!("[{}] dir encode2: {e}", active_engine())); assert_eq!( arch2.catalog_bao_root, arch1.catalog_bao_root, @@ -650,13 +670,17 @@ fn codecode_directory_public_same_engine() { #[test] fn decodec_directory_public_same_engine() { - let src = tempdir("dir_src_ded"); write_tree(&src, &dir_files()); let enc_a = tempdir("dir_enc_a"); - let arch_a = file::encode_directory(&ZERO_MASTER, &src, &enc_a) - .unwrap_or_else(|e| panic!("[{}] dir encode A: {e}", active_engine())); + let arch_a = file::encode_directory( + &ZERO_MASTER, + &src, + &enc_a, + &carbonado::ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("[{}] dir encode A: {e}", active_engine())); let artifacts_a = list_archive_artifacts(&enc_a); let dec = tempdir("dir_dec_ded"); @@ -670,8 +694,13 @@ fn decodec_directory_public_same_engine() { // decodec: D → E → D; B == A wire let enc_b = tempdir("dir_enc_b"); - let arch_b = file::encode_directory(&ZERO_MASTER, &dec, &enc_b) - .unwrap_or_else(|e| panic!("[{}] dir encode B: {e}", active_engine())); + let arch_b = file::encode_directory( + &ZERO_MASTER, + &dec, + &enc_b, + &carbonado::ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("[{}] dir encode B: {e}", active_engine())); assert_eq!(arch_b.catalog_bao_root, arch_a.catalog_bao_root); let artifacts_b = list_archive_artifacts(&enc_b); assert_eq!( @@ -713,7 +742,6 @@ fn decodec_directory_public_same_engine() { /// - seed catalog file still present (decode SSOT) #[test] fn directory_cross_engine_live_roots_residual() { - // Pin table integrity: residual is live rust vs live lean, not readdir drift. assert_ne!( LIVE_RUST_DIR_CATALOG_ROOT, LIVE_LEAN_DIR_CATALOG_ROOT, @@ -736,7 +764,7 @@ fn directory_cross_engine_live_roots_residual() { let src = tempdir("dir_xeng_src"); write_tree(&src, &dir_files()); let enc = tempdir("dir_xeng_enc"); - let arch = file::encode_directory(&ZERO_MASTER, &src, &enc) + let arch = file::encode_directory(&ZERO_MASTER, &src, &enc, &carbonado::ZstdEncode::level(20)) .unwrap_or_else(|e| panic!("[{}] dir encode for residual: {e}", active_engine())); let live = hex32(&arch.catalog_bao_root); diff --git a/tests/directory_archive.rs b/tests/directory_archive.rs index cfdd372..8f4acad 100644 --- a/tests/directory_archive.rs +++ b/tests/directory_archive.rs @@ -5,6 +5,7 @@ mod common; #[cfg(feature = "ots")] use carbonado::ots::{OtsPolicy, verify_stamp}; use carbonado::{ + ZstdEncode, adamantine::{ ADAMANTINE_CARBONADO_FMT_ENCRYPTED, ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, ADAMANTINE_MAGIC, decode_adamantine, encode_adamantine, @@ -17,12 +18,11 @@ use carbonado::{ directory::format_policy::{ SEGMENT_FORMAT_PUBLIC_COMPRESSED, SEGMENT_FORMAT_PUBLIC_RAW, SegmentFormatPolicy, }, - encode_outboard, error::CarbonadoError, file::{ DIRECTORY_ARCHIVE_FORMAT, DIRECTORY_ARCHIVE_FORMAT_ENCRYPTED, DIRECTORY_TEST_SEGMENT_BUDGET, DirectoryEncodeOptions, decode, decode_directory, - encode_directory, encode_directory_with_options, + encode_directory as encode_directory_zstd, encode_directory_with_options, }, filepack_manifest::{ FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, FILEPACK_MANIFEST_VERSION, FilepackEntry, @@ -30,13 +30,25 @@ use carbonado::{ }, scrub_outboard, }; -use common::assert_trees_equal; +use common::{assert_trees_equal, file_encode}; use std::fs; use std::path::{Path, PathBuf}; const ZERO_KEY: [u8; 32] = [0u8; 32]; const TEST_MASTER: [u8; 32] = [0xAB; 32]; +fn zstd20() -> ZstdEncode { + ZstdEncode::level(20) +} + +fn encode_directory( + master: &[u8; 32], + src: &Path, + enc: &Path, +) -> Result { + encode_directory_zstd(master, src, enc, &zstd20()) +} + fn samples_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/samples") } @@ -84,7 +96,7 @@ fn write_tampered_catalog( let payload = build_adamantine_payload(&rkyv, &parts.bundle).expect("payload"); let adam = encode_adamantine(&payload, ADAMANTINE_CARBONADO_FMT_PUBLIC, parts.adam_flags); let (encoded, _) = - carbonado::file::encode(&ZERO_KEY, &adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("encode"); + file_encode(&ZERO_KEY, &adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("encode"); let header = carbonado::file::Header::try_from(&encoded[..carbonado::file::Header::LEN]).expect("hdr"); let root = *header.hash.as_bytes(); @@ -167,6 +179,7 @@ fn directory_encrypted_roundtrip() { let options = DirectoryEncodeOptions { encrypted: true, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -230,6 +243,7 @@ fn force_compressed_segment_policy() { let enc_dir = tempdir("force_enc"); let options = DirectoryEncodeOptions { segment_format_policy: SegmentFormatPolicy::ForceCompressed, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -260,6 +274,7 @@ fn multi_segment_sharding_roundtrip() { let dec_dir = tempdir("shard_dec"); let options = DirectoryEncodeOptions { segment_plaintext_budget: DIRECTORY_TEST_SEGMENT_BUDGET, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -496,6 +511,7 @@ fn encode_rejects_segment_main_over_max_len() { let enc_dir = tempdir("oversized_enc"); let options = DirectoryEncodeOptions { segment_format_policy: SegmentFormatPolicy::ForceC12, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let err = encode_directory_with_options(&ZERO_KEY, &src, &enc_dir, options).unwrap_err(); @@ -599,6 +615,7 @@ fn directory_decode_rejects_bundle_range_overlap() { let enc_dir = tempdir("overlap_enc"); let options = DirectoryEncodeOptions { segment_plaintext_budget: DIRECTORY_TEST_SEGMENT_BUDGET, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -806,6 +823,7 @@ fn encode_rejects_zero_master_on_encrypted() { let enc_dir = tempdir("zero_key_enc"); let options = DirectoryEncodeOptions { encrypted: true, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let err = encode_directory_with_options(&ZERO_KEY, &src, &enc_dir, options).unwrap_err(); @@ -866,9 +884,8 @@ fn decode_rejects_path_traversal_writes_no_files() { let mal_rkyv = malicious.to_bytes().expect("malicious rkyv"); let mal_payload = build_adamantine_payload(&mal_rkyv, &bundle).expect("build payload"); let mal_adam = encode_adamantine(&mal_payload, ADAMANTINE_CARBONADO_FMT_PUBLIC, hdr.flags); - let (mal_encoded, _) = - carbonado::file::encode(&ZERO_KEY, &mal_adam, DIRECTORY_ARCHIVE_FORMAT, None) - .expect("encode malicious catalog"); + let (mal_encoded, _) = file_encode(&ZERO_KEY, &mal_adam, DIRECTORY_ARCHIVE_FORMAT, None) + .expect("encode malicious catalog"); let mal_header = carbonado::file::Header::try_from(&mal_encoded[..carbonado::file::Header::LEN]) .expect("header"); @@ -966,8 +983,7 @@ fn decode_rejects_content_blake3_mismatch() { let bad_payload = build_adamantine_payload(&bad_rkyv, &bundle).expect("payload"); let bad_adam = encode_adamantine(&bad_payload, ADAMANTINE_CARBONADO_FMT_PUBLIC, hdr.flags); let (bad_encoded, _) = - carbonado::file::encode(&ZERO_KEY, &bad_adam, DIRECTORY_ARCHIVE_FORMAT, None) - .expect("re-encode"); + file_encode(&ZERO_KEY, &bad_adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("re-encode"); let bad_header = carbonado::file::Header::try_from(&bad_encoded[..carbonado::file::Header::LEN]) .expect("header"); @@ -1005,11 +1021,11 @@ fn adamantine_rejects_dev_v2_magic() { #[test] fn adamantine_rejects_invalid_catalog_fmt() { let mut bytes = encode_adamantine(b"x", ADAMANTINE_CARBONADO_FMT_PUBLIC, 0); - bytes[13] = 6; + bytes[13] = 16; let err = decode_adamantine(&bytes).unwrap_err(); assert!(matches!( err, - CarbonadoError::InvalidAdamantineCarbonadoFormat(6) + CarbonadoError::InvalidAdamantineCarbonadoFormat(16) )); } @@ -1019,7 +1035,14 @@ fn single_file_encode_decode_regression() { let data = fs::read(&sample).expect("read sample"); let outdir = tempdir("single"); - let oenc = encode_outboard(&ZERO_KEY, &data, DIRECTORY_ARCHIVE_FORMAT).expect("encode"); + let oenc = carbonado::encode_outboard_with_zstd( + &ZERO_KEY, + &data, + DIRECTORY_ARCHIVE_FORMAT, + None, + &zstd20(), + ) + .expect("encode"); let root = *oenc.hash.as_bytes(); let hhex = hex32(&root); let main_path = outdir.join(format!("{}.c{:02x}", hhex, DIRECTORY_ARCHIVE_FORMAT)); @@ -1058,6 +1081,7 @@ fn ots_entry_and_catalog_wire_roundtrip() { stamp_entries: true, stamp_catalog: true, }), + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -1113,6 +1137,7 @@ fn decode_rejects_tampered_entry_ots_proof() { stamp_entries: true, stamp_catalog: false, }), + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -1140,8 +1165,7 @@ fn decode_rejects_tampered_entry_ots_proof() { hdr.flags, ); let (tampered_encoded, _) = - carbonado::file::encode(&ZERO_KEY, &tampered_adam, DIRECTORY_ARCHIVE_FORMAT, None) - .expect("re-encode"); + file_encode(&ZERO_KEY, &tampered_adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("re-encode"); let tampered_header = carbonado::file::Header::try_from(&tampered_encoded[..carbonado::file::Header::LEN]) .expect("header"); @@ -1193,7 +1217,7 @@ fn decode_rejects_invalid_adamantine_flags() { let (payload, _) = decode_adamantine(&body).expect("adam"); let wrapped = encode_adamantine(&payload, ADAMANTINE_CARBONADO_FMT_PUBLIC, 0x02); let (encoded, _) = - carbonado::file::encode(&ZERO_KEY, &wrapped, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); + file_encode(&ZERO_KEY, &wrapped, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); let header = carbonado::file::Header::try_from(&encoded[..carbonado::file::Header::LEN]).expect("hdr"); let root = *header.hash.as_bytes(); @@ -1237,7 +1261,7 @@ fn decode_rejects_adamantine_format_filename_mismatch() { let (payload, _) = decode_adamantine(&body).expect("adam"); let wrapped = encode_adamantine(&payload, ADAMANTINE_CARBONADO_FMT_ENCRYPTED, 0); let (encoded, _) = - carbonado::file::encode(&ZERO_KEY, &wrapped, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); + file_encode(&ZERO_KEY, &wrapped, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); let header = carbonado::file::Header::try_from(&encoded[..carbonado::file::Header::LEN]).expect("hdr"); let root = *header.hash.as_bytes(); @@ -1357,8 +1381,7 @@ fn decode_rejects_oversized_adamantine_bundle_len() { evil.extend_from_slice(&((MAX_BAO_BUNDLE_LEN as u32).wrapping_add(1)).to_le_bytes()); let evil_adam = encode_adamantine(&evil, ADAMANTINE_CARBONADO_FMT_PUBLIC, hdr.flags); let (encoded, _) = - carbonado::file::encode(&ZERO_KEY, &evil_adam, DIRECTORY_ARCHIVE_FORMAT, None) - .expect("enc"); + file_encode(&ZERO_KEY, &evil_adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); let header = carbonado::file::Header::try_from(&encoded[..carbonado::file::Header::LEN]).expect("hdr"); let root = *header.hash.as_bytes(); @@ -1395,8 +1418,7 @@ fn decode_rejects_missing_entry_ots_when_required() { ADAMANTINE_CARBONADO_FMT_PUBLIC, ADAMANTINE_FLAG_REQUIRE_OTS, ); - let (encoded, _) = - carbonado::file::encode(&ZERO_KEY, &adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); + let (encoded, _) = file_encode(&ZERO_KEY, &adam, DIRECTORY_ARCHIVE_FORMAT, None).expect("enc"); let header = carbonado::file::Header::try_from(&encoded[..carbonado::file::Header::LEN]).expect("hdr"); let root = *header.hash.as_bytes(); @@ -1433,8 +1455,7 @@ fn decode_rejects_headered_segment_main_layout() { entry.segment_format )); let (inboard, _) = - carbonado::file::encode(&ZERO_KEY, b"not valid segment", entry.segment_format, None) - .expect("inboard"); + file_encode(&ZERO_KEY, b"not valid segment", entry.segment_format, None).expect("inboard"); fs::write(&seg_path, &inboard).expect("overwrite segment with headered blob"); let err = decode_directory(&ZERO_KEY, &catalog_path, &tempdir("seg_layout_dec")).unwrap_err(); assert!( @@ -1511,8 +1532,14 @@ fn directory_segment_corruption_bao_bundle_extract_scrub_roundtrip() { ); let plaintext = fs::read(src.join("content.png")).expect("read source"); - let oenc = - encode_outboard(&ZERO_KEY, &plaintext, entry.segment_format).expect("encode_outboard"); + let oenc = carbonado::encode_outboard_with_zstd( + &ZERO_KEY, + &plaintext, + entry.segment_format, + None, + &zstd20(), + ) + .expect("encode_outboard"); assert_eq!( oenc.verification_outboard.as_deref(), Some(bao_ob), @@ -1635,6 +1662,7 @@ fn directory_fec_scrub_matrix_c12_c13_c14_c15() { let options = DirectoryEncodeOptions { segment_format_policy: policy, encrypted, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = encode_directory_with_options(&key, &src, &enc_dir, options).expect("encode"); @@ -1674,7 +1702,14 @@ fn directory_fec_scrub_matrix_c12_c13_c14_c15() { hex32(&seg.segment_bao_root), entry.segment_format )); - let oenc = encode_outboard(&key, &payload, entry.segment_format).expect("encode_outboard"); + let oenc = carbonado::encode_outboard_with_zstd( + &key, + &payload, + entry.segment_format, + None, + &zstd20(), + ) + .expect("encode_outboard"); let pristine = fs::read(&seg_path).expect("read segment main"); let mut seg_main = pristine.clone(); @@ -1787,6 +1822,7 @@ fn directory_multi_segment_fec_bundle_indices() { let enc_dir = tempdir("multi_seg_enc"); let options = DirectoryEncodeOptions { segment_plaintext_budget: DIRECTORY_TEST_SEGMENT_BUDGET, + zstd: zstd20(), ..DirectoryEncodeOptions::default() }; let archive = @@ -1846,8 +1882,14 @@ fn directory_multi_segment_fec_bundle_indices() { .expect("ver"); let fec_slice = fec_slice_from_bundle(&bundle, seg.fec_parity_offset, seg.fec_parity_len).expect("fec"); - let oenc = - encode_outboard(&ZERO_KEY, chunk, entry.segment_format).expect("encode_outboard"); + let oenc = carbonado::encode_outboard_with_zstd( + &ZERO_KEY, + chunk, + entry.segment_format, + None, + &zstd20(), + ) + .expect("encode_outboard"); assert_eq!(oenc.verification_outboard.as_deref(), Some(ver_slice)); if seg.main_len > 0 { assert_eq!(oenc.fec_parity.as_deref(), Some(fec_slice)); diff --git a/tests/fec_chaos.rs b/tests/fec_chaos.rs index ffca373..daf2813 100644 --- a/tests/fec_chaos.rs +++ b/tests/fec_chaos.rs @@ -1,33 +1,41 @@ -//! FEC chaos: distributed random byte knockout up to 50% shard budget (RS 4/8). +//! FEC stripe contracts: 16 KiB logical stripes of eight 4 KiB leaves (c12). //! -//! Validates scrub recovery when corruption is spread throughout the encoded stream -//! (multiple shards, random offsets) — not clustered in a single segment. +//! Deterministic (no `rand` / `getrandom` in the named contracts). Scrub nicks +//! Bao leaf payloads via [`common::corruption::wipe_inboard_leaves`]. mod common; use anyhow::Result; +use carbonado::constants::{FEC_M, SLICE_LEN}; +use carbonado::stream::{leaf_index_to_stripe_symbol, stripe_symbol_to_leaf_index}; use carbonado::{ - decode, decode_outboard, encode, encode_outboard, error::CarbonadoError, scrub, scrub_outboard, + decode, decode_outboard, error::CarbonadoError, extract_slice, scrub, scrub_outboard, structs::Encoded, }; use common::corruption::{ - InboardShardLayout, OutboardShardLayout, scattered_outboard_main_knockout, - scattered_stream_knockout, + InboardShardLayout, OutboardShardLayout, every_other_leaf, first_n_leaves, last_n_leaves, + leaves_with_symbol_slots, scattered_outboard_main_knockout, scattered_stream_knockout, + wipe_inboard_leaves, }; use common::format_matrix::{format_label, verification_fec_levels}; +use common::{encode, encode_outboard}; use proptest::prelude::*; use rand::Rng; const CHAOS_PAYLOAD_SIZES: [usize; 5] = [4096, 16_384, 65_536, 131_072, 262_144]; -/// Payload sizes spanning the 16 KiB RS stripe geometry edge (4 × 4 KiB data shards). -/// Carbonado buffer encode emits one RS stripe per blob, scaling `chunk_len` above 16 KiB -/// rather than multiple fixed 16 KiB stripes. +/// Payload sizes spanning the 16 KiB RS stripe geometry edge (4 × 4 KiB data leaves). +/// Encode emits one 32 KiB inboard stripe per 16 KiB of padded logical, then the next stripe. const STRIPE_BOUNDARY_SIZES: [usize; 5] = [16 * 1024 - 1, 16 * 1024, 16 * 1024 + 1, 32_768, 49_152]; +const C12: u8 = 12; +const ZERO_MASTER: [u8; 32] = [0u8; 32]; +const LEAF_FILL: u8 = 0xEE; + fn varied_payload(size: usize, seed: u8) -> Vec { + // Period 251, not 256, so 16 KiB stripe boundaries are not identical slices. (0..size) - .map(|i| (i.wrapping_mul(13).wrapping_add(seed as usize)) as u8) + .map(|i| (i % 251).wrapping_add(seed as usize) as u8) .collect() } @@ -374,22 +382,244 @@ fn stripe_boundary_outboard_scrub_chaos() -> Result<()> { #[test] fn five_shard_touch_fails_scrub_proves_fifty_percent_limit() -> Result<()> { - let payload = varied_payload(65_536, 12); - let Encoded(orig, hash, info) = encode(&[0u8; 32], &payload, 12)?; + // Five 4 KiB leaves in stripe 0 (symbols 0..5): above the RS 4/8 budget. + let payload = varied_payload(32_768, 12); + let Encoded(orig, hash, info) = encode(&ZERO_MASTER, &payload, C12)?; + let hash_bytes = hash.as_bytes(); + let mut corrupted = orig.clone(); + wipe_inboard_leaves(&mut corrupted, &[0, 1, 2, 3, 4], LEAF_FILL); + + let err = scrub(&corrupted, hash_bytes, &info, C12).unwrap_err(); + assert!( + matches!(err, CarbonadoError::InvalidScrubbedHash), + "5/8 leaf loss in one stripe must be irrecoverable, got {err:?}" + ); + Ok(()) +} + +/// c12, payload larger than one 16 KiB stripe: each Bao leaf is 4 KiB, not a tall +/// `padded_len / 4` column. +#[test] +fn c12_leaf_is_4kib_not_tall_column() -> Result<()> { + let payload = varied_payload(32_768, 7); + let Encoded(orig, hash, info) = encode(&ZERO_MASTER, &payload, C12)?; let hash_bytes = hash.as_bytes(); - let layout = InboardShardLayout::from_encode_info(orig.len(), info.chunk_len); + assert_eq!( + info.chunk_len, SLICE_LEN, + "RS symbol / Bao leaf is 4 KiB, not padded/4 tall columns (got {})", + info.chunk_len + ); + let leaf0 = extract_slice(&orig, 0, hash_bytes, C12)?; + assert_eq!(leaf0.len(), SLICE_LEN as usize, "leaf 0 size"); + assert_eq!(&leaf0[..], &payload[0..SLICE_LEN as usize]); + + // Stripe layout: leaves 0..3 are data of stripe 0; 4..7 are parity; leaf 8 + // is the first data leaf of stripe 1 (logical[16384..20480]). Tall columns + // put logical[16384..20480] at leaf 4 instead. + let leaf4 = extract_slice(&orig, 4, hash_bytes, C12)?; + assert_eq!(leaf4.len(), SLICE_LEN as usize); + assert_ne!( + &leaf4[..], + &payload[16_384..16_384 + SLICE_LEN as usize], + "leaf 4 must be stripe-0 parity, not the start of a tall column" + ); + let leaf8 = extract_slice(&orig, 8, hash_bytes, C12)?; + assert_eq!(&leaf8[..], &payload[16_384..16_384 + SLICE_LEN as usize]); + assert_eq!(leaf_index_to_stripe_symbol(4), (0, 4)); + assert_eq!(leaf_index_to_stripe_symbol(8), (1, 0)); + assert_eq!(stripe_symbol_to_leaf_index(1, 0), 8); + Ok(()) +} + +/// Four symbol slots nicked (one leaf in each of four positions, across stripes) recovers. +#[test] +fn four_symbol_slots_nicked_across_stripes_recovers() -> Result<()> { + let payload = varied_payload(49_152, 9); + let Encoded(orig, hash, info) = encode(&ZERO_MASTER, &payload, C12)?; + let hash_bytes = hash.as_bytes(); + // Three stripes. Nick slot 0 in stripe 0, slot 2 in stripe 1, slot 5 in + // stripe 2, slot 7 in stripe 0: four positions, spread across stripes. + let nicks = [ + stripe_symbol_to_leaf_index(0, 0), + stripe_symbol_to_leaf_index(1, 2), + stripe_symbol_to_leaf_index(2, 5), + stripe_symbol_to_leaf_index(0, 7), + ]; let mut corrupted = orig.clone(); - common::corruption::erase_shards(&mut corrupted, &layout, &[0, 1, 2, 3, 4]); + wipe_inboard_leaves(&mut corrupted, &nicks, LEAF_FILL); + let recovered = scrub(&corrupted, hash_bytes, &info, C12)?; + assert_eq!(recovered, orig); + let dec = decode(&ZERO_MASTER, hash_bytes, &recovered, info.padding_len, C12)?; + assert_eq!(dec, payload); + Ok(()) +} - let err = scrub(&corrupted, hash_bytes, &info, 12).unwrap_err(); +/// Five leaves bad in one stripe fails. +#[test] +fn five_leaves_bad_in_one_stripe_fails() -> Result<()> { + let payload = varied_payload(32_768, 11); + let Encoded(orig, hash, info) = encode(&ZERO_MASTER, &payload, C12)?; + let hash_bytes = hash.as_bytes(); + let bad: Vec = (0..5).map(|s| stripe_symbol_to_leaf_index(1, s)).collect(); + let mut corrupted = orig.clone(); + wipe_inboard_leaves(&mut corrupted, &bad, LEAF_FILL); + let err = scrub(&corrupted, hash_bytes, &info, C12).unwrap_err(); assert!( matches!(err, CarbonadoError::InvalidScrubbedHash), - "5/8 shard loss must be irrecoverable, got {err:?}" + "five bad leaves in stripe 1 must fail, got {err:?}" ); Ok(()) } +fn assert_fifty_percent_then_plus_one( + payload: &[u8], + mask: &[u32], + plus_one: u32, + label: &str, +) -> Result<()> { + let Encoded(orig, hash, info) = encode(&ZERO_MASTER, payload, C12)?; + let hash_bytes = hash.as_bytes(); + let leaf_count = info.verifiable_slice_count; + assert_eq!( + mask.len() * 2, + leaf_count as usize, + "{label}: mask must be exactly 50% of {leaf_count} leaves" + ); + + let mut half = orig.clone(); + wipe_inboard_leaves(&mut half, mask, LEAF_FILL); + let recovered = scrub(&half, hash_bytes, &info, C12).unwrap_or_else(|e| { + panic!("{label}: 50% leaf wipe must recover, got {e}"); + }); + assert_eq!(recovered, orig, "{label}: 50% recovered body"); + let dec = decode(&ZERO_MASTER, hash_bytes, &recovered, info.padding_len, C12)?; + assert_eq!(dec, payload, "{label}: 50% decoded payload"); + + let mut plus = orig.clone(); + let mut plus_mask = mask.to_vec(); + plus_mask.push(plus_one); + wipe_inboard_leaves(&mut plus, &plus_mask, LEAF_FILL); + let err = scrub(&plus, hash_bytes, &info, C12).unwrap_err(); + assert!( + matches!(err, CarbonadoError::InvalidScrubbedHash), + "{label}: 50%+1 must fail, got {err:?}" + ); + Ok(()) +} + +/// First four symbol slots of every stripe (`leaf % 8 < 4`). That is the first +/// half of each stripe's eight leaves, 50% of all leaves, 4 of 8 symbols per stripe. +#[test] +fn fifty_percent_first_half_of_leaves_then_plus_one() -> Result<()> { + let payload = varied_payload(65_536, 1); + let Encoded(_, _, info) = encode(&ZERO_MASTER, &payload, C12)?; + let n = info.verifiable_slice_count; + let mask = leaves_with_symbol_slots(n, &[0, 1, 2, 3]); + let plus_one = stripe_symbol_to_leaf_index(0, 4); + assert_fifty_percent_then_plus_one(&payload, &mask, plus_one, "first half") +} + +/// Last four symbol slots of every stripe (`leaf % 8 >= 4`). +#[test] +fn fifty_percent_last_half_of_leaves_then_plus_one() -> Result<()> { + let payload = varied_payload(65_536, 2); + let Encoded(_, _, info) = encode(&ZERO_MASTER, &payload, C12)?; + let n = info.verifiable_slice_count; + let mask = leaves_with_symbol_slots(n, &[4, 5, 6, 7]); + let plus_one = stripe_symbol_to_leaf_index(0, 0); + assert_fifty_percent_then_plus_one(&payload, &mask, plus_one, "last half") +} + +/// Even leaf indices: slots 0,2,4,6 of every stripe. +#[test] +fn fifty_percent_every_other_leaf_then_plus_one() -> Result<()> { + let payload = varied_payload(65_536, 3); + let Encoded(_, _, info) = encode(&ZERO_MASTER, &payload, C12)?; + let n = info.verifiable_slice_count; + let mask = every_other_leaf(n, 0); + let plus_one = 1; + assert_fifty_percent_then_plus_one(&payload, &mask, plus_one, "every other") +} + +/// Four-of-eight slots 0,3,4,6 in every stripe (mixed data and parity). +#[test] +fn fifty_percent_four_of_eight_slots_then_plus_one() -> Result<()> { + let payload = varied_payload(65_536, 4); + let Encoded(_, _, info) = encode(&ZERO_MASTER, &payload, C12)?; + let n = info.verifiable_slice_count; + let mask = leaves_with_symbol_slots(n, &[0, 3, 4, 6]); + let plus_one = stripe_symbol_to_leaf_index(0, 1); + assert_fifty_percent_then_plus_one(&payload, &mask, plus_one, "four-of-eight slots") +} + +/// Coffee-cup fill on an 8×8 leaf square. +/// +/// Payload is 128 KiB (8 × 16 KiB stripes → 64 inboard leaves). Leaves sit in +/// row-major order: `leaf = row * 8 + col`, `row` is the stripe, `col` is the +/// RS symbol (0..3 data, 4..7 parity). +/// +/// Cell centers are `(col + 0.5, row + 0.5)`. The circle is at the square +/// center `(4.0, 4.0)`. Rank every cell by Euclidean distance to that center, +/// then by leaf index. +/// +/// Grow the mask in that order. Skip a leaf when its stripe already has 4 +/// marked (RS 4/8 can take at most 4 erasures per stripe). Stop at 32 leaves +/// (exactly 50%). The +1 leaf is the next in rank order: the first that would +/// be a fifth erasure in some stripe. +/// +/// Without the per-stripe cap, the nearest 32 cells to center would mark 6 +/// leaves in the middle rows. The cap flattens the cup to the 4/8 budget +/// while still preferring the center. +fn coffee_cup_mask_128kib() -> (Vec, u32) { + const SIDE: u32 = 8; + const LEAVES: u32 = SIDE * SIDE; + let center = 4.0f64; + let mut ranked: Vec<(u32, u64, u32)> = (0..LEAVES) + .map(|leaf| { + let row = leaf / SIDE; + let col = leaf % SIDE; + let dx = (col as f64 + 0.5) - center; + let dy = (row as f64 + 0.5) - center; + let dist2_bits = (dx * dx + dy * dy).to_bits(); + (leaf, dist2_bits, leaf) + }) + .collect(); + ranked.sort_by(|a, b| a.1.cmp(&b.1).then(a.2.cmp(&b.2))); + + let mut mask = Vec::with_capacity(32); + let mut per_stripe = [0u8; FEC_M]; + let mut plus_one = None; + for &(leaf, _, _) in &ranked { + let stripe = (leaf / FEC_M as u32) as usize; + if mask.len() < 32 { + if per_stripe[stripe] < 4 { + mask.push(leaf); + per_stripe[stripe] += 1; + } + continue; + } + plus_one = Some(leaf); + break; + } + let plus_one = plus_one.expect("ranked list must supply a +1 leaf after 50%"); + (mask, plus_one) +} + +#[test] +fn fifty_percent_coffee_cup_then_plus_one() -> Result<()> { + let payload = varied_payload(128 * 1024, 5); + let (mask, plus_one) = coffee_cup_mask_128kib(); + assert_eq!(mask.len(), 32, "coffee-cup mask is 50% of 64 leaves"); + assert_eq!( + first_n_leaves(64, 32).len(), + 32, + "sanity: first-half helper" + ); + assert_eq!(last_n_leaves(64, 32).len(), 32); + assert_fifty_percent_then_plus_one(&payload, &mask, plus_one, "coffee-cup") +} + proptest! { #![proptest_config(ProptestConfig { cases: 32, diff --git a/tests/fec_scrub_matrix.rs b/tests/fec_scrub_matrix.rs index 7259548..5bb55c0 100644 --- a/tests/fec_scrub_matrix.rs +++ b/tests/fec_scrub_matrix.rs @@ -4,11 +4,11 @@ mod common; use anyhow::Result; use carbonado::{ - decode, decode_outboard, encode, encode_outboard, error::CarbonadoError, scrub, scrub_outboard, - structs::Encoded, + decode, decode_outboard, error::CarbonadoError, scrub, scrub_outboard, structs::Encoded, }; use common::corruption::{InboardShardLayout, flip_byte}; use common::format_matrix::{format_label, public_fec_levels, verification_fec_levels}; +use common::{encode, encode_outboard}; use rand::Rng; fn master_for(level: u8) -> [u8; 32] { diff --git a/tests/filepack_interop.rs b/tests/filepack_interop.rs index 764609b..00764c2 100644 --- a/tests/filepack_interop.rs +++ b/tests/filepack_interop.rs @@ -1,6 +1,6 @@ //! CBOR filepack ↔ rkyv FilepackManifest interop (Phase 4). //! -//! Cross-tool contract: rkyv `FilepackManifest` v2 wire inside Adamantine 1.0 plus +//! Cross-tool contract: rkyv `FilepackManifest` v3 wire inside Adamantine 1.0 plus //! Adamantine decimal on-disk segment naming (`{root}.c12` / `.c14` / `.adam.c14` / `.adam.c15`). use carbonado::{ @@ -46,7 +46,13 @@ fn adam_catalog_path(enc_dir: &Path, root: &[u8; 32], format: u8) -> PathBuf { } fn encode_samples_directory(enc_dir: &Path) -> DirectoryArchive { - encode_directory(&ZERO_KEY, &samples_dir(), enc_dir).expect("encode_directory") + encode_directory( + &ZERO_KEY, + &samples_dir(), + enc_dir, + &carbonado::ZstdEncode::level(20), + ) + .expect("encode_directory") } fn load_manifest_from_catalog(enc_dir: &Path, catalog_root: &[u8; 32]) -> FilepackManifest { @@ -108,6 +114,8 @@ fn mock_segment_ref(main_len: u64, chunk_index: u32, root_byte: u8) -> SegmentRe verification_outboard_len: ver_len, fec_parity_offset: ver_len, fec_parity_len: fec_len, + dict_offset: 0, + dict_len: 0, } } @@ -531,6 +539,8 @@ fn dump_interop_golden_fixture_values() { let enc_dir = tempdir("dump_golden"); let (_archive, manifest) = encode_samples_manifest(&enc_dir); let rkyv_bytes = manifest.to_bytes().expect("to_bytes"); + eprintln!("catalog_bao_root={}", hex32(&_archive.catalog_bao_root)); + eprintln!("manifest_version={}", manifest.version); eprintln!("manifest_rkyv_len={}", rkyv_bytes.len()); eprintln!("manifest_rkyv_sha256={}", sha256_hex(&rkyv_bytes)); let catalog_path = enc_dir.join(format!("{}.adam.c14", hex32(&_archive.catalog_bao_root))); @@ -546,12 +556,14 @@ fn dump_interop_golden_fixture_values() { ); for seg in &entry.segments { eprintln!( - " chunk {} ver_off={} ver_len={} fec_off={} fec_len={}", + " chunk {} ver_off={} ver_len={} fec_off={} fec_len={} dict_off={} dict_len={}", seg.chunk_index, seg.verification_outboard_offset, seg.verification_outboard_len, seg.fec_parity_offset, - seg.fec_parity_len + seg.fec_parity_len, + seg.dict_offset, + seg.dict_len ); } } diff --git a/tests/fixtures/directory_interop_golden.json b/tests/fixtures/directory_interop_golden.json index 125cf91..3ed5072 100644 --- a/tests/fixtures/directory_interop_golden.json +++ b/tests/fixtures/directory_interop_golden.json @@ -1,21 +1,21 @@ { "description": "Golden checksums for encode_directory(tests/samples, ZERO_KEY). Adamantine 1.0 uses decimal format suffixes in filenames (.c12/.c14/.adam.c14), not hex (.c0c). Canonical platform: Linux CI.", "generated_with": { - "carbonado": "2.1.0", + "carbonado": "0.7.1", "infer": "0.19.0", "platform": "linux", "master_key": "zero", - "command": "cargo run --bin carbonado --features cli -- encode tests/samples -o ", + "command": "cargo run --bin carbonado --features cli -- encode tests/samples --zstd-level 20 -o ", "regenerate": "cargo test --test filepack_interop dump_interop_golden_fixture_values -- --ignored --nocapture" }, - "catalog_bao_root": "3b91ff89d0215cc48c83529de78798f5890c04dbfa298e192dd2f07ce08c5b45", - "catalog_filename": "3b91ff89d0215cc48c83529de78798f5890c04dbfa298e192dd2f07ce08c5b45.adam.c14", - "manifest_version": 2, + "catalog_bao_root": "34e0e1e07e19f9a2face94d72fd0bfb3597215dae5e0647c70bcfb4a2a2fb216", + "catalog_filename": "34e0e1e07e19f9a2face94d72fd0bfb3597215dae5e0647c70bcfb4a2a2fb216.adam.c14", + "manifest_version": 3, "format_level": 14, "entry_count": 3, - "manifest_rkyv_len": 391, - "manifest_rkyv_sha256": "aed51d8b667f2c10f3e05faa03e92c3aaa74c89cd659aeb6ab6dcf32ba8a5b35", - "bundle_sha256": "d48c8db5b7f6b679c6d031a3cd542c62712254a3117417b344d252f2a21c0668", + "manifest_rkyv_len": 415, + "manifest_rkyv_sha256": "64ee0003bde24a0c9a8d369a47fd59321e11dc302bbfdfbe062f1e79ae981e25", + "bundle_sha256": "e505c0cc1bb19a474ccc647526cb40ef503c590b6ac6e82dbe591605afce7953", "entries": [ { "rel_path": "code.tar", @@ -25,7 +25,9 @@ "verification_outboard_offset": 0, "verification_outboard_len": 128, "fec_parity_offset": 128, - "fec_parity_len": 16384 + "fec_parity_len": 16384, + "dict_offset": 0, + "dict_len": 0 } ] }, @@ -37,7 +39,9 @@ "verification_outboard_offset": 16512, "verification_outboard_len": 9600, "fec_parity_offset": 26112, - "fec_parity_len": 622592 + "fec_parity_len": 622592, + "dict_offset": 0, + "dict_len": 0 } ] }, @@ -49,15 +53,17 @@ "verification_outboard_offset": 648704, "verification_outboard_len": 0, "fec_parity_offset": 648704, - "fec_parity_len": 16384 + "fec_parity_len": 16384, + "dict_offset": 0, + "dict_len": 0 } ] } ], "artifacts": { - "3b91ff89d0215cc48c83529de78798f5890c04dbfa298e192dd2f07ce08c5b45.adam.c14": { + "34e0e1e07e19f9a2face94d72fd0bfb3597215dae5e0647c70bcfb4a2a2fb216.adam.c14": { "len": 1331321, - "sha256": "9946a7a49218a3a4f1e677cf7aeda5f3099669afd717cf27a38bc1f9fcb1cbb5", + "sha256": "ac81339f61e1c5624332039f0114d759b84d76572e5731565fffe3fbef8840cd", "decimal_format_suffix": "14" }, "797216a438ea69c1c6885b31700e93a7e1ef492825a1c38d8f50c6250ab0a5f4.c14": { @@ -76,4 +82,4 @@ "decimal_format_suffix": "12" } } -} \ No newline at end of file +} diff --git a/tests/fixtures/phase3_g9_directory/README.txt b/tests/fixtures/phase3_g9_directory/README.txt index ac70ac0..ce35901 100644 --- a/tests/fixtures/phase3_g9_directory/README.txt +++ b/tests/fixtures/phase3_g9_directory/README.txt @@ -1,21 +1,9 @@ -G9 directory seed: rust-encoded Adamantine 1.0 archive (backend-rust / CLI). -Used by tests/lean_backend_phase3.rs for rust-encode → lean-decode extract. +Phase 3 G9 directory seed (public c14). -**Decode-only SSOT** for dual-suite interop. Catalog root is **not** a live -re-encode golden: current backend-rust re-encode of the same tree yields a -different catalog root (see LIVE_RUST_DIR_CATALOG_ROOT in -tests/determinism_roundtrip.rs) while segment mains may still match. Cross-engine -encode residual is live-rust vs live-lean (W2b), not lean-vs-this-seed. +Live rust encode of `a.txt` = "phase3 g9 hello" and `sub/b.bin` = "nested data" +with zstd level 20. Catalog Bao root is FilepackManifest v3 (dict offset fields). -Source tree (public c14 catalog, zero master): - a.txt = "phase3 g9 hello" - sub/b.bin = "nested data" - -Segment formats follow auto policy (both small texts → c14). -Catalog seed: 16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14 - -Regenerate (only when intentionally refreshing the decode seed; update -lean_backend_phase3 hard-coded path + this README): - printf 'phase3 g9 hello' > /tmp/src/a.txt - mkdir -p /tmp/src/sub && printf 'nested data' > /tmp/src/sub/b.bin - cargo run --features cli --bin carbonado -- encode /tmp/src -o tests/fixtures/phase3_g9_directory +Regenerate with: + carbonado encode --zstd-level 20 --output tests/fixtures/phase3_g9_directory +and update LIVE_RUST_DIR_CATALOG_ROOT / PHASE3_SEED_DIR_CATALOG_ROOT in +tests/determinism_roundtrip.rs. diff --git a/tests/fixtures/phase3_g9_directory/16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14 b/tests/fixtures/phase3_g9_directory/f14bfeb50f1d3072e5510d7eab42fea176868f091eec0d614a129c506125c114.adam.c14 similarity index 90% rename from tests/fixtures/phase3_g9_directory/16e2369f4f4465014e5e92740e3f76403f681cd60c01f7605ee11acd5423024f.adam.c14 rename to tests/fixtures/phase3_g9_directory/f14bfeb50f1d3072e5510d7eab42fea176868f091eec0d614a129c506125c114.adam.c14 index 8a0583ece4bbf5e7439f3574c8038cb8dced3ff4..c517d6da9b9063935b4acfd5df72e130f49e2343 100644 GIT binary patch delta 1068 zcmYL{eK6a19LK-E&@P^m33iC!CxR3iMi*5ip0j$4I_9*~qz&6;N^)h2NQ^Ab)9l+= z>Z#S<%w`Lsqo?VE&b3&3bF`YZ^HlY4SVnYLrkC4K`^P?iyzlkj>+}BHXSWi!TZ#MH zDLcMBqMCeRSCJ>sdK9Mwur{)o_6tMN#&O6iq+Gj_o+osmY#lTHLZtYKmQt)F-4Vu)LV1Ik^M=)_k4UY_r)a~r(BH(hvDK_R zCJg*P(zo~*(C~}%Z14dStkU72Dv^K9dd4jqqS(f$$W7dkSn&#q7kD&g`b6cIS-H*s zT6^guS%nALobX}rEOyq_tg8T=%o1qg9i@`CZ*>XDsRY)2c~?n4KBMR%Q^$?IO_<^g zc{T~348%fr#Li{*i)e|9Kx!vT32`fRA4GLS2R*1i1&#cjR(+>6Hy>soo(^?ZVB|e^ z6gz>Pc$||M?eAw!0FE*MNKl1%!x4mmu%H2<<6{s|jkPF-R2*vYu?dERuW(qxcw7f! z2lY4_rC_vSo0ABB^sbyYCFTU82Lf$1e%8rI^t$N=((i8)d1ATn;fE3* ze*F=#G#zO?YQFiMaRAP*t_569kGom6!uIOPT3ULpJ(=(QQueNqW`Rg=ShiJPBGbjO z53ck^%{}@xn7o&1;`-!L>+Cqts(ZlVE9G+~su-x5R(^r5Skdp{Vgzb2?~7pt>+pVa%g7GoDO5+&DCOy@xCBekrTKhRcu^tGDv}S+ ztPN1fyP?8T@_uuIOKh)=2^c$63#yb!)TFtjJ!8*axZu353mg~%Ur#H-U{ue}D< zD*!k^6%YFq9s_v^4LmwtMFK_Grv8x1M~TPwFZH@+zXA9M+npV04|DldNDkEVX_Sse za(*Emg#3B4Bu>9c%*`a8Yl?k;jHI54yGnX{H^+j)>wt0i1z4w*1((&8^_`yS^W9!i z=)mK90Gxv=5#|;#kWZmOM8{j(0s&?^0;yuuLV6TK-7POHVZ68lae#U;jnYhZ&YtN5 c^BxsQ@`Kxh>Zn00$_gjY*0(Kr%l3`)KRpzsW&i*H delta 1036 zcmYL{Yc!j89LN996P;%$9ji?^A(9+PX~XH7g`MSA8eJ=?xJ+m6WDYGVJy9jqkfPPJ zUH?*PW=K1ZT_dO&CY?Dh?WuNiOSPlZx>T_fHmPx(J*R!K->c8N&-eQ~-*pwbu0pSP zBxaAsW!e2DLzVVF=?v%$rCYYmxivvV&NEJP%Sw6Cb|r_rJD)E~zP|R@B%OM2=v<%{ zTsKE#XK)JNQeZuUuG6eqac~+Uiobf&kt^qM(>=x<5Nl%mpTv8S6jTiP1h3Dl-z*B{ zfdB98YxEmZmA?DJ@1>S(ei%R9JlEB*85UkROCTE>*Y)k|9mfC)1P#$@GDI{gcvaada$0p%~R8uF!(g zNCD6K*WJKwA27O8kW*Z?S9b+HX)8~A^dzI?OF(bo6WAnpgs`I}pUm?r`sV^6UH75a zR?vG@PO5R|tJ+Ke^rTT zO=LG0QX3O`Sx=%)1t=!DiJ&99pnAfRcQ;{1YbzQdaG!*M^?86# za7hPzDj{R@Y#bX`*w86Lk7%GVf<}^h>t$r%y9$DhnT(p1_?rdnqmfD4MTuXr6$oGX z5+sVi`@*=oOQYT|b}?0aPyd$V9~_M+t9Iff92r`&zRXkRId+a{qALPe{@8YxmP$-M z9NXD&?q?CveUtEU>B4;FML@~F1I~uecFl5TF{K>)AxhY~30mg>ph9gjzKmpi8OaQ+ z-WlZKA?F|^g^X2Gaont+LUD>735S*x8YyaTQLhO!*#PNh4_bb!I8iu2-8eHfqALjc z6$J~g|3f?>ZwGrao`ETvB2b}L7*qc+?p~cMH-qv%0O(Mgi~DkMUoHbX{5Fu{l0Hb0 zk+FU{j^=JU6wCAo4O(P0(rWnl{BsaL25z|VFa6v7$o4XQ^4MC{X(V(8h8W> literal 13 TcmZQ#U|`_;{r~@eAk6>(8fydz diff --git a/tests/fixtures/rkyv/multi_entry_ots.bin b/tests/fixtures/rkyv/multi_entry_ots.bin index 41eb5473495a12a55d4cd83a316ccaa21d0a41a1..5c6b92026973a6c2f7570ee07d8bd233b059638e 100644 GIT binary patch delta 81 zcmbQtw3umv4J!i(OtcJzGP(`KO#lD?&j=QV64QWm#>9&%a-mQe#tA?M3s969i22%q I93~(G00O@h;{X5v delta 69 zcmZ3?G?{6F%|w@=iSu;DT>t<7&&YrTrUBUn6CbL`#X@BmCjc2NKv5>}H%`!^!{x6D@V*CjbBcpApQ163jrF?;unZ0P|D|R{#J2 delta 40 gcmeBXY-XHbGtos;ZubBG{}~yO027eScMvKH0O?!`VE_OC diff --git a/tests/fixtures/rkyv/path_ool_9.bin b/tests/fixtures/rkyv/path_ool_9.bin index 905272afcb2367106e83abae381187c6b669a622..47248878995c519f6148048bc280d835cbf297a6 100644 GIT binary patch literal 148 zcmXpsGBz>}H%`!^!{x6D@V*CjbBcpApQ163jrF?;unZ0P|D|R{#J2 delta 40 gcmeBXY-XHbGtos;ZubBG{}~yO027eScMvKH0O?!`VE_OC diff --git a/tests/fixtures/rkyv/single_entry.bin b/tests/fixtures/rkyv/single_entry.bin index 113be42149581757724b23d66f3e535a5cbfab28..0b8db913ae82f95ea0988b0721a71e8629b0e1d8 100644 GIT binary patch delta 46 kcmZo>>}H%`!^!{x6D@V*CjbBcpApQ163jrF?;unZ0P|D|R{#J2 delta 40 gcmeBXY-XHbGtos;ZubBG{}~yO027eScMvKH0O?!`VE_OC diff --git a/tests/fixtures/rkyv/two_segments.bin b/tests/fixtures/rkyv/two_segments.bin index b03210083287440ee8afca2cb89c0e168ae7368c..3972c53c3758214f5d7d050b0eec904a9d985a3e 100644 GIT binary patch literal 207 zcmWd>#19yNAO%D~fdh~R;RYav0U-i_5vb4zs`vnqb^x(~f(;-V3KI27DoXzU|F1*< W@c9E3F~M|!Xl5YhI|vj7SpxvCuNTn( delta 54 vcmX@lxSw%?%|sU)jzqnZijx2T|4*DKub{*i4`eYhAOR4E?;uc=5y$`l9{CX< diff --git a/tests/format.rs b/tests/format.rs index aeeb59a..305207c 100644 --- a/tests/format.rs +++ b/tests/format.rs @@ -5,13 +5,14 @@ use std::{fs::OpenOptions, io::Write, path::PathBuf}; use anyhow::Result; use carbonado::{ constants::Format, - decode, decode_outboard, encode, encode_outboard, + decode, decode_outboard, error::CarbonadoError, file::{self, Header}, filepack, scrub, scrub_outboard, structs::{Encoded, OutboardEncoded}, }; use common::format_matrix::ALL_FORMAT_LEVELS; +use common::{encode, encode_outboard, file_encode_outboard}; use log::{debug, info, trace}; use rand::RngCore; use wasm_bindgen_test::wasm_bindgen_test_configure; @@ -175,19 +176,10 @@ fn scrub_specific_errors() -> Result<()> { assert!(matches!(err, CarbonadoError::ScrubRequiresVerification)); } - // For a Bao+Zfec, make irrecoverable (>4 shards) -> InvalidScrubbedHash + // For a Bao+Zfec, make irrecoverable (>4 leaves in one stripe) -> InvalidScrubbedHash let Encoded(e, h, ei) = encode(&key, input, 12)?; let mut too_bad = e.clone(); - // taint 5+ shard regions aggressively (large input ensures effective distributed hit) - let step = (too_bad.len().saturating_sub(8) / 8).max(16); - let clen = ei.chunk_len as usize; - for i in 0..5 { - let p = 8 + i * step; - let z = clen.min(too_bad.len().saturating_sub(p)); - if z > 0 { - too_bad[p..p + z].fill(0); - } - } + common::corruption::wipe_inboard_leaves(&mut too_bad, &[0, 1, 2, 3, 4], 0); let err = scrub(&too_bad, h.as_bytes(), &ei, 12).unwrap_err(); assert!( matches!(err, CarbonadoError::InvalidScrubbedHash), @@ -370,7 +362,7 @@ fn file_outboard_high_level_bare_and_header() -> Result<()> { // Public levels: bare main (no magic header in main), Some(header) for out-of-band, sidecars present for bits for &level in &[0u8, 2, 4, 6, 8, 10, 12, 14] { let (hdr_opt, oenc): (Option

, OutboardEncoded) = - file::encode_outboard(&key, input, level, Some(*b"testmeta"))?; + file_encode_outboard(&key, input, level, Some(*b"testmeta"))?; assert!( hdr_opt.is_some(), "out-of-band header for public file outboard level {}", @@ -433,7 +425,7 @@ fn file_outboard_high_level_bare_and_header() -> Result<()> { // 0-byte public via file outboard let empty: &[u8] = &[]; - let (h0, o0) = file::encode_outboard(&key, empty, 4, None)?; + let (h0, o0) = file_encode_outboard(&key, empty, 4, None)?; assert!(h0.is_some()); assert!(o0.main.is_empty()); let h0_bytes = h0.as_ref().map(|hh| hh.try_to_vec().unwrap()); @@ -450,15 +442,15 @@ fn file_outboard_high_level_bare_and_header() -> Result<()> { assert_eq!(r0, empty); // c# commitment still holds via file layer - let o4 = file::encode_outboard(&key, input, 4, None)?.1; - let o6 = file::encode_outboard(&key, input, 6, None)?.1; + let o4 = file_encode_outboard(&key, input, 4, None)?.1; + let o6 = file_encode_outboard(&key, input, 6, None)?.1; assert_ne!( o4.hash, o6.hash, "file outboard different c produce different keyed roots" ); // Encrypted outboard: bare main (no MAGIC), bao sidecar, nonce in out-of-band header - let (he_opt, oe) = file::encode_outboard(&key, input, 5, None)?; + let (he_opt, oe) = file_encode_outboard(&key, input, 5, None)?; assert!(he_opt.is_some()); let hdr_e = he_opt.unwrap(); assert!( @@ -505,7 +497,7 @@ fn file_outboard_high_level_bare_and_header() -> Result<()> { ); // error: missing sidecar for bao public via file decode_outboard (specific error) - let ob = file::encode_outboard(&key, input, 4, None)?.1; + let ob = file_encode_outboard(&key, input, 4, None)?.1; let err = file::decode_outboard( &key, ob.hash.as_bytes(), @@ -603,7 +595,7 @@ fn file_outboard_metadata_roundtrip_and_mac_binding() -> Result<()> { let input = b"metadata test for outboard header auth"; let meta = Some(*b"metameta"); - let (hdr_opt, oenc) = file::encode_outboard(&key, input, 4, meta)?; + let (hdr_opt, oenc) = file_encode_outboard(&key, input, 4, meta)?; let hdr = hdr_opt.expect("public outboard produces out-of-band header"); assert_eq!(hdr.metadata, meta, "metadata roundtrips in header"); diff --git a/tests/format_amplification.rs b/tests/format_amplification.rs index ae37dc0..e3d0d0a 100644 --- a/tests/format_amplification.rs +++ b/tests/format_amplification.rs @@ -8,10 +8,13 @@ //! //! Run: `cargo test --test format_amplification -- --nocapture` to print the matrix. +mod common; + use carbonado::{ constants::{FEC_K, FEC_M, SLICE_LEN}, file::{self, Header}, }; +use common::file_encode; /// ~1 MiB input (exactly 1_048_576 bytes). const INPUT_LEN: usize = 1_048_576; @@ -71,7 +74,7 @@ struct Row { } fn measure_row(format: u8, input: &[u8]) -> Row { - let (encoded, info) = file::encode(&MASTER, input, format, None).expect("encode"); + let (encoded, info) = file_encode(&MASTER, input, format, None).expect("encode"); let net_amp = info.output_len as f32 / info.input_len.max(1) as f32; Row { format, @@ -232,7 +235,7 @@ fn format_amplification_matrix_all_levels() { } fn roundtrip(format: u8, expected: &[u8]) { - let (encoded, _) = file::encode(&MASTER, expected, format, None).expect("encode"); + let (encoded, _) = file_encode(&MASTER, expected, format, None).expect("encode"); let (_hdr, decoded) = file::decode(&MASTER, &encoded).expect("decode"); assert_eq!(decoded, expected, "roundtrip failed for c{format:02X}"); } diff --git a/tests/g9_cross_backend.rs b/tests/g9_cross_backend.rs index 6fa68b3..80cc53f 100644 --- a/tests/g9_cross_backend.rs +++ b/tests/g9_cross_backend.rs @@ -14,7 +14,8 @@ use std::fs; use std::path::{Path, PathBuf}; use carbonado::{ - OutboardEncoded, constants::Format, decode, decode_outboard, encode_with_nonce, file, + OutboardEncoded, ZstdEncode, constants::Format, decode, decode_outboard, + encode_outboard_with_zstd, encode_with_nonce, encode_with_zstd, file, stream_encode_outboard_buffer, structs::Encoded, }; use serde::{Deserialize, Serialize}; @@ -139,7 +140,7 @@ fn encode_body(format: u8) -> (Vec, [u8; 32], u32) { None }; let carbonado::structs::Encoded(body, hash, info) = - encode_with_nonce(&MASTER, PLAINTEXT, format, nonce) + encode_with_zstd(&MASTER, PLAINTEXT, format, nonce, &ZstdEncode::level(20)) .unwrap_or_else(|e| panic!("encode body c{format}: {e}")); (body, *hash.as_bytes(), info.padding_len) } @@ -151,8 +152,15 @@ fn encode_headered(format: u8) -> (Vec, [u8; 32], u32, Option<[u8; 16]>) { } else { None }; - let (archive, info) = file::encode_with_nonce(&MASTER, PLAINTEXT, format, None, nonce) - .unwrap_or_else(|e| panic!("encode headered c{format}: {e}")); + let (archive, info) = file::encode_with_nonce_and_zstd( + &MASTER, + PLAINTEXT, + format, + None, + nonce, + &ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("encode headered c{format}: {e}")); let (hdr, _) = file::decode(&MASTER, &archive).expect("self-decode headered for hash"); let hash = *hdr.hash.as_bytes(); let nonce_out = if is_encrypted(format) { @@ -168,8 +176,14 @@ fn encode_outboard_fixture(format: u8) -> (OutboardEncoded, Option>, boo let encrypted = is_encrypted(format); if encrypted { // Header-path: stream_encode_outboard_buffer Some(nonce) + Header for file::decode_outboard. - let oenc = stream_encode_outboard_buffer(&MASTER, PLAINTEXT, format, Some(NONCE)) - .unwrap_or_else(|e| panic!("outboard header_path c{format}: {e}")); + let oenc = stream_encode_outboard_buffer( + &MASTER, + PLAINTEXT, + format, + Some(NONCE), + &carbonado::ZstdEncode::level(20), + ) + .unwrap_or_else(|e| panic!("outboard header_path c{format}: {e}")); let hdr = file::Header::new( &MASTER, NONCE, @@ -185,8 +199,9 @@ fn encode_outboard_fixture(format: u8) -> (OutboardEncoded, Option>, boo let hdr_bytes = hdr.try_to_vec().expect("hdr vec"); (oenc, Some(hdr_bytes), true) } else { - let oenc = carbonado::encode_outboard(&MASTER, PLAINTEXT, format) - .unwrap_or_else(|e| panic!("outboard public c{format}: {e}")); + let oenc = + encode_outboard_with_zstd(&MASTER, PLAINTEXT, format, None, &ZstdEncode::level(20)) + .unwrap_or_else(|e| panic!("outboard public c{format}: {e}")); (oenc, None, false) } } diff --git a/tests/header_tamper.rs b/tests/header_tamper.rs index 5667249..2a6240b 100644 --- a/tests/header_tamper.rs +++ b/tests/header_tamper.rs @@ -4,12 +4,12 @@ mod common; use carbonado::{ constants::Format, - encode, error::CarbonadoError, file::{self, Header}, structs::Encoded, }; use common::header_layout::{self, offsets}; +use common::{encode, file_encode, file_encode_outboard}; use rand::RngCore; fn random_master() -> [u8; 32] { @@ -21,7 +21,7 @@ fn random_master() -> [u8; 32] { fn valid_headered_archive(level: u8) -> ([u8; 32], Vec) { let key = random_master(); let input = b"header tamper matrix payload"; - let (encoded, _) = file::encode(&key, input, level, None).expect("encode"); + let (encoded, _) = file_encode(&key, input, level, None).expect("encode"); (key, encoded) } @@ -62,7 +62,7 @@ fn test_header_tamper_matrix() { // Non-zero metadata path: tamper must still fail MAC verify. let (encoded_meta, _) = - file::encode(&key, b"metadata tamper matrix", 14, Some(*b"metameta")).unwrap(); + file_encode(&key, b"metadata tamper matrix", 14, Some(*b"metameta")).unwrap(); let mut meta_tampered = encoded_meta.clone(); header_layout::flip_byte(&mut meta_tampered, offsets::METADATA); let err_meta = file::decode(&key, &meta_tampered).unwrap_err(); @@ -110,7 +110,7 @@ fn test_header_tamper_matrix() { fn test_decode_outboard_header_tamper_matrix() { let key = random_master(); let input = b"decode_outboard header tamper matrix"; - let (hdr_opt, oenc) = file::encode_outboard(&key, input, 14, Some(*b"metameta")).unwrap(); + let (hdr_opt, oenc) = file_encode_outboard(&key, input, 14, Some(*b"metameta")).unwrap(); let hdr = hdr_opt.unwrap(); let hdr_bytes = hdr.try_to_vec().unwrap(); @@ -219,7 +219,7 @@ fn test_chunk_index_nonzero_roundtrip_and_tamper() { #[test] fn decode_outboard_short_header_returns_invalid_header_length_not_panic() { let key = random_master(); - let (hdr_opt, oenc) = file::encode_outboard(&key, b"short header guard", 14, None).unwrap(); + let (hdr_opt, oenc) = file_encode_outboard(&key, b"short header guard", 14, None).unwrap(); let hdr = hdr_opt.unwrap(); let short = [0u8; 10]; @@ -262,7 +262,7 @@ fn decode_outboard_short_header_returns_invalid_header_length_not_panic() { fn decode_outboard_caller_header_mismatch_after_valid_mac() { let key = random_master(); let input = b"caller vs header mismatch"; - let (hdr_opt, oenc) = file::encode_outboard(&key, input, 14, None).unwrap(); + let (hdr_opt, oenc) = file_encode_outboard(&key, input, 14, None).unwrap(); let hdr = hdr_opt.unwrap(); let hbytes = hdr.try_to_vec().unwrap(); diff --git a/tests/parallel_determinism.rs b/tests/parallel_determinism.rs index 1e04f49..77fdb41 100644 --- a/tests/parallel_determinism.rs +++ b/tests/parallel_determinism.rs @@ -23,9 +23,10 @@ use carbonado::stream::parallel::{ ParallelConfig, encode_rs_parity_serial, encode_rs_parity_with_config, rs_parity_parallelism_active, }; -use carbonado::stream::{stream_decode_buffer, stream_encode_buffer}; -use carbonado::{decode, encode, scrub, structs::Encoded}; +use carbonado::stream::stream_decode_buffer; +use carbonado::{decode, scrub, structs::Encoded}; use common::corruption::{InboardShardLayout, flip_byte}; +use common::{encode, stream_encode_buffer}; use reed_solomon_erasure::galois_8::ReedSolomon; use common::inboard_parity::{assert_inboard_body_roundtrip, preprocess_and_body}; @@ -60,15 +61,18 @@ fn rs_parity_parallel_matches_serial_reference() { for logical_len in [4096usize, 16_384, 65_536, 262_144] { let input = patterned(logical_len); let mut enc = FecInboardEncoder::new(logical_len).expect("new"); - enc.feed(Cursor::new(&input)).expect("feed"); - let stripe = enc.finish().expect("finish").expect("stripe"); - let chunk_len = stripe.chunk_len as usize; - - let serial_shards = serial_parity_from_data_shards(&rs, &stripe.shards[..FEC_K], chunk_len); - assert_eq!( - stripe.shards, serial_shards, - "parity shards must match serial reference for logical_len={logical_len}" - ); + let mut stripes = enc.feed(Cursor::new(&input)).expect("feed"); + stripes.extend(enc.finish().expect("finish")); + assert!(!stripes.is_empty(), "logical_len={logical_len}"); + for stripe in &stripes { + let chunk_len = stripe.chunk_len as usize; + let serial_shards = + serial_parity_from_data_shards(&rs, &stripe.shards[..FEC_K], chunk_len); + assert_eq!( + stripe.shards, serial_shards, + "parity shards must match serial reference for logical_len={logical_len}" + ); + } } } @@ -82,19 +86,24 @@ fn stripe_boundary_parallel_fec_matches_serial_reference() { let (encoded_parallel, pl, cl) = encode_inboard_buffer(&input).expect("parallel encode"); let mut enc = FecInboardEncoder::new(logical_len).expect("new"); - enc.feed(Cursor::new(&input)).expect("feed"); - let stripe = enc.finish().expect("finish").expect("stripe"); - let serial_shards = - serial_parity_from_data_shards(&rs, &stripe.shards[..FEC_K], stripe.chunk_len as usize); + let mut stripes = enc.feed(Cursor::new(&input)).expect("feed"); + stripes.extend(enc.finish().expect("finish")); let mut encoded_serial = Vec::new(); - carbonado::stream::fec::write_inboard_stripe( - &FecStripe { - shards: serial_shards, - chunk_len: cl, - }, - &mut encoded_serial, - ) - .expect("flatten serial"); + for stripe in &stripes { + let serial_shards = serial_parity_from_data_shards( + &rs, + &stripe.shards[..FEC_K], + stripe.chunk_len as usize, + ); + carbonado::stream::fec::write_inboard_stripe( + &FecStripe { + shards: serial_shards, + chunk_len: cl, + }, + &mut encoded_serial, + ) + .expect("flatten serial"); + } assert_eq!( encoded_parallel, encoded_serial, @@ -147,19 +156,21 @@ fn outboard_parity_parallel_matches_serial_buffer_path() { let (pl, cl, parity_parallel) = encode_outboard_parity_buffer(&input).expect("parallel path"); let mut enc = FecInboardEncoder::new(input.len()).expect("new"); - enc.feed(Cursor::new(&input)).expect("feed"); - let stripe = enc.finish().expect("finish").expect("stripe"); - let serial_shards = - serial_parity_from_data_shards(&rs, &stripe.shards[..FEC_K], stripe.chunk_len as usize); + let mut stripes = enc.feed(Cursor::new(&input)).expect("feed"); + stripes.extend(enc.finish().expect("finish")); let mut parity_serial = Vec::new(); - write_outboard_parity( - &FecStripe { - shards: serial_shards, - chunk_len: cl, - }, - &mut parity_serial, - ) - .expect("write parity"); + for stripe in &stripes { + let serial_shards = + serial_parity_from_data_shards(&rs, &stripe.shards[..FEC_K], stripe.chunk_len as usize); + write_outboard_parity( + &FecStripe { + shards: serial_shards, + chunk_len: cl, + }, + &mut parity_serial, + ) + .expect("write parity"); + } assert_eq!(parity_parallel, parity_serial, "outboard parity bytes"); assert_eq!(pl, enc.padding_len()); @@ -224,8 +235,9 @@ fn parallel_config_max_threads_preserves_parity_bytes() { let input = patterned(16_384); let mut enc = FecInboardEncoder::new(input.len()).expect("new"); - enc.feed(Cursor::new(&input)).expect("feed"); - let stripe = enc.finish().expect("finish").expect("stripe"); + let mut stripes = enc.feed(Cursor::new(&input)).expect("feed"); + stripes.extend(enc.finish().expect("finish")); + let stripe = stripes.into_iter().next().expect("stripe"); let chunk_len = stripe.chunk_len as usize; let data = stripe.shards[..FEC_K].to_vec(); diff --git a/tests/rkyv_golden_lock.rs b/tests/rkyv_golden_lock.rs index d3b30e1..11ad073 100644 --- a/tests/rkyv_golden_lock.rs +++ b/tests/rkyv_golden_lock.rs @@ -1,7 +1,9 @@ //! W3 golden lock: fixture `.bin` files must match Rust `rkyv::to_bytes` **and** //! the Lean-embedded `golden*Hex` constants in `Carbonado/RkyvFilepack.lean`. //! -//! Regen (updates bins + prints hex to sync into Lean): +//! Regen (updates bins + prints hex for this file). Lean `golden*Hex` still +//! encodes FilepackManifest v2 SegmentRef (no dict fields); do not copy these +//! v3 strings into Lean until `Carbonado/Filepack.lean` grows dict offsets. //! ```bash //! cargo run --example dump_rkyv_r9 --features backend-rust //! ``` @@ -39,83 +41,95 @@ fn seg(root_fill: u8, main_len: u64, chunk: u32, vo: u32) -> SegmentRef { verification_outboard_len: 64, fec_parity_offset: vo + 64, fec_parity_len: 128, + dict_offset: 0, + dict_len: 0, } } /// Must stay bit-identical to `Carbonado/RkyvFilepack.lean` golden*Hex (Issue 3 lock). mod lean_hex { - pub const EMPTY: &str = "020000000efbffffff00000000"; + pub const EMPTY: &str = "030000000efbffffff00000000"; pub const SINGLE: &str = concat!( "1111111111111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", + "0000000000000000", "612e747874ffffff", "2222222222222222222222222222222222222222222222222222222222222222", - "0e9bffffff01000000000000000000000000", - "020000000ec1ffffff01000000", + "0e93ffffff01000000000000000000000000", + "030000000ec1ffffff01000000", ); pub const MULTI_OTS: &str = concat!( "1111111111111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", + "0000000000000000", "622f6c6f6e6765722d706174682d6e616d652e747874", "4444444444444444444444444444444444444444444444444444444444444444", "00000000c80000000000000000000000400000004000000080000000", + "0000000000000000", "abcdef01", "612e747874ffffff", "2222222222222222222222222222222222222222222222222222222222222222", - "0e45ffffff01000000000000000000000000", - "9600000070ffffff", + "0e35ffffff01000000000000000000000000", + "9600000068ffffff", "3333333333333333333333333333333333333333333333333333333333333333", - "0e5dffffff010000000190ffffff04000000", - "020000000e87ffffff02000000", + "0e55ffffff010000000190ffffff04000000", + "030000000e87ffffff02000000", ); pub const PATH_INLINE_8: &str = concat!( "1111111111111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", + "0000000000000000", "3132333435363738", "2222222222222222222222222222222222222222222222222222222222222222", - "0e9bffffff01000000000000000000000000", - "020000000ec1ffffff01000000", + "0e93ffffff01000000000000000000000000", + "030000000ec1ffffff01000000", ); pub const PATH_OOL_9: &str = concat!( "313233343536373839", "1111111111111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", - "89000000bbffffff", + "0000000000000000", + "89000000b3ffffff", "2222222222222222222222222222222222222222222222222222222222222222", - "0e9bffffff01000000000000000000000000", - "020000000ec1ffffff01000000", + "0e93ffffff01000000000000000000000000", + "030000000ec1ffffff01000000", ); pub const TWO_SEGMENTS: &str = concat!( "1111111111111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", + "0000000000000000", "1212121212121212121212121212121212121212121212121212121212121212", "010000003200000000000000c0000000400000000001000080000000", + "0000000000000000", "612e747874ffffff", "2222222222222222222222222222222222222222222222222222222222222222", - "0e5fffffff02000000000000000000000000", - "020000000ec1ffffff01000000", + "0e4fffffff02000000000000000000000000", + "030000000ec1ffffff01000000", ); pub const OTS_FIRST_ONLY: &str = concat!( "1111111111111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", + "0000000000000000", "dead", "4444444444444444444444444444444444444444444444444444444444444444", "00000000c80000000000000000000000400000004000000080000000", + "0000000000000000", "612e747874ffffff", "2222222222222222222222222222222222222222222222222222222222222222", - "0e5dffffff010000000190ffffff02000000", + "0e4dffffff010000000188ffffff02000000", "622e747874ffffff", "3333333333333333333333333333333333333333333333333333333333333333", - "0e61ffffff01000000000000000000000000", - "020000000e87ffffff02000000", + "0e59ffffff01000000000000000000000000", + "030000000e87ffffff02000000", ); pub const RKYV_CFP2_PREFIX: &str = concat!( "4346503211111111111111111111111111111111111111111111111111111111", "00000000640000000000000000000000400000004000000080000000", + "0000000000000000", "612e747874ffffff", "2222222222222222222222222222222222222222222222222222222222222222", - "0e9bffffff01000000000000000000000000", - "020000000ec1ffffff01000000", + "0e93ffffff01000000000000000000000000", + "030000000ec1ffffff01000000", ); } @@ -198,6 +212,8 @@ fn rust_rkyv_to_bytes_matches_fixtures() { verification_outboard_len: 64, fec_parity_offset: 64, fec_parity_len: 128, + dict_offset: 0, + dict_len: 0, }], ots_proof: None, }; @@ -211,6 +227,87 @@ fn rust_rkyv_to_bytes_matches_fixtures() { let b = m_cfp2.to_bytes().unwrap(); assert_eq!(&b[0..4], b"CFP2"); assert_eq!(b, fs::read(fixture("rkyv_cfp2_prefix.bin")).unwrap()); + + let path8 = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "12345678".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: None, + }], + }; + assert_eq!( + path8.to_bytes().unwrap(), + fs::read(fixture("path_inline_8.bin")).unwrap() + ); + + let path9 = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "123456789".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: None, + }], + }; + assert_eq!( + path9.to_bytes().unwrap(), + fs::read(fixture("path_ool_9.bin")).unwrap() + ); + + let two = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0), seg(0x12, 50, 1, 192)], + ots_proof: None, + }], + }; + assert_eq!( + two.to_bytes().unwrap(), + fs::read(fixture("two_segments.bin")).unwrap() + ); + + let ots_first = FilepackManifest { + version: FILEPACK_MANIFEST_VERSION, + format_level: FILEPACK_MANIFEST_FORMAT_LEVEL_PUBLIC, + catalog_bao_root: [0u8; 32], + catalog_ots_proof: None, + entries: vec![ + FilepackEntry { + rel_path: "a.txt".into(), + content_blake3: [0x22; 32], + segment_format: 0x0E, + segments: vec![seg(0x11, 100, 0, 0)], + ots_proof: Some(vec![0xDE, 0xAD]), + }, + FilepackEntry { + rel_path: "b.txt".into(), + content_blake3: [0x33; 32], + segment_format: 0x0E, + segments: vec![seg(0x44, 200, 0, 0)], + ots_proof: None, + }, + ], + }; + assert_eq!( + ots_first.to_bytes().unwrap(), + fs::read(fixture("ots_first_only.bin")).unwrap() + ); } #[test] diff --git a/tests/seekable_slices.rs b/tests/seekable_slices.rs index 8406037..60a675a 100644 --- a/tests/seekable_slices.rs +++ b/tests/seekable_slices.rs @@ -1,12 +1,15 @@ //! Seekable 4 KiB slice verification without full-stream materialization. +mod common; + use std::process::id; use anyhow::Result; use carbonado::{ - constants::SLICE_LEN, encode, encode_outboard, error::CarbonadoError, verify_slice, - verify_slice_inboard_seekable, verify_slice_outboard, + constants::SLICE_LEN, error::CarbonadoError, verify_slice, verify_slice_inboard_seekable, + verify_slice_outboard, }; +use common::{encode, encode_outboard}; use rand::RngCore; const C14: u8 = 0x0E; diff --git a/tests/serial_fec_path.rs b/tests/serial_fec_path.rs index 784b1a1..e9b4042 100644 --- a/tests/serial_fec_path.rs +++ b/tests/serial_fec_path.rs @@ -23,11 +23,13 @@ fn serial_fec_encoder_matches_buffer_path_at_stripe_boundaries() { let (buffer_encoded, pl, cl) = encode_inboard_buffer(&input).expect("buffer"); let mut enc = FecInboardEncoder::new(logical_len).expect("new"); - enc.feed(Cursor::new(&input)).expect("feed"); - let stripe = enc.finish().expect("finish").expect("stripe"); + let mut stripes = enc.feed(Cursor::new(&input)).expect("feed"); + stripes.extend(enc.finish().expect("finish")); let mut incremental = Vec::new(); - for shard in &stripe.shards { - incremental.extend_from_slice(shard); + for stripe in &stripes { + for shard in &stripe.shards { + incremental.extend_from_slice(shard); + } } assert_eq!(pl, enc.padding_len(), "padding len {logical_len}"); @@ -36,6 +38,7 @@ fn serial_fec_encoder_matches_buffer_path_at_stripe_boundaries() { incremental, buffer_encoded, "serial rs.encode path at logical_len={logical_len}" ); - assert_eq!(incremental.len(), FEC_M * cl as usize); + assert_eq!(incremental.len() % (FEC_M * cl as usize), 0); + assert_eq!(cl, 4096); } } diff --git a/tests/shard_fec_scrub.rs b/tests/shard_fec_scrub.rs index 9968250..94eed5d 100644 --- a/tests/shard_fec_scrub.rs +++ b/tests/shard_fec_scrub.rs @@ -8,9 +8,9 @@ use anyhow::Result; use carbonado::{ error::CarbonadoError, scrub, - stream::{ShardEncodeResult, ShardSource, decode_shards_stream, encode_shard_stream}, + stream::{ShardEncodeResult, ShardSource, decode_shards_stream}, }; -use common::corruption::InboardShardLayout; +use common::encode_shard_stream; const MASTER: [u8; 32] = [0x42; 32]; const FORMAT: u8 = 14; @@ -85,9 +85,11 @@ fn shard_body_corruption_scrub_one_segment() -> Result<()> { let info = result.encode_info.clone(); let mut corrupted_body = body.clone(); - let layout = InboardShardLayout::from_encode_info(corrupted_body.len(), info.chunk_len); - // Erase four shard stripes (50% RS budget) — guarantees Bao failure + scrub path. - common::corruption::erase_shards(&mut corrupted_body, &layout, &[0, 2, 4, 6]); + let n_leaves = carbonado::stream::inboard_leaf_data_ranges(&corrupted_body) + .expect("leaf ranges") + .len() as u32; + let mask = common::corruption::leaves_with_symbol_slots(n_leaves, &[0, 2, 4, 6]); + common::corruption::wipe_inboard_leaves(&mut corrupted_body, &mask, 0xEE); let recovered_body = scrub(&corrupted_body, hash_bytes, &info, FORMAT)?; assert_eq!(recovered_body, *body); diff --git a/tests/sharding.rs b/tests/sharding.rs index a94e33f..42da56f 100644 --- a/tests/sharding.rs +++ b/tests/sharding.rs @@ -9,9 +9,9 @@ use carbonado::{ error::CarbonadoError, stream::{ DEFAULT_SEGMENT_PLAINTEXT_BUDGET, ShardEncodeResult, ShardSource, decode_shards_stream, - encode_shard_stream, }, }; +use common::encode_shard_stream; use rand::RngCore; use common::{ diff --git a/tests/slh_outboard.rs b/tests/slh_outboard.rs index e31b4a8..2233500 100644 --- a/tests/slh_outboard.rs +++ b/tests/slh_outboard.rs @@ -5,6 +5,8 @@ #![cfg(feature = "pqc")] +mod common; + use std::fs; use carbonado::{ @@ -17,6 +19,7 @@ use carbonado::{ error::CarbonadoError, file::{self, Header}, }; +use common::file_encode_outboard; use getrandom::getrandom; use rand::RngCore; @@ -43,7 +46,7 @@ fn test_slh_outboard_sidecar_binds_header_public_key() { let key = random_master(); let input = b"SLH-DSA outboard E2E: sign keyed Bao root, verify via Header.slh_public_key"; - let (hdr_opt, oenc) = file::encode_outboard(&key, input, 14, None).unwrap(); + let (hdr_opt, oenc) = file_encode_outboard(&key, input, 14, None).unwrap(); let base_hdr = hdr_opt.unwrap(); let bao_root = base_hdr.hash.as_bytes(); diff --git a/tests/streaming.rs b/tests/streaming.rs index 694fcb4..76a3f83 100644 --- a/tests/streaming.rs +++ b/tests/streaming.rs @@ -1,16 +1,18 @@ //! Streaming encode/decode roundtrip vs buffer path + multi-MiB smoke. +mod common; + use std::fs::File; use std::io::{Cursor, Read, Write}; use carbonado::decode; -use carbonado::encode; -use carbonado::file::{decode_stream, encode_stream}; +use carbonado::file::decode_stream; use carbonado::stream::{ decode::stream_decode_outboard, - encode::{stream_encode_buffer, stream_encode_outboard, stream_encode_outboard_buffer}, + encode::{stream_encode_outboard, stream_encode_outboard_buffer}, stream_decode_buffer, stream_decode_outboard_buffer, }; +use common::{encode, file_encode_stream, stream_encode_buffer}; use proptest::prelude::*; use rand::RngCore; @@ -54,6 +56,7 @@ proptest! { has_zfec.then_some(&mut par_buf), &mut nonce, header_path, + &carbonado::ZstdEncode::level(20), )?; let buf = stream_encode_outboard_buffer( @@ -61,6 +64,7 @@ proptest! { &data, format, if encrypted { Some(nonce) } else { None }, + &carbonado::ZstdEncode::level(20), )?; prop_assert_eq!(hash, buf.hash); @@ -115,6 +119,7 @@ fn stream_outboard_empty_zfec_roundtrip() { Some(&mut par_buf), &mut nonce, false, + &carbonado::ZstdEncode::level(20), ) .expect("empty encode"); @@ -152,6 +157,7 @@ fn stream_outboard_encrypted_header_nonce_roundtrip() { None::<&mut Vec>, &mut nonce, true, + &carbonado::ZstdEncode::level(20), ) .expect("enc encode"); assert_ne!(nonce, [0u8; 16]); @@ -189,8 +195,8 @@ fn multi_mib_file_stream_smoke() { let mut in_f = File::open(&input_path).expect("open input"); let mut body_buf = Vec::new(); - let (header, _info) = - encode_stream(&MASTER, &mut in_f, 14, None, &mut body_buf).expect("encode_stream"); + let (header, _info) = file_encode_stream(&MASTER, &mut in_f, 14, None, &mut body_buf) + .expect("file_encode_stream"); let mut archive = header.try_to_vec().expect("header"); archive.extend_from_slice(&body_buf); @@ -230,8 +236,8 @@ fn decode_stream_codecode_decodec_public_c14() { let pt: Vec = (0..4096).map(|i| (i % 251) as u8).collect(); let mut body = Vec::new(); - let (h1, _) = - encode_stream(&MASTER, std::io::Cursor::new(&pt), FORMAT, None, &mut body).expect("enc1"); + let (h1, _) = file_encode_stream(&MASTER, std::io::Cursor::new(&pt), FORMAT, None, &mut body) + .expect("enc1"); let mut a = h1.try_to_vec().expect("hdr"); a.extend_from_slice(&body); @@ -242,7 +248,7 @@ fn decode_stream_codecode_decodec_public_c14() { // codecode: re-encode must match wire when public (deterministic) let mut body2 = Vec::new(); - let (h2, _) = encode_stream( + let (h2, _) = file_encode_stream( &MASTER, std::io::Cursor::new(&out), FORMAT, @@ -260,7 +266,7 @@ fn decode_stream_codecode_decodec_public_c14() { assert_eq!(out2, pt, "decodec: plaintext roundtrip"); } -/// `encode_stream` / `decode_stream` format sweep (~64 KiB) vs buffer path. +/// `file_encode_stream` / `decode_stream` format sweep (~64 KiB) vs buffer path. #[test] fn file_stream_format_sweep() { const PAYLOAD_LEN: usize = 64 * 1024; @@ -290,7 +296,8 @@ fn file_stream_format_sweep() { let mut in_f = File::open(&input_path).expect("open input"); let mut body_buf = Vec::new(); let (header, _stream_info) = - encode_stream(master, &mut in_f, format, None, &mut body_buf).expect("encode_stream"); + file_encode_stream(master, &mut in_f, format, None, &mut body_buf) + .expect("file_encode_stream"); let mut archive = header.try_to_vec().expect("header"); archive.extend_from_slice(&body_buf); @@ -362,6 +369,7 @@ fn stream_outboard_public_e2_codecode_decodec_c4_c12() { has_fec.then_some(&mut par1), &mut nonce, false, + &carbonado::ZstdEncode::level(20), ) .expect("encode1"); let main1_bytes = main1.into_inner(); @@ -396,6 +404,7 @@ fn stream_outboard_public_e2_codecode_decodec_c4_c12() { has_fec.then_some(&mut par2), &mut nonce2, false, + &carbonado::ZstdEncode::level(20), ) .expect("encode2"); let main2_bytes = main2.into_inner(); @@ -426,7 +435,14 @@ fn stream_outboard_public_e2_codecode_decodec_c4_c12() { assert_eq!(out2, pt, "c{format} decodec plaintext"); // Match buffer path - let buf = stream_encode_outboard_buffer(&MASTER, &pt, format, None).expect("buf encode"); + let buf = stream_encode_outboard_buffer( + &MASTER, + &pt, + format, + None, + &carbonado::ZstdEncode::level(20), + ) + .expect("buf encode"); assert_eq!(buf.hash, hash1, "c{format} stream vs buffer hash"); assert_eq!(buf.main, main1_bytes, "c{format} stream vs buffer main"); let buf_dec = stream_decode_outboard_buffer( diff --git a/tests/streaming_async.rs b/tests/streaming_async.rs index f31acb1..c46b5b5 100644 --- a/tests/streaming_async.rs +++ b/tests/streaming_async.rs @@ -13,7 +13,7 @@ use std::io::{Cursor, ErrorKind}; use carbonado::constants::FEC_M; use carbonado::error::CarbonadoError; use carbonado::stream::{stream_decode, stream_decode_async, stream_decode_buffer}; -use carbonado::stream_encode_buffer; +use common::stream_encode_buffer; use futures_lite::io::Cursor as AsyncCursor; use rand::RngCore; diff --git a/tests/streaming_limits.rs b/tests/streaming_limits.rs index 980d014..eaf4912 100644 --- a/tests/streaming_limits.rs +++ b/tests/streaming_limits.rs @@ -11,7 +11,7 @@ use std::io::Cursor; use carbonado::constants::FEC_M; use carbonado::decode as low_level_decode; use carbonado::error::CarbonadoError; -use carbonado::file::{Header, decode, decode_stream, encode, encode_stream}; +use carbonado::file::{Header, decode, decode_stream}; use carbonado::stream::crypto_stream::{ stream_decrypt, stream_decrypt_seek, stream_decrypt_with_nonce, stream_decrypt_with_nonce_seek, }; @@ -20,14 +20,14 @@ use carbonado::stream::encode::{PreprocessStats, stream_encode_inboard_body}; use carbonado::stream::fec::{FecInboardEncoder, encode_inboard_buffer}; use carbonado::stream::{ stream_decode, stream_decode_buffer, stream_decode_outboard, stream_decode_outboard_buffer, - stream_encode_buffer, }; -use carbonado::{encode_outboard, scrub, scrub_outboard, verify_inboard_keyed_oracle}; +use carbonado::{scrub, scrub_outboard, verify_inboard_keyed_oracle}; use rand::RngCore; use common::inboard_parity::{ BoundedReadSeek, assert_bounded_inboard_body_roundtrip, assert_inboard_body_roundtrip, }; +use common::{encode_outboard, file_encode, file_encode_stream, stream_encode_buffer}; const MASTER: [u8; 32] = [0x42; 32]; @@ -40,23 +40,28 @@ fn inboard_fec_encoder_incremental_feed_matches_buffer_path() { let mut enc = FecInboardEncoder::new(input.len()).expect("new"); let mut off = 0usize; + let mut stripes = Vec::new(); while off < input.len() { let step = 256.min(input.len() - off); - let _ = enc - .feed(Cursor::new(&input[off..off + step])) - .expect("feed"); + stripes.extend( + enc.feed(Cursor::new(&input[off..off + step])) + .expect("feed"), + ); off += step; } - let stripe = enc.finish().expect("finish").expect("stripe"); + stripes.extend(enc.finish().expect("finish")); let mut incremental = Vec::new(); - for shard in &stripe.shards { - incremental.extend_from_slice(shard); + for stripe in &stripes { + for shard in &stripe.shards { + incremental.extend_from_slice(shard); + } } assert_eq!(pl, enc.padding_len()); assert_eq!(cl, enc.chunk_len()); assert_eq!(incremental.len(), buffer_encoded.len()); assert_eq!(incremental, buffer_encoded); + assert_eq!(stripes.len(), 4); } /// `stream_encode_inboard_body` FEC path feeds post-preprocess data incrementally (S2). @@ -401,7 +406,7 @@ fn stream_decode_short_bao_body_invalid_header_length_all_entry_points() { "stream_decode_buffer: {err_buffer:?}" ); - let (archive, _) = encode(&master, b"hello", 12, None).expect("encode file"); + let (archive, _) = file_encode(&master, b"hello", 12, None).expect("encode file"); let truncated = &archive[..Header::LEN + 4]; let err_file = decode(&master, truncated).expect_err("file::decode"); @@ -471,7 +476,7 @@ fn decode_stream_rejects_bad_header_mac_before_body_read() { } } - let mut archive = encode(&[0u8; 32], b"mac-before-body", 14, None) + let mut archive = file_encode(&[0u8; 32], b"mac-before-body", 14, None) .expect("encode public c14") .0; assert!(archive.len() > Header::LEN); @@ -506,8 +511,8 @@ fn encode_stream_decode_stream_roundtrip_c14_c15() { for &(master, format) in &[(&[0u8; 32], 14u8), (&enc_master, 15u8)] { let input: Vec = (0..65_536).map(|i| (i % 251) as u8).collect(); let mut body = Vec::new(); - let (header, _) = - encode_stream(master, Cursor::new(&input), format, None, &mut body).expect("encode"); + let (header, _) = file_encode_stream(master, Cursor::new(&input), format, None, &mut body) + .expect("encode"); let mut archive = header.try_to_vec().expect("header"); archive.extend_from_slice(&body); @@ -532,7 +537,7 @@ fn stream_decrypt_rejects_tampered_tag_before_plaintext() { rand::thread_rng().fill_bytes(&mut enc_master); let (archive, _) = - encode(&enc_master, b"streaming etm mac-before-decrypt", 3, None).expect("encode c3"); + file_encode(&enc_master, b"streaming etm mac-before-decrypt", 3, None).expect("encode c3"); let mut tampered = archive; tampered[Header::LEN] ^= 0xFF; @@ -758,6 +763,7 @@ fn stream_decode_outboard_bounded_read_matches_buffer_path() { Some(&mut par_buf), &mut nonce, true, + &carbonado::ZstdEncode::level(20), ) .expect("header-path encode"); assert_stream_decode_outboard_parity( diff --git a/tests/udp_fec_sim.rs b/tests/udp_fec_sim.rs index 8f48b91..df606c7 100644 --- a/tests/udp_fec_sim.rs +++ b/tests/udp_fec_sim.rs @@ -1,50 +1,49 @@ //! UDP / FEC chaos injection model (RS 4/8) — **not** normative on-disk wire layout. //! -//! This crate simulates JBOD/UDP **shard erasure** using the same approximate -//! `InboardShardLayout` helper as `tests/common/corruption.rs` and `tests/fec_chaos.rs` -//! (`shard_byte_range` spaced by `chunk_len` from the 8-byte Bao prefix). That linear -//! model is for distributed knockout / `erase_shards` injection only; true inboard -//! c12/c14 wire is `[u64 LE content_len | keyed Bao response]` with FEC stripes inside -//! the Bao envelope (`src/stream/encode.rs`, `src/stream/bao.rs`). +//! One chaos datagram is every 4 KiB inboard Bao leaf with a given RS symbol +//! (`inboard_symbol_payload` / `erase_shards`). That is stripe geometry, not a +//! tall `chunk_len` column. True inboard c12/c14 wire is +//! `[u64 LE content_len | keyed Bao response]` with FEC stripes inside the Bao +//! envelope (`src/stream/encode.rs`, `src/stream/bao.rs`). //! -//! **Intended contract under test:** if a transport maps one logical RS shard column to -//! one datagram payload *at the scrub injection coordinates*, then dropping ≤4 datagrams -//! should match `erase_shards` + `scrub` recovery. At five drops: c12 is irrecoverable -//! (`InvalidScrubbedHash`); c14 may still recover (Snappy/geometry asymmetry — see -//! `five_datagram_drops_c12_irrecoverable_c14_documents_asymmetry`). Bao provides keyed -//! verification independent of datagram arrival order. +//! **Contract:** dropping ≤4 of 8 symbol datagrams matches `erase_shards` + `scrub` +//! recovery. Five symbol drops at c12 are irrecoverable (`InvalidScrubbedHash`). +//! c14 may still recover: zstd padding leaves are already zeros, so `fill(0)` is +//! not a Bao erasure. That is not a weaker 50% budget. Bao verifies independent +//! of arrival order. //! -//! Format coverage: 4-drop recovery uses c14 (Snappy+Bao+Zfec); 5-drop asymmetry is -//! exercised on both c12 (Bao+Zfec) and c14. +//! Format coverage: 4-drop recovery uses c14 (zstd+Bao+FEC); 5-drop c12 vs c14 +//! is `five_datagram_drops_c12_irrecoverable_c14_documents_asymmetry`. mod common; use anyhow::Result; use carbonado::{ - constants::{FEC_K, FEC_M}, - decode, encode, + constants::{FEC_K, FEC_M, SLICE_LEN}, + decode, error::CarbonadoError, scrub, structs::Encoded, }; -use common::corruption::{InboardShardLayout, OutboardShardLayout, erase_shards}; +use common::corruption::{ + InboardShardLayout, OutboardShardLayout, erase_shards, inboard_symbol_payload, + write_inboard_symbol_payload, +}; +use common::{encode, encode_outboard}; -/// Chaos-injection datagram: `shard_index` + payload at `InboardShardLayout` coordinates. +/// Chaos-injection datagram: one RS symbol's 4 KiB leaves concatenated. #[derive(Clone, Debug)] struct FecDatagram { shard_index: usize, payload: Vec, } -/// Split encoded buffer into eight chaos-injection shard slots (helper-internal geometry). +/// Split encoded buffer into eight symbol datagrams (stripe leaves, not tall columns). fn inboard_to_datagrams(encoded: &[u8], layout: &InboardShardLayout) -> Vec { (0..layout.num_shards) - .map(|shard_index| { - let range = layout.shard_byte_range(shard_index); - FecDatagram { - shard_index, - payload: encoded[range].to_vec(), - } + .map(|shard_index| FecDatagram { + shard_index, + payload: inboard_symbol_payload(encoded, layout, shard_index), }) .collect() } @@ -57,16 +56,16 @@ fn datagrams_to_inboard( buf: &mut [u8], ) -> Result<()> { for dgram in datagrams { - let range = layout.shard_byte_range(dgram.shard_index); - if range.len() != dgram.payload.len() { + if let Err((got, expected)) = + write_inboard_symbol_payload(buf, layout, dgram.shard_index, &dgram.payload) + { anyhow::bail!( "datagram shard {} payload len {} != layout range len {}", dgram.shard_index, - dgram.payload.len(), - range.len() + got, + expected ); } - buf[range].copy_from_slice(&dgram.payload); } Ok(()) } @@ -190,11 +189,23 @@ fn chaos_datagram_slots_align_with_inboard_shard_layout_helper() -> Result<()> { "RS 4/8 chaos model uses eight shard slots" ); + assert_eq!( + info.chunk_len, SLICE_LEN, + "RS symbol / Bao leaf is 4 KiB, not a tall column (got {})", + info.chunk_len + ); + // 32 KiB logical → two 16 KiB stripes → two 4 KiB leaves per symbol. + assert_eq!( + datagrams[0].payload.len(), + 2 * SLICE_LEN as usize, + "symbol datagram is concatenated stripe leaves, not one chunk_len column" + ); + for dgram in &datagrams { - let range = layout.shard_byte_range(dgram.shard_index); + let expected = inboard_symbol_payload(&encoded, &layout, dgram.shard_index); assert_eq!( - dgram.payload, encoded[range], - "chaos slot {} must match InboardShardLayout range (helper-internal)", + dgram.payload, expected, + "chaos slot {} must match stripe leaves for that RS symbol", dgram.shard_index ); } @@ -238,10 +249,9 @@ fn duplicate_datagram_shard_index_last_writer_wins() -> Result<()> { let mut reassembled = vec![0u8; encoded.len()]; datagrams_to_inboard(&[first, second], &layout, &mut reassembled)?; - let range = layout.shard_byte_range(1); assert_eq!( - &reassembled[range.clone()], - &encoded[range], + inboard_symbol_payload(&reassembled, &layout, 1), + inboard_symbol_payload(&encoded, &layout, 1), "duplicate shard_index: last datagram wins" ); Ok(()) @@ -304,8 +314,8 @@ fn five_datagram_drops_c12_irrecoverable_c14_documents_asymmetry() -> Result<()> "five datagram drops must be irrecoverable at c12, got {err:?}" ); } else { - // c14 + Snappy: at the approximate chaos coordinates, five erased stripes can - // still leave enough RS columns for scrub recovery (documented asymmetry vs c12). + // c14 + zstd: five symbol fills can be no-ops on already-zero padding + // leaves, so Bao still verifies those slots. Not a weaker 50% budget. let recovered = scrub_result.expect("c14 five-drop scrub recovery"); assert_eq!(recovered, orig); } @@ -321,7 +331,6 @@ fn directory_bundle_parity_outboard_scrub_recovery() -> Result<()> { fec_slice_from_bundle, split_adamantine_payload, verification_slice_from_bundle, }, directory::SegmentFormatPolicy, - encode_outboard, file::{ DIRECTORY_ARCHIVE_FORMAT, DirectoryEncodeOptions, decode, decode_directory, encode_directory_with_options, @@ -353,6 +362,7 @@ fn directory_bundle_parity_outboard_scrub_recovery() -> Result<()> { &enc_dir, DirectoryEncodeOptions { segment_format_policy: SegmentFormatPolicy::ForceC12, + zstd: carbonado::ZstdEncode::level(20), ..DirectoryEncodeOptions::default() }, )?; diff --git a/tests/zstd_frame_params.rs b/tests/zstd_frame_params.rs index 44a6975..a15086c 100644 --- a/tests/zstd_frame_params.rs +++ b/tests/zstd_frame_params.rs @@ -12,7 +12,7 @@ use std::fs; use std::path::{Path, PathBuf}; use carbonado::constants::{ - ZSTD_CONTENT_CHECKSUM, ZSTD_DICTIONARY_ID_FLAG, ZSTD_LEVEL, ZSTD_LEVEL20_WINDOW_LOG_LARGE, + ZSTD_CONTENT_CHECKSUM, ZSTD_DICTIONARY_ID_FLAG, ZSTD_LEVEL20, ZSTD_LEVEL20_WINDOW_LOG_LARGE, ZSTD_MAGIC, }; use carbonado::stream::compress::compress_buffer; @@ -60,7 +60,7 @@ fn assert_product_shared_flags(h: &ParsedZstdFrameHeader) { #[test] fn rust_constants_match_lean_spec() { const { - assert!(ZSTD_LEVEL == 20); + assert!(ZSTD_LEVEL20 == 20); assert!(matches!(ZSTD_MAGIC, [0x28, 0xb5, 0x2f, 0xfd])); assert!(!ZSTD_CONTENT_CHECKSUM); assert!(ZSTD_DICTIONARY_ID_FLAG == 0); @@ -90,7 +90,7 @@ fn parse_rejects_truncated_bad_magic_reserved() { #[test] fn bulk_level20_hello_matches_lean_aot_golden() { - let frame = zstd::bulk::Compressor::new(ZSTD_LEVEL) + let frame = zstd::bulk::Compressor::new(ZSTD_LEVEL20) .expect("compressor") .compress(b"hello") .expect("compress hello"); @@ -111,7 +111,7 @@ fn bulk_level20_hello_matches_lean_aot_golden() { #[test] fn bulk_level20_empty_matches_lean_aot_golden() { - let frame = zstd::bulk::Compressor::new(ZSTD_LEVEL) + let frame = zstd::bulk::Compressor::new(ZSTD_LEVEL20) .expect("compressor") .compress(b"") .expect("compress empty"); @@ -129,7 +129,7 @@ fn bulk_level20_empty_matches_lean_aot_golden() { #[test] fn product_compress_buffer_frame_params() { - let frame = compress_buffer(b"hello").expect("compress_buffer hello"); + let frame = compress_buffer(b"hello", ZSTD_LEVEL20).expect("compress_buffer hello"); assert_eq!(&frame[..4], &ZSTD_MAGIC); let h = parse_zstd_frame_header(&frame).expect("parse product hello"); assert_product_shared_flags(&h); @@ -195,7 +195,7 @@ fn g9_outboard_c14_fixtures_match_lean_named_params() { #[test] fn stream_copy_encode_unknown_size_window_log_25() { let mut frame = Vec::new(); - zstd::stream::copy_encode(b"hello" as &[u8], &mut frame, ZSTD_LEVEL).expect("copy_encode"); + zstd::stream::copy_encode(b"hello" as &[u8], &mut frame, ZSTD_LEVEL20).expect("copy_encode"); let h = parse_zstd_frame_header(&frame).expect("parse copy_encode"); assert_eq!(&frame[..4], &ZSTD_MAGIC); assert_product_shared_flags(&h);