From 569c0868bc5fb1d03b99d5e7d93bd02a7f490b4d Mon Sep 17 00:00:00 2001 From: thedancingdeveloper <306930456+thedancingdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:27:45 +0000 Subject: [PATCH] test(nzb-postproc): emit IFSC slice checksums in the PAR2 fixture (WI-151) The synthesised PAR2 index only carried Main + FileDesc packets, so a fixture set had no per-slice checksums and rust_par2 reported every file as damaged. Emit IFSC (Input File Slice Checksum) packets too: per file, the File ID followed by (MD5[16] + CRC32[4]) for each slice, with the final partial slice zero-padded to the slice size exactly as PAR2 and rust_par2's verifier compute them. This is the first piece of the WI-151 deterministic PAR2 fixtures and is what the in-flight slice verifier (WI-144) will check decoded slices against. Two tests prove the checksums are byte-exact: an intact file verifies through the slice checksums, and a single corrupted slice is pinpointed by its block index (which only works if the per-slice MD5/CRC are correct). Full nzb-postproc suite stays green. RecoverySlice packets remain out of scope here (they need the encoder and only matter for repair fixtures, WI-146/148). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01TpoFnScdkRq7qxQLQZe1we --- Cargo.lock | 1 + crates/nzb-postproc/Cargo.toml | 1 + .../nzb-postproc/tests/par2_slice_fixture.rs | 59 +++++++++++++++++++ .../tests/support/par2_fixture.rs | 46 +++++++++++++-- 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 crates/nzb-postproc/tests/par2_slice_fixture.rs diff --git a/Cargo.lock b/Cargo.lock index 6209acb..9226f03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1877,6 +1877,7 @@ name = "nzb-postproc" version = "0.2.7" dependencies = [ "anyhow", + "crc32fast", "md-5 0.11.0", "nzb-core", "opentelemetry", diff --git a/crates/nzb-postproc/Cargo.toml b/crates/nzb-postproc/Cargo.toml index dcc368c..8f820db 100644 --- a/crates/nzb-postproc/Cargo.toml +++ b/crates/nzb-postproc/Cargo.toml @@ -29,6 +29,7 @@ tempfile = "3" # `tests/support/par2_fixture.rs`), which requires writing spec-correct # packet MD5s. Version tracks what `rust-par2` itself parses with. md-5 = "0.11" +crc32fast = "1" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints.clippy] diff --git a/crates/nzb-postproc/tests/par2_slice_fixture.rs b/crates/nzb-postproc/tests/par2_slice_fixture.rs new file mode 100644 index 0000000..cc9f4bc --- /dev/null +++ b/crates/nzb-postproc/tests/par2_slice_fixture.rs @@ -0,0 +1,59 @@ +//! WI-151: the PAR2 fixture emits IFSC slice checksums that `rust_par2` +//! accepts. A correct file verifies intact through those per-slice checksums, +//! and a single corrupted slice is pinpointed by its block index — which only +//! works if the fixture's per-slice MD5/CRC32 are byte-exact. + +mod support; + +use std::fs; + +use support::par2_fixture::Par2Fixture; + +fn payload(len: usize) -> Vec { + (0..len).map(|i| (i % 251) as u8).collect() +} + +#[test] +fn intact_file_verifies_through_slice_checksums() { + let dir = tempfile::tempdir().unwrap(); + // 10_000 bytes at the fixture's 4096-byte slice size -> 3 slices, the last + // one partial (zero-padded for hashing). + let data = payload(10_000); + let name = "movie.mkv"; + Par2Fixture::new() + .add_file(name, &data) + .write_index(&dir.path().join("recovery.par2")); + fs::write(dir.path().join(name), &data).unwrap(); + + let set = rust_par2::parse(&dir.path().join("recovery.par2")).unwrap(); + let file = set.files.values().next().unwrap(); + assert_eq!(file.slices.len(), 3, "three slices including one partial"); + + let result = rust_par2::verify(&set, dir.path()); + assert!(result.all_correct(), "intact fixture must verify: {result}"); +} + +#[test] +fn corrupt_slice_is_located_by_its_block_index() { + let dir = tempfile::tempdir().unwrap(); + let data = payload(10_000); + let name = "movie.mkv"; + Par2Fixture::new() + .add_file(name, &data) + .write_index(&dir.path().join("recovery.par2")); + + // Flip a byte inside slice index 1 (bytes 4096..8192). + let mut corrupt = data.clone(); + corrupt[5000] ^= 0xff; + fs::write(dir.path().join(name), &corrupt).unwrap(); + + let set = rust_par2::parse(&dir.path().join("recovery.par2")).unwrap(); + let result = rust_par2::verify(&set, dir.path()); + + assert_eq!(result.damaged.len(), 1, "one damaged file: {result}"); + assert_eq!( + result.damaged[0].damaged_block_indices, + vec![1], + "IFSC checksums must pinpoint the corrupted slice" + ); +} diff --git a/crates/nzb-postproc/tests/support/par2_fixture.rs b/crates/nzb-postproc/tests/support/par2_fixture.rs index 73e85da..c5db01d 100644 --- a/crates/nzb-postproc/tests/support/par2_fixture.rs +++ b/crates/nzb-postproc/tests/support/par2_fixture.rs @@ -13,10 +13,16 @@ //! and the *expected* filename. The 16K hash is what `rename_to_par2_names` //! matches obfuscated files against. //! -//! Recovery (`RecvSlic`) and slice-checksum (`IFSC`) packets are omitted: they -//! only matter for actual repair, which these tests never reach. Verification -//! will therefore report files as damaged — that is fine and expected, because -//! the assertions are about *filenames on disk*, not repair outcomes. +//! * **IFSC** — one per file: the per-slice MD5 and CRC32 checksums, with the +//! final partial slice zero-padded to the slice size, exactly as PAR2 (and +//! `rust_par2`'s verifier) compute them. This is what lets `verify` report a +//! correct file as intact and what the in-flight slice verifier (WI-144) +//! checks decoded slices against. +//! +//! Recovery (`RecvSlic`) packets are still omitted: they only matter for actual +//! repair, which these fixtures do not exercise. `verify` reports the recovery +//! set as unrepairable (zero recovery blocks), which is fine — the assertions +//! here are about slice checksums and filenames, not repair. //! //! Packet layout implemented here (little-endian), per the PAR 2.0 spec: //! @@ -36,6 +42,7 @@ use md5::{Digest, Md5}; const MAGIC: &[u8; 8] = b"PAR2\x00PKT"; const TYPE_MAIN: &[u8; 16] = b"PAR 2.0\x00Main\x00\x00\x00\x00"; const TYPE_FILE_DESC: &[u8; 16] = b"PAR 2.0\x00FileDesc"; +const TYPE_IFSC: &[u8; 16] = b"PAR 2.0\x00IFSC\x00\x00\x00\x00"; /// One file recorded in the recovery set. struct FileEntry { @@ -46,6 +53,8 @@ struct FileEntry { hash: [u8; 16], hash_16k: [u8; 16], size: u64, + /// Per-slice (MD5, CRC32) checksums, in slice order. + slices: Vec<([u8; 16], u32)>, } /// Builds a PAR2 index file describing a set of files by content. @@ -84,12 +93,29 @@ impl Par2Fixture { id_input.extend_from_slice(expected_name.as_bytes()); let file_id: [u8; 16] = Md5::digest(&id_input).into(); + // Per-slice checksums for the IFSC packet. Each slice is hashed + // zero-padded to the slice size, matching PAR2 and rust_par2's verifier. + let slice_size = self.slice_size as usize; + let mut slices = Vec::new(); + let mut off = 0; + while off < contents.len() { + let end = (off + slice_size).min(contents.len()); + let mut chunk = contents[off..end].to_vec(); + chunk.resize(slice_size, 0); + let slice_md5: [u8; 16] = Md5::digest(&chunk).into(); + let mut crc = crc32fast::Hasher::new(); + crc.update(&chunk); + slices.push((slice_md5, crc.finalize())); + off = end; + } + self.files.push(FileEntry { expected_name: expected_name.to_string(), file_id, hash, hash_16k, size, + slices, }); self } @@ -123,6 +149,18 @@ impl Par2Fixture { out.extend_from_slice(&self.packet(TYPE_FILE_DESC, &body)); } + // One IFSC packet per file: file ID then (MD5[16] + CRC32[4]) per slice. + // 16 + 20*n is always 4-aligned, so the packet length stays valid. + for file in &self.files { + let mut body = Vec::new(); + body.extend_from_slice(&file.file_id); + for (slice_md5, crc32) in &file.slices { + body.extend_from_slice(slice_md5); + body.extend_from_slice(&crc32.to_le_bytes()); + } + out.extend_from_slice(&self.packet(TYPE_IFSC, &body)); + } + std::fs::write(path, &out).unwrap(); }