From ba7dc60681ffbd0ea55563f4ae31390d2f44164e Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 07:05:18 -0500 Subject: [PATCH 1/4] refactor(real): route AudioV4 through serial tables --- src/parsers/audio/real_audio.rs | 443 +++++++++++++++++++------------- 1 file changed, 259 insertions(+), 184 deletions(-) diff --git a/src/parsers/audio/real_audio.rs b/src/parsers/audio/real_audio.rs index 0cdfdfdee..38791220f 100644 --- a/src/parsers/audio/real_audio.rs +++ b/src/parsers/audio/real_audio.rs @@ -1,94 +1,85 @@ -//! RealAudio (.ra) binary metadata parser. +//! RealAudio (`.ra`) metadata parser. //! -//! ExifTool routes a `.ra` file through `Image::ExifTool::Real::ProcessReal` -//! (`Real.pm:516-587`). The file opens with an 8-byte record: the signature -//! `".ra\xfd"`, a big-endian `u16` version, and a big-endian `u16` "extra" -//! field (`Real.pm:565`, `unpack('x4nn', $buff)`). The version selects one of -//! three sub-tables -- `Real::AudioV3`, `Real::AudioV4`, `Real::AudioV5` -//! (`Real.pm:72-74`) -- each walked by `Canon::ProcessSerialData`: fields -//! decode in declared order at cumulative byte offsets, several of them -//! (`TitleLen`/`Title`, `ArtistLen`/`Artist`, `CopyrightLen`/`Copyright`, -//! `CommentLen`/`Comment`) a length-prefixed pair where the earlier field's -//! *value* -- not a fixed offset -- sizes the later one -//! (`Real.pm:313-322`, `Format => 'string[$val{N}]'`). +//! ExifTool's `Image::ExifTool::Real::ProcessReal` first reads an eight-byte +//! `.ra` header, then chooses `Real::AudioV3`, `::AudioV4`, or `::AudioV5` +//! from its version (`Real.pm:563-587`). This caller deliberately enables +//! only the source-generated `Real::AudioV4` serial descriptor. The other +//! versions remain inactive until their carrier semantics have equivalent +//! native proof; a descriptor's presence alone is not a route. //! -//! This parser reproduces `Real::AudioV4` (`Real.pm:289-322`), the only -//! version the pinned test corpus exercises. `Real::AudioV3`/`AudioV5` -//! (`Real.pm:270-286`, `327-346`) are declared but not implemented: per -//! AGENTS.md, an unimplemented version is left absent (just the header, no -//! `Real-RA3`/`Real-RA5` tags) rather than approximated against the wrong -//! table -- the same outcome ExifTool itself produces for a version with no -//! matching `.ra$vers` tag at all (`Real.pm:568-577`, "Unsupported -//! RealAudio version"). +//! `ProcessReal` asks its file handle for 512 bytes after the header, but Perl +//! accepts any nonzero short read. The carrier therefore passes at most the +//! first 512 available bytes into the shared `ProcessSerialData` reader rather +//! than requiring a complete 512-byte audio header. The shared reader owns +//! AudioV4's 31 serial slots, dynamic string lengths, `Unknown` cursor rows, +//! NUL handling, and source groups. This module retains only carrier +//! signature/version selection and that bounded file read. //! -//! `ProcessSerialData`'s per-field `Unknown => 1` flag (`Real.pm`'s own -//! table) hides a field from ExifTool's default (non-`-u`) output; this -//! parser skips those fields' bytes without emitting a tag, the same -//! visibility ExifTool's default run has. -//! -//! # References -//! -//! - ExifTool source: `lib/Image/ExifTool/Real.pm` +//! References: pinned ExifTool 13.59 `lib/Image/ExifTool/Real.pm:289-322, +//! 559-587`; serial descriptor generated from `Canon::ProcessSerialData`. -use crate::core::{FileReader, MetadataMap, TagValue}; +use std::collections::HashMap; -/// `Real.pm:523`, `$buff =~ m{^(\.RMF|\.ra\xfd|pnm://|rtsp://|http://)}`. -const RA_SIGNATURE: &[u8] = b".ra\xfd"; +use crate::core::{FileReader, Instance, MetadataMap}; +use crate::exiftool_tables::{ + Ctx, Emitted, SerialDir, SerialEmissionSink, SerialTable, find_serial_table, + process_serial_directory, +}; +use crate::io::ByteOrder; -/// `Real.pm:565`, `$raf->Read($buff, 512)` -- the body read following the -/// 8-byte header, once the version is known. +/// `Real.pm:523`, `$buff =~ m{^(.RMF|.ra\\xfd|...)}`. +const RA_SIGNATURE: &[u8] = b".ra\xfd"; +/// `Real.pm:566`, `$raf->Read($buff, 512)`. const BODY_READ_LEN: usize = 512; -/// Sequential cursor over `Real::AudioV4`'s big-endian fields -/// (`Real.pm:289-322`), mirroring `Canon::ProcessSerialData`'s cumulative -/// offset tracking. -struct FieldCursor<'a> { - data: &'a [u8], - pos: usize, +/// Project source-described `FoundTag` rows into the parser's existing +/// occurrence store. The map's visible key is ExifTool group 1 plus name, +/// as the old RealAudio parser used; priority and `-n` form remain attached to +/// the occurrence. `MetadataMap` has no public group-2 projection seam, so +/// the source group-2 fact remains on the generated row and is not claimed by +/// this carrier's visible map. +struct MetadataSink<'a> { + metadata: &'a mut MetadataMap, } -impl<'a> FieldCursor<'a> { - fn new(data: &'a [u8]) -> Self { - Self { data, pos: 0 } - } - - fn take(&mut self, len: usize) -> Option<&'a [u8]> { - let bytes = self.data.get(self.pos..self.pos + len)?; - self.pos += len; - Some(bytes) - } - - fn u8(&mut self) -> Option { - self.take(1).map(|b| b[0]) - } - - fn u16(&mut self) -> Option { - self.take(2).map(|b| u16::from_be_bytes([b[0], b[1]])) - } - - fn u32(&mut self) -> Option { - self.take(4) - .map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]])) +impl SerialEmissionSink for MetadataSink<'_> { + fn emit(&mut self, row: Emitted) { + let key = format!("{}:{}", row.group1, row.name); + let priority = u8::from(!(row.low_priority || row.avoid)); + match row.value_conv { + Some(value_conv) => { + self.metadata.insert_occurrence_with_raw( + key, + row.value, + value_conv, + priority, + row.group1, + Instance::default(), + ); + } + None => { + self.metadata.insert_occurrence( + key, + row.value, + priority, + row.group1, + Instance::default(), + ); + } + } } - /// Skip a length-prefixed string field (`Format => 'string[$val{N}]'`) - /// whose length was captured by an earlier `*Len` field, returning the - /// bytes when the length is non-zero. - /// - /// A zero-length field is a decline, not an empty value: ExifTool's own - /// `Real.ra` sample carries `ArtistLen: 0` and `CommentLen: 0`, and - /// `Artist`/`Comment` do not appear in its `-a -G1 -s` output at all. - fn string(&mut self, len: u8) -> Option<&'a [u8]> { - if len == 0 { - return None; - } - self.take(usize::from(len)) + fn serial_enabled(&self, _table: &'static SerialTable) -> bool { + true } } -/// Extract RealAudio (`.ra`) metadata using ExifTool's `Real::AudioV4` -/// sequential layout (version 4 only -- see the module doc for why other -/// versions are left unimplemented rather than approximated). +/// Extract RealAudio metadata through the source-generated AudioV4 descriptor. +/// +/// The handwritten carrier selection is intentionally narrow: ExifTool's +/// version-four route is the only Real Audio route whose generated reader has +/// native replay coverage. Unsupported versions retain the existing empty +/// audio-tag result rather than borrowing a nearby serial layout. pub fn parse_real_audio_metadata( reader: &dyn FileReader, ) -> std::result::Result { @@ -96,126 +87,105 @@ pub fn parse_real_audio_metadata( if !header.starts_with(RA_SIGNATURE) { return Err("missing RealAudio '.ra\\xfd' signature".to_string()); } - // Real.pm:565: `unpack('x4nn', $buff)` -- big-endian u16 version at - // offset 4, "extra" (unused here) at offset 6. + // Real.pm:565: `unpack('x4nn', $buff)`. let version = u16::from_be_bytes([header[4], header[5]]); - let mut metadata = MetadataMap::new(); if version != 4 { - // Real.pm:568-577: no `.ra$vers` tag table for this version -- - // ExifTool warns and stops after `SetFileType`, reporting no audio - // tags. `File:FileType`/`MIMEType` still come from - // `add_identity_tags` in `core::operations`. return Ok(metadata); } + // Perl IO's Read succeeds with a positive short read. FileReader is + // exact-length, so request only the bytes actually available while still + // preserving ProcessReal's 512-byte upper bound. let available = reader.size().saturating_sub(8).min(BODY_READ_LEN as u64) as usize; let body = reader .read(8, available) .map_err(|error| error.to_string())?; - - let mut cursor = FieldCursor::new(body); - let _four_cc1 = cursor.take(4); // Unknown => 1 - let _audio_file_size = cursor.u32(); // Unknown => 1 - let _version2 = cursor.u16(); // Unknown => 1 - let _header_size = cursor.u32(); // Unknown => 1 - let _codec_flavor_id = cursor.u16(); // Unknown => 1 - let _coded_frame_size = cursor.u32(); // Unknown => 1 - let audio_bytes = cursor.u32(); - let bytes_per_minute = cursor.u32(); - let _unknown8 = cursor.u32(); // Unknown => 1 - let _sub_packet_h = cursor.u16(); // Unknown => 1 - let audio_frame_size = cursor.u16(); - let _sub_packet_size = cursor.u16(); // Unknown => 1 - let _unknown12 = cursor.u16(); // Unknown => 1 - let sample_rate = cursor.u16(); - let _unknown14 = cursor.u16(); // Unknown => 1 - let bits_per_sample = cursor.u16(); - let channels = cursor.u16(); - let _four_cc2_len = cursor.u8(); // Unknown => 1 - let _four_cc2 = cursor.take(4); // Unknown => 1 - let _four_cc3_len = cursor.u8(); // Unknown => 1 - let _four_cc3 = cursor.take(4); // Unknown => 1 - let _unknown21 = cursor.u8(); // Unknown => 1 - let _unknown22 = cursor.u16(); // Unknown => 1 - let title_len = cursor.u8(); - let title = title_len.and_then(|len| cursor.string(len)); - let artist_len = cursor.u8(); - let artist = artist_len.and_then(|len| cursor.string(len)); - let copyright_len = cursor.u8(); - let copyright = copyright_len.and_then(|len| cursor.string(len)); - let comment_len = cursor.u8(); - let comment = comment_len.and_then(|len| cursor.string(len)); - - const GROUP: &str = "Real-RA4"; - if let Some(value) = audio_bytes { - metadata.insert( - format!("{GROUP}:AudioBytes"), - TagValue::Integer(i64::from(value)), - ); - } - if let Some(value) = bytes_per_minute { - metadata.insert( - format!("{GROUP}:BytesPerMinute"), - TagValue::Integer(i64::from(value)), - ); - } - if let Some(value) = audio_frame_size { - metadata.insert( - format!("{GROUP}:AudioFrameSize"), - TagValue::Integer(i64::from(value)), - ); - } - if let Some(value) = sample_rate { - metadata.insert( - format!("{GROUP}:SampleRate"), - TagValue::Integer(i64::from(value)), - ); - } - if let Some(value) = bits_per_sample { - metadata.insert( - format!("{GROUP}:BitsPerSample"), - TagValue::Integer(i64::from(value)), - ); - } - if let Some(value) = channels { - metadata.insert( - format!("{GROUP}:Channels"), - TagValue::Integer(i64::from(value)), - ); - } - if let Some(bytes) = title { - metadata.insert( - format!("{GROUP}:Title"), - TagValue::new_string(String::from_utf8_lossy(bytes).into_owned()), - ); - } - if let Some(bytes) = artist { - metadata.insert( - format!("{GROUP}:Artist"), - TagValue::new_string(String::from_utf8_lossy(bytes).into_owned()), - ); - } - if let Some(bytes) = copyright { - metadata.insert( - format!("{GROUP}:Copyright"), - TagValue::new_string(String::from_utf8_lossy(bytes).into_owned()), - ); - } - if let Some(bytes) = comment { - metadata.insert( - format!("{GROUP}:Comment"), - TagValue::new_string(String::from_utf8_lossy(bytes).into_owned()), - ); + if body.is_empty() { + // Native warns and returns after a zero-byte body read. This parser's + // Result has no warning channel; preserving its prior output behavior + // means returning the header-only map without invented AudioV4 rows. + return Ok(metadata); } + let table = find_serial_table("Real", "AudioV4") + .expect("generated Real::AudioV4 descriptor must accompany its opted-in carrier"); + let mut members = HashMap::new(); + let mut ctx = Ctx::new(&mut members); + let mut sink = MetadataSink { + metadata: &mut metadata, + }; + let _result = process_serial_directory( + table, + SerialDir { + data: body, + dir_start: 0, + dir_len: body.len(), + base: 0, + data_pos: 8, + byte_order: ByteOrder::Big, + }, + &mut ctx, + &mut sink, + ); Ok(metadata) } #[cfg(test)] mod tests { use super::*; - use crate::test_support::pinned_fixture_reader; + use crate::core::{FileReader, TagValue}; + use crate::test_support::{TestReader, pinned_fixture_reader}; + use std::cell::RefCell; + use std::io; + + fn u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_be_bytes()); + } + + fn u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_be_bytes()); + } + + /// A carrier-level AudioV4 sample. The descriptor, rather than this + /// fixture helper, defines slot order and names. + fn audio_v4(title: &[u8], artist: &[u8], copyright: &[u8], comment: &[u8]) -> Vec { + let mut body = Vec::new(); + body.extend_from_slice(b".ra4"); // FourCC1, Unknown + u32(&mut body, 0); // AudioFileSize, Unknown + u16(&mut body, 4); // Version2, Unknown + u32(&mut body, 0); // HeaderSize, Unknown + u16(&mut body, 0); // CodecFlavorID, Unknown + u32(&mut body, 0); // CodedFrameSize, Unknown + u32(&mut body, 9_000); // AudioBytes + u32(&mut body, 1_200); // BytesPerMinute + u32(&mut body, 0); // Unknown + u16(&mut body, 0); // SubPacketH, Unknown + u16(&mut body, 256); // AudioFrameSize + u16(&mut body, 0); // SubPacketSize, Unknown + u16(&mut body, 0); // Unknown + u16(&mut body, 44_100); // SampleRate + u16(&mut body, 0); // Unknown + u16(&mut body, 16); // BitsPerSample + u16(&mut body, 2); // Channels + body.push(4); // FourCC2Len, Unknown + body.extend_from_slice(b"cook"); // FourCC2, Unknown + body.push(4); // FourCC3Len, Unknown + body.extend_from_slice(b"genr"); // FourCC3, Unknown + body.push(0); // Unknown + u16(&mut body, 0); // Unknown + for text in [title, artist, copyright, comment] { + body.push(u8::try_from(text.len()).expect("fixture string fits native int8u length")); + body.extend_from_slice(text); + } + + let mut file = Vec::with_capacity(8 + body.len()); + file.extend_from_slice(RA_SIGNATURE); + u16(&mut file, 4); + u16(&mut file, 0); + file.extend_from_slice(&body); + file + } #[test] fn matches_exiftool_13_59_on_the_real_fixture() { @@ -224,15 +194,13 @@ mod tests { }; let metadata = parse_real_audio_metadata(&reader).expect("parses"); - // Cross-checked against `exiftool -a -G1 -s` (pinned 13.59) on the - // same fixture. assert_eq!( metadata.get("Real-RA4:AudioBytes"), - Some(&TagValue::Integer(704352)) + Some(&TagValue::Integer(704_352)) ); assert_eq!( metadata.get("Real-RA4:BytesPerMinute"), - Some(&TagValue::Integer(299743)) + Some(&TagValue::Integer(299_743)) ); assert_eq!( metadata.get("Real-RA4:AudioFrameSize"), @@ -240,7 +208,7 @@ mod tests { ); assert_eq!( metadata.get("Real-RA4:SampleRate"), - Some(&TagValue::Integer(22050)) + Some(&TagValue::Integer(22_050)) ); assert_eq!( metadata.get("Real-RA4:BitsPerSample"), @@ -254,8 +222,115 @@ mod tests { metadata.get("Real-RA4:Title"), Some(&TagValue::new_string("The Sewing Girls")) ); - // ArtistLen/CommentLen are 0 in this fixture -- absent, not empty. + // ArtistLen/CommentLen are zero in this fixture: absent, not empty. assert_eq!(metadata.get("Real-RA4:Artist"), None); assert_eq!(metadata.get("Real-RA4:Comment"), None); } + + #[test] + fn generated_v4_reader_preserves_short_body_and_string_boundaries() { + let bytes = audio_v4(b"\0tail", b"", b"\xe9", b"ok"); + let metadata = parse_real_audio_metadata(&TestReader::new(bytes)).expect("parses"); + + assert_eq!( + metadata.get("Real-RA4:AudioBytes"), + Some(&TagValue::Integer(9_000)) + ); + // Positive count with a leading/trailing NUL is reported as the + // NUL-truncated string; count zero does not call FoundTag. + assert_eq!( + metadata.get("Real-RA4:Title"), + Some(&TagValue::new_string("")) + ); + assert_eq!(metadata.get("Real-RA4:Artist"), None); + // Native's output repair occurs only at reporting time. + assert_eq!( + metadata.get("Real-RA4:Copyright"), + Some(&TagValue::new_string("?")) + ); + assert_eq!( + metadata.get("Real-RA4:Comment"), + Some(&TagValue::new_string("ok")) + ); + } + + #[test] + fn one_byte_or_header_only_body_invents_no_v4_rows() { + let mut one_byte = Vec::from(RA_SIGNATURE); + u16(&mut one_byte, 4); + u16(&mut one_byte, 0); + one_byte.push(0); + assert!( + parse_real_audio_metadata(&TestReader::new(one_byte)) + .expect("native accepts a nonzero short body read") + .is_empty() + ); + + let mut header_only = Vec::from(RA_SIGNATURE); + u16(&mut header_only, 4); + u16(&mut header_only, 0); + assert!( + parse_real_audio_metadata(&TestReader::new(header_only)) + .expect("parser has no native warning channel") + .is_empty() + ); + } + + #[test] + fn overlarge_dynamic_length_stops_later_rows_but_keeps_prior_scalars() { + let mut bytes = audio_v4(b"title", b"artist", b"copy", b"comment"); + // AudioV4's TitleLen is the byte immediately before the title. Make + // it extend beyond this bounded body: ProcessSerialData retains the + // numeric slots already read and stops before Title/remaining rows. + let title_len = 8 + 61; + bytes[title_len] = u8::MAX; + bytes.truncate(title_len + 4); + let metadata = parse_real_audio_metadata(&TestReader::new(bytes)).expect("parses"); + assert_eq!( + metadata.get("Real-RA4:SampleRate"), + Some(&TagValue::Integer(44_100)) + ); + assert_eq!(metadata.get("Real-RA4:Title"), None); + assert_eq!(metadata.get("Real-RA4:Artist"), None); + } + + struct RecordingReader { + data: Vec, + reads: RefCell>, + } + + impl FileReader for RecordingReader { + fn read(&self, offset: u64, length: usize) -> io::Result<&[u8]> { + self.reads.borrow_mut().push((offset, length)); + let start = usize::try_from(offset).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "offset does not fit usize") + })?; + let end = start.checked_add(length).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "read range overflows") + })?; + self.data + .get(start..end) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "read past fixture")) + } + + fn size(&self) -> u64 { + self.data.len() as u64 + } + } + + #[test] + fn carrier_reads_at_most_native_512_byte_window() { + let mut bytes = audio_v4(b"title", b"artist", b"copy", b"comment"); + bytes.resize(8 + BODY_READ_LEN + 88, 0xa5); + let reader = RecordingReader { + data: bytes, + reads: RefCell::new(Vec::new()), + }; + let metadata = parse_real_audio_metadata(&reader).expect("parses"); + assert_eq!( + metadata.get("Real-RA4:Title"), + Some(&TagValue::new_string("title")) + ); + assert_eq!(reader.reads.into_inner(), vec![(0, 8), (8, BODY_READ_LEN)]); + } } From 4f01db974ec83d8562589bb900c84072dc86d87d Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 07:07:41 -0500 Subject: [PATCH 2/4] docs(real): record occurrence group projection limit --- src/parsers/audio/real_audio.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/parsers/audio/real_audio.rs b/src/parsers/audio/real_audio.rs index 38791220f..0e2e71104 100644 --- a/src/parsers/audio/real_audio.rs +++ b/src/parsers/audio/real_audio.rs @@ -33,11 +33,13 @@ const RA_SIGNATURE: &[u8] = b".ra\xfd"; const BODY_READ_LEN: usize = 512; /// Project source-described `FoundTag` rows into the parser's existing -/// occurrence store. The map's visible key is ExifTool group 1 plus name, -/// as the old RealAudio parser used; priority and `-n` form remain attached to -/// the occurrence. `MetadataMap` has no public group-2 projection seam, so -/// the source group-2 fact remains on the generated row and is not claimed by -/// this carrier's visible map. +/// occurrence store. The visible key remains ExifTool group 1 plus name, as +/// the old RealAudio parser used; priority and the `-n` form remain attached +/// to the occurrence. This insertion API derives occurrence group 0 from that +/// visible key (`Real-RA4`), keeps generated group 1 (`Real-RA4`), and has no +/// group-2 parameter. Native AudioV4 group 0 (`Real`) and group 2 (including +/// per-row `Author`) therefore remain descriptor facts, not occurrence-level +/// metadata claims in this narrow carrier migration. struct MetadataSink<'a> { metadata: &'a mut MetadataMap, } From d3f513250b1cac0245a0612b5545106c9c1ac16f Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 07:53:23 -0500 Subject: [PATCH 3/4] Pin native RealAudio string repair and incomplete-field stopping --- src/parsers/audio/real_audio.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/parsers/audio/real_audio.rs b/src/parsers/audio/real_audio.rs index 0e2e71104..9e21c5f66 100644 --- a/src/parsers/audio/real_audio.rs +++ b/src/parsers/audio/real_audio.rs @@ -224,6 +224,12 @@ mod tests { metadata.get("Real-RA4:Title"), Some(&TagValue::new_string("The Sewing Girls")) ); + assert_eq!( + metadata.get("Real-RA4:Copyright"), + Some(&TagValue::new_string( + "** institut f?r universelle zusammenh?nge" + )) + ); // ArtistLen/CommentLen are zero in this fixture: absent, not empty. assert_eq!(metadata.get("Real-RA4:Artist"), None); assert_eq!(metadata.get("Real-RA4:Comment"), None); @@ -296,6 +302,25 @@ mod tests { assert_eq!(metadata.get("Real-RA4:Artist"), None); } + #[test] + fn incomplete_artist_is_not_reinterpreted_as_copyright_or_comment() { + // Native ProcessSerialData stops when the declared Artist span does + // not fit in ProcessReal's 512-byte read. The former manual cursor + // stayed at that span after failure, then reused its bytes as later + // string lengths and invented Copyright and Comment values. + let title = vec![b'A'; 255]; + let artist = vec![b'B'; 255]; + let bytes = audio_v4(&title, &artist, b"copyright", b"comment"); + let metadata = parse_real_audio_metadata(&TestReader::new(bytes)).expect("parses"); + assert_eq!( + metadata.get("Real-RA4:Title"), + Some(&TagValue::new_string("A".repeat(255))) + ); + for name in ["Artist", "Copyright", "Comment"] { + assert_eq!(metadata.get(&format!("Real-RA4:{name}")), None); + } + } + struct RecordingReader { data: Vec, reads: RefCell>, From 4bcda9d930a1ab7d4bee4d966a76994e9907a3f0 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 07:58:22 -0500 Subject: [PATCH 4/4] Record RealAudio retirement proof and remaining acceptance checks --- docs/AUTOGENERATION-PLAN.md | 7 +- docs/AUTOGENERATION-PROGRESS.md | 38 ++++++++--- docs/UPGRADE-NEXT-STEPS.md | 2 +- docs/reference/real-audio-v4-retirement.md | 73 +++++++++++++++++++++ docs/reference/serial-runtime-checkpoint.md | 14 ++-- 5 files changed, 116 insertions(+), 18 deletions(-) create mode 100644 docs/reference/real-audio-v4-retirement.md diff --git a/docs/AUTOGENERATION-PLAN.md b/docs/AUTOGENERATION-PLAN.md index 380268f4b..86c4a9c15 100644 --- a/docs/AUTOGENERATION-PLAN.md +++ b/docs/AUTOGENERATION-PLAN.md @@ -37,7 +37,7 @@ areas remain unfinished; they cannot disappear from the denominator. | Shared binary strings | Merged in PR #747 at `8f0fdaf4`. It preserves raw bytes in saved state, distinguishes a one-byte default from a remainder string, and carries the bounded CameraInfo repair. | This is a shared capability, not a Canon migration. CameraInfo remains a legacy text-domain adapter, and no Canon manual reader has been removed. The prior full-pair supervisor-status limitation remains recorded. | | Keyed-directory schema and compiler | Merged in PR #748 at `ebbe1ece`; all final hosted checks passed at `a422e8de`. | Native parent facts and expression declarations are checked. Shared reporting policy merged in #750; the inactive reader merged in #752 at `634e5616`. No production route is active. | | Shared word-directory processor | Merged in PR #754 at `1138a880`; nine tables, 132 rows, 698 Python tests and all five hosted jobs pass. | Four of five unsupported child processors now have generated descriptors. Canon production routing and manual-reader retirement remain unfinished. | -| Shared serial processor | Native probe and inventory merged in #755/#756. The next combined checkpoint compiles eight generated tables, accounts for 106 emitted alternatives and 26 omissions, and passes Real AudioV3/V4 native/Rust replay. | Final hosted acceptance and merge remain. A separate V4 draft replaces its manual 31-slot sequence; carrier/corpus acceptance is still required before counting retirement. | +| Shared serial processor | Merged in #757 at `58849bc7`, after #755/#756. Eight tables account for 106 emitted alternatives and 26 omissions; all five hosted checks pass. | The V4 migration replaces its manual 31-slot sequence and passes bounded native output plus source-change execution proof. Full corpus and hosted acceptance remain before counting retirement. | | Recorded source inventory | Merged in PR #749 at `18a8ef17`; all final hosted checks passed at `72e8e664`. The report accounts for 1,512 table identities and retains 119 tables with no named rows. | This establishes the captured source population. Classifying which rules are generated, manual, unsupported or unclassified remains open; source shape is not automation. | | Sony plain generator recovery | PR #745 merged; six tables and 193 rows reproduced | These tables can be rebuilt. This alone does not prove that their behavior is fully automatic. | | Sony enciphered recovery | Producer and independent verifier preserved; M4 review found five blockers; not landed | The draft still has a Sony-specific translation layer. Its review remains useful, but it is not the architecture target. | @@ -151,8 +151,9 @@ Canonical artifacts and ledgers were regenerated successfully with local Perl the native serial processor. The [native probe](reference/serial-processor-checkpoint.md) is merged in #755. The reviewed JSON inventory captures all eight selected tables and preserves every refusal. The combined Rust emitter and reader - compile and pass native Real AudioV3/V4 replay. Complete their final hosted - gate, then validate the V4 carrier and remove its manual sequence. + merged in #757 after native Real AudioV3/V4 replay and all final hosted + checks. The V4 carrier passes bounded/native and supported source-change + execution proof; complete its corpus and hosted gate, then land retirement. Account for every condition and conversion before reducing the final unsupported-child count from one to zero. 2. Resolve the four omitted parent rows using shared byte handling. An explicit diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index cff219a31..b9497e0fc 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -65,8 +65,11 @@ chronological callback trace. The combined checkpoint at `66c430c6` passes **37 failures and zero skips**, on both threaded and non-threaded Perl 5.38.2 against pinned ExifTool 13.59. The full hosted checks then passed for #756. -The shared serial reader and Rust emitter are now combined on -`codex/shared-serial-integration-20260913`, with pipeline checkpoint `9693ef9e`. +The shared serial reader and Rust emitter merged in PR #757 as `58849bc7` +at 12:47 UTC. All five hosted checks passed at `72fbebb1`, including 757 +canonical Python tests with zero failures/skips in 641.913 seconds, 5,756 +nextest tests passing (59 intentionally skipped), the complete Cargo test/doc +invocation, and the explicitly invoked native serial replay. The [runtime checkpoint](reference/serial-runtime-checkpoint.md) records the proof and its limits. All eight tables remain accounted for: **106 emitted alternatives and 26 explicit omissions**, with zero independent verifier @@ -86,14 +89,29 @@ local absolute invocation paths. Re-running its producer with the portable invocation reproduces the committed ledger exactly. All 607 expressions agree on 16,789 applicable comparisons; 14 inputs are inapplicable. -This checkpoint still needs its final hosted gate and merge. No production -route is enabled; **one unsupported Canon child processor, four omitted parent -rows and zero Canon manual readers retired** remain the production status. -The Real AudioV4 retirement draft is separately committed and published at -`118afac7`. Source review accepts its narrow behavior, but it still needs a -fresh Rust build, native output comparison, full-corpus pair and upgrade-flow -proof. Its metadata occurrence API retains the existing group-0/group-2 -limitations; native warning output and V3/V5 activation also remain unfinished. +That merged checkpoint enabled no production caller. **One unsupported Canon +child processor, four omitted parent rows and zero Canon manual readers +retired** remain the Canon status. + +The next [Real AudioV4 migration](reference/real-audio-v4-retirement.md) is +implemented on `codex/real-v4-retirement-integration-20260913`, runtime +`4f01db97` with additional regression assertions at `d3f51325`. Its generated +reader replaces the manual 31-slot sequence. All six carrier tests, full +Clippy and formatting pass. A fresh 19-file native/control/candidate comparison +has 15 scored cases matching native output and four explicitly retained scope +or diagnostic cases. Ten files correct existing legacy behavior: UTF-8 repair, +NUL truncation, or later fields invented after an incomplete string. + +A supported source mutation also reaches actual output: changing AudioV4's +`Title` name to `UpgradeTitle` in a copied pinned source, regenerating and +compiling with the unchanged carrier, changes exactly that output key. No +tag-specific Python/Rust rule is edited. This proves that supported change; +it does not substitute for a real release upgrade or broader semantics. + +The full 4,238-file paired comparison and hosted acceptance are required before +landing and counting the 31-slot retirement. The current occurrence API retains +its existing native group-0/group-2 limitations; native warning output and +V3/V5 activation remain unfinished. The first additional retirement candidate is Real AudioV4's manually specified 31-entry sequence. It uses the same native serial processor and can prove reuse diff --git a/docs/UPGRADE-NEXT-STEPS.md b/docs/UPGRADE-NEXT-STEPS.md index 17614bd10..475a7357f 100644 --- a/docs/UPGRADE-NEXT-STEPS.md +++ b/docs/UPGRADE-NEXT-STEPS.md @@ -19,7 +19,7 @@ populations. They are implementation history, not work to restart. | Priority | Remaining work | State | Completion evidence | | --- | --- | --- | --- | -| 1 | Extend the source inventory and complete the next shared directory capability | Sony Tag202a migration/retirement merged in #746; source inventory and artifact join merged in #749/#751. Directory validation and nine word tables merged in #753/#754; canonical regeneration and all final gates pass. One child processor and four parent rows remain unsupported. Serial probe/inventory merged in #755/#756. Combined Rust emission/reading passes native V3/V4 replay; eight tables account for 106 emitted alternatives and 26 omissions. Final hosted gate is pending. Real AudioV4 retirement is published separately and still needs carrier/corpus/upgrade-flow acceptance. | Classify runtime/manual rules against the recorded inventory; resolve named processor blockers, verify both carriers, then retire the duplicate readers. Details in the [scoreboard](./AUTOGENERATION-PROGRESS.md). | +| 1 | Extend the source inventory and complete the next shared directory capability | Sony Tag202a migration/retirement merged in #746; source inventory and artifact join merged in #749/#751. Directory validation and nine word tables merged in #753/#754; canonical regeneration and all final gates pass. One child processor and four parent rows remain unsupported. Serial probe/inventory/runtime merged in #755–#757; eight tables account for 106 emitted alternatives and 26 omissions. Real AudioV4 migration passes six carrier tests, bounded native output and supported source-change execution proof. Complete its paired corpus and hosted acceptance before landing the 31-slot retirement. | Classify runtime/manual rules against the recorded inventory; resolve named processor blockers, verify both carriers, then retire the duplicate readers. Details in the [scoreboard](./AUTOGENERATION-PROGRESS.md). | | 2 | Continue generated EXIF directory migration | IFD1, InteropIFD and ExifIFD E-2 landed; Claude's handoff reserves E-3, fallback retirement, IFD0 and numeric-coercion work | Refresh Claude's ownership before taking work; preserve occurrence behavior and measure each activation against its own control | | 3 | Join producer and upgrade accounting | Source-to-artifact join merged; runtime and manually maintained rule classification remain open | Generated facts, manual rules, unsupported rules and actual execution remain distinct; ordinary source changes need no new tag rules | | 4 | Preserve Sony/Nikon producer recovery evidence | Sony plain landed; enciphered/encrypted recovery preserved and unlanded | Recovery is labeled maintenance; no numbering-only reconstruction or duplicate per-vendor interpreter is promoted as the target | diff --git a/docs/reference/real-audio-v4-retirement.md b/docs/reference/real-audio-v4-retirement.md new file mode 100644 index 000000000..3a5cb14d0 --- /dev/null +++ b/docs/reference/real-audio-v4-retirement.md @@ -0,0 +1,73 @@ +# Real AudioV4 manual-sequence retirement + +The shared serial capability merged in #757 at `58849bc7`. This next change +uses it in the Real AudioV4 carrier and removes the manually copied 31-slot +field sequence. It adds no new tag-specific generator or interpreter. + +## What changes + +`real_audio.rs` retains signature/version selection and the native 512-byte +maximum body read. Generated `Real::AudioV4` supplies names, formats, field +order, dynamic lengths, visibility and groups to the common serial reader. +The carrier projects emitted rows through the existing metadata insertion API. + +Fresh native comparisons also identify corrections to the old parser: + +- Invalid UTF-8 uses native question-mark repair. This fixes Copyright on the + real corpus sample, not only synthetic input. +- Strings stop at NUL while consuming their full declared byte span. +- An incomplete Artist span stops the serial reader. The old cursor stayed in + place after the failed read and could invent Copyright and Comment from the + same bytes. Cases with 511, 512 and 600 body bytes prove the correction. + +AudioV3/V5 activation, native warnings and complete occurrence groups remain +outside this migration. The current API derives stored group 0 from the visible +Real-RA4 key and cannot store group 2; this pre-existing limitation remains +explicit. The descriptor retains the native group facts. No new project +percentage or Canon retirement is claimed. + +## Evidence before the full acceptance gate + +Control is `58849bc7`; candidate runtime is `4f01db97`. `d3f51325` adds only +regression assertions. The six carrier unit tests, full all-feature Clippy and +formatting pass. The source builds reuse one coordinator-owned local cache: +control 1.486 seconds, candidate 7.443 seconds; the final six-test build/run is +61.502 seconds. Workers perform source/evidence work without Cargo. + +The native carrier contract covers 19 fixtures against pinned ExifTool 13.59 +and Perl 5.38.2. All 15 scored candidate projections match native Real-* output. +Ten fixtures improve legacy output; five are unchanged. Four pre-existing +scope/diagnostic cases remain recorded. Control/candidate exit codes agree. +Original failed assumptions that five control projections were already correct +are preserved alongside raw runs and the independently corrected contract. +These counts are bounded fixture evidence, not a full-corpus result. + +A separate copied-source proof changes exactly one supported native property: +`Real::AudioV4[24].Name` from `Title` to `UpgradeTitle`. Fresh dump, generation, +independent verification and formatting change one generated Rust name. A +source archive compiled with that artifact and the unchanged carrier changes +exactly `Real-RA4:Title` to `Real-RA4:UpgradeTitle` on the real sample, retaining +its value and matching the modified native source. Compilation takes 15.461 +seconds. Tag-specific Python/Rust edits: zero. This proves one supported source +change, not compatibility with an entire new release. + +## Acceptance and retirement accounting + +Before landing, require the complete paired 4,238-file census and final hosted +checks. The paired runner reuses `conformance.py` scoring, authenticates both +binary/build manifests and unchanged scoring/oracle helpers, probes DOCX oracle +capability, and preserves file hashes, raw outputs, exits and per-file results. +It caps concurrency at two and persists progress; interrupted or vacuous runs +cannot be reported as a pass. Record any pre-existing diagnostic exits apart +from new failures. Exact per-file changes, rather than equal totals alone, +determine acceptance. + +After acceptance and squash merge, count one manual serial reader and its +31-slot sequence as retired. This is not 31 newly emitted tags: hidden fields +still advance the cursor, and visible output is separately measured. The +remaining Canon child processor and four parent omissions are separate work. + +Evidence is under `$OXIDEX_WORK_EVIDENCE/shared-pilot/real-v4-integration-20260913/`: +`native-carrier-comparison.json`, `native-carrier-accepted.json`, build/test +records, `source-upgrade-proof/runtime-replay.json` and `full-pair-20260913/`. +Native fixture generation/contracts are in sibling `real-audio-retirement-20260913/`. diff --git a/docs/reference/serial-runtime-checkpoint.md b/docs/reference/serial-runtime-checkpoint.md index aa8b7b70d..01ca53910 100644 --- a/docs/reference/serial-runtime-checkpoint.md +++ b/docs/reference/serial-runtime-checkpoint.md @@ -10,7 +10,11 @@ The base is merged PR #756 (`eb700430`), pinned to ExifTool 13.59. Runtime, emitter and independent verifier are integrated at `9693ef9e` on `codex/shared-serial-integration-20260913`. The earlier compiled checkpoint `ff383b8e` is published. Its Rust source is unchanged by the later verifier and -pipeline additions. The final checkpoint still needs hosted acceptance. +pipeline additions. Final head `72fbebb1` passed all five hosted checks and +merged as `58849bc7` in PR #757 at 12:47 UTC on September 13. Canonical Python +passed 757 tests with zero failures/skips in 641.913 seconds; nextest passed +5,756 tests with 59 intentionally skipped. Full Cargo test/doc and explicit +serial native replay also passed. Native inventory selects all tables whose processor is `ProcessSerialData` before examining generated output. Eight tables contain 130 entries and 132 @@ -95,10 +99,12 @@ No i7 job or daemon change is part of this checkpoint. ## What remains -- Complete final hosted checks and merge this inactive runtime checkpoint. +- The inactive runtime checkpoint is merged. Complete acceptance of its first + production carrier, described in the [V4 retirement record](real-audio-v4-retirement.md). - Validate the separately published AudioV4 carrier draft (`118afac7`) with a - fresh candidate/control, real and constructed files, source-change flow and - the full corpus, then remove its manual 31-slot sequence. + full corpus and hosted checks, then land its manual 31-slot retirement. + Fresh bounded candidate/control/native output and supported source-change + execution proof now pass. - Preserve the carrier's known output-model gaps: native group 0/group 2 are not stored by the current occurrence API; header-only native warning output is absent; AudioV3/V5 are not activated. Do not call these complete parity.