diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index ef3b531da..3a9ddde7b 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -106,8 +106,40 @@ Portable readiness tests replace private-path dependencies with controlled archived sources and CLI subprocesses. The real selected 11.78 and 12.64 releases separately pass native readiness. This advances the rehearsal instrument; it does not establish generated OxiDex conformance for either -release. Shared WriteValue translation, encoding, helper composition and -public generated routing remain unfinished. +release. Encoding, helper composition and public generated routing remain +unfinished. + +The helper-capture and native-readiness integration **merged in PR #762** as +`54206ecb`. All five required hosted checks passed on `d091f251`, including +generated-table verification. The selected historical releases still have no +completed generated OxiDex/native conformance run. + +The next work-branch checkpoint compiles captured CheckValue and WriteValue +rules into Rust operands through normal regeneration, adding two declared +artifacts (34 total). Unsupported helper semantics produce an explicit ledger +gap and no admitted rule; they cannot silently reuse the older release's +operands. CI's native suite checks both committed artifacts against fresh +generation. The shared Rust executor matches 512 native cases on canonical +source, 512 on a copied WriteValue count-bound change and 512 on a copied +CheckValue comparison change. These are three helper probes, not three release +upgrades. The tests found and corrected UTF8 substring storage and negative +repetition behavior. Source-reference count checks are separate from native +return checks. + +Native capture and oracle startup now explicitly disable ambient ExifTool +configuration. Clean and hostile-home native captures are byte-identical in +the focused probe. This prevents personal configuration from being mistaken +for the selected release's rules. Full 34-artifact regeneration passed in +339.975 seconds. The first full Python run reached all 900 tests and failed two +regeneration-shell controls because their simulated leaf executable did not +recognize the new scalar helper producer. The repaired controls now pass all +five tests, including selected-source/output routing and failure propagation +for the new producer. Preserve the failed full run; its retry and full Rust +validation are pending. The only regeneration difference was the recorded +library path; regenerating the expression ledger through the canonical relative +source path restores identical committed artifacts. No public writer route +is enabled, no manual tag rule has been retired in this checkpoint, and no +project-wide generation percentage has been remeasured. ### Previous merged definitions checkpoint diff --git a/src/writers/generated_scalar.rs b/src/writers/generated_scalar.rs new file mode 100644 index 000000000..419f4eb56 --- /dev/null +++ b/src/writers/generated_scalar.rs @@ -0,0 +1,369 @@ +//! Inactive source-derived scalar CheckValue and WriteValue execution. + +use crate::error::{ExifToolError, Result}; + +/// Perl's scalar storage state at the boundary of the proven helper branch. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum Scalar { + Undefined, + Bytes(Vec), + Utf8(String), +} + +/// One comparison operator carried from a source-derived recipe. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Comparison { + Lt, + Le, + Eq, + Ne, + Ge, + Gt, +} + +impl Comparison { + fn applies(self, left: i64, right: i64) -> bool { + match self { + Self::Lt => left < right, + Self::Le => left <= right, + Self::Eq => left == right, + Self::Ne => left != right, + Self::Ge => left >= right, + Self::Gt => left > right, + } + } +} + +/// The complete admitted scalar early-return branch of native `CheckValue`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ScalarCheckRecipe { + pub formats: [&'static str; 2], + pub positive_count_operator: Comparison, + pub positive_count_bound: i64, + pub first_format: &'static str, + pub first_limit_operator: Comparison, + pub first_error: &'static str, + pub second_limit_operator: Comparison, + pub second_error: &'static str, + pub padding_operator: Comparison, + /// The compiler admits only the native NUL operand. + pub padding_character: u8, +} + +/// The complete admitted scalar early-return branch of native `WriteValue`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ScalarWriteRecipe { + pub formats: [&'static str; 2], + pub terminated_format: &'static str, + pub count_positive_operator: Comparison, + pub count_positive_bound: i64, + pub diff_negative_operator: Comparison, + /// The compiler admits only the native NUL operand. + pub terminator: u8, +} + +/// A CheckValue result: its possibly padded scalar and native source error. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CheckedScalar { + pub value: Scalar, + pub error: Option<&'static str>, +} + +/// A WriteValue result before optional data-pointer mutation or CharsetEXIF. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SerializedScalar { + pub value: Scalar, + /// The source branch's scalar count. TIFF's u32 field-count admission is later. + pub count: usize, +} + +fn refused(reason: &str) -> ExifToolError { + ExifToolError::unsupported_format(format!("generated scalar recipe refused: {reason}")) +} + +fn ensure_format(formats: [&'static str; 2], format_name: &str) -> Result<()> { + if formats.contains(&format_name) { + Ok(()) + } else { + Err(refused("format is outside the proven scalar branch")) + } +} + +fn scalar_length(value: &Scalar) -> Result { + match value { + Scalar::Undefined => Ok(0), + Scalar::Bytes(bytes) => Ok(bytes.len()), + // Perl's length/substr operate on characters for a UTF8-flagged scalar. + Scalar::Utf8(text) => Ok(text.chars().count()), + } +} + +fn i64_length(value: &Scalar) -> Result { + i64::try_from(scalar_length(value)?).map_err(|_| refused("scalar length overflows i64")) +} + +fn positive_count(count: Option, operator: Comparison, bound: i64) -> bool { + count.is_some_and(|value| value != 0 && operator.applies(value, bound)) +} + +fn checked_count(value: i64) -> Result { + usize::try_from(value).map_err(|_| refused("count cannot be represented on this platform")) +} + +fn append_byte(value: Scalar, byte: u8, amount: usize) -> Result { + match value { + Scalar::Undefined => { + let mut bytes = Vec::new(); + bytes + .try_reserve(amount) + .map_err(|_| refused("scalar padding allocation failed"))?; + bytes.resize(amount, byte); + Ok(Scalar::Bytes(bytes)) + } + Scalar::Bytes(mut bytes) => { + bytes + .try_reserve(amount) + .map_err(|_| refused("scalar padding allocation failed"))?; + bytes.resize( + bytes + .len() + .checked_add(amount) + .ok_or_else(|| refused("scalar length overflow"))?, + byte, + ); + Ok(Scalar::Bytes(bytes)) + } + Scalar::Utf8(mut text) => { + if byte > 0x7f { + return Err(refused("non-ASCII scalar padding is unsupported")); + } + text.try_reserve(amount) + .map_err(|_| refused("scalar padding allocation failed"))?; + text.extend(std::iter::repeat(char::from(byte)).take(amount)); + Ok(Scalar::Utf8(text)) + } + } +} + +fn truncate(value: Scalar, length: usize) -> Result { + match value { + Scalar::Undefined => Ok(Scalar::Undefined), + Scalar::Bytes(mut bytes) => { + bytes.truncate(length); + Ok(Scalar::Bytes(bytes)) + } + Scalar::Utf8(text) => { + let byte_end = text + .char_indices() + .nth(length) + .map_or(text.len(), |(index, _)| index); + let truncated = &text[..byte_end]; + // Perl's `substr` downgrades an upgraded ASCII scalar, including + // one containing embedded NULs. A source scalar that contained a + // non-ASCII character retains its UTF8 flag even when the selected + // substring happens to be ASCII or empty. + if text.is_ascii() { + Ok(Scalar::Bytes(truncated.as_bytes().to_vec())) + } else { + Ok(Scalar::Utf8(truncated.to_owned())) + } + } + } +} + +/// Execute the proven scalar `CheckValue` branch without tag lookup or conversion. +pub(crate) fn validate_scalar( + recipe: &ScalarCheckRecipe, + mut value: Scalar, + format_name: &str, + count: Option, +) -> Result { + ensure_format(recipe.formats, format_name)?; + if recipe.padding_character != 0 { + return Err(refused("scalar CheckValue padding is not NUL")); + } + if !positive_count( + count, + recipe.positive_count_operator, + recipe.positive_count_bound, + ) { + return Ok(CheckedScalar { value, error: None }); + } + let count = count.expect("positive_count requires Some"); + let length = i64_length(&value)?; + let (limit, error) = if format_name == recipe.first_format { + (recipe.first_limit_operator, recipe.first_error) + } else { + (recipe.second_limit_operator, recipe.second_error) + }; + if limit.applies(length, count) { + return Ok(CheckedScalar { + value, + error: Some(error), + }); + } + if recipe.padding_operator.applies(length, count) { + // Perl repetition with a non-positive count produces the empty string. + // Source changes may select this branch when length exceeds count. + let padding = if count <= length { + 0 + } else { + checked_count(count - length)? + }; + value = append_byte(value, recipe.padding_character, padding)?; + } + Ok(CheckedScalar { value, error: None }) +} + +/// Execute the proven scalar `WriteValue` branch without data-pointer mutation. +pub(crate) fn serialize_scalar( + recipe: &ScalarWriteRecipe, + mut value: Scalar, + format_name: &str, + count: Option, +) -> Result { + ensure_format(recipe.formats, format_name)?; + if recipe.terminator != 0 { + return Err(refused("scalar WriteValue terminator is not NUL")); + } + if format_name == recipe.terminated_format { + value = append_byte(value, recipe.terminator, 1)?; + } + let final_count = if positive_count( + count, + recipe.count_positive_operator, + recipe.count_positive_bound, + ) { + let count = count.expect("positive_count requires Some"); + let length = i64_length(&value)?; + let difference = count + .checked_sub(length) + .ok_or_else(|| refused("scalar count subtraction overflow"))?; + if difference != 0 { + if recipe.diff_negative_operator.applies(difference, 0) { + let target = checked_count(count)?; + value = if format_name == recipe.terminated_format { + let before_terminator = target + .checked_sub(1) + .ok_or_else(|| refused("terminated scalar count is zero"))?; + append_byte(truncate(value, before_terminator)?, recipe.terminator, 1)? + } else { + truncate(value, target)? + }; + } else { + value = append_byte(value, recipe.terminator, checked_count(difference)?)?; + } + } + checked_count(count)? + } else { + scalar_length(&value)? + }; + Ok(SerializedScalar { + value, + count: final_count, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CHECK: ScalarCheckRecipe = ScalarCheckRecipe { + formats: ["string", "undef"], + positive_count_operator: Comparison::Gt, + positive_count_bound: 0, + first_format: "string", + first_limit_operator: Comparison::Ge, + first_error: "String too long", + second_limit_operator: Comparison::Gt, + second_error: "Data too long", + padding_operator: Comparison::Lt, + padding_character: 0, + }; + + const WRITE: ScalarWriteRecipe = ScalarWriteRecipe { + formats: ["string", "undef"], + terminated_format: "string", + count_positive_operator: Comparison::Gt, + count_positive_bound: 0, + diff_negative_operator: Comparison::Lt, + terminator: 0, + }; + + #[test] + fn changed_check_comparisons_preserve_negative_repeat_semantics() { + let changed = ScalarCheckRecipe { + first_limit_operator: Comparison::Eq, + second_limit_operator: Comparison::Eq, + padding_operator: Comparison::Gt, + ..CHECK + }; + let result = validate_scalar(&changed, Scalar::Bytes(b"abc".to_vec()), "string", Some(2)) + .expect("negative repeat appends an empty string"); + assert_eq!(result.value, Scalar::Bytes(b"abc".to_vec())); + assert_eq!(result.error, None); + } + + #[test] + fn checkvalue_uses_character_lengths_and_preserves_utf8() { + let checked = validate_scalar(&CHECK, Scalar::Utf8("é".into()), "string", Some(3)) + .expect("scalar recipe accepts string"); + assert_eq!(checked.error, None); + assert_eq!(checked.value, Scalar::Utf8("é\0\0".into())); + + let too_long = validate_scalar(&CHECK, Scalar::Bytes(b"abc".to_vec()), "string", Some(3)) + .expect("validated scalar returns source error"); + assert_eq!(too_long.error, Some("String too long")); + } + + #[test] + fn writevalue_nul_terminates_then_truncates_and_reports_encoded_count() { + let encoded = serialize_scalar(&WRITE, Scalar::Bytes(b"abcd".to_vec()), "string", Some(3)) + .expect("scalar recipe serializes string"); + assert_eq!(encoded.value, Scalar::Bytes(b"ab\0".to_vec())); + assert_eq!(encoded.count, 3); + } + + #[test] + fn writevalue_substr_downgrades_utf8_scalars_that_become_ascii() { + let nul_only = serialize_scalar(&WRITE, Scalar::Utf8("\0".into()), "string", Some(1)) + .expect("scalar recipe serializes string"); + assert_eq!(nul_only.value, Scalar::Bytes(vec![0])); + + let ascii_with_nuls = + serialize_scalar(&WRITE, Scalar::Utf8("a\0\0".into()), "undef", Some(2)) + .expect("scalar recipe serializes undef"); + assert_eq!(ascii_with_nuls.value, Scalar::Bytes(b"a\0".to_vec())); + } + + #[test] + fn absent_zero_and_negative_counts_follow_the_native_scalar_branches() { + for count in [None, Some(0), Some(-1)] { + let checked = validate_scalar(&CHECK, Scalar::Undefined, "undef", count) + .expect("check accepts scalar count"); + assert_eq!( + checked, + CheckedScalar { + value: Scalar::Undefined, + error: None + } + ); + + let encoded = serialize_scalar(&WRITE, Scalar::Undefined, "string", count) + .expect("write accepts scalar count"); + assert_eq!( + encoded, + SerializedScalar { + value: Scalar::Bytes(vec![0]), + count: 1 + } + ); + } + } + + #[test] + fn unproven_format_and_count_overflow_refuse() { + assert!(validate_scalar(&CHECK, Scalar::Bytes(vec![]), "int8u", Some(1)).is_err()); + assert!(serialize_scalar(&WRITE, Scalar::Bytes(vec![]), "string", Some(i64::MAX)).is_err()); + } +} diff --git a/src/writers/generated_scalar_rules.rs b/src/writers/generated_scalar_rules.rs new file mode 100644 index 000000000..d0e3d7aed --- /dev/null +++ b/src/writers/generated_scalar_rules.rs @@ -0,0 +1,44 @@ +// @generated by tools/exiftool-tables/scalar_helper_codegen.py. +// Source-derived helper operands; no public tag or writer admission. +#![allow(dead_code, unused_imports)] +use crate::writers::generated_scalar::{Comparison, ScalarCheckRecipe, ScalarWriteRecipe}; + +pub(crate) const CHECK_VALUE: Option = Some(ScalarCheckRecipe { + formats: ["string", "undef"], + + positive_count_operator: Comparison::Gt, + + positive_count_bound: 0, + + first_format: "string", + + first_limit_operator: Comparison::Ge, + + first_error: "String too long", + + second_limit_operator: Comparison::Gt, + + second_error: "Data too long", + + padding_operator: Comparison::Lt, + + padding_character: 0, +}); + +pub(crate) const CHECK_VALUE_PROVENANCE: &str = "{\"actual_name\": \"Image::ExifTool::CheckValue\", \"body_sha256\": \"30460d5f4bb56fce39ef411a19595eb2aa77ad99282a5bcbdc6b735242c24956\", \"dependencies\": [{\"binding\": \"Image::ExifTool::IsFloat\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsFloat\", \"body_sha256\": \"0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsHex\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsHex\", \"body_sha256\": \"79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsInt\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsInt\", \"body_sha256\": \"2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}], \"requested_binding\": \"Image::ExifTool::CheckValue\", \"source_file\": \"Image/ExifTool/Writer.pl\", \"source_sha256\": \"cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44\"}"; + +pub(crate) const WRITE_VALUE: Option = Some(ScalarWriteRecipe { + formats: ["string", "undef"], + + terminated_format: "string", + + count_positive_operator: Comparison::Gt, + + count_positive_bound: 0, + + diff_negative_operator: Comparison::Lt, + + terminator: 0, +}); + +pub(crate) const WRITE_VALUE_PROVENANCE: &str = "{\"actual_name\": \"Image::ExifTool::WriteValue\", \"body_sha256\": \"4b72f386df682e76ca7dd0262bd89bdfa10f9e964042777e6b5f7009f2de3c0a\", \"dependencies\": [{\"binding\": \"Image::ExifTool::IsFloat\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsFloat\", \"body_sha256\": \"0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsHex\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsHex\", \"body_sha256\": \"79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsInt\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsInt\", \"body_sha256\": \"2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsRational\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsRational\", \"body_sha256\": \"ecfabc808b12990e7c72603f11c23cab462d4d52b29f88fe9a521d2ef4574356\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}], \"requested_binding\": \"Image::ExifTool::WriteValue\", \"source_file\": \"Image/ExifTool/Writer.pl\", \"source_sha256\": \"cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44\"}"; diff --git a/src/writers/mod.rs b/src/writers/mod.rs index 05fdae5ee..62f0152c3 100644 --- a/src/writers/mod.rs +++ b/src/writers/mod.rs @@ -7,6 +7,10 @@ pub mod atomic_writer; pub mod exif_inplace; pub mod exif_surgical; +// Shared source-derived helpers are not yet connected to public writes. +#[allow(dead_code)] +pub(crate) mod generated_scalar; +pub(crate) mod generated_scalar_rules; pub mod jpeg_writer; pub mod pdf_writer; pub mod png_writer; diff --git a/tools/exiftool-tables/README.md b/tools/exiftool-tables/README.md index 72437f5fc..499b24f2b 100644 --- a/tools/exiftool-tables/README.md +++ b/tools/exiftool-tables/README.md @@ -8,7 +8,7 @@ the pin remains 13.59. Conservative IFD-aware upgrade classification, one generated-output inventory, verified Canon CODE references and isolated upgrade orchestration are implemented. -The inventory contains 32 outputs, including inactive keyed and serial definitions and the +The inventory contains 34 outputs, including inactive scalar helper, keyed and serial definitions and the [Sony plain producer recovery](../../docs/reference/sony-plain-generator-recovery.md). See the [execution plan](../../docs/UPGRADE-NEXT-STEPS.md) for original validation evidence and the remaining work. The @@ -38,7 +38,7 @@ definitions and compiled Composite expressions, FITS names, and expression/value conversion ledgers. `regen-all.sh` adds vendor subdirectory tables, Nikon AF-point grids, bespoke transcriptions, recovered Sony/Minolta/Nikon generators, Macintosh CJK charset tables, GeoTIFF key maps, DICOM dictionaries and lens alternatives. -`artifacts.py` is the single output inventory: 10 tier-1 and 22 tier-2 artifacts. +`artifacts.py` is the single output inventory: 12 tier-1 and 22 tier-2 artifacts. Both scripts resolve their output paths and formatting sets from it; the bump's promotion/recovery sets and CI's tier-2 comparison use the same inventory. The bump classifier also @@ -113,7 +113,7 @@ for the restored field, bounded validation and remaining real-file acceptance. ## Generated-output inventory and write checks ```sh -python3 tools/exiftool-tables/artifacts.py paths # all 32 outputs +python3 tools/exiftool-tables/artifacts.py paths # all 34 outputs python3 tools/exiftool-tables/artifacts.py paths --tier 2 # downstream outputs python3 tools/exiftool-tables/artifacts.py paths --tier 1 --kind rust --absolute python3 tools/exiftool-tables/artifacts.py path composite-compute diff --git a/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md b/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md new file mode 100644 index 000000000..4f40d635e --- /dev/null +++ b/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md @@ -0,0 +1,69 @@ +# Source-derived scalar WriteValue execution + +`writevalue_recipes.compile_scalar_write` consumes the final captured +`native_write_helpers.write_value` CODE fact. It authenticates the exact +entry bindings, the lexical `%writeValueProc` lookup, and the `if ($proc) … +elsif (string/undef)` control-flow. Admission requires the live captured +lexical hash to be resolved and to have no entry for either source-derived +scalar format. Thus an installed dispatch entry cannot silently take the +numeric branch before scalar handling. + +`evaluate_scalar_write` preserves undefined, byte and UTF8-flagged scalar +states, source-derived terminator/truncation/padding behaviour, and the final +native count. It deliberately refuses all formats outside the captured scalar +branch and all non-integer counts. It does not model optional `$dataPt`/ +`$offset` mutation, numeric packing, tag lookup, CharsetEXIF encoding, or a +public writer route. + +This is an inactive reference mechanism. The captured `%writeValueProc` is +read as loaded state, so platform-time dispatch removal is observed. A missing +or unresolved pad never means an empty dispatch map. + +## Validation and current integration boundary + +Nine tests pass with explicit pinned native inputs and no skips. The native +scalar differential covers 272 cases, including wide characters outside the +selected substring. Independent review exercised actual native lexical +shadowing; a local that aliases the input value must refuse. The regression +suite also covers source-derived format and count-bound changes. + +A separate actual-source rehearsal changes only WriteValue's count guard +from `> 0` to `> 2` in a copied native Writer.pl. Fresh capture and compilation +match all 240 original native cases; 43 native outcomes change, with zero +handwritten tag-rule or generated-output edits. Evidence relative to the +continuation root is +`shared-pilot/write-upgrade-integration-20260913/writevalue-source-replay-20260914/`. +The native test observes return bytes, defined/UTF8 state and length. Its count +observation is the unchanged caller argument, not the helper-local final count; +the latter remains a reference result checked by source/unit tests. + +## Generated Rust execution + +`scalar_helper_codegen.py` compiles the captured CheckValue and WriteValue +operands into `src/writers/generated_scalar_rules.rs`. Normal `regen.sh` owns +that artifact and `scalar_helper_ledger.json`; the inventory now has 34 outputs. +The shared `generated_scalar.rs` executor consumes those operands. Supported +source changes replace the operands; unsupported source emits `None` and a +named ledger gap instead of retaining an old rule. The native suite also +compares both committed artifacts with generation from CI's fresh pinned dump, +so omitting regeneration cannot leave stale rules behind a green helper test. +The public writer remains inactive for these definitions. + +The actual compiled Rust executor matches 512 authenticated native helper cases +(240 validation and 272 serialization) for the canonical source and again for +the copied WriteValue count-guard change. A third actual-source replay changes +CheckValue's comparisons, including a branch that executes a negative Perl +repetition; all 512 cases match that native source too. The native return +records supply the expected values, defined/UTF8 state and validation errors. +Rust's helper-local count is separately compared with the Python source +reference; this is not a native observation of that local variable. + +The first Rust replay exposed an ASCII substring's UTF8 storage downgrade. +The corrected executor preserves native behavior when non-ASCII characters +occur outside the selected prefix as well. These are helper-level checks, +not evidence of a public generated write route. + +Composition with CheckExif/CheckValue, CharsetEXIF encoding, public tag routing +and complete file write/read-back remain the next steps. Numeric packing and +the optional native data-target mutation are also unfinished. This checkpoint +does not establish full release-upgrade or read/write conformance. diff --git a/tools/exiftool-tables/artifacts.py b/tools/exiftool-tables/artifacts.py index af360a234..b43513c7b 100644 --- a/tools/exiftool-tables/artifacts.py +++ b/tools/exiftool-tables/artifacts.py @@ -37,6 +37,8 @@ class Artifact: Artifact("serial", 1, "serial_directory", "src/exiftool_tables/serial_tables.rs"), Artifact("expr-ledger", 1, "verify_exprs", "tools/exiftool-tables/expr_oracle_ledger.json"), Artifact("value-ledger", 1, "codegen", "tools/exiftool-tables/value_conv_ledger.json"), + Artifact("scalar-helpers", 1, "scalar_helper_codegen", "src/writers/generated_scalar_rules.rs"), + Artifact("scalar-helper-ledger", 1, "scalar_helper_codegen", "tools/exiftool-tables/scalar_helper_ledger.json"), Artifact("filetypes", 1, "codegen_filetypes", "src/filetype/tables.rs"), Artifact("composite", 1, "codegen_composite", "src/composite/tables.rs"), Artifact("composite-compute", 1, "codegen_composite", "src/composite/generated_compute.rs"), diff --git a/tools/exiftool-tables/dump_binary_reader_contract.pl b/tools/exiftool-tables/dump_binary_reader_contract.pl index 6fd27d53f..51fd6465e 100755 --- a/tools/exiftool-tables/dump_binary_reader_contract.pl +++ b/tools/exiftool-tables/dump_binary_reader_contract.pl @@ -27,6 +27,7 @@ # dump regeneration; the caller records this non-zero child as unresolved. $SIG{ALRM} = sub { die "native_reader_contract_timeout\n" }; alarm $timeout; +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } require Image::ExifTool; my $contract = capture_in_process($EXIFTOOL_LIB_ABS); alarm 0; diff --git a/tools/exiftool-tables/dump_tables.pl b/tools/exiftool-tables/dump_tables.pl index 38fe0b058..7a4b90425 100755 --- a/tools/exiftool-tables/dump_tables.pl +++ b/tools/exiftool-tables/dump_tables.pl @@ -250,6 +250,10 @@ sub to_text { my $EXIFTOOL_LIB_ABS = abs_path($EXIFTOOL_LIB) or die "invalid exiftool lib: $EXIFTOOL_LIB\n"; +# Table facts are a property of the selected native tree, never of the +# account running the dump. ExifTool loads $EXIFTOOL_HOME/.ExifTool_config +# during this require unless its configFile global is the empty string. +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } require Image::ExifTool; # Keys that describe the table itself rather than a tag within it. diff --git a/tools/exiftool-tables/oracle.pl b/tools/exiftool-tables/oracle.pl index 718881e55..9cb7ef5fb 100755 --- a/tools/exiftool-tables/oracle.pl +++ b/tools/exiftool-tables/oracle.pl @@ -140,6 +140,7 @@ my $LIB = shift @ARGV or die "usage: $0 \n"; unshift @INC, $LIB; +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } require Image::ExifTool; binmode(STDOUT, ':encoding(UTF-8)'); diff --git a/tools/exiftool-tables/regen.sh b/tools/exiftool-tables/regen.sh index 061206c09..bf2662b82 100755 --- a/tools/exiftool-tables/regen.sh +++ b/tools/exiftool-tables/regen.sh @@ -91,6 +91,12 @@ echo ">> generating inactive serial-directory facts" python3 "$HERE/serial_directory.py" "$JSON" \ --output "$CACHE/serial-$VERSION.json" --rust-output "$SERIAL_OUT" +echo +echo ">> generating source-derived scalar writer helper operands" +python3 "$HERE/scalar_helper_codegen.py" "$JSON" \ + --output "$(artifact_path scalar-helpers)" \ + --report "$(artifact_path scalar-helper-ledger)" + echo echo ">> extracting file-identification tables" "$PERL" "$HERE/dump_filetypes.pl" "$LIB" > "$CACHE/filetypes-$VERSION.json" diff --git a/tools/exiftool-tables/scalar_helper_codegen.py b/tools/exiftool-tables/scalar_helper_codegen.py new file mode 100644 index 000000000..8a950e54d --- /dev/null +++ b/tools/exiftool-tables/scalar_helper_codegen.py @@ -0,0 +1,122 @@ +"""Generate Rust scalar helper operands from captured native executable rules. + +This joins the source compilers to the shared Rust executor. It does not claim +that a tag, its input conversions, encoding or carrier route is admitted. +""" +from dataclasses import asdict +import json +from pathlib import Path +from typing import Any + +from checkexif_recipes import RecipeMalformed, RecipeRefused +from checkvalue_recipes import compile_scalar_check +from writevalue_recipes import compile_scalar_write + +COMPARISONS = {'<': 'Lt', '<=': 'Le', '==': 'Eq', '!=': 'Ne', '>=': 'Ge', '>': 'Gt'} + + +def rust_string(value: str) -> str: + """Escape Rust literals, including control characters JSON spells differently.""" + chunks = ['"'] + for char in value: + if char in ('"', '\\'): + chunks.append('\\' + char) + elif ord(char) < 32 or ord(char) == 127: + chunks.append('\\u{' + format(ord(char), 'x') + '}') + else: + chunks.append(char) + return ''.join(chunks) + '"' + + +def _bound(value: int) -> str: + if type(value) is not int or not -(1 << 63) <= value < (1 << 63): + raise RecipeRefused('source count bound is outside the Rust i64 operand') + return str(value) + + +def _comparison(value: str) -> str: + if value not in COMPARISONS: + raise RecipeRefused('source comparison has no Rust operation') + return 'Comparison::' + COMPARISONS[value] + + +def _byte(value: str) -> str: + # Both closed source parsers currently admit only the native NUL literal. + if value != '\0': + raise RecipeRefused('source scalar terminator/padding is not admitted') + return '0' + + +def _check_fields(recipe) -> dict[str, str]: + return { + 'formats': '[' + ', '.join(map(rust_string, recipe.formats)) + ']', + 'positive_count_operator': _comparison(recipe.positive_count_operator), + 'positive_count_bound': _bound(recipe.positive_count_bound), + 'first_format': rust_string(recipe.first_format), + 'first_limit_operator': _comparison(recipe.first_limit_operator), + 'first_error': rust_string(recipe.first_error), + 'second_limit_operator': _comparison(recipe.second_limit_operator), + 'second_error': rust_string(recipe.second_error), + 'padding_operator': _comparison(recipe.padding_operator), + 'padding_character': _byte(recipe.padding_character), + } + + +def _write_fields(recipe) -> dict[str, str]: + return { + 'formats': '[' + ', '.join(map(rust_string, recipe.formats)) + ']', + 'terminated_format': rust_string(recipe.terminated_format), + 'count_positive_operator': _comparison(recipe.count_positive_operator), + 'count_positive_bound': _bound(recipe.count_positive_bound), + 'diff_negative_operator': _comparison(recipe.diff_negative_operator), + 'terminator': _byte(recipe.terminator), + } + + +def generate(document: dict[str, Any]) -> tuple[str, dict[str, Any]]: + helpers = document.get('native_write_helpers') + if not isinstance(helpers, dict): + raise RecipeMalformed('native_write_helpers is not an object') + chunks = [ + '// @generated by tools/exiftool-tables/scalar_helper_codegen.py.\n' + '// Source-derived helper operands; no public tag or writer admission.\n' + '#![allow(dead_code, unused_imports)]\n' + 'use crate::writers::generated_scalar::{Comparison, ScalarCheckRecipe, ScalarWriteRecipe};\n' + ] + report = {'scope': 'generated Rust scalar helpers; no tag/carrier/encoding admission', + 'helpers': {}} + for key, symbol, typ, compiler, fields in ( + ('check_value', 'CHECK_VALUE', 'ScalarCheckRecipe', compile_scalar_check, _check_fields), + ('write_value', 'WRITE_VALUE', 'ScalarWriteRecipe', compile_scalar_write, _write_fields), + ): + try: + recipe = compiler(helpers.get(key)) + operands = fields(recipe) + except RecipeRefused as exc: + chunks.append(f'pub(crate) const {symbol}: Option<{typ}> = None;\n') + report['helpers'][key] = {'state': 'unsupported', 'reason': str(exc)} + continue + provenance = asdict(recipe.provenance) + report['helpers'][key] = {'state': 'compiled', 'recipe': asdict(recipe)} + chunks.append(f'pub(crate) const {symbol}: Option<{typ}> = Some({typ} {{\n') + chunks.extend(f' {name}: {value},\n' for name, value in operands.items()) + chunks.append('});\n') + chunks.append(f'pub(crate) const {symbol}_PROVENANCE: &str = ' + + rust_string(json.dumps(provenance, sort_keys=True)) + ';\n') + return '\n'.join(chunks), report + + +def main() -> None: + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('tables', type=Path) + parser.add_argument('-o', '--output', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + args = parser.parse_args() + source, report = generate(json.loads(args.tables.read_text())) + args.output.write_text(source) + args.report.write_text(json.dumps(report, sort_keys=True, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/exiftool-tables/scalar_helper_ledger.json b/tools/exiftool-tables/scalar_helper_ledger.json new file mode 100644 index 000000000..c39a7dc04 --- /dev/null +++ b/tools/exiftool-tables/scalar_helper_ledger.json @@ -0,0 +1,133 @@ +{ + "helpers": { + "check_value": { + "recipe": { + "first_error": "String too long", + "first_format": "string", + "first_limit_operator": ">=", + "formats": [ + "string", + "undef" + ], + "padding_character": "\u0000", + "padding_operator": "<", + "positive_count_bound": 0, + "positive_count_operator": ">", + "provenance": { + "actual_name": "Image::ExifTool::CheckValue", + "body_sha256": "30460d5f4bb56fce39ef411a19595eb2aa77ad99282a5bcbdc6b735242c24956", + "dependencies": [ + { + "binding": "Image::ExifTool::IsFloat", + "fact": { + "actual_name": "Image::ExifTool::IsFloat", + "body_sha256": "0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsHex", + "fact": { + "actual_name": "Image::ExifTool::IsHex", + "body_sha256": "79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsInt", + "fact": { + "actual_name": "Image::ExifTool::IsInt", + "body_sha256": "2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + } + ], + "requested_binding": "Image::ExifTool::CheckValue", + "source_file": "Image/ExifTool/Writer.pl", + "source_sha256": "cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44" + }, + "second_error": "Data too long", + "second_limit_operator": ">" + }, + "state": "compiled" + }, + "write_value": { + "recipe": { + "count_positive_bound": 0, + "count_positive_operator": ">", + "diff_negative_operator": "<", + "dispatch_lexical": "%writeValueProc", + "formats": [ + "string", + "undef" + ], + "provenance": { + "actual_name": "Image::ExifTool::WriteValue", + "body_sha256": "4b72f386df682e76ca7dd0262bd89bdfa10f9e964042777e6b5f7009f2de3c0a", + "dependencies": [ + { + "binding": "Image::ExifTool::IsFloat", + "fact": { + "actual_name": "Image::ExifTool::IsFloat", + "body_sha256": "0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsHex", + "fact": { + "actual_name": "Image::ExifTool::IsHex", + "body_sha256": "79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsInt", + "fact": { + "actual_name": "Image::ExifTool::IsInt", + "body_sha256": "2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsRational", + "fact": { + "actual_name": "Image::ExifTool::IsRational", + "body_sha256": "ecfabc808b12990e7c72603f11c23cab462d4d52b29f88fe9a521d2ef4574356", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + } + ], + "requested_binding": "Image::ExifTool::WriteValue", + "source_file": "Image/ExifTool/Writer.pl", + "source_sha256": "cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44" + }, + "terminated_format": "string", + "terminator": "\u0000" + }, + "state": "compiled" + } + }, + "scope": "generated Rust scalar helpers; no tag/carrier/encoding admission" +} diff --git a/tools/exiftool-tables/test_artifacts.py b/tools/exiftool-tables/test_artifacts.py index 5585d505f..b4136779f 100644 --- a/tools/exiftool-tables/test_artifacts.py +++ b/tools/exiftool-tables/test_artifacts.py @@ -13,8 +13,8 @@ class ManifestTests(unittest.TestCase): def test_unique_valid_partition_and_selectors(self): artifacts.validate() all_items = artifacts.select() - self.assertEqual(len(all_items), 32) - self.assertEqual(len(artifacts.select(1)), 10) + self.assertEqual(len(all_items), 34) + self.assertEqual(len(artifacts.select(1)), 12) self.assertEqual(len(artifacts.select(2)), 22) self.assertEqual(set(all_items), set(artifacts.select(1) + artifacts.select(2))) self.assertTrue(all(a.path.endswith('.rs') for a in artifacts.select(kind='rust'))) diff --git a/tools/exiftool-tables/test_checkvalue_recipes.py b/tools/exiftool-tables/test_checkvalue_recipes.py index 77dc1b172..74ed6701a 100644 --- a/tools/exiftool-tables/test_checkvalue_recipes.py +++ b/tools/exiftool-tables/test_checkvalue_recipes.py @@ -121,7 +121,7 @@ def test_actual_native_helper_matches_all_scalar_states_and_counts(self): values += [NativeScalar("utf8", value) for value in ("", "é", "Ā", "😀", "a\0b", "a\n")] cases, expected = [], [] for value in values: - for fmt in ("string", "undef"): + for fmt in recipe.formats: for count in (None, 0, -1, 1, 2, 3, 4, 8): cases.append({"kind": value.kind, "value": value.value.hex() if value.kind == "bytes" else value.value, @@ -135,7 +135,7 @@ def test_actual_native_helper_matches_all_scalar_states_and_counts(self): "error": error}) perl = r''' use strict; use warnings; use JSON::PP; use Encode (); use B (); use B::Deparse; -use Digest::SHA qw(sha256_hex); use Image::ExifTool; +use Digest::SHA qw(sha256_hex); BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } use Image::ExifTool; require 'Image/ExifTool/Writer.pl'; local $/; my $rows=JSON::PP->new->utf8->decode(); my @results; my $cv=\&Image::ExifTool::CheckValue; diff --git a/tools/exiftool-tables/test_exiftool_config_isolation.py b/tools/exiftool-tables/test_exiftool_config_isolation.py new file mode 100644 index 000000000..2ef87499a --- /dev/null +++ b/tools/exiftool-tables/test_exiftool_config_isolation.py @@ -0,0 +1,102 @@ +"""Native table instruments must not inherit a user's .ExifTool_config.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import textwrap +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DUMP = REPO_ROOT / "tools/exiftool-tables/dump_tables.pl" +ORACLE = REPO_ROOT / "tools/exiftool-tables/oracle.pl" +READER_CONTRACT = REPO_ROOT / "tools/exiftool-tables/dump_binary_reader_contract.pl" +PERL = os.environ.get("EXIFTOOL_PERL", "/usr/bin/perl") + + +class ExifToolConfigIsolation(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + root = Path(self.tmp.name) + self.home = root / "hostile-home" + self.home.mkdir() + self.lib = root / "lib" + package = self.lib / "Image/ExifTool" + package.mkdir(parents=True) + (self.home / ".ExifTool_config").write_text( + "package Image::ExifTool; warn \"HOSTILE_CONFIG_EXECUTED\\n\"; $VERSION = 'HOSTILE_CONFIG'; 1;\n", + encoding="utf-8", + ) + (self.lib / "Image/ExifTool.pm").write_text(textwrap.dedent("""\ + package Image::ExifTool; + use strict; + our $VERSION = 'clean'; + our $configFile; + our $currentByteOrder = 'II'; + our %unpackStd = ( S => 'v' ); + our %specialTags = map { $_ => 1 } qw(GROUPS FORMAT FIRST_ENTRY); + sub SetByteOrder { $currentByteOrder = shift; $unpackStd{S} = $currentByteOrder eq 'II' ? 'v' : 'n'; return 1; } + sub GetByteOrder { return $currentByteOrder; } + sub Get16u { my ($data, $offset) = @_; return undef if length($$data) < $offset + 2; return unpack($unpackStd{S}, substr($$data, $offset, 2)); } + sub DoUnpackStd { return Get16u(@_); } + if (!defined $configFile) { + my $file = ($ENV{EXIFTOOL_HOME} || $ENV{HOME} || '.') . '/.ExifTool_config'; + require $file if -r $file; + } elsif (length $configFile) { + require $configFile; + } + 1; + """), encoding="utf-8") + (package / "Fixture.pm").write_text(textwrap.dedent("""\ + package Image::ExifTool::Fixture; + our %Main = ( GROUPS => { 0 => 'EXIF' }, 1 => { Name => 'CleanTag' } ); + 1; + """), encoding="utf-8") + + def run_dump(self, script=DUMP): + env = os.environ.copy() + env["EXIFTOOL_HOME"] = str(self.home) + result = subprocess.run([PERL, str(script), str(self.lib), "Fixture"], + check=True, text=True, capture_output=True, env=env) + return json.loads(result.stdout) + + def run_instrument(self, script, *arguments): + env = os.environ.copy() + env["EXIFTOOL_HOME"] = str(self.home) + return subprocess.run([PERL, str(script), str(self.lib), *arguments], + text=True, capture_output=True, env=env) + + def unfenced_control(self, instrument): + directory = Path(tempfile.mkdtemp(prefix="unfenced-" + instrument.stem + "-", dir=self.tmp.name)) + control = directory / instrument.name + fence = "BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; }\n" + text = instrument.read_text(encoding="utf-8") + self.assertIn(fence, text) + control.write_text(text.replace(fence, "", 1), encoding="utf-8") + shutil.copytree(DUMP.parent / "OxiDex", directory / "OxiDex") + return control + + def test_config_control_proves_the_dump_fence(self): + # The copied control has only the canonical early config assignment + # removed. It proves the hostile home config is a real input, rather + # than merely asserting a line of Perl text exists. + control = self.unfenced_control(DUMP) + self.assertEqual(self.run_dump(control)["exiftool_version"], "HOSTILE_CONFIG") + self.assertEqual(self.run_dump()["exiftool_version"], "clean") + + def test_each_capture_instrument_blocks_hostile_config_at_load(self): + for instrument, arguments in ((DUMP, ("Fixture",)), (ORACLE, ()), (READER_CONTRACT, ())): + with self.subTest(instrument=instrument.name): + fenced = self.run_instrument(instrument, *arguments) + self.assertEqual(fenced.returncode, 0, fenced.stderr) + self.assertNotIn("HOSTILE_CONFIG_EXECUTED", fenced.stdout + fenced.stderr) + control = self.run_instrument(self.unfenced_control(instrument), *arguments) + self.assertEqual(control.returncode, 0, control.stderr) + self.assertIn("HOSTILE_CONFIG_EXECUTED", control.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/test_regen_shell.py b/tools/exiftool-tables/test_regen_shell.py index 594b6b9d9..6885f93ee 100644 --- a/tools/exiftool-tables/test_regen_shell.py +++ b/tools/exiftool-tables/test_regen_shell.py @@ -74,6 +74,11 @@ def artifact(producer): dump(args[0]);output(flag('--rust-output'),'serial') assert flag('--rust-output')==artifact('serial_directory') output(flag('--output'),'serial-report') + elif name=='scalar_helper_codegen.py': + dump(args[0]) + selected={root/item.path for item in artifacts.select(producer='scalar_helper_codegen')} + assert {flag('--output'),flag('--report')}==selected + output(flag('--output'),'scalar-helpers');output(flag('--report'),'scalar-helper-ledger') elif name=='verify_serial_directory.py': assert pathlib.Path(args[0]).resolve()==artifact('serial_directory') assert pathlib.Path(args[0]).read_text()=='generated explicit-A serial\n' @@ -213,9 +218,11 @@ def test_both_tiers_and_tier2_use_selected_source_and_complete_checks(self): self.assertEqual(names.count(name), 1, names) self.assertEqual(names.count('verify_exprs.py'), int(full)) self.assertEqual(names.count('serial_directory.py'), int(full)) + self.assertEqual(names.count('scalar_helper_codegen.py'), int(full)) self.assertEqual(names.count('verify_serial_directory.py'), int(full)) if full: self.assertLess(names.index('serial_directory.py'), names.index('rustfmt')) + self.assertLess(names.index('scalar_helper_codegen.py'), names.index('rustfmt')) self.assertGreater(names.index('verify_serial_directory.py'), names.index('rustfmt')) self.assertEqual(names.count('rustfmt'), 2 if full else 1) format_calls = [c for c in calls if c['tool'] == 'rustfmt'] @@ -241,8 +248,8 @@ def test_each_new_producer_or_verifier_failure_survives_exit_guard(self): self.assertIn('regeneration write-set PASS', result.stdout) self.assertNotIn('>> done:', result.stdout) - def test_serial_producer_and_verifier_failures_survive_tier_one_guard(self): - for leaf in ('serial_directory.py', 'verify_serial_directory.py'): + def test_tier_one_producer_and_verifier_failures_survive_exit_guard(self): + for leaf in ('serial_directory.py', 'scalar_helper_codegen.py', 'verify_serial_directory.py'): with self.subTest(leaf=leaf): result, calls = self.run_regeneration(full=True, env={'CONTROL_FAIL': leaf}) self.assertEqual(result.returncode, 47, result.stdout + result.stderr) diff --git a/tools/exiftool-tables/test_scalar_helper_codegen.py b/tools/exiftool-tables/test_scalar_helper_codegen.py new file mode 100644 index 000000000..29bec300d --- /dev/null +++ b/tools/exiftool-tables/test_scalar_helper_codegen.py @@ -0,0 +1,92 @@ +"""Generated Rust operands must follow source semantics, not just fingerprints.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +from checkexif_recipes import RecipeMalformed +import scalar_helper_codegen as generator +from test_checkvalue_recipes import fact as check_fact +from test_writevalue_recipes import fact as write_fact + + +def document(): + return {'native_write_helpers': {'check_value': check_fact(), 'write_value': write_fact()}} + + +class ScalarHelperCodegenTests(unittest.TestCase): + def test_source_operands_change_in_generated_rust(self): + doc = document() + check = doc['native_write_helpers']['check_value'] + before, after = check['__deparse'].rsplit("'string'", 1) + check['__deparse'] = before + "'undef'" + after + write = doc['native_write_helpers']['write_value'] + write['__deparse'] = write['__deparse'].replace('$count > 0', '$count > 2') + output, report = generator.generate(doc) + self.assertIn('first_format: "undef"', output) + self.assertIn('count_positive_bound: 2', output) + self.assertEqual(report['helpers']['check_value']['state'], 'compiled') + self.assertEqual(report['helpers']['write_value']['state'], 'compiled') + + def test_unknown_source_rule_emits_none_and_named_gap(self): + doc = document() + doc['native_write_helpers']['check_value']['__deparse'] = 'unknown upstream check' + output, report = generator.generate(doc) + self.assertIn('CHECK_VALUE: Option = None', output) + self.assertNotIn('CHECK_VALUE: Option = Some', output) + self.assertEqual(report['helpers']['check_value']['state'], 'unsupported') + self.assertTrue(report['helpers']['check_value']['reason']) + self.assertEqual(report['helpers']['write_value']['state'], 'compiled') + + def test_live_dispatch_override_cannot_emit_old_scalar_rule(self): + doc = document() + helper = doc['native_write_helpers']['write_value'] + helper['lexical_hashes']['bindings']['%writeValueProc']['entries']['string'] = {'__perl': 'CODE'} + output, report = generator.generate(doc) + self.assertIn('WRITE_VALUE: Option = None', output) + self.assertIn('intercepted', report['helpers']['write_value']['reason']) + + def test_unrepresentable_source_count_bound_is_an_explicit_gap(self): + doc = document() + helper = doc['native_write_helpers']['write_value'] + helper['__deparse'] = helper['__deparse'].replace('$count > 0', '$count > ' + str(1 << 63)) + output, report = generator.generate(doc) + self.assertIn('WRITE_VALUE: Option = None', output) + self.assertIn('i64', report['helpers']['write_value']['reason']) + + def test_missing_capture_is_malformed_not_ordinary_unsupported_source(self): + with self.assertRaises(RecipeMalformed): + generator.generate({}) + doc = document() + del doc['native_write_helpers']['check_value'] + with self.assertRaises(RecipeMalformed): + generator.generate(doc) + + def test_rust_literals_preserve_unicode_controls_quotes_and_backslashes(self): + self.assertEqual(generator.rust_string('é\0\n"\\'), '"é\\u{0}\\u{a}\\"\\\\"') + + +@unittest.skipUnless(os.environ.get('OXIDEX_TABLES_JSON'), + 'requires fresh captured helper facts from the pinned release') +class NativeArtifactFreshnessTests(unittest.TestCase): + def test_committed_rules_and_ledger_equal_fresh_native_generation(self): + # CI supplies a fresh dump from .exiftool-version, not an artifact's + # own version stamp. Check the files we ship as well as temporary Rust. + root = Path(__file__).resolve().parents[2] + source, report = generator.generate(json.loads(Path(os.environ['OXIDEX_TABLES_JSON']).read_text())) + with tempfile.TemporaryDirectory(prefix='oxidex-scalar-freshness-') as directory: + output = Path(directory) / 'rules.rs' + output.write_text(source) + subprocess.run(['rustfmt', '--edition', '2024', '--config-path', str(root / 'rustfmt.toml'), + str(output)], check=True, capture_output=True, timeout=30) + self.assertEqual(output.read_text(), (root / 'src/writers/generated_scalar_rules.rs').read_text(), + 'committed scalar rules are stale; run official regeneration') + self.assertEqual(json.dumps(report, sort_keys=True, indent=2) + '\n', + (Path(__file__).parent / 'scalar_helper_ledger.json').read_text(), + 'committed scalar helper ledger is stale; run official regeneration') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/exiftool-tables/test_scalar_helper_rust.py b/tools/exiftool-tables/test_scalar_helper_rust.py new file mode 100644 index 000000000..eec2971b9 --- /dev/null +++ b/tools/exiftool-tables/test_scalar_helper_rust.py @@ -0,0 +1,102 @@ +"""Execute freshly generated helper operands in the real Rust scalar executor. + +The native return records, not the Python reference evaluator, supply the Rust +expectations. The native fixture runners additionally authenticate loaded CV +source/body hashes and check their own Python references before returning data. +The helper-local write count is checked separately against the source reference; +native return records cannot observe that local variable. +""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import scalar_helper_codegen as generator +import test_checkvalue_recipes as native_check +import test_writevalue_recipes as native_write +from checkvalue_recipes import NativeScalar +from writevalue_recipes import compile_scalar_write, evaluate_scalar_write + +ROOT = Path(__file__).resolve().parents[2] + + +def _scalar(value): + if value.get('kind') == 'undefined' or value.get('defined') is False: + return 'Scalar::Undefined' + if value.get('kind') == 'bytes' or value.get('utf8') is False: + raw = bytes.fromhex(value.get('value', value.get('hex'))) + return 'Scalar::Bytes(vec![' + ','.join(str(byte) for byte in raw) + '])' + text = value['value'] if 'kind' in value else bytes.fromhex(value['hex']).decode('utf8') + return 'Scalar::Utf8(' + generator.rust_string(text) + '.to_owned())' + + +@unittest.skipUnless(os.environ.get('OXIDEX_PINNED_EXIFTOOL') and os.environ.get('OXIDEX_TABLES_JSON'), + 'requires explicit native source and matching captured helper facts') +class GeneratedScalarRustTests(unittest.TestCase): + def test_generated_rust_matches_authenticated_native_helper_returns(self): + document = json.loads(Path(os.environ['OXIDEX_TABLES_JSON']).read_text()) + source, report = generator.generate(document) + self.assertTrue(all(row['state'] == 'compiled' for row in report['helpers'].values()), report) + write_recipe = compile_scalar_write(document['native_write_helpers']['write_value']) + real_run = subprocess.run + observations = {} + for kind, test in ( + ('check', native_check.NativeScalarDifferential().test_actual_native_helper_matches_all_scalar_states_and_counts), + ('write', native_write.NativeScalarDifferential().test_actual_native_helper_matches_scalar_states_and_counts), + ): + def capture(*args, **kwargs): + result = real_run(*args, **kwargs) + observations[kind] = (json.loads(kwargs['input']), json.loads(result.stdout)['results']) + return result + with patch.object(subprocess, 'run', capture): + test() + with tempfile.TemporaryDirectory(prefix='oxidex-generated-scalar-') as directory: + root = Path(directory) + rules = root / 'rules.rs' + rules.write_text(source) + harness = [ + '#![allow(dead_code)]', + '#[path=' + generator.rust_string(str(ROOT / 'src/error/mod.rs')) + '] mod error;', + 'mod writers { #[path=' + generator.rust_string(str(ROOT / 'src/writers/generated_scalar.rs')) + + '] pub(crate) mod generated_scalar; }', + '#[path=' + generator.rust_string(str(rules)) + '] mod rules;', + 'use writers::generated_scalar::*;', + ] + for kind, (cases, results) in observations.items(): + self.assertGreaterEqual(len(cases), 240) + self.assertEqual(len(cases), len(results)) + harness.append('#[test] fn native_' + kind + '() {') + for index, (case, result) in enumerate(zip(cases, results, strict=True)): + count = 'None' if case['count'] is None else 'Some(' + str(case['count']) + ')' + function = 'validate_scalar' if kind == 'check' else 'serialize_scalar' + rule = 'CHECK_VALUE' if kind == 'check' else 'WRITE_VALUE' + harness.append(f'let value = {function}(&rules::{rule}.unwrap(), {_scalar(case)}, ' + f'{generator.rust_string(case["format"])}, {count}).unwrap();') + harness.append(f'assert_eq!(value.value, {_scalar(result)}, "{kind} case {index}");') + if kind == 'check': + error = 'None' if result['error'] is None else 'Some(' + generator.rust_string(result['error']) + ')' + harness.append(f'assert_eq!(value.error, {error}, "check error {index}");') + else: + # Native JSON's count is the unchanged caller argument. + # This distinct assertion checks only reference agreement. + raw = bytes.fromhex(case['value']) if case['kind'] == 'bytes' else case['value'] + reference = evaluate_scalar_write( + write_recipe, NativeScalar(case['kind'], raw), case['format'], case['count']) + harness.append(f'assert_eq!(value.count, {reference.count}, ' + f'"write source-reference local count {index}");') + harness.append('}') + program = root / 'proof.rs' + program.write_text('\n'.join(harness)) + binary = root / 'proof' + compile_result = real_run(['rustc', '--edition=2024', '--test', str(program), '-o', str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(compile_result.returncode, 0, compile_result.stdout + compile_result.stderr) + execution = real_run([str(binary), '--nocapture'], capture_output=True, text=True, timeout=60) + self.assertEqual(execution.returncode, 0, execution.stdout + execution.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/exiftool-tables/test_writevalue_recipes.py b/tools/exiftool-tables/test_writevalue_recipes.py new file mode 100644 index 000000000..5407841db --- /dev/null +++ b/tools/exiftool-tables/test_writevalue_recipes.py @@ -0,0 +1,171 @@ +"""Source-change admission and scalar reference execution for WriteValue.""" +from pathlib import Path +import json +import os +import subprocess +import unittest + +from checkexif_recipes import RecipeRefused +from checkvalue_recipes import NativeScalar +from writevalue_recipes import compile_scalar_write, evaluate_scalar_write + +BODY = (Path(__file__).parent / "testdata/writevalue_scalar_body.txt").read_text() + + +def fact(body=BODY, entries=None): + return {"__perl": "CODE", "resolved": True, + "requested_binding": "Image::ExifTool::WriteValue", + "__name": "Image::ExifTool::WriteValue", + "source_file": "Image/ExifTool/Writer.pl", "source_sha256": "b" * 64, + "__deparse": body, "dependencies": {}, + "lexical_hashes": {"resolved": True, "bindings": { + "%writeValueProc": {"resolved": True, "entries": entries or {}}}}} + + +class ScalarWriteTests(unittest.TestCase): + def setUp(self): + self.recipe = compile_scalar_write(fact()) + + def test_count_and_scalar_kind_are_preserved(self): + cases = ( + (NativeScalar("undefined", None), "undef", None, NativeScalar("undefined", None), 0), + (NativeScalar("undefined", None), "string", None, NativeScalar("bytes", b"\0"), 1), + (NativeScalar("bytes", b"a\0b"), "undef", 0, NativeScalar("bytes", b"a\0b"), 3), + (NativeScalar("utf8", "é"), "string", -1, NativeScalar("utf8", "é\0"), 2), + (NativeScalar("bytes", b"a"), "string", 4, NativeScalar("bytes", b"a\0\0\0"), 4), + (NativeScalar("utf8", "é"), "undef", 3, NativeScalar("utf8", "é\0\0"), 3), + ) + for value, format_name, count, expected_value, expected_count in cases: + with self.subTest(value=value, format=format_name, count=count): + actual = evaluate_scalar_write(self.recipe, value, format_name, count) + self.assertEqual((actual.value, actual.count), (expected_value, expected_count)) + + def test_source_truncation_uses_terminated_format_and_count(self): + self.assertEqual(evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"abcd"), "string", 3).value, + NativeScalar("bytes", b"ab\0")) + self.assertEqual(evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"abcd"), "undef", 3).value, + NativeScalar("bytes", b"abc")) + + def test_source_format_change_changes_admission_and_execution(self): + changed = compile_scalar_write(fact(BODY.replace("'string'", "'wide-string'"))) + with self.assertRaises(RecipeRefused): + evaluate_scalar_write(changed, NativeScalar("bytes", b"a"), "string", 3) + self.assertEqual(evaluate_scalar_write(changed, NativeScalar("bytes", b"a"), "wide-string", 3).value, + NativeScalar("bytes", b"a\0\0")) + + def test_changed_source_count_bound_changes_serialization(self): + changed = compile_scalar_write(fact(BODY.replace("$count > 0", "$count > 2"))) + value = NativeScalar("bytes", b"abcd") + self.assertEqual(evaluate_scalar_write(self.recipe, value, "string", 2).value, + NativeScalar("bytes", b"a\0")) + result = evaluate_scalar_write(changed, value, "string", 2) + self.assertEqual((result.value, result.count), (NativeScalar("bytes", b"abcd\0"), 5)) + + def test_live_dispatch_interception_or_unresolved_capture_refuses(self): + with self.assertRaises(RecipeRefused): + compile_scalar_write(fact(entries={"string": {"__perl": "CODE"}})) + missing = fact() + missing["lexical_hashes"]["resolved"] = False + with self.assertRaises(RecipeRefused): + compile_scalar_write(missing) + unresolved = fact() + unresolved["lexical_hashes"]["bindings"]["%writeValueProc"]["resolved"] = False + with self.assertRaises(RecipeRefused): + compile_scalar_write(unresolved) + + def test_control_flow_entry_and_scalar_branch_mutations_refuse(self): + mutations = ( + BODY.replace("if ($proc)", "if (1)"), + BODY.replace(" elsif", " if", 1), + BODY.replace("my($proc)", "my($format)"), + BODY.replace("my($diff)", "my($count)").replace("$diff", "$count"), + BODY.replace("$diff", "$val"), + BODY.replace("(return $val);", "do_something(); (return $val);"), + BODY.replace("$count > 0", "$count >= 0"), + ) + for body in mutations: + with self.subTest(body=body), self.assertRaises(RecipeRefused): + compile_scalar_write(fact(body)) + + def test_non_scalar_formats_and_noninteger_counts_refuse(self): + with self.assertRaises(RecipeRefused): + evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"1"), "int8u", 1) + for count in (True, 1.5, "3"): + with self.subTest(count=count), self.assertRaises(RecipeRefused): + evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"a"), "string", count) + + +@unittest.skipUnless(os.environ.get("OXIDEX_PINNED_EXIFTOOL") and + os.environ.get("OXIDEX_TABLES_JSON"), + "requires explicit pinned native source and matching captured tables") +class NativeScalarDifferential(unittest.TestCase): + def test_actual_capture_refuses_a_mutated_live_dispatch(self): + document = json.loads(Path(os.environ["OXIDEX_TABLES_JSON"]).read_text()) + helper = document["native_write_helpers"]["write_value"] + self.assertEqual(compile_scalar_write(helper).dispatch_lexical, "%writeValueProc") + mutated = json.loads(json.dumps(helper)) + mutated["lexical_hashes"]["bindings"]["%writeValueProc"]["entries"]["string"] = {"__perl": "CODE"} + with self.assertRaises(RecipeRefused): + compile_scalar_write(mutated) + + def test_actual_native_helper_matches_scalar_states_and_counts(self): + document = json.loads(Path(os.environ["OXIDEX_TABLES_JSON"]).read_text()) + recipe = compile_scalar_write(document["native_write_helpers"]["write_value"]) + values = [NativeScalar("undefined", None)] + values += [NativeScalar("bytes", value) for value in + (b"", b"A", b"abc", b"a\0b", b"\0", b"\xff", b"\xc3\xa9", b"line\n")] + values += [NativeScalar("utf8", value) for value in + ("", "é", "Ā", "😀", "a\0b", "a\n", "Aé", "AĀ")] + cases, expected = [], [] + for value in values: + for format_name in recipe.formats: + for count in (None, 0, -1, 1, 2, 3, 4, 8): + cases.append({"kind": value.kind, + "value": value.value.hex() if value.kind == "bytes" else value.value, + "format": format_name, "count": count}) + result = evaluate_scalar_write(recipe, value, format_name, count) + raw = result.value.value.encode("utf8") if result.value.kind == "utf8" else result.value.value + expected.append({"defined": result.value.kind != "undefined", + "utf8": result.value.kind == "utf8", + "hex": None if raw is None else raw.hex(), + "length": None if result.value.value is None else len(result.value.value), + # Perl's lexical ``my $count`` is not an + # alias of the caller's scalar. The + # helper's final count is retained by the + # reference result, while this oracle can + # only observe that the caller's count is + # unchanged. + "count": count}) + perl = r''' +use strict; use warnings; use JSON::PP; use Encode (); use B (); use B::Deparse; +use Digest::SHA qw(sha256_hex); +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } +use Image::ExifTool; +require 'Image/ExifTool/Writer.pl'; local $/; +my $rows=JSON::PP->new->utf8->decode(); my @results; my $cv=\&Image::ExifTool::WriteValue; +my $body=B::Deparse->new('-p','-sC')->coderef2text($cv); +open my $source, '<:raw', B::svref_2object($cv)->FILE or die $!; my $source_sha=sha256_hex(<$source>); close $source; +for my $r (@$rows) { + my $value=$r->{kind} eq 'undefined' ? undef : $r->{kind} eq 'bytes' ? pack('H*',$r->{value}) : $r->{value}; + utf8::upgrade($value) if $r->{kind} eq 'utf8'; my $count=$r->{count}; + my $out=Image::ExifTool::WriteValue($value,$r->{format},$count); my $flag=utf8::is_utf8($out)?JSON::PP::true:JSON::PP::false; + my $bytes=defined($out)?($flag ? Encode::encode('UTF-8',$out) : $out):undef; + push @results,{defined=>defined($out)?JSON::PP::true:JSON::PP::false,utf8=>$flag,hex=>defined($bytes)?unpack('H*',$bytes):undef,length=>defined($out)?length($out):undef,count=>$count}; +} +print JSON::PP->new->canonical->utf8->encode({results=>\@results,source_sha256=>$source_sha,body_sha256=>sha256_hex($body)}); +''' + env = os.environ.copy() + for name in ("PERL5LIB", "PERLLIB", "PERL5OPT"): + env.pop(name, None) + result = subprocess.run([env.get("EXIFTOOL_PERL", "/usr/bin/perl"), + "-I" + str(Path(os.environ["OXIDEX_PINNED_EXIFTOOL"]) / "lib"), "-e", perl], + input=json.dumps(cases, ensure_ascii=False).encode("utf8"), env=env, + capture_output=True, timeout=30, check=True) + native = json.loads(result.stdout) + self.assertEqual(native["source_sha256"], recipe.provenance.source_sha256) + self.assertEqual(native["body_sha256"], recipe.provenance.body_sha256) + self.assertEqual(native["results"], expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/testdata/writevalue_scalar_body.txt b/tools/exiftool-tables/testdata/writevalue_scalar_body.txt new file mode 100644 index 000000000..b5243bba9 --- /dev/null +++ b/tools/exiftool-tables/testdata/writevalue_scalar_body.txt @@ -0,0 +1,31 @@ +($$;$$$$) { + package Image::ExifTool; + use strict; + (my($val, $format, $count, $dataPt, $offset) = @_); + (my($proc) = $writeValueProc{$format}); + my($packed); + if ($proc) { die('numeric dispatch must not execute'); } + elsif ((($format eq 'string') or ($format eq 'undef'))) { + (($format eq 'string') and ($val .= "\000")); + if (($count and ($count > 0))) { + (my($diff) = ($count - length($val))); + if ($diff) { + if (($diff < 0)) { + if (($format eq 'string')) { + ($count or (return (undef))); + ($val = substr($val, 0, ($count - 1)) . "\000"); + } else { + ($val = substr($val, 0, $count)); + } + } else { + ($val .= ("\000" x $diff)); + } + } + } else { + ($count = length($val)); + } + ($dataPt and substr($$dataPt, $offset, $count) = $val); + (return $val); + } else { return undef; } + return $packed; +} diff --git a/tools/exiftool-tables/writevalue_recipes.py b/tools/exiftool-tables/writevalue_recipes.py new file mode 100644 index 000000000..61ed7cf54 --- /dev/null +++ b/tools/exiftool-tables/writevalue_recipes.py @@ -0,0 +1,241 @@ +"""Compile and evaluate the scalar ``WriteValue`` branch from captured Perl. + +This is source-derived helper execution only. It neither selects a tag nor +writes a file. Numeric packing and the optional ``$dataPt`` mutation remain +outside this closed step. +""" +from dataclasses import dataclass +import operator +from typing import Any, Mapping + +from checkexif_recipes import CodeProvenance, RecipeMalformed, RecipeRefused, _Body, _fact +from checkvalue_recipes import NativeScalar + + +_COMPARE = {">": operator.gt, ">=": operator.ge, "<": operator.lt, + "<=": operator.le, "==": operator.eq, "!=": operator.ne} + + +@dataclass(frozen=True) +class ScalarWriteRecipe: + provenance: CodeProvenance + dispatch_lexical: str + formats: tuple[str, str] + terminated_format: str + count_positive_operator: str + count_positive_bound: int + diff_negative_operator: str + terminator: str + + +@dataclass(frozen=True) +class ScalarWriteResult: + value: NativeScalar + count: int + + +def _comparison(body: _Body) -> str: + for symbol in (">=", "<=", "==", "!=", ">", "<"): + if body.tokens[body.at:body.at + len(symbol)] == list(symbol): + body.at += len(symbol) + return symbol + raise RecipeRefused("unsupported WriteValue comparison") + + +def _literal(body: _Body) -> str: + token = body.tokens[body.at] if body.at < len(body.tokens) else "" + if token.startswith('"') and any(char in token[1:-1] for char in "$@"): + raise RecipeRefused("interpolating WriteValue literal is unsupported") + return body.quoted() + + +def _skip_block(body: _Body) -> None: + """Skip only the proven-unreachable dispatch body, checking brace balance.""" + depth = 1 + while body.at < len(body.tokens) and depth: + token = body.tokens[body.at] + body.at += 1 + if token == "{": + depth += 1 + elif token == "}": + depth -= 1 + if depth: + raise RecipeRefused("WriteValue dispatch block is incomplete") + + +def _dispatch_is_absent(fact: Mapping[str, Any], lexical: str, formats: tuple[str, str]) -> None: + hashes = fact.get("lexical_hashes") + if not isinstance(hashes, Mapping) or hashes.get("resolved") is not True: + raise RecipeRefused("WriteValue lexical hash capture is unresolved") + bindings = hashes.get("bindings") + if not isinstance(bindings, Mapping): + raise RecipeMalformed("WriteValue lexical hash bindings are malformed") + captured = bindings.get(lexical) + if not isinstance(captured, Mapping) or captured.get("resolved") is not True: + raise RecipeRefused("WriteValue dispatch lexical hash is unresolved") + entries = captured.get("entries") + if not isinstance(entries, Mapping): + raise RecipeMalformed("WriteValue dispatch lexical entries are malformed") + if any(not isinstance(key, str) for key in entries): + raise RecipeMalformed("WriteValue dispatch lexical key is malformed") + for format_name in formats: + if format_name in entries: + raise RecipeRefused("WriteValue scalar format is intercepted by live dispatch") + + +def compile_scalar_write(fact: dict[str, Any]) -> ScalarWriteRecipe: + provenance = _fact(fact, "native_write_helpers.write_value", require_binding=True) + if provenance.requested_binding != "Image::ExifTool::WriteValue": + raise RecipeRefused("WriteValue requested binding is stale") + body = _Body(fact.get("__deparse")) + body.take("(", "$", "$", ";", "$", "$", "$", "$", ")", "{", "package") + package = body.tokens[body.at] if body.at < len(body.tokens) else "" + if not package or "::" not in package: + raise RecipeRefused("WriteValue package unavailable") + body.at += 1 + body.take(";", "use", "strict", ";", "(", "my", "(") + value = body.word("$") + body.take(",") + format_name = body.word("$") + body.take(",") + count = body.word("$") + body.take(",") + data = body.word("$") + body.take(",") + offset = body.word("$") + body.take(")", "=", "@", "_", ")", ";", "(", "my") + proc = body.word("$") + body.take("=", "$") + # ``word`` owns sigils, but this lookup is a hash sigil followed by a + # variable key and must retain both names for the live-pad join. + if body.at >= len(body.tokens) or not body.tokens[body.at].isidentifier(): + raise RecipeRefused("WriteValue dispatch is not a lexical hash lookup") + lexical = "%" + body.tokens[body.at] + body.at += 1 + body.take("{", "$") + if body.at >= len(body.tokens) or body.tokens[body.at] != format_name: + raise RecipeRefused("WriteValue dispatch key is not the format argument") + body.at += 1 + body.take("}", ")", ";", "my", "$") + if body.at >= len(body.tokens) or not body.tokens[body.at].isidentifier(): + raise RecipeRefused("WriteValue packed local is unavailable") + packed = body.tokens[body.at] + body.at += 1 + body.take(";", "if", "(", "$", proc, ")", "{") + if len({value, format_name, count, data, offset, proc, packed}) != 7: + raise RecipeRefused("WriteValue local aliases an argument") + _skip_block(body) + body.take("elsif", "(", "(", "(", "$", format_name, "eq") + first = _literal(body) + body.take(")", "or", "(", "$", format_name, "eq") + second = _literal(body) + body.take(")", ")", ")", "{") + body.take("(", "(", "$", format_name, "eq") + terminated = _literal(body) + body.take(")", "and", "(", "$", value, ".", "=") + token = body.tokens[body.at] if body.at < len(body.tokens) else "" + if token != '"\\000"': + raise RecipeRefused("unsupported WriteValue terminator") + body.at += 1 + body.take(")", ")", ";", "if", "(", "(", "$", count, "and", "(", "$", count) + positive = _comparison(body) + if positive != ">": + raise RecipeRefused("unsupported WriteValue count guard operator") + token = body.tokens[body.at] if body.at < len(body.tokens) else "" + if not token.isdecimal(): + raise RecipeRefused("noninteger WriteValue count guard bound") + bound = int(token) + body.at += 1 + body.take(")", ")", ")", "{") + body.take("(", "my", "$") + if body.at >= len(body.tokens) or not body.tokens[body.at].isidentifier(): + raise RecipeRefused("WriteValue diff local is unavailable") + diff = body.tokens[body.at] + body.at += 1 + if diff in {value, format_name, count, data, offset, proc, packed}: + raise RecipeRefused("WriteValue diff local aliases an existing local") + body.take("=", "(", "$", count, "-", "length", "(", "$", value, ")", ")", ")", ";", "if", "(", "$", diff, ")", "{") + body.take("if", "(", "(", "$", diff) + negative = _comparison(body) + if negative != "<": + raise RecipeRefused("unsupported WriteValue truncation operator") + body.take("0", ")", ")", "{") + body.take("if", "(", "(", "$", format_name, "eq") + truncate_terminated = _literal(body) + body.take(")", ")", "{") + body.take("(", "$", count, "or", "(", "return", "(", "undef", ")", ")", ")", ";") + body.take("(", "$", value, "=", "substr", "(", "$", value, ",", "0", ",", "(", "$", count, "-", "1", ")", ")", ".") + if body.at >= len(body.tokens) or body.tokens[body.at] != '"\\000"': + raise RecipeRefused("unsupported WriteValue truncation terminator") + body.at += 1 + body.take(")", ";", "}", "else", "{") + body.take("(", "$", value, "=", "substr", "(", "$", value, ",", "0", ",", "$", count, ")", ")", ";", "}", "}", "else", "{") + body.take("(", "$", value, ".", "=", "(") + if body.at >= len(body.tokens) or body.tokens[body.at] != '"\\000"': + raise RecipeRefused("unsupported WriteValue padding literal") + body.at += 1 + body.take("x", "$", diff, ")", ")", ";", "}", "}", "}", "else", "{") + body.take("(", "$", count, "=", "length", "(", "$", value, ")", ")", ";", "}") + # ``$dataPt`` mutation is deliberately not executed, but its exact + # placement before the scalar return proves this branch has no other call. + body.take("(", "$", data, "and", "substr", "(", "$", "$", data, ",", "$", offset, ",", "$", count, ")", "=", "$", value, ")", ";", "(", "return", "$", value, ")", ";", "}") + if body.at >= len(body.tokens): + raise RecipeRefused("WriteValue fallback is incomplete") + # The scalar return must be followed by a separate fallback and the numeric + # tail; we do not execute either. Balanced completion catches an inserted + # scalar-side statement while avoiding a numerical grammar claim. + body.take("else", "{") + _skip_block(body) + if body.at >= len(body.tokens) or body.tokens[-1] != "}": + raise RecipeRefused("WriteValue function tail is incomplete") + # The remaining numeric-tail return is unreachable after the scalar + # branch's proven return. It is deliberately not translated here. + if first == second or terminated not in {first, second} or truncate_terminated != terminated: + raise RecipeRefused("WriteValue scalar format controls are inconsistent") + _dispatch_is_absent(fact, lexical, (first, second)) + return ScalarWriteRecipe(provenance, lexical, (first, second), terminated, + positive, bound, negative, "\0") + + +def _append(value: NativeScalar, text: str) -> NativeScalar: + if value.kind == "utf8": + return NativeScalar("utf8", value.value + text) + raw = b"" if value.kind == "undefined" else value.value + return NativeScalar("bytes", raw + text.encode("latin1")) + + +def _truncate(value: NativeScalar, count: int) -> NativeScalar: + if value.kind == "undefined": + return value + truncated = value.value[:count] + if value.kind == "utf8" and not any(ord(char) > 0x7f for char in value.value): + # Perl's ``substr`` may downgrade an upgraded ASCII scalar. Keep that + # state transition explicit instead of pretending all Python ``str`` + # results retain the UTF8 flag; a non-ASCII source retains it even + # when the selected substring is empty. + return NativeScalar("bytes", truncated.encode("latin1")) + return NativeScalar(value.kind, truncated) + + +def evaluate_scalar_write(recipe: ScalarWriteRecipe, value: NativeScalar, + format_name: str, count: int | None) -> ScalarWriteResult: + """Execute the admitted scalar return branch without ``$dataPt`` mutation.""" + if format_name not in recipe.formats: + raise RecipeRefused("format is outside the proven WriteValue scalar branch") + if count is not None and type(count) is not int: + raise RecipeRefused("noninteger native count is unsupported") + if format_name == recipe.terminated_format: + value = _append(value, recipe.terminator) + length = 0 if value.kind == "undefined" else len(value.value) + if count and _COMPARE[recipe.count_positive_operator](count, recipe.count_positive_bound): + diff = count - length + if diff: + if _COMPARE[recipe.diff_negative_operator](diff, 0): + if format_name == recipe.terminated_format: + value = _append(_truncate(value, count - 1), recipe.terminator) + else: + value = _truncate(value, count) + else: + value = _append(value, recipe.terminator * diff) + return ScalarWriteResult(value, count) + return ScalarWriteResult(value, length)