From 987a630c5db104d1fb100865ea129173424dc6be Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 20:41:43 -0500 Subject: [PATCH 1/3] feat(tables): derive scalar WriteValue execution from native source and dispatch --- .../exiftool-tables/WRITEVALUE_RECIPE_API.md | 42 +++ .../test_writevalue_recipes.py | 169 ++++++++++++ .../testdata/writevalue_scalar_body.txt | 31 +++ tools/exiftool-tables/writevalue_recipes.py | 241 ++++++++++++++++++ 4 files changed, 483 insertions(+) create mode 100644 tools/exiftool-tables/WRITEVALUE_RECIPE_API.md create mode 100644 tools/exiftool-tables/test_writevalue_recipes.py create mode 100644 tools/exiftool-tables/testdata/writevalue_scalar_body.txt create mode 100644 tools/exiftool-tables/writevalue_recipes.py diff --git a/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md b/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md new file mode 100644 index 000000000..d574339a7 --- /dev/null +++ b/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md @@ -0,0 +1,42 @@ +# Source-derived scalar WriteValue execution + +`writevalue_recipes.compile_scalar_write` consumes the final captured +`native_write_helpers.write_value` CODE fact. It authenticates the exact +entry bindings, the lexical `%writeValueProc` lookup, and the `if ($proc) … +elsif (string/undef)` control-flow. Admission requires the live captured +lexical hash to be resolved and to have no entry for either source-derived +scalar format. Thus an installed dispatch entry cannot silently take the +numeric branch before scalar handling. + +`evaluate_scalar_write` preserves undefined, byte and UTF8-flagged scalar +states, source-derived terminator/truncation/padding behaviour, and the final +native count. It deliberately refuses all formats outside the captured scalar +branch and all non-integer counts. It does not model optional `$dataPt`/ +`$offset` mutation, numeric packing, tag lookup, CharsetEXIF encoding, or a +public writer route. + +This is an inactive reference mechanism. The captured `%writeValueProc` is +read as loaded state, so platform-time dispatch removal is observed. A missing +or unresolved pad never means an empty dispatch map. + +## Validation and current integration boundary + +Nine tests pass with explicit pinned native inputs and no skips. The native +scalar differential covers 272 cases, including wide characters outside the +selected substring. Independent review exercised actual native lexical +shadowing; a local that aliases the input value must refuse. The regression +suite also covers source-derived format and count-bound changes. + +A separate actual-source rehearsal changes only WriteValue's count guard +from `> 0` to `> 2` in a copied native Writer.pl. Fresh capture and compilation +match all 240 original native cases; 43 native outcomes change, with zero +handwritten tag-rule or generated-output edits. Evidence relative to the +continuation root is +`shared-pilot/write-upgrade-integration-20260913/writevalue-source-replay-20260914/`. +The native test observes return bytes, defined/UTF8 state and length. Its count +observation is the unchanged caller argument, not the helper-local final count; +the latter remains a reference result checked by source/unit tests. + +Composition with CheckExif/CheckValue and generated Rust execution, encoding, +and public tag routing remain the next steps. This helper does not activate +production writes and is not a full release-upgrade or conformance result. diff --git a/tools/exiftool-tables/test_writevalue_recipes.py b/tools/exiftool-tables/test_writevalue_recipes.py new file mode 100644 index 000000000..760e56e13 --- /dev/null +++ b/tools/exiftool-tables/test_writevalue_recipes.py @@ -0,0 +1,169 @@ +"""Source-change admission and scalar reference execution for WriteValue.""" +from pathlib import Path +import json +import os +import subprocess +import unittest + +from checkexif_recipes import RecipeRefused +from checkvalue_recipes import NativeScalar +from writevalue_recipes import compile_scalar_write, evaluate_scalar_write + +BODY = (Path(__file__).parent / "testdata/writevalue_scalar_body.txt").read_text() + + +def fact(body=BODY, entries=None): + return {"__perl": "CODE", "resolved": True, + "requested_binding": "Image::ExifTool::WriteValue", + "__name": "Image::ExifTool::WriteValue", + "source_file": "Image/ExifTool/Writer.pl", "source_sha256": "b" * 64, + "__deparse": body, "dependencies": {}, + "lexical_hashes": {"resolved": True, "bindings": { + "%writeValueProc": {"resolved": True, "entries": entries or {}}}}} + + +class ScalarWriteTests(unittest.TestCase): + def setUp(self): + self.recipe = compile_scalar_write(fact()) + + def test_count_and_scalar_kind_are_preserved(self): + cases = ( + (NativeScalar("undefined", None), "undef", None, NativeScalar("undefined", None), 0), + (NativeScalar("undefined", None), "string", None, NativeScalar("bytes", b"\0"), 1), + (NativeScalar("bytes", b"a\0b"), "undef", 0, NativeScalar("bytes", b"a\0b"), 3), + (NativeScalar("utf8", "é"), "string", -1, NativeScalar("utf8", "é\0"), 2), + (NativeScalar("bytes", b"a"), "string", 4, NativeScalar("bytes", b"a\0\0\0"), 4), + (NativeScalar("utf8", "é"), "undef", 3, NativeScalar("utf8", "é\0\0"), 3), + ) + for value, format_name, count, expected_value, expected_count in cases: + with self.subTest(value=value, format=format_name, count=count): + actual = evaluate_scalar_write(self.recipe, value, format_name, count) + self.assertEqual((actual.value, actual.count), (expected_value, expected_count)) + + def test_source_truncation_uses_terminated_format_and_count(self): + self.assertEqual(evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"abcd"), "string", 3).value, + NativeScalar("bytes", b"ab\0")) + self.assertEqual(evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"abcd"), "undef", 3).value, + NativeScalar("bytes", b"abc")) + + def test_source_format_change_changes_admission_and_execution(self): + changed = compile_scalar_write(fact(BODY.replace("'string'", "'wide-string'"))) + with self.assertRaises(RecipeRefused): + evaluate_scalar_write(changed, NativeScalar("bytes", b"a"), "string", 3) + self.assertEqual(evaluate_scalar_write(changed, NativeScalar("bytes", b"a"), "wide-string", 3).value, + NativeScalar("bytes", b"a\0\0")) + + def test_changed_source_count_bound_changes_serialization(self): + changed = compile_scalar_write(fact(BODY.replace("$count > 0", "$count > 2"))) + value = NativeScalar("bytes", b"abcd") + self.assertEqual(evaluate_scalar_write(self.recipe, value, "string", 2).value, + NativeScalar("bytes", b"a\0")) + result = evaluate_scalar_write(changed, value, "string", 2) + self.assertEqual((result.value, result.count), (NativeScalar("bytes", b"abcd\0"), 5)) + + def test_live_dispatch_interception_or_unresolved_capture_refuses(self): + with self.assertRaises(RecipeRefused): + compile_scalar_write(fact(entries={"string": {"__perl": "CODE"}})) + missing = fact() + missing["lexical_hashes"]["resolved"] = False + with self.assertRaises(RecipeRefused): + compile_scalar_write(missing) + unresolved = fact() + unresolved["lexical_hashes"]["bindings"]["%writeValueProc"]["resolved"] = False + with self.assertRaises(RecipeRefused): + compile_scalar_write(unresolved) + + def test_control_flow_entry_and_scalar_branch_mutations_refuse(self): + mutations = ( + BODY.replace("if ($proc)", "if (1)"), + BODY.replace(" elsif", " if", 1), + BODY.replace("my($proc)", "my($format)"), + BODY.replace("my($diff)", "my($count)").replace("$diff", "$count"), + BODY.replace("$diff", "$val"), + BODY.replace("(return $val);", "do_something(); (return $val);"), + BODY.replace("$count > 0", "$count >= 0"), + ) + for body in mutations: + with self.subTest(body=body), self.assertRaises(RecipeRefused): + compile_scalar_write(fact(body)) + + def test_non_scalar_formats_and_noninteger_counts_refuse(self): + with self.assertRaises(RecipeRefused): + evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"1"), "int8u", 1) + for count in (True, 1.5, "3"): + with self.subTest(count=count), self.assertRaises(RecipeRefused): + evaluate_scalar_write(self.recipe, NativeScalar("bytes", b"a"), "string", count) + + +@unittest.skipUnless(os.environ.get("OXIDEX_PINNED_EXIFTOOL") and + os.environ.get("OXIDEX_TABLES_JSON"), + "requires explicit pinned native source and matching captured tables") +class NativeScalarDifferential(unittest.TestCase): + def test_actual_capture_refuses_a_mutated_live_dispatch(self): + document = json.loads(Path(os.environ["OXIDEX_TABLES_JSON"]).read_text()) + helper = document["native_write_helpers"]["write_value"] + self.assertEqual(compile_scalar_write(helper).dispatch_lexical, "%writeValueProc") + mutated = json.loads(json.dumps(helper)) + mutated["lexical_hashes"]["bindings"]["%writeValueProc"]["entries"]["string"] = {"__perl": "CODE"} + with self.assertRaises(RecipeRefused): + compile_scalar_write(mutated) + + def test_actual_native_helper_matches_scalar_states_and_counts(self): + document = json.loads(Path(os.environ["OXIDEX_TABLES_JSON"]).read_text()) + recipe = compile_scalar_write(document["native_write_helpers"]["write_value"]) + values = [NativeScalar("undefined", None)] + values += [NativeScalar("bytes", value) for value in + (b"", b"A", b"abc", b"a\0b", b"\0", b"\xff", b"\xc3\xa9", b"line\n")] + values += [NativeScalar("utf8", value) for value in + ("", "é", "Ā", "😀", "a\0b", "a\n", "Aé", "AĀ")] + cases, expected = [], [] + for value in values: + for format_name in recipe.formats: + for count in (None, 0, -1, 1, 2, 3, 4, 8): + cases.append({"kind": value.kind, + "value": value.value.hex() if value.kind == "bytes" else value.value, + "format": format_name, "count": count}) + result = evaluate_scalar_write(recipe, value, format_name, count) + raw = result.value.value.encode("utf8") if result.value.kind == "utf8" else result.value.value + expected.append({"defined": result.value.kind != "undefined", + "utf8": result.value.kind == "utf8", + "hex": None if raw is None else raw.hex(), + "length": None if result.value.value is None else len(result.value.value), + # Perl's lexical ``my $count`` is not an + # alias of the caller's scalar. The + # helper's final count is retained by the + # reference result, while this oracle can + # only observe that the caller's count is + # unchanged. + "count": count}) + perl = r''' +use strict; use warnings; use JSON::PP; use Encode (); use B (); use B::Deparse; +use Digest::SHA qw(sha256_hex); use Image::ExifTool; +require 'Image/ExifTool/Writer.pl'; local $/; +my $rows=JSON::PP->new->utf8->decode(); my @results; my $cv=\&Image::ExifTool::WriteValue; +my $body=B::Deparse->new('-p','-sC')->coderef2text($cv); +open my $source, '<:raw', B::svref_2object($cv)->FILE or die $!; my $source_sha=sha256_hex(<$source>); close $source; +for my $r (@$rows) { + my $value=$r->{kind} eq 'undefined' ? undef : $r->{kind} eq 'bytes' ? pack('H*',$r->{value}) : $r->{value}; + utf8::upgrade($value) if $r->{kind} eq 'utf8'; my $count=$r->{count}; + my $out=Image::ExifTool::WriteValue($value,$r->{format},$count); my $flag=utf8::is_utf8($out)?JSON::PP::true:JSON::PP::false; + my $bytes=defined($out)?($flag ? Encode::encode('UTF-8',$out) : $out):undef; + push @results,{defined=>defined($out)?JSON::PP::true:JSON::PP::false,utf8=>$flag,hex=>defined($bytes)?unpack('H*',$bytes):undef,length=>defined($out)?length($out):undef,count=>$count}; +} +print JSON::PP->new->canonical->utf8->encode({results=>\@results,source_sha256=>$source_sha,body_sha256=>sha256_hex($body)}); +''' + env = os.environ.copy() + for name in ("PERL5LIB", "PERLLIB", "PERL5OPT"): + env.pop(name, None) + result = subprocess.run([env.get("EXIFTOOL_PERL", "/usr/bin/perl"), + "-I" + str(Path(os.environ["OXIDEX_PINNED_EXIFTOOL"]) / "lib"), "-e", perl], + input=json.dumps(cases, ensure_ascii=False).encode("utf8"), env=env, + capture_output=True, timeout=30, check=True) + native = json.loads(result.stdout) + self.assertEqual(native["source_sha256"], recipe.provenance.source_sha256) + self.assertEqual(native["body_sha256"], recipe.provenance.body_sha256) + self.assertEqual(native["results"], expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/testdata/writevalue_scalar_body.txt b/tools/exiftool-tables/testdata/writevalue_scalar_body.txt new file mode 100644 index 000000000..b5243bba9 --- /dev/null +++ b/tools/exiftool-tables/testdata/writevalue_scalar_body.txt @@ -0,0 +1,31 @@ +($$;$$$$) { + package Image::ExifTool; + use strict; + (my($val, $format, $count, $dataPt, $offset) = @_); + (my($proc) = $writeValueProc{$format}); + my($packed); + if ($proc) { die('numeric dispatch must not execute'); } + elsif ((($format eq 'string') or ($format eq 'undef'))) { + (($format eq 'string') and ($val .= "\000")); + if (($count and ($count > 0))) { + (my($diff) = ($count - length($val))); + if ($diff) { + if (($diff < 0)) { + if (($format eq 'string')) { + ($count or (return (undef))); + ($val = substr($val, 0, ($count - 1)) . "\000"); + } else { + ($val = substr($val, 0, $count)); + } + } else { + ($val .= ("\000" x $diff)); + } + } + } else { + ($count = length($val)); + } + ($dataPt and substr($$dataPt, $offset, $count) = $val); + (return $val); + } else { return undef; } + return $packed; +} diff --git a/tools/exiftool-tables/writevalue_recipes.py b/tools/exiftool-tables/writevalue_recipes.py new file mode 100644 index 000000000..61ed7cf54 --- /dev/null +++ b/tools/exiftool-tables/writevalue_recipes.py @@ -0,0 +1,241 @@ +"""Compile and evaluate the scalar ``WriteValue`` branch from captured Perl. + +This is source-derived helper execution only. It neither selects a tag nor +writes a file. Numeric packing and the optional ``$dataPt`` mutation remain +outside this closed step. +""" +from dataclasses import dataclass +import operator +from typing import Any, Mapping + +from checkexif_recipes import CodeProvenance, RecipeMalformed, RecipeRefused, _Body, _fact +from checkvalue_recipes import NativeScalar + + +_COMPARE = {">": operator.gt, ">=": operator.ge, "<": operator.lt, + "<=": operator.le, "==": operator.eq, "!=": operator.ne} + + +@dataclass(frozen=True) +class ScalarWriteRecipe: + provenance: CodeProvenance + dispatch_lexical: str + formats: tuple[str, str] + terminated_format: str + count_positive_operator: str + count_positive_bound: int + diff_negative_operator: str + terminator: str + + +@dataclass(frozen=True) +class ScalarWriteResult: + value: NativeScalar + count: int + + +def _comparison(body: _Body) -> str: + for symbol in (">=", "<=", "==", "!=", ">", "<"): + if body.tokens[body.at:body.at + len(symbol)] == list(symbol): + body.at += len(symbol) + return symbol + raise RecipeRefused("unsupported WriteValue comparison") + + +def _literal(body: _Body) -> str: + token = body.tokens[body.at] if body.at < len(body.tokens) else "" + if token.startswith('"') and any(char in token[1:-1] for char in "$@"): + raise RecipeRefused("interpolating WriteValue literal is unsupported") + return body.quoted() + + +def _skip_block(body: _Body) -> None: + """Skip only the proven-unreachable dispatch body, checking brace balance.""" + depth = 1 + while body.at < len(body.tokens) and depth: + token = body.tokens[body.at] + body.at += 1 + if token == "{": + depth += 1 + elif token == "}": + depth -= 1 + if depth: + raise RecipeRefused("WriteValue dispatch block is incomplete") + + +def _dispatch_is_absent(fact: Mapping[str, Any], lexical: str, formats: tuple[str, str]) -> None: + hashes = fact.get("lexical_hashes") + if not isinstance(hashes, Mapping) or hashes.get("resolved") is not True: + raise RecipeRefused("WriteValue lexical hash capture is unresolved") + bindings = hashes.get("bindings") + if not isinstance(bindings, Mapping): + raise RecipeMalformed("WriteValue lexical hash bindings are malformed") + captured = bindings.get(lexical) + if not isinstance(captured, Mapping) or captured.get("resolved") is not True: + raise RecipeRefused("WriteValue dispatch lexical hash is unresolved") + entries = captured.get("entries") + if not isinstance(entries, Mapping): + raise RecipeMalformed("WriteValue dispatch lexical entries are malformed") + if any(not isinstance(key, str) for key in entries): + raise RecipeMalformed("WriteValue dispatch lexical key is malformed") + for format_name in formats: + if format_name in entries: + raise RecipeRefused("WriteValue scalar format is intercepted by live dispatch") + + +def compile_scalar_write(fact: dict[str, Any]) -> ScalarWriteRecipe: + provenance = _fact(fact, "native_write_helpers.write_value", require_binding=True) + if provenance.requested_binding != "Image::ExifTool::WriteValue": + raise RecipeRefused("WriteValue requested binding is stale") + body = _Body(fact.get("__deparse")) + body.take("(", "$", "$", ";", "$", "$", "$", "$", ")", "{", "package") + package = body.tokens[body.at] if body.at < len(body.tokens) else "" + if not package or "::" not in package: + raise RecipeRefused("WriteValue package unavailable") + body.at += 1 + body.take(";", "use", "strict", ";", "(", "my", "(") + value = body.word("$") + body.take(",") + format_name = body.word("$") + body.take(",") + count = body.word("$") + body.take(",") + data = body.word("$") + body.take(",") + offset = body.word("$") + body.take(")", "=", "@", "_", ")", ";", "(", "my") + proc = body.word("$") + body.take("=", "$") + # ``word`` owns sigils, but this lookup is a hash sigil followed by a + # variable key and must retain both names for the live-pad join. + if body.at >= len(body.tokens) or not body.tokens[body.at].isidentifier(): + raise RecipeRefused("WriteValue dispatch is not a lexical hash lookup") + lexical = "%" + body.tokens[body.at] + body.at += 1 + body.take("{", "$") + if body.at >= len(body.tokens) or body.tokens[body.at] != format_name: + raise RecipeRefused("WriteValue dispatch key is not the format argument") + body.at += 1 + body.take("}", ")", ";", "my", "$") + if body.at >= len(body.tokens) or not body.tokens[body.at].isidentifier(): + raise RecipeRefused("WriteValue packed local is unavailable") + packed = body.tokens[body.at] + body.at += 1 + body.take(";", "if", "(", "$", proc, ")", "{") + if len({value, format_name, count, data, offset, proc, packed}) != 7: + raise RecipeRefused("WriteValue local aliases an argument") + _skip_block(body) + body.take("elsif", "(", "(", "(", "$", format_name, "eq") + first = _literal(body) + body.take(")", "or", "(", "$", format_name, "eq") + second = _literal(body) + body.take(")", ")", ")", "{") + body.take("(", "(", "$", format_name, "eq") + terminated = _literal(body) + body.take(")", "and", "(", "$", value, ".", "=") + token = body.tokens[body.at] if body.at < len(body.tokens) else "" + if token != '"\\000"': + raise RecipeRefused("unsupported WriteValue terminator") + body.at += 1 + body.take(")", ")", ";", "if", "(", "(", "$", count, "and", "(", "$", count) + positive = _comparison(body) + if positive != ">": + raise RecipeRefused("unsupported WriteValue count guard operator") + token = body.tokens[body.at] if body.at < len(body.tokens) else "" + if not token.isdecimal(): + raise RecipeRefused("noninteger WriteValue count guard bound") + bound = int(token) + body.at += 1 + body.take(")", ")", ")", "{") + body.take("(", "my", "$") + if body.at >= len(body.tokens) or not body.tokens[body.at].isidentifier(): + raise RecipeRefused("WriteValue diff local is unavailable") + diff = body.tokens[body.at] + body.at += 1 + if diff in {value, format_name, count, data, offset, proc, packed}: + raise RecipeRefused("WriteValue diff local aliases an existing local") + body.take("=", "(", "$", count, "-", "length", "(", "$", value, ")", ")", ")", ";", "if", "(", "$", diff, ")", "{") + body.take("if", "(", "(", "$", diff) + negative = _comparison(body) + if negative != "<": + raise RecipeRefused("unsupported WriteValue truncation operator") + body.take("0", ")", ")", "{") + body.take("if", "(", "(", "$", format_name, "eq") + truncate_terminated = _literal(body) + body.take(")", ")", "{") + body.take("(", "$", count, "or", "(", "return", "(", "undef", ")", ")", ")", ";") + body.take("(", "$", value, "=", "substr", "(", "$", value, ",", "0", ",", "(", "$", count, "-", "1", ")", ")", ".") + if body.at >= len(body.tokens) or body.tokens[body.at] != '"\\000"': + raise RecipeRefused("unsupported WriteValue truncation terminator") + body.at += 1 + body.take(")", ";", "}", "else", "{") + body.take("(", "$", value, "=", "substr", "(", "$", value, ",", "0", ",", "$", count, ")", ")", ";", "}", "}", "else", "{") + body.take("(", "$", value, ".", "=", "(") + if body.at >= len(body.tokens) or body.tokens[body.at] != '"\\000"': + raise RecipeRefused("unsupported WriteValue padding literal") + body.at += 1 + body.take("x", "$", diff, ")", ")", ";", "}", "}", "}", "else", "{") + body.take("(", "$", count, "=", "length", "(", "$", value, ")", ")", ";", "}") + # ``$dataPt`` mutation is deliberately not executed, but its exact + # placement before the scalar return proves this branch has no other call. + body.take("(", "$", data, "and", "substr", "(", "$", "$", data, ",", "$", offset, ",", "$", count, ")", "=", "$", value, ")", ";", "(", "return", "$", value, ")", ";", "}") + if body.at >= len(body.tokens): + raise RecipeRefused("WriteValue fallback is incomplete") + # The scalar return must be followed by a separate fallback and the numeric + # tail; we do not execute either. Balanced completion catches an inserted + # scalar-side statement while avoiding a numerical grammar claim. + body.take("else", "{") + _skip_block(body) + if body.at >= len(body.tokens) or body.tokens[-1] != "}": + raise RecipeRefused("WriteValue function tail is incomplete") + # The remaining numeric-tail return is unreachable after the scalar + # branch's proven return. It is deliberately not translated here. + if first == second or terminated not in {first, second} or truncate_terminated != terminated: + raise RecipeRefused("WriteValue scalar format controls are inconsistent") + _dispatch_is_absent(fact, lexical, (first, second)) + return ScalarWriteRecipe(provenance, lexical, (first, second), terminated, + positive, bound, negative, "\0") + + +def _append(value: NativeScalar, text: str) -> NativeScalar: + if value.kind == "utf8": + return NativeScalar("utf8", value.value + text) + raw = b"" if value.kind == "undefined" else value.value + return NativeScalar("bytes", raw + text.encode("latin1")) + + +def _truncate(value: NativeScalar, count: int) -> NativeScalar: + if value.kind == "undefined": + return value + truncated = value.value[:count] + if value.kind == "utf8" and not any(ord(char) > 0x7f for char in value.value): + # Perl's ``substr`` may downgrade an upgraded ASCII scalar. Keep that + # state transition explicit instead of pretending all Python ``str`` + # results retain the UTF8 flag; a non-ASCII source retains it even + # when the selected substring is empty. + return NativeScalar("bytes", truncated.encode("latin1")) + return NativeScalar(value.kind, truncated) + + +def evaluate_scalar_write(recipe: ScalarWriteRecipe, value: NativeScalar, + format_name: str, count: int | None) -> ScalarWriteResult: + """Execute the admitted scalar return branch without ``$dataPt`` mutation.""" + if format_name not in recipe.formats: + raise RecipeRefused("format is outside the proven WriteValue scalar branch") + if count is not None and type(count) is not int: + raise RecipeRefused("noninteger native count is unsupported") + if format_name == recipe.terminated_format: + value = _append(value, recipe.terminator) + length = 0 if value.kind == "undefined" else len(value.value) + if count and _COMPARE[recipe.count_positive_operator](count, recipe.count_positive_bound): + diff = count - length + if diff: + if _COMPARE[recipe.diff_negative_operator](diff, 0): + if format_name == recipe.terminated_format: + value = _append(_truncate(value, count - 1), recipe.terminator) + else: + value = _truncate(value, count) + else: + value = _append(value, recipe.terminator * diff) + return ScalarWriteResult(value, count) + return ScalarWriteResult(value, length) From bc487c6952b71182bf2544bd612b90475bdc2087 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 21:07:38 -0500 Subject: [PATCH 2/3] Generate scalar writer helper operands from captured native rules Execute captured CheckValue and WriteValue operands in inactive shared Rust. Regeneration owns both artifacts; fresh native CI checks reject stale committed rules and ledger. Isolate native capture from ambient ExifTool config. Focused native Python, compiled Rust replay, six scalar unit tests and required Clippy pass; full regeneration and full Python suite pending. --- docs/AUTOGENERATION-PROGRESS.md | 29 +- src/writers/generated_scalar.rs | 369 ++++++++++++++++++ src/writers/generated_scalar_rules.rs | 44 +++ src/writers/mod.rs | 4 + tools/exiftool-tables/README.md | 6 +- .../exiftool-tables/WRITEVALUE_RECIPE_API.md | 33 +- tools/exiftool-tables/artifacts.py | 2 + .../dump_binary_reader_contract.pl | 1 + tools/exiftool-tables/dump_tables.pl | 4 + tools/exiftool-tables/oracle.pl | 1 + tools/exiftool-tables/regen.sh | 6 + .../exiftool-tables/scalar_helper_codegen.py | 122 ++++++ .../exiftool-tables/scalar_helper_ledger.json | 133 +++++++ tools/exiftool-tables/test_artifacts.py | 4 +- .../test_checkvalue_recipes.py | 4 +- .../test_exiftool_config_isolation.py | 102 +++++ .../test_scalar_helper_codegen.py | 92 +++++ .../test_scalar_helper_rust.py | 102 +++++ .../test_writevalue_recipes.py | 4 +- 19 files changed, 1049 insertions(+), 13 deletions(-) create mode 100644 src/writers/generated_scalar.rs create mode 100644 src/writers/generated_scalar_rules.rs create mode 100644 tools/exiftool-tables/scalar_helper_codegen.py create mode 100644 tools/exiftool-tables/scalar_helper_ledger.json create mode 100644 tools/exiftool-tables/test_exiftool_config_isolation.py create mode 100644 tools/exiftool-tables/test_scalar_helper_codegen.py create mode 100644 tools/exiftool-tables/test_scalar_helper_rust.py diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index ef3b531da..b94dad438 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -106,8 +106,33 @@ Portable readiness tests replace private-path dependencies with controlled archived sources and CLI subprocesses. The real selected 11.78 and 12.64 releases separately pass native readiness. This advances the rehearsal instrument; it does not establish generated OxiDex conformance for either -release. Shared WriteValue translation, encoding, helper composition and -public generated routing remain unfinished. +release. Encoding, helper composition and public generated routing remain +unfinished. + +The helper-capture and native-readiness integration **merged in PR #762** as +`54206ecb`. All five required hosted checks passed on `d091f251`, including +generated-table verification. The selected historical releases still have no +completed generated OxiDex/native conformance run. + +The next work-branch checkpoint compiles captured CheckValue and WriteValue +rules into Rust operands through normal regeneration, adding two declared +artifacts (34 total). Unsupported helper semantics produce an explicit ledger +gap and no admitted rule; they cannot silently reuse the older release's +operands. CI's native suite checks both committed artifacts against fresh +generation. The shared Rust executor matches 512 native cases on canonical +source, 512 on a copied WriteValue count-bound change and 512 on a copied +CheckValue comparison change. These are three helper probes, not three release +upgrades. The tests found and corrected UTF8 substring storage and negative +repetition behavior. Source-reference count checks are separate from native +return checks. + +Native capture and oracle startup now explicitly disable ambient ExifTool +configuration. Clean and hostile-home native captures are byte-identical in +the focused probe. This prevents personal configuration from being mistaken +for the selected release's rules. Full 34-artifact regeneration and the full +Python suite for this combined checkpoint are pending. No public writer route +is enabled, no manual tag rule has been retired in this checkpoint, and no +project-wide generation percentage has been remeasured. ### Previous merged definitions checkpoint diff --git a/src/writers/generated_scalar.rs b/src/writers/generated_scalar.rs new file mode 100644 index 000000000..419f4eb56 --- /dev/null +++ b/src/writers/generated_scalar.rs @@ -0,0 +1,369 @@ +//! Inactive source-derived scalar CheckValue and WriteValue execution. + +use crate::error::{ExifToolError, Result}; + +/// Perl's scalar storage state at the boundary of the proven helper branch. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum Scalar { + Undefined, + Bytes(Vec), + Utf8(String), +} + +/// One comparison operator carried from a source-derived recipe. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Comparison { + Lt, + Le, + Eq, + Ne, + Ge, + Gt, +} + +impl Comparison { + fn applies(self, left: i64, right: i64) -> bool { + match self { + Self::Lt => left < right, + Self::Le => left <= right, + Self::Eq => left == right, + Self::Ne => left != right, + Self::Ge => left >= right, + Self::Gt => left > right, + } + } +} + +/// The complete admitted scalar early-return branch of native `CheckValue`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ScalarCheckRecipe { + pub formats: [&'static str; 2], + pub positive_count_operator: Comparison, + pub positive_count_bound: i64, + pub first_format: &'static str, + pub first_limit_operator: Comparison, + pub first_error: &'static str, + pub second_limit_operator: Comparison, + pub second_error: &'static str, + pub padding_operator: Comparison, + /// The compiler admits only the native NUL operand. + pub padding_character: u8, +} + +/// The complete admitted scalar early-return branch of native `WriteValue`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ScalarWriteRecipe { + pub formats: [&'static str; 2], + pub terminated_format: &'static str, + pub count_positive_operator: Comparison, + pub count_positive_bound: i64, + pub diff_negative_operator: Comparison, + /// The compiler admits only the native NUL operand. + pub terminator: u8, +} + +/// A CheckValue result: its possibly padded scalar and native source error. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CheckedScalar { + pub value: Scalar, + pub error: Option<&'static str>, +} + +/// A WriteValue result before optional data-pointer mutation or CharsetEXIF. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SerializedScalar { + pub value: Scalar, + /// The source branch's scalar count. TIFF's u32 field-count admission is later. + pub count: usize, +} + +fn refused(reason: &str) -> ExifToolError { + ExifToolError::unsupported_format(format!("generated scalar recipe refused: {reason}")) +} + +fn ensure_format(formats: [&'static str; 2], format_name: &str) -> Result<()> { + if formats.contains(&format_name) { + Ok(()) + } else { + Err(refused("format is outside the proven scalar branch")) + } +} + +fn scalar_length(value: &Scalar) -> Result { + match value { + Scalar::Undefined => Ok(0), + Scalar::Bytes(bytes) => Ok(bytes.len()), + // Perl's length/substr operate on characters for a UTF8-flagged scalar. + Scalar::Utf8(text) => Ok(text.chars().count()), + } +} + +fn i64_length(value: &Scalar) -> Result { + i64::try_from(scalar_length(value)?).map_err(|_| refused("scalar length overflows i64")) +} + +fn positive_count(count: Option, operator: Comparison, bound: i64) -> bool { + count.is_some_and(|value| value != 0 && operator.applies(value, bound)) +} + +fn checked_count(value: i64) -> Result { + usize::try_from(value).map_err(|_| refused("count cannot be represented on this platform")) +} + +fn append_byte(value: Scalar, byte: u8, amount: usize) -> Result { + match value { + Scalar::Undefined => { + let mut bytes = Vec::new(); + bytes + .try_reserve(amount) + .map_err(|_| refused("scalar padding allocation failed"))?; + bytes.resize(amount, byte); + Ok(Scalar::Bytes(bytes)) + } + Scalar::Bytes(mut bytes) => { + bytes + .try_reserve(amount) + .map_err(|_| refused("scalar padding allocation failed"))?; + bytes.resize( + bytes + .len() + .checked_add(amount) + .ok_or_else(|| refused("scalar length overflow"))?, + byte, + ); + Ok(Scalar::Bytes(bytes)) + } + Scalar::Utf8(mut text) => { + if byte > 0x7f { + return Err(refused("non-ASCII scalar padding is unsupported")); + } + text.try_reserve(amount) + .map_err(|_| refused("scalar padding allocation failed"))?; + text.extend(std::iter::repeat(char::from(byte)).take(amount)); + Ok(Scalar::Utf8(text)) + } + } +} + +fn truncate(value: Scalar, length: usize) -> Result { + match value { + Scalar::Undefined => Ok(Scalar::Undefined), + Scalar::Bytes(mut bytes) => { + bytes.truncate(length); + Ok(Scalar::Bytes(bytes)) + } + Scalar::Utf8(text) => { + let byte_end = text + .char_indices() + .nth(length) + .map_or(text.len(), |(index, _)| index); + let truncated = &text[..byte_end]; + // Perl's `substr` downgrades an upgraded ASCII scalar, including + // one containing embedded NULs. A source scalar that contained a + // non-ASCII character retains its UTF8 flag even when the selected + // substring happens to be ASCII or empty. + if text.is_ascii() { + Ok(Scalar::Bytes(truncated.as_bytes().to_vec())) + } else { + Ok(Scalar::Utf8(truncated.to_owned())) + } + } + } +} + +/// Execute the proven scalar `CheckValue` branch without tag lookup or conversion. +pub(crate) fn validate_scalar( + recipe: &ScalarCheckRecipe, + mut value: Scalar, + format_name: &str, + count: Option, +) -> Result { + ensure_format(recipe.formats, format_name)?; + if recipe.padding_character != 0 { + return Err(refused("scalar CheckValue padding is not NUL")); + } + if !positive_count( + count, + recipe.positive_count_operator, + recipe.positive_count_bound, + ) { + return Ok(CheckedScalar { value, error: None }); + } + let count = count.expect("positive_count requires Some"); + let length = i64_length(&value)?; + let (limit, error) = if format_name == recipe.first_format { + (recipe.first_limit_operator, recipe.first_error) + } else { + (recipe.second_limit_operator, recipe.second_error) + }; + if limit.applies(length, count) { + return Ok(CheckedScalar { + value, + error: Some(error), + }); + } + if recipe.padding_operator.applies(length, count) { + // Perl repetition with a non-positive count produces the empty string. + // Source changes may select this branch when length exceeds count. + let padding = if count <= length { + 0 + } else { + checked_count(count - length)? + }; + value = append_byte(value, recipe.padding_character, padding)?; + } + Ok(CheckedScalar { value, error: None }) +} + +/// Execute the proven scalar `WriteValue` branch without data-pointer mutation. +pub(crate) fn serialize_scalar( + recipe: &ScalarWriteRecipe, + mut value: Scalar, + format_name: &str, + count: Option, +) -> Result { + ensure_format(recipe.formats, format_name)?; + if recipe.terminator != 0 { + return Err(refused("scalar WriteValue terminator is not NUL")); + } + if format_name == recipe.terminated_format { + value = append_byte(value, recipe.terminator, 1)?; + } + let final_count = if positive_count( + count, + recipe.count_positive_operator, + recipe.count_positive_bound, + ) { + let count = count.expect("positive_count requires Some"); + let length = i64_length(&value)?; + let difference = count + .checked_sub(length) + .ok_or_else(|| refused("scalar count subtraction overflow"))?; + if difference != 0 { + if recipe.diff_negative_operator.applies(difference, 0) { + let target = checked_count(count)?; + value = if format_name == recipe.terminated_format { + let before_terminator = target + .checked_sub(1) + .ok_or_else(|| refused("terminated scalar count is zero"))?; + append_byte(truncate(value, before_terminator)?, recipe.terminator, 1)? + } else { + truncate(value, target)? + }; + } else { + value = append_byte(value, recipe.terminator, checked_count(difference)?)?; + } + } + checked_count(count)? + } else { + scalar_length(&value)? + }; + Ok(SerializedScalar { + value, + count: final_count, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CHECK: ScalarCheckRecipe = ScalarCheckRecipe { + formats: ["string", "undef"], + positive_count_operator: Comparison::Gt, + positive_count_bound: 0, + first_format: "string", + first_limit_operator: Comparison::Ge, + first_error: "String too long", + second_limit_operator: Comparison::Gt, + second_error: "Data too long", + padding_operator: Comparison::Lt, + padding_character: 0, + }; + + const WRITE: ScalarWriteRecipe = ScalarWriteRecipe { + formats: ["string", "undef"], + terminated_format: "string", + count_positive_operator: Comparison::Gt, + count_positive_bound: 0, + diff_negative_operator: Comparison::Lt, + terminator: 0, + }; + + #[test] + fn changed_check_comparisons_preserve_negative_repeat_semantics() { + let changed = ScalarCheckRecipe { + first_limit_operator: Comparison::Eq, + second_limit_operator: Comparison::Eq, + padding_operator: Comparison::Gt, + ..CHECK + }; + let result = validate_scalar(&changed, Scalar::Bytes(b"abc".to_vec()), "string", Some(2)) + .expect("negative repeat appends an empty string"); + assert_eq!(result.value, Scalar::Bytes(b"abc".to_vec())); + assert_eq!(result.error, None); + } + + #[test] + fn checkvalue_uses_character_lengths_and_preserves_utf8() { + let checked = validate_scalar(&CHECK, Scalar::Utf8("é".into()), "string", Some(3)) + .expect("scalar recipe accepts string"); + assert_eq!(checked.error, None); + assert_eq!(checked.value, Scalar::Utf8("é\0\0".into())); + + let too_long = validate_scalar(&CHECK, Scalar::Bytes(b"abc".to_vec()), "string", Some(3)) + .expect("validated scalar returns source error"); + assert_eq!(too_long.error, Some("String too long")); + } + + #[test] + fn writevalue_nul_terminates_then_truncates_and_reports_encoded_count() { + let encoded = serialize_scalar(&WRITE, Scalar::Bytes(b"abcd".to_vec()), "string", Some(3)) + .expect("scalar recipe serializes string"); + assert_eq!(encoded.value, Scalar::Bytes(b"ab\0".to_vec())); + assert_eq!(encoded.count, 3); + } + + #[test] + fn writevalue_substr_downgrades_utf8_scalars_that_become_ascii() { + let nul_only = serialize_scalar(&WRITE, Scalar::Utf8("\0".into()), "string", Some(1)) + .expect("scalar recipe serializes string"); + assert_eq!(nul_only.value, Scalar::Bytes(vec![0])); + + let ascii_with_nuls = + serialize_scalar(&WRITE, Scalar::Utf8("a\0\0".into()), "undef", Some(2)) + .expect("scalar recipe serializes undef"); + assert_eq!(ascii_with_nuls.value, Scalar::Bytes(b"a\0".to_vec())); + } + + #[test] + fn absent_zero_and_negative_counts_follow_the_native_scalar_branches() { + for count in [None, Some(0), Some(-1)] { + let checked = validate_scalar(&CHECK, Scalar::Undefined, "undef", count) + .expect("check accepts scalar count"); + assert_eq!( + checked, + CheckedScalar { + value: Scalar::Undefined, + error: None + } + ); + + let encoded = serialize_scalar(&WRITE, Scalar::Undefined, "string", count) + .expect("write accepts scalar count"); + assert_eq!( + encoded, + SerializedScalar { + value: Scalar::Bytes(vec![0]), + count: 1 + } + ); + } + } + + #[test] + fn unproven_format_and_count_overflow_refuse() { + assert!(validate_scalar(&CHECK, Scalar::Bytes(vec![]), "int8u", Some(1)).is_err()); + assert!(serialize_scalar(&WRITE, Scalar::Bytes(vec![]), "string", Some(i64::MAX)).is_err()); + } +} diff --git a/src/writers/generated_scalar_rules.rs b/src/writers/generated_scalar_rules.rs new file mode 100644 index 000000000..d0e3d7aed --- /dev/null +++ b/src/writers/generated_scalar_rules.rs @@ -0,0 +1,44 @@ +// @generated by tools/exiftool-tables/scalar_helper_codegen.py. +// Source-derived helper operands; no public tag or writer admission. +#![allow(dead_code, unused_imports)] +use crate::writers::generated_scalar::{Comparison, ScalarCheckRecipe, ScalarWriteRecipe}; + +pub(crate) const CHECK_VALUE: Option = Some(ScalarCheckRecipe { + formats: ["string", "undef"], + + positive_count_operator: Comparison::Gt, + + positive_count_bound: 0, + + first_format: "string", + + first_limit_operator: Comparison::Ge, + + first_error: "String too long", + + second_limit_operator: Comparison::Gt, + + second_error: "Data too long", + + padding_operator: Comparison::Lt, + + padding_character: 0, +}); + +pub(crate) const CHECK_VALUE_PROVENANCE: &str = "{\"actual_name\": \"Image::ExifTool::CheckValue\", \"body_sha256\": \"30460d5f4bb56fce39ef411a19595eb2aa77ad99282a5bcbdc6b735242c24956\", \"dependencies\": [{\"binding\": \"Image::ExifTool::IsFloat\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsFloat\", \"body_sha256\": \"0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsHex\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsHex\", \"body_sha256\": \"79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsInt\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsInt\", \"body_sha256\": \"2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}], \"requested_binding\": \"Image::ExifTool::CheckValue\", \"source_file\": \"Image/ExifTool/Writer.pl\", \"source_sha256\": \"cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44\"}"; + +pub(crate) const WRITE_VALUE: Option = Some(ScalarWriteRecipe { + formats: ["string", "undef"], + + terminated_format: "string", + + count_positive_operator: Comparison::Gt, + + count_positive_bound: 0, + + diff_negative_operator: Comparison::Lt, + + terminator: 0, +}); + +pub(crate) const WRITE_VALUE_PROVENANCE: &str = "{\"actual_name\": \"Image::ExifTool::WriteValue\", \"body_sha256\": \"4b72f386df682e76ca7dd0262bd89bdfa10f9e964042777e6b5f7009f2de3c0a\", \"dependencies\": [{\"binding\": \"Image::ExifTool::IsFloat\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsFloat\", \"body_sha256\": \"0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsHex\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsHex\", \"body_sha256\": \"79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsInt\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsInt\", \"body_sha256\": \"2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}, {\"binding\": \"Image::ExifTool::IsRational\", \"fact\": {\"actual_name\": \"Image::ExifTool::IsRational\", \"body_sha256\": \"ecfabc808b12990e7c72603f11c23cab462d4d52b29f88fe9a521d2ef4574356\", \"dependencies\": [], \"requested_binding\": null, \"source_file\": \"Image/ExifTool.pm\", \"source_sha256\": \"95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508\"}}], \"requested_binding\": \"Image::ExifTool::WriteValue\", \"source_file\": \"Image/ExifTool/Writer.pl\", \"source_sha256\": \"cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44\"}"; diff --git a/src/writers/mod.rs b/src/writers/mod.rs index 05fdae5ee..62f0152c3 100644 --- a/src/writers/mod.rs +++ b/src/writers/mod.rs @@ -7,6 +7,10 @@ pub mod atomic_writer; pub mod exif_inplace; pub mod exif_surgical; +// Shared source-derived helpers are not yet connected to public writes. +#[allow(dead_code)] +pub(crate) mod generated_scalar; +pub(crate) mod generated_scalar_rules; pub mod jpeg_writer; pub mod pdf_writer; pub mod png_writer; diff --git a/tools/exiftool-tables/README.md b/tools/exiftool-tables/README.md index 72437f5fc..499b24f2b 100644 --- a/tools/exiftool-tables/README.md +++ b/tools/exiftool-tables/README.md @@ -8,7 +8,7 @@ the pin remains 13.59. Conservative IFD-aware upgrade classification, one generated-output inventory, verified Canon CODE references and isolated upgrade orchestration are implemented. -The inventory contains 32 outputs, including inactive keyed and serial definitions and the +The inventory contains 34 outputs, including inactive scalar helper, keyed and serial definitions and the [Sony plain producer recovery](../../docs/reference/sony-plain-generator-recovery.md). See the [execution plan](../../docs/UPGRADE-NEXT-STEPS.md) for original validation evidence and the remaining work. The @@ -38,7 +38,7 @@ definitions and compiled Composite expressions, FITS names, and expression/value conversion ledgers. `regen-all.sh` adds vendor subdirectory tables, Nikon AF-point grids, bespoke transcriptions, recovered Sony/Minolta/Nikon generators, Macintosh CJK charset tables, GeoTIFF key maps, DICOM dictionaries and lens alternatives. -`artifacts.py` is the single output inventory: 10 tier-1 and 22 tier-2 artifacts. +`artifacts.py` is the single output inventory: 12 tier-1 and 22 tier-2 artifacts. Both scripts resolve their output paths and formatting sets from it; the bump's promotion/recovery sets and CI's tier-2 comparison use the same inventory. The bump classifier also @@ -113,7 +113,7 @@ for the restored field, bounded validation and remaining real-file acceptance. ## Generated-output inventory and write checks ```sh -python3 tools/exiftool-tables/artifacts.py paths # all 32 outputs +python3 tools/exiftool-tables/artifacts.py paths # all 34 outputs python3 tools/exiftool-tables/artifacts.py paths --tier 2 # downstream outputs python3 tools/exiftool-tables/artifacts.py paths --tier 1 --kind rust --absolute python3 tools/exiftool-tables/artifacts.py path composite-compute diff --git a/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md b/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md index d574339a7..4f40d635e 100644 --- a/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md +++ b/tools/exiftool-tables/WRITEVALUE_RECIPE_API.md @@ -37,6 +37,33 @@ The native test observes return bytes, defined/UTF8 state and length. Its count observation is the unchanged caller argument, not the helper-local final count; the latter remains a reference result checked by source/unit tests. -Composition with CheckExif/CheckValue and generated Rust execution, encoding, -and public tag routing remain the next steps. This helper does not activate -production writes and is not a full release-upgrade or conformance result. +## Generated Rust execution + +`scalar_helper_codegen.py` compiles the captured CheckValue and WriteValue +operands into `src/writers/generated_scalar_rules.rs`. Normal `regen.sh` owns +that artifact and `scalar_helper_ledger.json`; the inventory now has 34 outputs. +The shared `generated_scalar.rs` executor consumes those operands. Supported +source changes replace the operands; unsupported source emits `None` and a +named ledger gap instead of retaining an old rule. The native suite also +compares both committed artifacts with generation from CI's fresh pinned dump, +so omitting regeneration cannot leave stale rules behind a green helper test. +The public writer remains inactive for these definitions. + +The actual compiled Rust executor matches 512 authenticated native helper cases +(240 validation and 272 serialization) for the canonical source and again for +the copied WriteValue count-guard change. A third actual-source replay changes +CheckValue's comparisons, including a branch that executes a negative Perl +repetition; all 512 cases match that native source too. The native return +records supply the expected values, defined/UTF8 state and validation errors. +Rust's helper-local count is separately compared with the Python source +reference; this is not a native observation of that local variable. + +The first Rust replay exposed an ASCII substring's UTF8 storage downgrade. +The corrected executor preserves native behavior when non-ASCII characters +occur outside the selected prefix as well. These are helper-level checks, +not evidence of a public generated write route. + +Composition with CheckExif/CheckValue, CharsetEXIF encoding, public tag routing +and complete file write/read-back remain the next steps. Numeric packing and +the optional native data-target mutation are also unfinished. This checkpoint +does not establish full release-upgrade or read/write conformance. diff --git a/tools/exiftool-tables/artifacts.py b/tools/exiftool-tables/artifacts.py index af360a234..b43513c7b 100644 --- a/tools/exiftool-tables/artifacts.py +++ b/tools/exiftool-tables/artifacts.py @@ -37,6 +37,8 @@ class Artifact: Artifact("serial", 1, "serial_directory", "src/exiftool_tables/serial_tables.rs"), Artifact("expr-ledger", 1, "verify_exprs", "tools/exiftool-tables/expr_oracle_ledger.json"), Artifact("value-ledger", 1, "codegen", "tools/exiftool-tables/value_conv_ledger.json"), + Artifact("scalar-helpers", 1, "scalar_helper_codegen", "src/writers/generated_scalar_rules.rs"), + Artifact("scalar-helper-ledger", 1, "scalar_helper_codegen", "tools/exiftool-tables/scalar_helper_ledger.json"), Artifact("filetypes", 1, "codegen_filetypes", "src/filetype/tables.rs"), Artifact("composite", 1, "codegen_composite", "src/composite/tables.rs"), Artifact("composite-compute", 1, "codegen_composite", "src/composite/generated_compute.rs"), diff --git a/tools/exiftool-tables/dump_binary_reader_contract.pl b/tools/exiftool-tables/dump_binary_reader_contract.pl index 6fd27d53f..51fd6465e 100755 --- a/tools/exiftool-tables/dump_binary_reader_contract.pl +++ b/tools/exiftool-tables/dump_binary_reader_contract.pl @@ -27,6 +27,7 @@ # dump regeneration; the caller records this non-zero child as unresolved. $SIG{ALRM} = sub { die "native_reader_contract_timeout\n" }; alarm $timeout; +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } require Image::ExifTool; my $contract = capture_in_process($EXIFTOOL_LIB_ABS); alarm 0; diff --git a/tools/exiftool-tables/dump_tables.pl b/tools/exiftool-tables/dump_tables.pl index 38fe0b058..7a4b90425 100755 --- a/tools/exiftool-tables/dump_tables.pl +++ b/tools/exiftool-tables/dump_tables.pl @@ -250,6 +250,10 @@ sub to_text { my $EXIFTOOL_LIB_ABS = abs_path($EXIFTOOL_LIB) or die "invalid exiftool lib: $EXIFTOOL_LIB\n"; +# Table facts are a property of the selected native tree, never of the +# account running the dump. ExifTool loads $EXIFTOOL_HOME/.ExifTool_config +# during this require unless its configFile global is the empty string. +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } require Image::ExifTool; # Keys that describe the table itself rather than a tag within it. diff --git a/tools/exiftool-tables/oracle.pl b/tools/exiftool-tables/oracle.pl index 718881e55..9cb7ef5fb 100755 --- a/tools/exiftool-tables/oracle.pl +++ b/tools/exiftool-tables/oracle.pl @@ -140,6 +140,7 @@ my $LIB = shift @ARGV or die "usage: $0 \n"; unshift @INC, $LIB; +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } require Image::ExifTool; binmode(STDOUT, ':encoding(UTF-8)'); diff --git a/tools/exiftool-tables/regen.sh b/tools/exiftool-tables/regen.sh index 061206c09..bf2662b82 100755 --- a/tools/exiftool-tables/regen.sh +++ b/tools/exiftool-tables/regen.sh @@ -91,6 +91,12 @@ echo ">> generating inactive serial-directory facts" python3 "$HERE/serial_directory.py" "$JSON" \ --output "$CACHE/serial-$VERSION.json" --rust-output "$SERIAL_OUT" +echo +echo ">> generating source-derived scalar writer helper operands" +python3 "$HERE/scalar_helper_codegen.py" "$JSON" \ + --output "$(artifact_path scalar-helpers)" \ + --report "$(artifact_path scalar-helper-ledger)" + echo echo ">> extracting file-identification tables" "$PERL" "$HERE/dump_filetypes.pl" "$LIB" > "$CACHE/filetypes-$VERSION.json" diff --git a/tools/exiftool-tables/scalar_helper_codegen.py b/tools/exiftool-tables/scalar_helper_codegen.py new file mode 100644 index 000000000..8a950e54d --- /dev/null +++ b/tools/exiftool-tables/scalar_helper_codegen.py @@ -0,0 +1,122 @@ +"""Generate Rust scalar helper operands from captured native executable rules. + +This joins the source compilers to the shared Rust executor. It does not claim +that a tag, its input conversions, encoding or carrier route is admitted. +""" +from dataclasses import asdict +import json +from pathlib import Path +from typing import Any + +from checkexif_recipes import RecipeMalformed, RecipeRefused +from checkvalue_recipes import compile_scalar_check +from writevalue_recipes import compile_scalar_write + +COMPARISONS = {'<': 'Lt', '<=': 'Le', '==': 'Eq', '!=': 'Ne', '>=': 'Ge', '>': 'Gt'} + + +def rust_string(value: str) -> str: + """Escape Rust literals, including control characters JSON spells differently.""" + chunks = ['"'] + for char in value: + if char in ('"', '\\'): + chunks.append('\\' + char) + elif ord(char) < 32 or ord(char) == 127: + chunks.append('\\u{' + format(ord(char), 'x') + '}') + else: + chunks.append(char) + return ''.join(chunks) + '"' + + +def _bound(value: int) -> str: + if type(value) is not int or not -(1 << 63) <= value < (1 << 63): + raise RecipeRefused('source count bound is outside the Rust i64 operand') + return str(value) + + +def _comparison(value: str) -> str: + if value not in COMPARISONS: + raise RecipeRefused('source comparison has no Rust operation') + return 'Comparison::' + COMPARISONS[value] + + +def _byte(value: str) -> str: + # Both closed source parsers currently admit only the native NUL literal. + if value != '\0': + raise RecipeRefused('source scalar terminator/padding is not admitted') + return '0' + + +def _check_fields(recipe) -> dict[str, str]: + return { + 'formats': '[' + ', '.join(map(rust_string, recipe.formats)) + ']', + 'positive_count_operator': _comparison(recipe.positive_count_operator), + 'positive_count_bound': _bound(recipe.positive_count_bound), + 'first_format': rust_string(recipe.first_format), + 'first_limit_operator': _comparison(recipe.first_limit_operator), + 'first_error': rust_string(recipe.first_error), + 'second_limit_operator': _comparison(recipe.second_limit_operator), + 'second_error': rust_string(recipe.second_error), + 'padding_operator': _comparison(recipe.padding_operator), + 'padding_character': _byte(recipe.padding_character), + } + + +def _write_fields(recipe) -> dict[str, str]: + return { + 'formats': '[' + ', '.join(map(rust_string, recipe.formats)) + ']', + 'terminated_format': rust_string(recipe.terminated_format), + 'count_positive_operator': _comparison(recipe.count_positive_operator), + 'count_positive_bound': _bound(recipe.count_positive_bound), + 'diff_negative_operator': _comparison(recipe.diff_negative_operator), + 'terminator': _byte(recipe.terminator), + } + + +def generate(document: dict[str, Any]) -> tuple[str, dict[str, Any]]: + helpers = document.get('native_write_helpers') + if not isinstance(helpers, dict): + raise RecipeMalformed('native_write_helpers is not an object') + chunks = [ + '// @generated by tools/exiftool-tables/scalar_helper_codegen.py.\n' + '// Source-derived helper operands; no public tag or writer admission.\n' + '#![allow(dead_code, unused_imports)]\n' + 'use crate::writers::generated_scalar::{Comparison, ScalarCheckRecipe, ScalarWriteRecipe};\n' + ] + report = {'scope': 'generated Rust scalar helpers; no tag/carrier/encoding admission', + 'helpers': {}} + for key, symbol, typ, compiler, fields in ( + ('check_value', 'CHECK_VALUE', 'ScalarCheckRecipe', compile_scalar_check, _check_fields), + ('write_value', 'WRITE_VALUE', 'ScalarWriteRecipe', compile_scalar_write, _write_fields), + ): + try: + recipe = compiler(helpers.get(key)) + operands = fields(recipe) + except RecipeRefused as exc: + chunks.append(f'pub(crate) const {symbol}: Option<{typ}> = None;\n') + report['helpers'][key] = {'state': 'unsupported', 'reason': str(exc)} + continue + provenance = asdict(recipe.provenance) + report['helpers'][key] = {'state': 'compiled', 'recipe': asdict(recipe)} + chunks.append(f'pub(crate) const {symbol}: Option<{typ}> = Some({typ} {{\n') + chunks.extend(f' {name}: {value},\n' for name, value in operands.items()) + chunks.append('});\n') + chunks.append(f'pub(crate) const {symbol}_PROVENANCE: &str = ' + + rust_string(json.dumps(provenance, sort_keys=True)) + ';\n') + return '\n'.join(chunks), report + + +def main() -> None: + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('tables', type=Path) + parser.add_argument('-o', '--output', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + args = parser.parse_args() + source, report = generate(json.loads(args.tables.read_text())) + args.output.write_text(source) + args.report.write_text(json.dumps(report, sort_keys=True, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/exiftool-tables/scalar_helper_ledger.json b/tools/exiftool-tables/scalar_helper_ledger.json new file mode 100644 index 000000000..c39a7dc04 --- /dev/null +++ b/tools/exiftool-tables/scalar_helper_ledger.json @@ -0,0 +1,133 @@ +{ + "helpers": { + "check_value": { + "recipe": { + "first_error": "String too long", + "first_format": "string", + "first_limit_operator": ">=", + "formats": [ + "string", + "undef" + ], + "padding_character": "\u0000", + "padding_operator": "<", + "positive_count_bound": 0, + "positive_count_operator": ">", + "provenance": { + "actual_name": "Image::ExifTool::CheckValue", + "body_sha256": "30460d5f4bb56fce39ef411a19595eb2aa77ad99282a5bcbdc6b735242c24956", + "dependencies": [ + { + "binding": "Image::ExifTool::IsFloat", + "fact": { + "actual_name": "Image::ExifTool::IsFloat", + "body_sha256": "0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsHex", + "fact": { + "actual_name": "Image::ExifTool::IsHex", + "body_sha256": "79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsInt", + "fact": { + "actual_name": "Image::ExifTool::IsInt", + "body_sha256": "2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + } + ], + "requested_binding": "Image::ExifTool::CheckValue", + "source_file": "Image/ExifTool/Writer.pl", + "source_sha256": "cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44" + }, + "second_error": "Data too long", + "second_limit_operator": ">" + }, + "state": "compiled" + }, + "write_value": { + "recipe": { + "count_positive_bound": 0, + "count_positive_operator": ">", + "diff_negative_operator": "<", + "dispatch_lexical": "%writeValueProc", + "formats": [ + "string", + "undef" + ], + "provenance": { + "actual_name": "Image::ExifTool::WriteValue", + "body_sha256": "4b72f386df682e76ca7dd0262bd89bdfa10f9e964042777e6b5f7009f2de3c0a", + "dependencies": [ + { + "binding": "Image::ExifTool::IsFloat", + "fact": { + "actual_name": "Image::ExifTool::IsFloat", + "body_sha256": "0a55b27b2b165b3bc43fa97f4b4efbb3d3114734c1fd76b1765ddaec96532015", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsHex", + "fact": { + "actual_name": "Image::ExifTool::IsHex", + "body_sha256": "79755a02b4648a6bd56cdbf318b7064b6f6f5f2b5e553648eaf2a9c528cca81e", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsInt", + "fact": { + "actual_name": "Image::ExifTool::IsInt", + "body_sha256": "2fa127fbeffa5d459927879babad932c9040a30fc69775744d3ba570c3f7acec", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + }, + { + "binding": "Image::ExifTool::IsRational", + "fact": { + "actual_name": "Image::ExifTool::IsRational", + "body_sha256": "ecfabc808b12990e7c72603f11c23cab462d4d52b29f88fe9a521d2ef4574356", + "dependencies": [], + "requested_binding": null, + "source_file": "Image/ExifTool.pm", + "source_sha256": "95fa4ec3cc3603866dd6e37bfe52ad019ff50a23bbd5cc87ce40f949cf49a508" + } + } + ], + "requested_binding": "Image::ExifTool::WriteValue", + "source_file": "Image/ExifTool/Writer.pl", + "source_sha256": "cfe916df77f7b37a4fc62e22f0c03de93ca727f0fa22a155ad0eecf95c050d44" + }, + "terminated_format": "string", + "terminator": "\u0000" + }, + "state": "compiled" + } + }, + "scope": "generated Rust scalar helpers; no tag/carrier/encoding admission" +} diff --git a/tools/exiftool-tables/test_artifacts.py b/tools/exiftool-tables/test_artifacts.py index 5585d505f..b4136779f 100644 --- a/tools/exiftool-tables/test_artifacts.py +++ b/tools/exiftool-tables/test_artifacts.py @@ -13,8 +13,8 @@ class ManifestTests(unittest.TestCase): def test_unique_valid_partition_and_selectors(self): artifacts.validate() all_items = artifacts.select() - self.assertEqual(len(all_items), 32) - self.assertEqual(len(artifacts.select(1)), 10) + self.assertEqual(len(all_items), 34) + self.assertEqual(len(artifacts.select(1)), 12) self.assertEqual(len(artifacts.select(2)), 22) self.assertEqual(set(all_items), set(artifacts.select(1) + artifacts.select(2))) self.assertTrue(all(a.path.endswith('.rs') for a in artifacts.select(kind='rust'))) diff --git a/tools/exiftool-tables/test_checkvalue_recipes.py b/tools/exiftool-tables/test_checkvalue_recipes.py index 77dc1b172..74ed6701a 100644 --- a/tools/exiftool-tables/test_checkvalue_recipes.py +++ b/tools/exiftool-tables/test_checkvalue_recipes.py @@ -121,7 +121,7 @@ def test_actual_native_helper_matches_all_scalar_states_and_counts(self): values += [NativeScalar("utf8", value) for value in ("", "é", "Ā", "😀", "a\0b", "a\n")] cases, expected = [], [] for value in values: - for fmt in ("string", "undef"): + for fmt in recipe.formats: for count in (None, 0, -1, 1, 2, 3, 4, 8): cases.append({"kind": value.kind, "value": value.value.hex() if value.kind == "bytes" else value.value, @@ -135,7 +135,7 @@ def test_actual_native_helper_matches_all_scalar_states_and_counts(self): "error": error}) perl = r''' use strict; use warnings; use JSON::PP; use Encode (); use B (); use B::Deparse; -use Digest::SHA qw(sha256_hex); use Image::ExifTool; +use Digest::SHA qw(sha256_hex); BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } use Image::ExifTool; require 'Image/ExifTool/Writer.pl'; local $/; my $rows=JSON::PP->new->utf8->decode(); my @results; my $cv=\&Image::ExifTool::CheckValue; diff --git a/tools/exiftool-tables/test_exiftool_config_isolation.py b/tools/exiftool-tables/test_exiftool_config_isolation.py new file mode 100644 index 000000000..2ef87499a --- /dev/null +++ b/tools/exiftool-tables/test_exiftool_config_isolation.py @@ -0,0 +1,102 @@ +"""Native table instruments must not inherit a user's .ExifTool_config.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import textwrap +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DUMP = REPO_ROOT / "tools/exiftool-tables/dump_tables.pl" +ORACLE = REPO_ROOT / "tools/exiftool-tables/oracle.pl" +READER_CONTRACT = REPO_ROOT / "tools/exiftool-tables/dump_binary_reader_contract.pl" +PERL = os.environ.get("EXIFTOOL_PERL", "/usr/bin/perl") + + +class ExifToolConfigIsolation(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + root = Path(self.tmp.name) + self.home = root / "hostile-home" + self.home.mkdir() + self.lib = root / "lib" + package = self.lib / "Image/ExifTool" + package.mkdir(parents=True) + (self.home / ".ExifTool_config").write_text( + "package Image::ExifTool; warn \"HOSTILE_CONFIG_EXECUTED\\n\"; $VERSION = 'HOSTILE_CONFIG'; 1;\n", + encoding="utf-8", + ) + (self.lib / "Image/ExifTool.pm").write_text(textwrap.dedent("""\ + package Image::ExifTool; + use strict; + our $VERSION = 'clean'; + our $configFile; + our $currentByteOrder = 'II'; + our %unpackStd = ( S => 'v' ); + our %specialTags = map { $_ => 1 } qw(GROUPS FORMAT FIRST_ENTRY); + sub SetByteOrder { $currentByteOrder = shift; $unpackStd{S} = $currentByteOrder eq 'II' ? 'v' : 'n'; return 1; } + sub GetByteOrder { return $currentByteOrder; } + sub Get16u { my ($data, $offset) = @_; return undef if length($$data) < $offset + 2; return unpack($unpackStd{S}, substr($$data, $offset, 2)); } + sub DoUnpackStd { return Get16u(@_); } + if (!defined $configFile) { + my $file = ($ENV{EXIFTOOL_HOME} || $ENV{HOME} || '.') . '/.ExifTool_config'; + require $file if -r $file; + } elsif (length $configFile) { + require $configFile; + } + 1; + """), encoding="utf-8") + (package / "Fixture.pm").write_text(textwrap.dedent("""\ + package Image::ExifTool::Fixture; + our %Main = ( GROUPS => { 0 => 'EXIF' }, 1 => { Name => 'CleanTag' } ); + 1; + """), encoding="utf-8") + + def run_dump(self, script=DUMP): + env = os.environ.copy() + env["EXIFTOOL_HOME"] = str(self.home) + result = subprocess.run([PERL, str(script), str(self.lib), "Fixture"], + check=True, text=True, capture_output=True, env=env) + return json.loads(result.stdout) + + def run_instrument(self, script, *arguments): + env = os.environ.copy() + env["EXIFTOOL_HOME"] = str(self.home) + return subprocess.run([PERL, str(script), str(self.lib), *arguments], + text=True, capture_output=True, env=env) + + def unfenced_control(self, instrument): + directory = Path(tempfile.mkdtemp(prefix="unfenced-" + instrument.stem + "-", dir=self.tmp.name)) + control = directory / instrument.name + fence = "BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; }\n" + text = instrument.read_text(encoding="utf-8") + self.assertIn(fence, text) + control.write_text(text.replace(fence, "", 1), encoding="utf-8") + shutil.copytree(DUMP.parent / "OxiDex", directory / "OxiDex") + return control + + def test_config_control_proves_the_dump_fence(self): + # The copied control has only the canonical early config assignment + # removed. It proves the hostile home config is a real input, rather + # than merely asserting a line of Perl text exists. + control = self.unfenced_control(DUMP) + self.assertEqual(self.run_dump(control)["exiftool_version"], "HOSTILE_CONFIG") + self.assertEqual(self.run_dump()["exiftool_version"], "clean") + + def test_each_capture_instrument_blocks_hostile_config_at_load(self): + for instrument, arguments in ((DUMP, ("Fixture",)), (ORACLE, ()), (READER_CONTRACT, ())): + with self.subTest(instrument=instrument.name): + fenced = self.run_instrument(instrument, *arguments) + self.assertEqual(fenced.returncode, 0, fenced.stderr) + self.assertNotIn("HOSTILE_CONFIG_EXECUTED", fenced.stdout + fenced.stderr) + control = self.run_instrument(self.unfenced_control(instrument), *arguments) + self.assertEqual(control.returncode, 0, control.stderr) + self.assertIn("HOSTILE_CONFIG_EXECUTED", control.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/test_scalar_helper_codegen.py b/tools/exiftool-tables/test_scalar_helper_codegen.py new file mode 100644 index 000000000..29bec300d --- /dev/null +++ b/tools/exiftool-tables/test_scalar_helper_codegen.py @@ -0,0 +1,92 @@ +"""Generated Rust operands must follow source semantics, not just fingerprints.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +from checkexif_recipes import RecipeMalformed +import scalar_helper_codegen as generator +from test_checkvalue_recipes import fact as check_fact +from test_writevalue_recipes import fact as write_fact + + +def document(): + return {'native_write_helpers': {'check_value': check_fact(), 'write_value': write_fact()}} + + +class ScalarHelperCodegenTests(unittest.TestCase): + def test_source_operands_change_in_generated_rust(self): + doc = document() + check = doc['native_write_helpers']['check_value'] + before, after = check['__deparse'].rsplit("'string'", 1) + check['__deparse'] = before + "'undef'" + after + write = doc['native_write_helpers']['write_value'] + write['__deparse'] = write['__deparse'].replace('$count > 0', '$count > 2') + output, report = generator.generate(doc) + self.assertIn('first_format: "undef"', output) + self.assertIn('count_positive_bound: 2', output) + self.assertEqual(report['helpers']['check_value']['state'], 'compiled') + self.assertEqual(report['helpers']['write_value']['state'], 'compiled') + + def test_unknown_source_rule_emits_none_and_named_gap(self): + doc = document() + doc['native_write_helpers']['check_value']['__deparse'] = 'unknown upstream check' + output, report = generator.generate(doc) + self.assertIn('CHECK_VALUE: Option = None', output) + self.assertNotIn('CHECK_VALUE: Option = Some', output) + self.assertEqual(report['helpers']['check_value']['state'], 'unsupported') + self.assertTrue(report['helpers']['check_value']['reason']) + self.assertEqual(report['helpers']['write_value']['state'], 'compiled') + + def test_live_dispatch_override_cannot_emit_old_scalar_rule(self): + doc = document() + helper = doc['native_write_helpers']['write_value'] + helper['lexical_hashes']['bindings']['%writeValueProc']['entries']['string'] = {'__perl': 'CODE'} + output, report = generator.generate(doc) + self.assertIn('WRITE_VALUE: Option = None', output) + self.assertIn('intercepted', report['helpers']['write_value']['reason']) + + def test_unrepresentable_source_count_bound_is_an_explicit_gap(self): + doc = document() + helper = doc['native_write_helpers']['write_value'] + helper['__deparse'] = helper['__deparse'].replace('$count > 0', '$count > ' + str(1 << 63)) + output, report = generator.generate(doc) + self.assertIn('WRITE_VALUE: Option = None', output) + self.assertIn('i64', report['helpers']['write_value']['reason']) + + def test_missing_capture_is_malformed_not_ordinary_unsupported_source(self): + with self.assertRaises(RecipeMalformed): + generator.generate({}) + doc = document() + del doc['native_write_helpers']['check_value'] + with self.assertRaises(RecipeMalformed): + generator.generate(doc) + + def test_rust_literals_preserve_unicode_controls_quotes_and_backslashes(self): + self.assertEqual(generator.rust_string('é\0\n"\\'), '"é\\u{0}\\u{a}\\"\\\\"') + + +@unittest.skipUnless(os.environ.get('OXIDEX_TABLES_JSON'), + 'requires fresh captured helper facts from the pinned release') +class NativeArtifactFreshnessTests(unittest.TestCase): + def test_committed_rules_and_ledger_equal_fresh_native_generation(self): + # CI supplies a fresh dump from .exiftool-version, not an artifact's + # own version stamp. Check the files we ship as well as temporary Rust. + root = Path(__file__).resolve().parents[2] + source, report = generator.generate(json.loads(Path(os.environ['OXIDEX_TABLES_JSON']).read_text())) + with tempfile.TemporaryDirectory(prefix='oxidex-scalar-freshness-') as directory: + output = Path(directory) / 'rules.rs' + output.write_text(source) + subprocess.run(['rustfmt', '--edition', '2024', '--config-path', str(root / 'rustfmt.toml'), + str(output)], check=True, capture_output=True, timeout=30) + self.assertEqual(output.read_text(), (root / 'src/writers/generated_scalar_rules.rs').read_text(), + 'committed scalar rules are stale; run official regeneration') + self.assertEqual(json.dumps(report, sort_keys=True, indent=2) + '\n', + (Path(__file__).parent / 'scalar_helper_ledger.json').read_text(), + 'committed scalar helper ledger is stale; run official regeneration') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/exiftool-tables/test_scalar_helper_rust.py b/tools/exiftool-tables/test_scalar_helper_rust.py new file mode 100644 index 000000000..eec2971b9 --- /dev/null +++ b/tools/exiftool-tables/test_scalar_helper_rust.py @@ -0,0 +1,102 @@ +"""Execute freshly generated helper operands in the real Rust scalar executor. + +The native return records, not the Python reference evaluator, supply the Rust +expectations. The native fixture runners additionally authenticate loaded CV +source/body hashes and check their own Python references before returning data. +The helper-local write count is checked separately against the source reference; +native return records cannot observe that local variable. +""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import scalar_helper_codegen as generator +import test_checkvalue_recipes as native_check +import test_writevalue_recipes as native_write +from checkvalue_recipes import NativeScalar +from writevalue_recipes import compile_scalar_write, evaluate_scalar_write + +ROOT = Path(__file__).resolve().parents[2] + + +def _scalar(value): + if value.get('kind') == 'undefined' or value.get('defined') is False: + return 'Scalar::Undefined' + if value.get('kind') == 'bytes' or value.get('utf8') is False: + raw = bytes.fromhex(value.get('value', value.get('hex'))) + return 'Scalar::Bytes(vec![' + ','.join(str(byte) for byte in raw) + '])' + text = value['value'] if 'kind' in value else bytes.fromhex(value['hex']).decode('utf8') + return 'Scalar::Utf8(' + generator.rust_string(text) + '.to_owned())' + + +@unittest.skipUnless(os.environ.get('OXIDEX_PINNED_EXIFTOOL') and os.environ.get('OXIDEX_TABLES_JSON'), + 'requires explicit native source and matching captured helper facts') +class GeneratedScalarRustTests(unittest.TestCase): + def test_generated_rust_matches_authenticated_native_helper_returns(self): + document = json.loads(Path(os.environ['OXIDEX_TABLES_JSON']).read_text()) + source, report = generator.generate(document) + self.assertTrue(all(row['state'] == 'compiled' for row in report['helpers'].values()), report) + write_recipe = compile_scalar_write(document['native_write_helpers']['write_value']) + real_run = subprocess.run + observations = {} + for kind, test in ( + ('check', native_check.NativeScalarDifferential().test_actual_native_helper_matches_all_scalar_states_and_counts), + ('write', native_write.NativeScalarDifferential().test_actual_native_helper_matches_scalar_states_and_counts), + ): + def capture(*args, **kwargs): + result = real_run(*args, **kwargs) + observations[kind] = (json.loads(kwargs['input']), json.loads(result.stdout)['results']) + return result + with patch.object(subprocess, 'run', capture): + test() + with tempfile.TemporaryDirectory(prefix='oxidex-generated-scalar-') as directory: + root = Path(directory) + rules = root / 'rules.rs' + rules.write_text(source) + harness = [ + '#![allow(dead_code)]', + '#[path=' + generator.rust_string(str(ROOT / 'src/error/mod.rs')) + '] mod error;', + 'mod writers { #[path=' + generator.rust_string(str(ROOT / 'src/writers/generated_scalar.rs')) + + '] pub(crate) mod generated_scalar; }', + '#[path=' + generator.rust_string(str(rules)) + '] mod rules;', + 'use writers::generated_scalar::*;', + ] + for kind, (cases, results) in observations.items(): + self.assertGreaterEqual(len(cases), 240) + self.assertEqual(len(cases), len(results)) + harness.append('#[test] fn native_' + kind + '() {') + for index, (case, result) in enumerate(zip(cases, results, strict=True)): + count = 'None' if case['count'] is None else 'Some(' + str(case['count']) + ')' + function = 'validate_scalar' if kind == 'check' else 'serialize_scalar' + rule = 'CHECK_VALUE' if kind == 'check' else 'WRITE_VALUE' + harness.append(f'let value = {function}(&rules::{rule}.unwrap(), {_scalar(case)}, ' + f'{generator.rust_string(case["format"])}, {count}).unwrap();') + harness.append(f'assert_eq!(value.value, {_scalar(result)}, "{kind} case {index}");') + if kind == 'check': + error = 'None' if result['error'] is None else 'Some(' + generator.rust_string(result['error']) + ')' + harness.append(f'assert_eq!(value.error, {error}, "check error {index}");') + else: + # Native JSON's count is the unchanged caller argument. + # This distinct assertion checks only reference agreement. + raw = bytes.fromhex(case['value']) if case['kind'] == 'bytes' else case['value'] + reference = evaluate_scalar_write( + write_recipe, NativeScalar(case['kind'], raw), case['format'], case['count']) + harness.append(f'assert_eq!(value.count, {reference.count}, ' + f'"write source-reference local count {index}");') + harness.append('}') + program = root / 'proof.rs' + program.write_text('\n'.join(harness)) + binary = root / 'proof' + compile_result = real_run(['rustc', '--edition=2024', '--test', str(program), '-o', str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(compile_result.returncode, 0, compile_result.stdout + compile_result.stderr) + execution = real_run([str(binary), '--nocapture'], capture_output=True, text=True, timeout=60) + self.assertEqual(execution.returncode, 0, execution.stdout + execution.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/exiftool-tables/test_writevalue_recipes.py b/tools/exiftool-tables/test_writevalue_recipes.py index 760e56e13..5407841db 100644 --- a/tools/exiftool-tables/test_writevalue_recipes.py +++ b/tools/exiftool-tables/test_writevalue_recipes.py @@ -138,7 +138,9 @@ def test_actual_native_helper_matches_scalar_states_and_counts(self): "count": count}) perl = r''' use strict; use warnings; use JSON::PP; use Encode (); use B (); use B::Deparse; -use Digest::SHA qw(sha256_hex); use Image::ExifTool; +use Digest::SHA qw(sha256_hex); +BEGIN { no warnings 'once'; $Image::ExifTool::configFile = ''; } +use Image::ExifTool; require 'Image/ExifTool/Writer.pl'; local $/; my $rows=JSON::PP->new->utf8->decode(); my @results; my $cv=\&Image::ExifTool::WriteValue; my $body=B::Deparse->new('-p','-sC')->coderef2text($cv); From b7b5faeb125e1cb226c1c24a7bc9cf1d241904dd Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 21:29:41 -0500 Subject: [PATCH 3/3] Cover scalar helper generation in regeneration shell controls Teach the synthetic leaf harness the two declared helper outputs and check invocation order and failure propagation. Five shell-control tests and required Clippy pass. Preserve the initial 900-test full-suite failure and queue its retry; no production code or generated operands changed. --- docs/AUTOGENERATION-PROGRESS.md | 11 +++++++++-- tools/exiftool-tables/test_regen_shell.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index b94dad438..3a9ddde7b 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -129,8 +129,15 @@ return checks. Native capture and oracle startup now explicitly disable ambient ExifTool configuration. Clean and hostile-home native captures are byte-identical in the focused probe. This prevents personal configuration from being mistaken -for the selected release's rules. Full 34-artifact regeneration and the full -Python suite for this combined checkpoint are pending. No public writer route +for the selected release's rules. Full 34-artifact regeneration passed in +339.975 seconds. The first full Python run reached all 900 tests and failed two +regeneration-shell controls because their simulated leaf executable did not +recognize the new scalar helper producer. The repaired controls now pass all +five tests, including selected-source/output routing and failure propagation +for the new producer. Preserve the failed full run; its retry and full Rust +validation are pending. The only regeneration difference was the recorded +library path; regenerating the expression ledger through the canonical relative +source path restores identical committed artifacts. No public writer route is enabled, no manual tag rule has been retired in this checkpoint, and no project-wide generation percentage has been remeasured. diff --git a/tools/exiftool-tables/test_regen_shell.py b/tools/exiftool-tables/test_regen_shell.py index 594b6b9d9..6885f93ee 100644 --- a/tools/exiftool-tables/test_regen_shell.py +++ b/tools/exiftool-tables/test_regen_shell.py @@ -74,6 +74,11 @@ def artifact(producer): dump(args[0]);output(flag('--rust-output'),'serial') assert flag('--rust-output')==artifact('serial_directory') output(flag('--output'),'serial-report') + elif name=='scalar_helper_codegen.py': + dump(args[0]) + selected={root/item.path for item in artifacts.select(producer='scalar_helper_codegen')} + assert {flag('--output'),flag('--report')}==selected + output(flag('--output'),'scalar-helpers');output(flag('--report'),'scalar-helper-ledger') elif name=='verify_serial_directory.py': assert pathlib.Path(args[0]).resolve()==artifact('serial_directory') assert pathlib.Path(args[0]).read_text()=='generated explicit-A serial\n' @@ -213,9 +218,11 @@ def test_both_tiers_and_tier2_use_selected_source_and_complete_checks(self): self.assertEqual(names.count(name), 1, names) self.assertEqual(names.count('verify_exprs.py'), int(full)) self.assertEqual(names.count('serial_directory.py'), int(full)) + self.assertEqual(names.count('scalar_helper_codegen.py'), int(full)) self.assertEqual(names.count('verify_serial_directory.py'), int(full)) if full: self.assertLess(names.index('serial_directory.py'), names.index('rustfmt')) + self.assertLess(names.index('scalar_helper_codegen.py'), names.index('rustfmt')) self.assertGreater(names.index('verify_serial_directory.py'), names.index('rustfmt')) self.assertEqual(names.count('rustfmt'), 2 if full else 1) format_calls = [c for c in calls if c['tool'] == 'rustfmt'] @@ -241,8 +248,8 @@ def test_each_new_producer_or_verifier_failure_survives_exit_guard(self): self.assertIn('regeneration write-set PASS', result.stdout) self.assertNotIn('>> done:', result.stdout) - def test_serial_producer_and_verifier_failures_survive_tier_one_guard(self): - for leaf in ('serial_directory.py', 'verify_serial_directory.py'): + def test_tier_one_producer_and_verifier_failures_survive_exit_guard(self): + for leaf in ('serial_directory.py', 'scalar_helper_codegen.py', 'verify_serial_directory.py'): with self.subTest(leaf=leaf): result, calls = self.run_regeneration(full=True, env={'CONTROL_FAIL': leaf}) self.assertEqual(result.returncode, 47, result.stdout + result.stderr)