From f1cd97d8c6b2688f8c0ec477de7fb0e30a81e093 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 10:54:52 -0500 Subject: [PATCH 01/22] feat(tables): capture native write facts --- tools/exiftool-tables/dump_tables.pl | 268 +++++++++++++++++- .../exiftool-tables/test_dump_write_facts.py | 179 ++++++++++++ 2 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 tools/exiftool-tables/test_dump_write_facts.py diff --git a/tools/exiftool-tables/dump_tables.pl b/tools/exiftool-tables/dump_tables.pl index 0e61479b5..af4c4edb3 100755 --- a/tools/exiftool-tables/dump_tables.pl +++ b/tools/exiftool-tables/dump_tables.pl @@ -295,6 +295,18 @@ sub to_text { FixFormat SubIFD ); +# Write routing is deliberately captured beside, rather than inside, the read +# projection above. Read codegen has a closed grammar and must not become more +# permissive merely because a source row names a write-only control. The +# sidecar below retains the raw source values for a later write compiler to +# authenticate and model independently. +my @WRITE_CONTROL_KEYS = qw( + Writable WriteGroup RawConvInv ValueConvInv PrintConvInv + Validate Mandatory DelValue Deletable WriteAlso WriteCheck WriteCondition + WriteHook WriteLast WritePseudo CanCreate +); +my %WRITE_KNOWN_TAG_KEY = map { $_ => 1 } (@TAG_KEYS, @WRITE_CONTROL_KEYS); + sub scrub { my ($v, $depth) = @_; $depth //= 0; @@ -362,6 +374,227 @@ sub classify_conv { return { kind => 'other', dump => scrub($v) }; } +# The live table graph contains runtime references such as Composite tags' +# `Table` back-pointer. They are not serializable source operands and may be +# cyclic. Preserve their reference kind explicitly while keeping ordinary +# source hashes, arrays, scalar refs, CODE, undef, zero, and empty strings +# distinct. This is intentionally separate from read-side `scrub`, whose +# historical output must remain byte-for-byte stable. +sub write_scrub { + my ($v, $depth, $seen) = @_; + $depth //= 0; + $seen //= {}; + return undef unless defined $v; + return { __deep => 1 } if $depth > 12; + my $kind = ref($v); + return to_text($v) unless $kind; + if ($kind eq 'CODE') { + my $name = code_name($v); + my $source = deparse($v); + return { + __perl => 'CODE', __opaque => JSON::PP::true, + (defined $name ? (__name => $name) : ()), + (defined $source ? (__deparse => $source) : ()), + }; + } + return write_scrub($$v, $depth + 1, $seen) if $kind eq 'SCALAR'; + my $id = refaddr($v); + return { __ref => $kind, __cycle => JSON::PP::true } + if defined $id && $seen->{$id}; + $seen->{$id} = 1 if defined $id; + my $out; + if ($kind eq 'ARRAY') { + $out = [ map { write_scrub($_, $depth + 1, $seen) } @$v ]; + } elsif ($kind eq 'HASH') { + my %hash; + for my $key (sort keys %$v) { + $hash{to_text($key)} = write_scrub($v->{$key}, $depth + 1, $seen); + } + $out = \%hash; + } else { + $out = { __ref => $kind }; + } + delete $seen->{$id} if defined $id; + return $out; +} + +# Write facts preserve the original value even for a conversion. The +# classification is a convenience for a future compiler, never a replacement +# for the raw source operand. In particular, a present-but-undef value is +# distinguishable from an absent key through the surrounding `present` flag. +sub write_source_property { + my ($hash, $key) = @_; + return { present => JSON::PP::false } unless exists $hash->{$key}; + my %property = ( + present => JSON::PP::true, + value => write_scrub($hash->{$key}), + ); + if ($key =~ /^(?:RawConvInv|ValueConvInv|PrintConvInv)$/) { + $property{classification} = classify_conv($hash->{$key}); + } + return \%property; +} + +sub is_table_property { + my ($key) = @_; + no warnings 'once'; + return $TABLE_META{$key} || $Image::ExifTool::specialTags{$key}; +} + +sub dump_write_entry { + my ($entry) = @_; + my $kind = ref($entry) || 'SCALAR'; + if (!$entry || !ref($entry)) { + return { + entry_kind => $kind, + value => scrub($entry), + }; + } + if ($kind eq 'ARRAY') { + return { + entry_kind => 'ARRAY', + # Array order is native variant-selection order. Do not sort it. + alternatives => [ map { dump_write_entry($_) } @$entry ], + }; + } + return { entry_kind => $kind, value => scrub($entry) } unless $kind eq 'HASH'; + + my %properties; + my %unknown; + for my $key (sort keys %$entry) { + my $property = write_source_property($entry, $key); + $properties{to_text($key)} = $property; + $unknown{to_text($key)} = $property unless $WRITE_KNOWN_TAG_KEY{$key}; + } + my %controls = map { $_ => write_source_property($entry, $_) } @WRITE_CONTROL_KEYS; + return { + entry_kind => 'HASH', + properties => \%properties, + write_controls => \%controls, + unknown_properties => \%unknown, + }; +} + +sub dump_write_table { + my ($module, $table, $full_name, $hash) = @_; + my (%properties, %unknown); + for my $key (sort keys %$hash) { + next unless is_table_property($key); + my $property = write_source_property($hash, $key); + $properties{to_text($key)} = $property; + $unknown{to_text($key)} = $property unless $TABLE_META{$key}; + } + my %controls = map { $_ => write_source_property($hash, $_) } + qw(WRITABLE WRITE_GROUP WRITE_PROC CHECK_PROC); + my %rows; + for my $key (sort keys %$hash) { + next if is_table_property($key) || $key =~ /^_/; + $rows{to_text($key)} = dump_write_entry($hash->{$key}); + } + return { + module => $module, + table => $table, + full_name => $full_name, + table_properties => \%properties, + write_controls => \%controls, + unknown_table_properties => \%unknown, + rows => \%rows, + row_count => scalar(keys %rows), + }; +} + +# ExifTool's write functions are often only prototype declarations while an +# input table is being loaded. Its AUTOLOAD convention derives the file from +# the fully-qualified function name. Requiring that file is a load operation, +# not a call: it cannot run an arbitrary writer against dummy metadata. We do +# this only for a prototype-only CV; normal loaded procedures keep their actual +# table-owned binding unchanged. +sub prototype_only_code { + my ($cv) = @_; + my $body = deparse($cv); + return 0 unless defined $body; + return $body =~ /^\s*\([^{};]*\)\s*;\s*$/s ? 1 : 0; +} + +sub write_autoload_file { + my ($name) = @_; + return undef unless defined $name && $name =~ /^(?:[A-Za-z_]\w*::)+[A-Za-z_]\w*$/; + my @part = split /::/, $name; + return undef unless @part >= 3 && $part[0] eq 'Image' && $part[1] eq 'ExifTool'; + return "Image/ExifTool/Write$part[2].pl" if @part == 4; + return 'Image/ExifTool/Shift.pl' if $part[-1] eq 'ShiftTime'; + return 'Image/ExifTool/Writer.pl'; +} + +sub hydrate_write_procedures { + my ($tables) = @_; + my %status; + for my $full_name (sort keys %$tables) { + my $hash = $tables->{$full_name}{hash}; + for my $key (qw(WRITE_PROC CHECK_PROC)) { + next unless ref($hash->{$key}) eq 'CODE'; + my $cv = $hash->{$key}; + next unless prototype_only_code($cv); + my $name = code_name($cv); + my $file = write_autoload_file($name); + my $id = refaddr($cv); + if (!defined $file) { + $status{$id} = { reason => 'autoload_target_unavailable' }; + next; + } + my $loaded = eval { require $file; 1 }; + if (!$loaded) { + $status{$id} = { reason => 'autoload_load_failed', autoload_file => $file }; + next; + } + # A successful require which leaves the stored CV as a prototype + # is still not an implementation fact. + if (prototype_only_code($cv)) { + $status{$id} = { reason => 'autoload_did_not_define', autoload_file => $file }; + } else { + $status{$id} = { autoload_file => $file }; + } + } + } + return \%status; +} + +sub effective_write_code_fact { + my ($hash, $key, $fallback_name, $lib_abs, $status) = @_; + return { present => JSON::PP::false } unless exists $hash->{$key}; + my $value = $hash->{$key}; + my $declared = write_source_property($hash, $key); + if (ref($value) ne 'CODE') { + return { + present => JSON::PP::true, + declared => $declared, + effective => unresolved_code_fact($fallback_name, 'write_proc_not_code'), + }; + } + my $load = $status->{refaddr($value)}; + if ($load && $load->{reason}) { + my $fact = unresolved_code_fact(code_name($value) // $fallback_name, $load->{reason}); + $fact->{autoload_file} = $load->{autoload_file} if exists $load->{autoload_file}; + return { present => JSON::PP::true, declared => $declared, effective => $fact }; + } + if (prototype_only_code($value)) { + return { + present => JSON::PP::true, + declared => $declared, + effective => unresolved_code_fact(code_name($value) // $fallback_name, + 'prototype_only_write_proc'), + }; + } + # Capture only the bounded package-local reader dependency, as for + # PROCESS_PROC. Writer admission still needs a distinct body recognizer. + return { + present => JSON::PP::true, + declared => $declared, + effective => code_ref_fact($value, $fallback_name, $lib_abs, undef, undef, 0), + ($load && exists $load->{autoload_file} ? (autoload_file => $load->{autoload_file}) : ()), + }; +} + sub dump_tag_entry { my ($entry) = @_; my $r = ref $entry; @@ -402,7 +635,7 @@ sub dump_tag_entry { } sub dump_module { - my ($module, $validate_function_names, $processor_tables) = @_; + my ($module, $validate_function_names, $processor_tables, $write_tables) = @_; my $pkg = "Image::ExifTool::$module"; eval "require $pkg; 1" or do { return { module => $module, error => "$@" }; @@ -448,6 +681,13 @@ sub dump_module { cv => $hash->{PROCESS_PROC}, module => $module, table => $sym, }; } + # The write sidecar owns a raw reference to every live tag table, not + # only tables which currently look writable. A later compiler must be + # able to tell absent controls from an unsupported or newly introduced + # one, and zero-row tables are source facts too. + $write_tables->{"${pkg}::${sym}"} = { + hash => $hash, module => $module, table => $sym, + }; $tables{$sym} = { full_name => "${pkg}::${sym}", @@ -517,9 +757,11 @@ sub dump_module { my %subdirectory_validate_function_names; my %subdirectory_validate_functions; my %processor_tables; +my %write_tables; +my %native_write_tables; my ($ok, $failed) = (0, 0); for my $m (@modules) { - my $r = dump_module($m, \%subdirectory_validate_function_names, \%processor_tables); + my $r = dump_module($m, \%subdirectory_validate_function_names, \%processor_tables, \%write_tables); if ($r->{error}) { $failed++; warn "SKIP $m: $r->{error}"; @@ -548,6 +790,24 @@ sub dump_module { $out{$entry->{module}}{tables}{$entry->{table}}{meta}{PROCESS_PROC} = $fact; } +# Resolve prototype-only writer declarations after every selected module is +# loaded. This is intentionally after the read dump was built: writer module +# loading must not alter the existing read projection. The sidecar gets the +# table's stored CV, its final implementation source fact, and raw source +# controls separately. +my $write_autoload_status = hydrate_write_procedures(\%write_tables); +for my $full_name (sort keys %write_tables) { + my $entry = $write_tables{$full_name}; + my $fact = dump_write_table($entry->{module}, $entry->{table}, $full_name, $entry->{hash}); + $fact->{effective_write_proc} = effective_write_code_fact( + $entry->{hash}, 'WRITE_PROC', "${full_name}::WRITE_PROC", + $EXIFTOOL_LIB_ABS, $write_autoload_status); + $fact->{effective_check_proc} = effective_write_code_fact( + $entry->{hash}, 'CHECK_PROC', "${full_name}::CHECK_PROC", + $EXIFTOOL_LIB_ABS, $write_autoload_status); + $native_write_tables{$entry->{module}}{$entry->{table}} = $fact; +} + # The child captures byte-order state without changing this table-walking # process. These facts prove that the final loaded CODE refs are the same ones # the child observed; a later module override makes the contract unresolved. @@ -567,6 +827,10 @@ sub dump_module { modules_ok => $ok, modules_failed => $failed, modules => \%out, + # Facts only: no reader or writer consumes this sidecar yet. Keeping it + # separate prevents write-only properties from becoming accidental read + # admission or changing the existing module/table/tag projection. + native_write_tables => \%native_write_tables, subdirectory_validate_functions => \%subdirectory_validate_functions, native_reader_contracts => { unsigned16 => $unsigned_reader_contract }, }); diff --git a/tools/exiftool-tables/test_dump_write_facts.py b/tools/exiftool-tables/test_dump_write_facts.py new file mode 100644 index 000000000..fff638e74 --- /dev/null +++ b/tools/exiftool-tables/test_dump_write_facts.py @@ -0,0 +1,179 @@ +"""Native write-fact sidecar coverage. + +The sidecar is source evidence only. These tests prove it preserves the +write-relevant native values and effective procedure provenance without +changing the existing read-table projection or admitting any writer. +""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DUMP = REPO_ROOT / "tools/exiftool-tables/dump_tables.pl" +PERL = os.environ.get("EXIFTOOL_PERL", "/usr/bin/perl") + + +class NativeWriteFacts(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.lib = Path(self.tmp.name) / "lib" + package = self.lib / "Image/ExifTool" + package.mkdir(parents=True) + (self.lib / "Image/ExifTool.pm").write_text( + "package Image::ExifTool; our $VERSION = 'fixture'; " + "our %specialTags = map { $_ => 1 } " + "qw(GROUPS WRITE_PROC CHECK_PROC WRITABLE WRITE_GROUP FORMAT FIRST_ENTRY); 1;\n", + encoding="utf-8", + ) + self.exif = package / "Exif.pm" + self.writer = package / "WriteExif.pl" + self.later = package / "Later.pm" + self.write_exif("IFD0") + self.write_writer("return 1;") + self.write_later("return 'late';") + + def write_exif(self, host_group: str): + self.exif.write_text(textwrap.dedent(f"""\ + package Image::ExifTool::Exif; + sub WriteExif($$$); + sub CheckExif($$$); + sub DeferredWrite($$$); + our %Main = ( + GROUPS => {{ 0 => 'EXIF', 1 => 'IFD0', 2 => 'Image' }}, + WRITE_PROC => \\&WriteExif, + CHECK_PROC => \\&CheckExif, + WRITE_GROUP => 'ExifIFD', + WRITABLE => 0, + 0x13c => {{ + Name => 'HostComputer', Writable => 'string', + WriteGroup => '{host_group}', RawConvInv => '$val', + ValueConvInv => 0, PrintConvInv => '', Mandatory => 0, + DelValue => '', Validate => undef, + FutureWriterSwitch => {{ zero => 0, empty => '', undef => undef }}, + }}, + 0x140 => [ + {{ Name => 'First', WriteLast => 0, WriteGroup => 'IFD0' }}, + {{ Name => 'Second', WriteLast => 1, WriteGroup => undef }}, + ], + ); + our %Deferred = ( + WRITE_PROC => \\&DeferredWrite, + 1 => {{ Name => 'DeferredTag', Writable => 1 }}, + ); + 1; + """), encoding="utf-8") + + def write_writer(self, body: str): + self.writer.write_text(textwrap.dedent(f"""\ + package Image::ExifTool::Exif; + sub WriteExif($$$) {{ {body} }} + sub CheckExif($$$) {{ return undef; }} + 1; + """), encoding="utf-8") + + def write_later(self, body: str): + # This module is loaded after Exif. It fills the prototype stored by + # `%Deferred`, so the final source binding must describe this file. + self.later.write_text(textwrap.dedent(f"""\ + package Image::ExifTool::Exif; + sub DeferredWrite($$$) {{ {body} }} + 1; + """), encoding="utf-8") + + def dump(self): + result = subprocess.run( + [PERL, str(DUMP), str(self.lib), "Exif", "Later"], + check=True, text=True, capture_output=True, + ) + return json.loads(result.stdout) + + @staticmethod + def sidecar(doc, table="Main"): + return doc["native_write_tables"]["Exif"][table] + + def test_preserves_complete_controls_variants_and_unknown_values(self): + doc = self.dump() + table = self.sidecar(doc) + host = table["rows"]["316"] + controls = host["write_controls"] + self.assertEqual(controls["Writable"], {"present": True, "value": "string"}) + self.assertEqual(controls["WriteGroup"], {"present": True, "value": "IFD0"}) + # Perl table scalars have no numeric/string type bit. The source + # spelling survives as the distinct non-empty scalar "0", which is + # what matters against absence, undef, and the empty scalar below. + self.assertEqual(controls["Mandatory"], {"present": True, "value": "0"}) + self.assertEqual(controls["DelValue"], {"present": True, "value": ""}) + self.assertEqual(controls["Validate"], {"present": True, "value": None}) + self.assertFalse(controls["WriteAlso"]["present"]) + self.assertEqual(host["properties"]["RawConvInv"]["classification"], + {"kind": "expr", "expr": "$val"}) + self.assertEqual(host["unknown_properties"]["FutureWriterSwitch"]["value"], + {"empty": "", "undef": None, "zero": "0"}) + + variants = table["rows"]["320"]["alternatives"] + self.assertEqual([item["properties"]["Name"]["value"] for item in variants], + ["First", "Second"]) + self.assertEqual(variants[0]["write_controls"]["WriteLast"]["value"], "0") + self.assertEqual(variants[1]["write_controls"]["WriteGroup"], + {"present": True, "value": None}) + # Table defaults and row overrides remain separate source facts. + self.assertEqual(table["write_controls"]["WRITE_GROUP"], + {"present": True, "value": "ExifIFD"}) + + def test_forward_declarations_capture_loaded_writer_bodies(self): + doc = self.dump() + table = self.sidecar(doc) + for key in ("effective_write_proc", "effective_check_proc"): + effective = table[key]["effective"] + self.assertTrue(effective["resolved"]) + self.assertEqual(effective["source_file"], "Image/ExifTool/WriteExif.pl") + self.assertEqual(effective["source_sha256"], + hashlib.sha256(self.writer.read_bytes()).hexdigest()) + self.assertNotEqual(effective["__deparse"].strip(), "($$$) ;") + deferred = self.sidecar(doc, "Deferred")["effective_write_proc"]["effective"] + self.assertTrue(deferred["resolved"]) + self.assertEqual(deferred["source_file"], "Image/ExifTool/Later.pm") + self.assertEqual(deferred["source_sha256"], + hashlib.sha256(self.later.read_bytes()).hexdigest()) + + def test_write_only_source_mutation_changes_sidecar_not_read_projection(self): + before = self.dump() + self.write_exif("ExifIFD") + after = self.dump() + self.assertEqual(before["modules"], after["modules"]) + first = self.sidecar(before)["rows"]["316"] + second = self.sidecar(after)["rows"]["316"] + self.assertEqual(first["write_controls"]["WriteGroup"]["value"], "IFD0") + self.assertEqual(second["write_controls"]["WriteGroup"]["value"], "ExifIFD") + + def test_writer_body_mutation_changes_authenticated_fact(self): + before = self.dump() + self.write_writer("return 2;") + after = self.dump() + first = self.sidecar(before)["effective_write_proc"]["effective"] + second = self.sidecar(after)["effective_write_proc"]["effective"] + self.assertTrue(first["resolved"]) + self.assertTrue(second["resolved"]) + self.assertNotEqual(first["__deparse"], second["__deparse"]) + self.assertNotEqual(first["source_sha256"], second["source_sha256"]) + self.assertEqual(before["modules"], after["modules"]) + + def test_missing_autoload_implementation_is_explicitly_unresolved(self): + self.writer.unlink() + table = self.sidecar(self.dump()) + effective = table["effective_write_proc"]["effective"] + self.assertFalse(effective["resolved"]) + self.assertEqual(effective["reason"], "autoload_load_failed") + self.assertEqual(effective["autoload_file"], "Image/ExifTool/WriteExif.pl") + + +if __name__ == "__main__": + unittest.main() From ec6214c15d950fe096eef835451c183b88cceac5 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:04:38 -0500 Subject: [PATCH 02/22] fix(tables): preserve write source indirection --- tools/exiftool-tables/dump_tables.pl | 67 ++++++++++++++++++- .../exiftool-tables/test_dump_write_facts.py | 24 ++++++- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/tools/exiftool-tables/dump_tables.pl b/tools/exiftool-tables/dump_tables.pl index af4c4edb3..4b39c1ac7 100755 --- a/tools/exiftool-tables/dump_tables.pl +++ b/tools/exiftool-tables/dump_tables.pl @@ -397,8 +397,15 @@ sub write_scrub { (defined $source ? (__deparse => $source) : ()), }; } - return write_scrub($$v, $depth + 1, $seen) if $kind eq 'SCALAR'; my $id = refaddr($v); + if ($kind eq 'SCALAR') { + return { __ref => 'SCALAR', __cycle => JSON::PP::true } + if defined $id && $seen->{$id}; + $seen->{$id} = 1 if defined $id; + my $out = { __ref => 'SCALAR', value => write_scrub($$v, $depth + 1, $seen) }; + delete $seen->{$id} if defined $id; + return $out; + } return { __ref => $kind, __cycle => JSON::PP::true } if defined $id && $seen->{$id}; $seen->{$id} = 1 if defined $id; @@ -526,8 +533,52 @@ sub write_autoload_file { return 'Image/ExifTool/Writer.pl'; } +# ExifTool routes prototype writer declarations through DoAutoLoad. The +# dumper may load an implementation only if it sees the closed, generic routing +# body below in the *actual loaded* core CV. This protects the sidecar from +# claiming an implementation after an upstream routing change; it does not +# treat the routing fact as writer admission. +sub write_autoload_router_fact { + my ($lib_abs) = @_; + no strict 'refs'; + my $cv = *{'Image::ExifTool::DoAutoLoad'}{CODE}; + return unresolved_code_fact('Image::ExifTool::DoAutoLoad', 'code_ref_unavailable') unless $cv; + return code_ref_fact($cv, 'Image::ExifTool::DoAutoLoad', $lib_abs, undef, undef, 0); +} + +sub write_autoload_router_supported { + my ($fact) = @_; + return 0 unless $fact->{resolved} && ($fact->{__name} // '') eq 'Image::ExifTool::DoAutoLoad'; + my $flat = $fact->{__deparse}; + return 0 unless defined $flat; + $flat =~ s/\s+//g; + # Ordered executable fragments from the generic native dispatcher: split + # name, reject DESTROY, choose the four-part/ShiftTime/default file, load + # it, then tail-call the resolved routine. Names and file construction are + # generic ExifTool mechanism, never table or vendor selection. + my @parts = ( + 'split(/::/,$autoload,0)', + "'Image/ExifTool/Write'", + "\$callInfo[\$#callInfo]eq'DESTROY'", + '@callInfo==4', + '$file.="$callInfo[2].pl"', + "\$callInfo[-1]eq'ShiftTime'", + "\$file='Image/ExifTool/Shift.pl'", + "\$file.='r.pl'", + 'require$file', + 'return&$autoload(@_)', + ); + my $at = -1; + for my $part (@parts) { + my $next = index($flat, $part, $at + 1); + return 0 if $next < 0; + $at = $next; + } + return 1; +} + sub hydrate_write_procedures { - my ($tables) = @_; + my ($tables, $router) = @_; my %status; for my $full_name (sort keys %$tables) { my $hash = $tables->{$full_name}{hash}; @@ -538,6 +589,10 @@ sub hydrate_write_procedures { my $name = code_name($cv); my $file = write_autoload_file($name); my $id = refaddr($cv); + if (!$router->{supported}) { + $status{$id} = { reason => 'autoload_router_unsupported' }; + next; + } if (!defined $file) { $status{$id} = { reason => 'autoload_target_unavailable' }; next; @@ -795,7 +850,12 @@ sub dump_module { # loading must not alter the existing read projection. The sidecar gets the # table's stored CV, its final implementation source fact, and raw source # controls separately. -my $write_autoload_status = hydrate_write_procedures(\%write_tables); +my $write_autoload_router = write_autoload_router_fact($EXIFTOOL_LIB_ABS); +my $write_autoload_router_status = { + router => $write_autoload_router, + supported => write_autoload_router_supported($write_autoload_router) ? JSON::PP::true : JSON::PP::false, +}; +my $write_autoload_status = hydrate_write_procedures(\%write_tables, $write_autoload_router_status); for my $full_name (sort keys %write_tables) { my $entry = $write_tables{$full_name}; my $fact = dump_write_table($entry->{module}, $entry->{table}, $full_name, $entry->{hash}); @@ -830,6 +890,7 @@ sub dump_module { # Facts only: no reader or writer consumes this sidecar yet. Keeping it # separate prevents write-only properties from becoming accidental read # admission or changing the existing module/table/tag projection. + native_write_autoload => $write_autoload_router_status, native_write_tables => \%native_write_tables, subdirectory_validate_functions => \%subdirectory_validate_functions, native_reader_contracts => { unsigned16 => $unsigned_reader_contract }, diff --git a/tools/exiftool-tables/test_dump_write_facts.py b/tools/exiftool-tables/test_dump_write_facts.py index fff638e74..d2f33b50b 100644 --- a/tools/exiftool-tables/test_dump_write_facts.py +++ b/tools/exiftool-tables/test_dump_write_facts.py @@ -30,7 +30,8 @@ def setUp(self): (self.lib / "Image/ExifTool.pm").write_text( "package Image::ExifTool; our $VERSION = 'fixture'; " "our %specialTags = map { $_ => 1 } " - "qw(GROUPS WRITE_PROC CHECK_PROC WRITABLE WRITE_GROUP FORMAT FIRST_ENTRY); 1;\n", + "qw(GROUPS WRITE_PROC CHECK_PROC WRITABLE WRITE_GROUP FORMAT FIRST_ENTRY);\n" + """sub DoAutoLoad { my $autoload = shift; my @callInfo = split /::/, $autoload, 0; my $file = 'Image/ExifTool/Write'; return if $callInfo[$#callInfo] eq 'DESTROY'; if (@callInfo == 4) { $file .= "$callInfo[2].pl"; } elsif ($callInfo[-1] eq 'ShiftTime') { $file = 'Image/ExifTool/Shift.pl'; } else { $file .= 'r.pl'; } require $file; no strict 'refs'; return &$autoload(@_); } 1;\n""", encoding="utf-8", ) self.exif = package / "Exif.pm" @@ -40,7 +41,8 @@ def setUp(self): self.write_writer("return 1;") self.write_later("return 'late';") - def write_exif(self, host_group: str): + def write_exif(self, host_group: str, host_group_expr: str | None = None): + write_group = host_group_expr if host_group_expr is not None else f"'{host_group}'" self.exif.write_text(textwrap.dedent(f"""\ package Image::ExifTool::Exif; sub WriteExif($$$); @@ -54,7 +56,7 @@ def write_exif(self, host_group: str): WRITABLE => 0, 0x13c => {{ Name => 'HostComputer', Writable => 'string', - WriteGroup => '{host_group}', RawConvInv => '$val', + WriteGroup => {write_group}, RawConvInv => '$val', ValueConvInv => 0, PrintConvInv => '', Mandatory => 0, DelValue => '', Validate => undef, FutureWriterSwitch => {{ zero => 0, empty => '', undef => undef }}, @@ -128,6 +130,22 @@ def test_preserves_complete_controls_variants_and_unknown_values(self): self.assertEqual(table["write_controls"]["WRITE_GROUP"], {"present": True, "value": "ExifIFD"}) + def test_scalar_reference_is_not_collapsed_to_literal(self): + self.write_exif("unused", host_group_expr="\\'IFD0'") + value = self.sidecar(self.dump())["rows"]["316"]["write_controls"]["WriteGroup"]["value"] + self.assertEqual(value, {"__ref": "SCALAR", "value": "IFD0"}) + + def test_mutated_autoload_router_refuses_forced_writer_loading(self): + core = self.lib / "Image/ExifTool.pm" + text = core.read_text(encoding="utf-8") + core.write_text(text.replace("Image/ExifTool/Write", "Image/ExifTool/Broken", 1), encoding="utf-8") + doc = self.dump() + router = doc["native_write_autoload"] + self.assertFalse(router["supported"]) + effective = self.sidecar(doc)["effective_write_proc"]["effective"] + self.assertFalse(effective["resolved"]) + self.assertEqual(effective["reason"], "autoload_router_unsupported") + def test_forward_declarations_capture_loaded_writer_bodies(self): doc = self.dump() table = self.sidecar(doc) From 468a114b306de1fda1f72e1c23061ffe33723bbe Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:13:09 -0500 Subject: [PATCH 03/22] fix(tables): close native writer autoload grammar --- tools/exiftool-tables/dump_tables.pl | 95 ++++++++++++++----- .../exiftool-tables/test_dump_write_facts.py | 55 ++++++++--- 2 files changed, 115 insertions(+), 35 deletions(-) diff --git a/tools/exiftool-tables/dump_tables.pl b/tools/exiftool-tables/dump_tables.pl index 4b39c1ac7..8fece5c5d 100755 --- a/tools/exiftool-tables/dump_tables.pl +++ b/tools/exiftool-tables/dump_tables.pl @@ -546,33 +546,80 @@ sub write_autoload_router_fact { return code_ref_fact($cv, 'Image::ExifTool::DoAutoLoad', $lib_abs, undef, undef, 0); } +sub write_autoload_router_tokens { + my ($body) = @_; + return undef unless defined $body; + my @tokens; + pos($body) = 0; + while (pos($body) < length($body)) { + if ($body =~ /\G\s+/gc) { + next; + } elsif ($body =~ /\G((?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'))/gc) { + push @tokens, $1; + } elsif ($body =~ /\G(\/(?:\\.|[^\/\\])*\/)/gc) { + push @tokens, $1; + } elsif ($body =~ /\G(\$\#?[A-Za-z_]\w*|\$\@|\@(?:[A-Za-z_]\w*|_))/gc) { + push @tokens, $1; + } elsif ($body =~ /\G([A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)/gc) { + push @tokens, $1; + } elsif ($body =~ /\G(\d+)/gc) { + push @tokens, $1; + } elsif ($body =~ /\G(\.=|==)/gc) { + push @tokens, $1; + } elsif ($body =~ /\G([(){}\[\];,.=&@-])/gc) { + push @tokens, $1; + } else { + return undef; + } + } + return \@tokens; +} + +sub write_autoload_router_expected_tokens { + # Full B::Deparse token grammar for the generic native DoAutoLoad route: + # DESTROY guard; 4-part, ShiftTime, and Writer file paths; guarded require; + # implementation check; and tail call. Whitespace is immaterial, but every + # executable token is consumed, so an inserted assignment or early return + # is not mistaken for the native dispatcher. + my $canonical = <<'END_AUTOLOAD'; +(@) { +package Image::ExifTool; +use strict; +(my($autoload) = (shift())); +(my @callInfo = split(/::/, $autoload, 0)); +(my($file) = 'Image/ExifTool/Write'); +(($callInfo[$#callInfo] eq 'DESTROY') and (return)); +if ((@callInfo == 4)) { +($file .= "$callInfo[2].pl"); +} elsif (($callInfo[-1] eq 'ShiftTime')) { +($file = 'Image/ExifTool/Shift.pl'); +} else { +($file .= 'r.pl'); +} +(eval { +do { +(require $file) +} +} or die(("Error while attempting to call $autoload\n$@\n"))); +unless (defined(&$autoload)) { +(my(@caller) = caller(0)); +die(("Undefined subroutine $autoload called at $caller[1] line $caller[2]\n")); +} +no strict 'refs'; +(return &$autoload(@_)); +} +END_AUTOLOAD + return write_autoload_router_tokens($canonical); +} + sub write_autoload_router_supported { my ($fact) = @_; return 0 unless $fact->{resolved} && ($fact->{__name} // '') eq 'Image::ExifTool::DoAutoLoad'; - my $flat = $fact->{__deparse}; - return 0 unless defined $flat; - $flat =~ s/\s+//g; - # Ordered executable fragments from the generic native dispatcher: split - # name, reject DESTROY, choose the four-part/ShiftTime/default file, load - # it, then tail-call the resolved routine. Names and file construction are - # generic ExifTool mechanism, never table or vendor selection. - my @parts = ( - 'split(/::/,$autoload,0)', - "'Image/ExifTool/Write'", - "\$callInfo[\$#callInfo]eq'DESTROY'", - '@callInfo==4', - '$file.="$callInfo[2].pl"', - "\$callInfo[-1]eq'ShiftTime'", - "\$file='Image/ExifTool/Shift.pl'", - "\$file.='r.pl'", - 'require$file', - 'return&$autoload(@_)', - ); - my $at = -1; - for my $part (@parts) { - my $next = index($flat, $part, $at + 1); - return 0 if $next < 0; - $at = $next; + my $actual = write_autoload_router_tokens($fact->{__deparse}); + my $expected = write_autoload_router_expected_tokens(); + return 0 unless $actual && $expected && @$actual == @$expected; + for my $i (0 .. $#$expected) { + return 0 unless $actual->[$i] eq $expected->[$i]; } return 1; } diff --git a/tools/exiftool-tables/test_dump_write_facts.py b/tools/exiftool-tables/test_dump_write_facts.py index d2f33b50b..12cc1c9d0 100644 --- a/tools/exiftool-tables/test_dump_write_facts.py +++ b/tools/exiftool-tables/test_dump_write_facts.py @@ -28,10 +28,35 @@ def setUp(self): package = self.lib / "Image/ExifTool" package.mkdir(parents=True) (self.lib / "Image/ExifTool.pm").write_text( - "package Image::ExifTool; our $VERSION = 'fixture'; " - "our %specialTags = map { $_ => 1 } " - "qw(GROUPS WRITE_PROC CHECK_PROC WRITABLE WRITE_GROUP FORMAT FIRST_ENTRY);\n" - """sub DoAutoLoad { my $autoload = shift; my @callInfo = split /::/, $autoload, 0; my $file = 'Image/ExifTool/Write'; return if $callInfo[$#callInfo] eq 'DESTROY'; if (@callInfo == 4) { $file .= "$callInfo[2].pl"; } elsif ($callInfo[-1] eq 'ShiftTime') { $file = 'Image/ExifTool/Shift.pl'; } else { $file .= 'r.pl'; } require $file; no strict 'refs'; return &$autoload(@_); } 1;\n""", + textwrap.dedent("""\ + package Image::ExifTool; + use strict; + our $VERSION = 'fixture'; + our %specialTags = map { $_ => 1 } + qw(GROUPS WRITE_PROC CHECK_PROC WRITABLE WRITE_GROUP FORMAT FIRST_ENTRY); + sub DoAutoLoad(@) { + my ($autoload) = shift; + my @callInfo = split /::/, $autoload, 0; + my ($file) = 'Image/ExifTool/Write'; + ($callInfo[$#callInfo] eq 'DESTROY') and return; + if (@callInfo == 4) { + $file .= "$callInfo[2].pl"; + } elsif ($callInfo[-1] eq 'ShiftTime') { + $file = 'Image/ExifTool/Shift.pl'; + } else { + $file .= 'r.pl'; + } + eval { require $file } + or die("Error while attempting to call $autoload\n$@\n"); + unless (defined &$autoload) { + my @caller = caller(0); + die("Undefined subroutine $autoload called at $caller[1] line $caller[2]\n"); + } + no strict 'refs'; + return &$autoload(@_); + } + 1; + """), encoding="utf-8", ) self.exif = package / "Exif.pm" @@ -138,13 +163,21 @@ def test_scalar_reference_is_not_collapsed_to_literal(self): def test_mutated_autoload_router_refuses_forced_writer_loading(self): core = self.lib / "Image/ExifTool.pm" text = core.read_text(encoding="utf-8") - core.write_text(text.replace("Image/ExifTool/Write", "Image/ExifTool/Broken", 1), encoding="utf-8") - doc = self.dump() - router = doc["native_write_autoload"] - self.assertFalse(router["supported"]) - effective = self.sidecar(doc)["effective_write_proc"]["effective"] - self.assertFalse(effective["resolved"]) - self.assertEqual(effective["reason"], "autoload_router_unsupported") + for label, changed in { + "changed_route": text.replace("Image/ExifTool/Write", "Image/ExifTool/Broken", 1), + "injected_route": text.replace( + "eval { require $file }", "$file = 'Image/ExifTool/WriteNoSuchWriter.pl';\n eval { require $file }"), + "early_return": text.replace("eval { require $file }", "return;\n eval { require $file }"), + }.items(): + with self.subTest(label=label): + core.write_text(changed, encoding="utf-8") + doc = self.dump() + router = doc["native_write_autoload"] + self.assertFalse(router["supported"]) + effective = self.sidecar(doc)["effective_write_proc"]["effective"] + self.assertFalse(effective["resolved"]) + self.assertEqual(effective["reason"], "autoload_router_unsupported") + core.write_text(text, encoding="utf-8") def test_forward_declarations_capture_loaded_writer_bodies(self): doc = self.dump() From 2c9519cc2285001fa3cf417cae43ab7bb358e569 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 10:57:12 -0500 Subject: [PATCH 04/22] tools: add seeded version rehearsal planning --- .../exiftool-tables/test_version_rehearsal.py | 178 +++++++ tools/exiftool-tables/version_rehearsal.py | 439 ++++++++++++++++++ 2 files changed, 617 insertions(+) create mode 100644 tools/exiftool-tables/test_version_rehearsal.py create mode 100755 tools/exiftool-tables/version_rehearsal.py diff --git a/tools/exiftool-tables/test_version_rehearsal.py b/tools/exiftool-tables/test_version_rehearsal.py new file mode 100644 index 000000000..5f490bdfe --- /dev/null +++ b/tools/exiftool-tables/test_version_rehearsal.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Offline contract tests for version_rehearsal.py; never use the network.""" +from __future__ import annotations + +import copy +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("version_rehearsal", HERE / "version_rehearsal.py") +assert spec and spec.loader +vr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(vr) + +OID_A = "a" * 40 +OID_B = "b" * 40 +OID_C = "c" * 40 +OID_D = "d" * 40 +SHA_A = "1" * 64 +SHA_B = "2" * 64 +SHA_C = "3" * 64 + + +def release(name, oid, sha): + return {"name": name, "tag_object": oid, "peeled_commit": oid, "archive": {"url": f"https://github.com/exiftool/exiftool/archive/refs/tags/{name}.tar.gz", "sha256": sha}} + + +def catalog_entries(): + return { + "catalog_source": {"kind": "official_exiftool_tag_catalog", "pages": [{"url": "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=1", "sha256": "0" * 64}]}, + "captured_at": "2026-09-13T00:00:00Z", + "entries": [ + release("13.59", OID_C, SHA_C), + {"name": "v13.58", "tag_object": OID_B}, + release("13.57", OID_A, SHA_A), + {"name": "13.58", "tag_object": OID_B, "peeled_commit": OID_B, "archive": {"url": "https://github.com/exiftool/exiftool/archive/refs/tags/13.58.tar.gz"}}, + ], + } + + +class VersionRehearsalTests(unittest.TestCase): + def normalized(self): + return vr.normalize_catalog(catalog_entries()) + + def plan(self, seed=7, sample_index=0, pair_count=1): + return vr.make_plan(self.normalized(), seed, sample_index, pair_count, "e" * 40) + + def test_catalog_preserves_unclassified_and_excluded_entries(self): + catalog = self.normalized() + self.assertEqual(len(catalog["entries"]), 4) + self.assertEqual(catalog["entries"][1]["classification"], {"state": "excluded", "reason": "tag_name_not_numeric_release"}) + self.assertEqual(catalog["entries"][3]["classification"], {"state": "unclassified", "reason": "missing_or_invalid_archive_sha256"}) + self.assertEqual([row["name"] for row in vr.eligible_releases(catalog)], ["13.57", "13.59"]) + + def test_deterministic_replay_and_ordered_distinct_pairs(self): + first = self.plan(seed=99, sample_index=4) + second = self.plan(seed=99, sample_index=4) + self.assertEqual(first, second) + old, new = first["pairs"][0]["old"]["release"], first["pairs"][0]["new"]["release"] + self.assertLess(vr.release_key(old), vr.release_key(new)) + self.assertNotEqual(old, new) + self.assertEqual(first["execution"]["read"], "unrun") + self.assertEqual(first["execution"]["write"], "unrun") + + def test_same_version_and_ambiguous_identity_are_refused(self): + catalog = self.normalized() + duplicate = copy.deepcopy(catalog_entries()) + duplicate["entries"].append(release("13.59", OID_D, SHA_B)) + duplicate_normalized = vr.normalize_catalog(duplicate) + with self.assertRaisesRegex(vr.Refused, "at least two"): + vr.eligible_releases(duplicate_normalized) + bad_plan = self.plan() + bad_plan["pairs"][0]["new"]["release"] = bad_plan["pairs"][0]["old"]["release"] + # The immutable checksum catches mutation before the semantic check. + with self.assertRaisesRegex(vr.Refused, "identity changed"): + vr.verify_plan(bad_plan) + + def test_changed_catalog_identity_refuses_plan_reuse(self): + catalog = self.normalized() + plan = vr.make_plan(catalog, 3, 0, 1, "e" * 40) + changed_raw = catalog_entries() + changed_raw["entries"][0]["archive"]["sha256"] = "f" * 64 + changed = vr.normalize_catalog(changed_raw) + with self.assertRaisesRegex(vr.Refused, "catalog identity differs"): + vr.verify_plan(plan, changed) + + def test_unselected_eligible_release_is_explicitly_untested(self): + raw = catalog_entries() + raw["entries"].append(release("13.60", OID_D, SHA_B)) + catalog = vr.normalize_catalog(raw) + plan = vr.make_plan(catalog, 12, 0, 1, "e" * 40) + selected = {side["release"] for pair in plan["pairs"] for side in (pair["old"], pair["new"])} + untested = {row["release"]: row["reason"] for row in plan["untested_eligible_releases"]} + self.assertEqual(selected | set(untested), {"13.57", "13.59", "13.60"}) + self.assertTrue(all(reason == "not_selected_by_seeded_pair_plan" for reason in untested.values())) + + def test_duplicate_run_outputs_are_refused_and_initial_status_is_all_unrun(self): + plan = self.plan() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + _, status = vr.create_run(plan, self.normalized(), run_dir) + journal = json.loads(status.read_text()) + self.assertEqual(journal["phase"], "planned") + self.assertTrue(all(v["read"] == v["write"] == v["state"] == "unrun" for v in journal["releases"].values())) + with self.assertRaisesRegex(vr.Refused, "already exists"): + vr.create_run(plan, self.normalized(), run_dir) + + def test_interrupted_recovery_preserves_unrun_and_failure_is_accounted(self): + plan = self.plan() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + vr.start_pair(run_dir, 0) + recovered = vr.recover_interrupted(run_dir, self.normalized()) + self.assertEqual(recovered["phase"], "interrupted") + self.assertTrue(all(v["read"] == v["write"] == "unrun" for v in recovered["releases"].values())) + selected = plan["pairs"][0]["old"]["release"] + failed = vr.record_failure(run_dir, 0, selected, "write", "native validation failed") + self.assertEqual(failed["phase"], "failed") + self.assertEqual(failed["releases"][selected]["write"], "failed") + self.assertEqual(failed["pairs"][0]["failure"]["detail"], "native validation failed") + + def test_mutated_plan_and_journal_identity_are_refused(self): + plan = self.plan() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + plan_path = run_dir / "run-plan.json" + altered = json.loads(plan_path.read_text()) + altered["repository_commit"] = "f" * 40 + plan_path.write_text(json.dumps(altered)) + with self.assertRaisesRegex(vr.Refused, "plan identity changed"): + vr.load_verified_run(run_dir) + + def test_mutated_saved_catalog_is_refused_before_journal_use(self): + plan = self.plan() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + catalog_path = run_dir / "catalog.normalized.json" + catalog = json.loads(catalog_path.read_text()) + catalog["entries"][0]["raw"]["archive"]["sha256"] = "f" * 64 + catalog_path.write_text(json.dumps(catalog)) + with self.assertRaisesRegex(vr.Refused, "catalog identity changed"): + vr.load_verified_run(run_dir) + + def test_cli_plan_writes_no_success_state(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + catalog_path = base / "catalog.json" + catalog_path.write_text(json.dumps(catalog_entries())) + run_dir = base / "run" + rc = vr.main(["plan", "--catalog", str(catalog_path), "--seed", "4", "--repository-commit", "e" * 40, "--run-dir", str(run_dir)]) + self.assertEqual(rc, 0) + _, journal = vr.load_verified_run(run_dir) + self.assertEqual(journal["phase"], "planned") + self.assertTrue(all(item["read"] == item["write"] == "unrun" for item in journal["releases"].values())) + + def test_planning_journal_cannot_be_tampered_to_claim_success(self): + plan = self.plan() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + status_path = run_dir / "status.json" + journal = json.loads(status_path.read_text()) + release = next(iter(journal["releases"])) + journal["releases"][release]["read"] = "passed" + journal["releases"][release]["state"] = "passed" + status_path.write_text(json.dumps(journal)) + with self.assertRaisesRegex(vr.Refused, "cannot claim read/write success"): + vr.load_verified_run(run_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py new file mode 100755 index 000000000..8a5798245 --- /dev/null +++ b/tools/exiftool-tables/version_rehearsal.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""Offline planning and durable journaling for sampled ExifTool release rehearsals. + +This module deliberately does not fetch a tag, download an archive, modify the +ExifTool pin, generate tables, build Rust, run a native oracle, or promote +anything. It turns an already captured official tag catalog into an immutable, +seeded pair plan and an all-unrun journal. A later runner must verify every +identity recorded here before it is allowed to perform those expensive stages. +""" +from __future__ import annotations + +import argparse +import hashlib +import itertools +import json +import os +import random +import re +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +SCHEMA = 1 +SELECTOR = "python-random-mt19937-v1" +RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+$") +GIT_OID_RE = re.compile(r"^[0-9a-f]{40,64}$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class Refused(ValueError): + """The input cannot support an attributable rehearsal.""" + + +def canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + + +def sha256_json(value: Any) -> str: + return hashlib.sha256(canonical_json(value)).hexdigest() + + +def atomic_json(path: Path, document: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + encoded = json.dumps(document, indent=2, sort_keys=True) + "\n" + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False) as fh: + fh.write(encoded) + temp = Path(fh.name) + try: + os.replace(temp, path) + finally: + if temp.exists(): + temp.unlink() + + +def read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise Refused(f"invalid JSON: {path}") from exc + if not isinstance(value, dict): + raise Refused(f"JSON object required: {path}") + return value + + +def release_key(tag: str) -> tuple[int, int]: + if not isinstance(tag, str) or not RELEASE_RE.fullmatch(tag): + raise Refused(f"not a numeric ExifTool release: {tag!r}") + major, minor = tag.split(".") + return int(major), int(minor) + + +def _identity_reason(raw: dict[str, Any]) -> str | None: + name = raw.get("name") + tag_object = raw.get("tag_object") + peeled_commit = raw.get("peeled_commit") + archive = raw.get("archive") + if not isinstance(tag_object, str) or not GIT_OID_RE.fullmatch(tag_object): + return "missing_or_invalid_tag_object" + if not isinstance(peeled_commit, str) or not GIT_OID_RE.fullmatch(peeled_commit): + return "missing_or_invalid_peeled_commit" + if not isinstance(archive, dict): + return "missing_archive_identity" + expected_url = f"https://github.com/exiftool/exiftool/archive/refs/tags/{name}.tar.gz" + if archive.get("url") != expected_url: + return "missing_or_invalid_archive_url" + if not isinstance(archive.get("sha256"), str) or not SHA256_RE.fullmatch(archive["sha256"]): + return "missing_or_invalid_archive_sha256" + return None + + +def normalize_catalog(raw: dict[str, Any]) -> dict[str, Any]: + """Normalize an offline official-tag catalog without silently dropping tags. + + Input requires an ``entries`` list. Every original entry is preserved under + ``raw`` with either ``eligible`` or a concrete ``unclassified`` reason. + Duplicate numeric names are deliberately unclassified rather than choosing + whichever source happened to arrive first. + """ + source = raw.get("catalog_source") + pages = source.get("pages") if isinstance(source, dict) else None + if (not isinstance(source, dict) or source.get("kind") != "official_exiftool_tag_catalog" + or not isinstance(pages, list) or not pages + or any(not isinstance(page, dict) or not isinstance(page.get("url"), str) + or not page["url"].startswith("https://api.github.com/repos/exiftool/exiftool/tags") + or not isinstance(page.get("sha256"), str) or not SHA256_RE.fullmatch(page["sha256"]) + for page in pages) + or not isinstance(raw.get("captured_at"), str)): + raise Refused("catalog_source must identify an official ExifTool tag catalog") + entries = raw.get("entries") + if not isinstance(entries, list): + raise Refused("catalog requires an entries list") + names: dict[str, int] = {} + for entry in entries: + if isinstance(entry, dict) and isinstance(entry.get("name"), str) and RELEASE_RE.fullmatch(entry["name"]): + names[entry["name"]] = names.get(entry["name"], 0) + 1 + normalized: list[dict[str, Any]] = [] + for index, raw_entry in enumerate(entries): + if not isinstance(raw_entry, dict): + normalized.append({"source_index": index, "raw": raw_entry, "classification": {"state": "unclassified", "reason": "entry_not_object"}}) + continue + name = raw_entry.get("name") + if not isinstance(name, str) or not RELEASE_RE.fullmatch(name): + classification = {"state": "excluded", "reason": "tag_name_not_numeric_release"} + elif names[name] > 1: + classification = {"state": "unclassified", "reason": "ambiguous_duplicate_release"} + else: + reason = _identity_reason(raw_entry) + classification = {"state": "eligible"} if reason is None else {"state": "unclassified", "reason": reason} + normalized.append({"source_index": index, "raw": raw_entry, "classification": classification}) + payload = { + "schema": SCHEMA, + "catalog_source": source, + "captured_at": raw.get("captured_at"), + "entries": normalized, + } + return {**payload, "catalog_sha256": sha256_json(payload)} + + +def verify_catalog(catalog: dict[str, Any]) -> None: + if catalog.get("schema") != SCHEMA or not isinstance(catalog.get("entries"), list): + raise Refused("unsupported normalized catalog") + expected = catalog.get("catalog_sha256") + payload = {k: v for k, v in catalog.items() if k != "catalog_sha256"} + if not isinstance(expected, str) or expected != sha256_json(payload): + raise Refused("catalog identity changed or is malformed") + + +def eligible_releases(catalog: dict[str, Any]) -> list[dict[str, Any]]: + verify_catalog(catalog) + releases = [entry["raw"] for entry in catalog["entries"] if entry["classification"]["state"] == "eligible"] + releases.sort(key=lambda entry: release_key(entry["name"])) + if len(releases) < 2: + raise Refused("at least two fully identified numeric releases are required") + return releases + + +def parse_seed(value: str) -> int: + try: + seed = int(value, 10) + except ValueError as exc: + raise Refused("seed must be an unsigned decimal integer") from exc + if not 0 <= seed < 2**64: + raise Refused("seed must fit unsigned 64-bit range") + return seed + + +def select_pairs(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: int) -> list[tuple[dict[str, Any], dict[str, Any]]]: + if sample_index < 0 or pair_count < 1: + raise Refused("sample index must be nonnegative and pair count positive") + releases = eligible_releases(catalog) + pairs = list(itertools.combinations(releases, 2)) + if pair_count > len(pairs): + raise Refused(f"pair count {pair_count} exceeds {len(pairs)} available unordered pairs") + rng = random.Random(f"{SELECTOR}:{seed}:{sample_index}") + selected = rng.sample(pairs, pair_count) + # itertools combinations has each pair in increasing release order; verify + # it rather than relying on implementation details in a future refactor. + for old, new in selected: + if release_key(old["name"]) >= release_key(new["name"]): + raise Refused("selected same-version or reversed pair") + return selected + + +def _release_identity(entry: dict[str, Any]) -> dict[str, Any]: + archive = entry["archive"] + return { + "release": entry["name"], + "tag_object": entry["tag_object"], + "peeled_commit": entry["peeled_commit"], + "archive": {"url": archive["url"], "sha256": archive["sha256"]}, + } + + +def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: int, repository_commit: str) -> dict[str, Any]: + verify_catalog(catalog) + if not isinstance(repository_commit, str) or not GIT_OID_RE.fullmatch(repository_commit): + raise Refused("repository commit must be a full git object id") + selected = select_pairs(catalog, seed, sample_index, pair_count) + chosen_names = {entry["name"] for pair in selected for entry in pair} + untested = [] + for entry in catalog["entries"]: + raw = entry["raw"] + if entry["classification"]["state"] == "eligible" and raw["name"] not in chosen_names: + untested.append({"release": raw["name"], "reason": "not_selected_by_seeded_pair_plan"}) + payload = { + "schema": SCHEMA, + "kind": "oxidex_exiftool_version_rehearsal_plan", + "selector": SELECTOR, + "repository_commit": repository_commit, + "catalog_sha256": catalog["catalog_sha256"], + "seed": seed, + "sample_index": sample_index, + "pair_count": pair_count, + "pairs": [ + {"pair_index": index, "old": _release_identity(old), "new": _release_identity(new)} + for index, (old, new) in enumerate(selected) + ], + "untested_eligible_releases": sorted(untested, key=lambda row: release_key(row["release"])), + "catalog_noneligible_entries": [ + {"source_index": entry["source_index"], "name": entry["raw"].get("name") if isinstance(entry["raw"], dict) else None, + "classification": entry["classification"]} + for entry in catalog["entries"] if entry["classification"]["state"] != "eligible" + ], + "execution": { + "state": "plan_only", + "read": "unrun", + "write": "unrun", + "explicit_limit": "selection is not native read/write proof or upgrade success", + }, + } + return {**payload, "plan_sha256": sha256_json(payload)} + + +def verify_plan(plan: dict[str, Any], catalog: dict[str, Any] | None = None) -> None: + if plan.get("schema") != SCHEMA or plan.get("kind") != "oxidex_exiftool_version_rehearsal_plan": + raise Refused("unsupported run plan") + expected = plan.get("plan_sha256") + payload = {k: v for k, v in plan.items() if k != "plan_sha256"} + if not isinstance(expected, str) or expected != sha256_json(payload): + raise Refused("plan identity changed or is malformed") + eligible_by_release: dict[str, dict[str, Any]] | None = None + if catalog is not None: + verify_catalog(catalog) + if plan.get("catalog_sha256") != catalog["catalog_sha256"]: + raise Refused("catalog identity differs from recorded plan") + eligible_by_release = {entry["raw"]["name"]: entry["raw"] for entry in catalog["entries"] + if entry["classification"]["state"] == "eligible"} + seen: set[str] = set() + for pair in plan.get("pairs", []): + if not isinstance(pair, dict): + raise Refused("malformed pair") + old, new = pair.get("old"), pair.get("new") + if not isinstance(old, dict) or not isinstance(new, dict): + raise Refused("pair lacks release identities") + for release in (old, new): + # Reapply the same exact requirements before future execution. + reason = _identity_reason({"name": release.get("release"), **release}) + if not isinstance(release.get("release"), str) or not RELEASE_RE.fullmatch(release["release"]) or reason: + raise Refused("pair has missing or ambiguous release identity") + if eligible_by_release is not None: + actual = eligible_by_release.get(release["release"]) + expected = {"release": actual["name"], "tag_object": actual["tag_object"], + "peeled_commit": actual["peeled_commit"], + "archive": {"url": actual["archive"]["url"], "sha256": actual["archive"]["sha256"]}} if actual else None + if expected != release: + raise Refused("pair release identity differs from catalog") + if release_key(old["release"]) >= release_key(new["release"]): + raise Refused("pair must have distinct old/new releases in numeric order") + pair_key = f"{old['release']}->{new['release']}" + if pair_key in seen: + raise Refused("duplicate pair in plan") + seen.add(pair_key) + + +def _run_id(plan: dict[str, Any]) -> str: + return f"rehearsal-{plan['plan_sha256'][:16]}" + + +def initial_journal(plan: dict[str, Any]) -> dict[str, Any]: + verify_plan(plan) + variants: dict[str, dict[str, str]] = {} + for pair in plan["pairs"]: + for release in (pair["old"], pair["new"]): + variants.setdefault(release["release"], {"read": "unrun", "write": "unrun", "state": "unrun"}) + return { + "schema": SCHEMA, + "kind": "oxidex_exiftool_version_rehearsal_journal", + "run_id": _run_id(plan), + "plan_sha256": plan["plan_sha256"], + "phase": "planned", + "active": None, + "events": [{"event": "plan_created", "at_unix": time.time()}], + "pairs": [{"pair_index": pair["pair_index"], "state": "unrun", "failure": None} for pair in plan["pairs"]], + "releases": variants, + "untested_eligible_releases": plan["untested_eligible_releases"], + "execution_limit": "all read/write states are unrun; planning is not proof", + } + + +def create_run(plan: dict[str, Any], catalog: dict[str, Any], run_dir: Path) -> tuple[Path, Path]: + verify_plan(plan, catalog) + try: + run_dir.mkdir(parents=True, exist_ok=False) + except FileExistsError as exc: + raise Refused(f"run output already exists: {run_dir}") from exc + catalog_path = run_dir / "catalog.normalized.json" + plan_path = run_dir / "run-plan.json" + journal_path = run_dir / "status.json" + atomic_json(catalog_path, catalog) + atomic_json(plan_path, plan) + atomic_json(journal_path, initial_journal(plan)) + return plan_path, journal_path + + +def load_verified_run(run_dir: Path, catalog: dict[str, Any] | None = None) -> tuple[dict[str, Any], dict[str, Any]]: + if catalog is None: + catalog = read_json(run_dir / "catalog.normalized.json") + plan = read_json(run_dir / "run-plan.json") + verify_plan(plan, catalog) + journal = read_json(run_dir / "status.json") + if journal.get("schema") != SCHEMA or journal.get("kind") != "oxidex_exiftool_version_rehearsal_journal": + raise Refused("unsupported journal") + if journal.get("plan_sha256") != plan["plan_sha256"] or journal.get("run_id") != _run_id(plan): + raise Refused("journal does not belong to immutable plan") + if journal.get("phase") not in {"planned", "running", "interrupted", "failed"}: + raise Refused("journal has unsupported phase") + for release, state in journal.get("releases", {}).items(): + if not isinstance(release, str) or not isinstance(state, dict): + raise Refused("journal has malformed release state") + if state.get("read") not in {"unrun", "failed"} or state.get("write") not in {"unrun", "failed"}: + raise Refused("planning journal cannot claim read/write success") + expected_state = "failed" if "failed" in {state.get("read"), state.get("write")} else "unrun" + if state.get("state") != expected_state: + raise Refused("journal release aggregate does not match read/write states") + for pair in journal.get("pairs", []): + if not isinstance(pair, dict) or pair.get("state") not in {"unrun", "failed"}: + raise Refused("planning journal cannot claim pair success") + return plan, journal + + +def start_pair(run_dir: Path, pair_index: int) -> dict[str, Any]: + """Record an active pair for a future runner; it never marks a pass.""" + plan, journal = load_verified_run(run_dir) + if journal["phase"] not in {"planned", "interrupted"}: + raise Refused("journal is not available to start a pair") + if not any(item["pair_index"] == pair_index and item["state"] == "unrun" for item in journal["pairs"]): + raise Refused("requested unrun pair not found") + journal["phase"] = "running" + journal["active"] = {"pair_index": pair_index, "stage": "not_started"} + journal["events"].append({"event": "pair_started", "pair_index": pair_index, "at_unix": time.time()}) + atomic_json(run_dir / "status.json", journal) + return journal + + +def record_failure(run_dir: Path, pair_index: int, release: str, operation: str, detail: str) -> dict[str, Any]: + """Record an observed future-run failure without allowing a synthetic pass.""" + if operation not in {"read", "write"}: + raise Refused("failure operation must be read or write") + plan, journal = load_verified_run(run_dir) + if journal["phase"] not in {"running", "interrupted"}: + raise Refused("only a running or recovered pair can record a failure") + if release not in journal["releases"]: + raise Refused("release is not selected by this run") + journal["releases"][release][operation] = "failed" + journal["releases"][release]["state"] = "failed" + selected_pair = next((pair for pair in plan["pairs"] if pair["pair_index"] == pair_index), None) + if selected_pair is None or release not in {selected_pair["old"]["release"], selected_pair["new"]["release"]}: + raise Refused("release does not belong to selected pair") + for pair in journal["pairs"]: + if pair["pair_index"] == pair_index: + pair.update(state="failed", failure={"release": release, "operation": operation, "detail": detail}) + break + else: + raise Refused("pair is not selected by this run") + journal["phase"] = "failed" + journal["active"] = None + journal["events"].append({"event": "failure", "pair_index": pair_index, "release": release, "operation": operation, "detail": detail, "at_unix": time.time()}) + atomic_json(run_dir / "status.json", journal) + return journal + + +def recover_interrupted(run_dir: Path, catalog: dict[str, Any] | None = None) -> dict[str, Any]: + """Close an interrupted pre-execution journal while preserving all unrun states.""" + _, journal = load_verified_run(run_dir, catalog) + if journal["phase"] != "running" or not isinstance(journal.get("active"), dict): + raise Refused("only an active interrupted journal can be recovered") + active = journal["active"] + journal["phase"] = "interrupted" + journal["active"] = None + journal["events"].append({"event": "interrupted_recovered", "previous_active": active, "at_unix": time.time()}) + atomic_json(run_dir / "status.json", journal) + return journal + + +def _cmd_plan(args: argparse.Namespace) -> int: + catalog = normalize_catalog(read_json(Path(args.catalog))) + seed = parse_seed(args.seed) + plan = make_plan(catalog, seed, args.sample_index, args.pair_count, args.repository_commit) + run_dir = Path(args.run_dir).resolve() + plan_path, status_path = create_run(plan, catalog, run_dir) + print(json.dumps({"run_id": _run_id(plan), "plan": str(plan_path), "status": str(status_path), "selected_pairs": len(plan["pairs"]), "read": "unrun", "write": "unrun"}, sort_keys=True)) + return 0 + + +def _cmd_recover(args: argparse.Namespace) -> int: + run_dir = Path(args.run_dir).resolve() + catalog = normalize_catalog(read_json(Path(args.catalog))) if args.catalog else None + journal = recover_interrupted(run_dir, catalog) + print(json.dumps({"run_id": journal["run_id"], "phase": journal["phase"], "read": "unrun", "write": "unrun"}, sort_keys=True)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + plan = sub.add_parser("plan", help="write an immutable plan and all-unrun journal; performs no remote or build work") + plan.add_argument("--catalog", required=True, help="offline captured official-tag catalog JSON") + plan.add_argument("--seed", required=True, help="unsigned 64-bit decimal seed") + plan.add_argument("--sample-index", type=int, default=0) + plan.add_argument("--pair-count", type=int, default=1) + plan.add_argument("--repository-commit", required=True) + plan.add_argument("--run-dir", required=True, help="must not already exist") + plan.set_defaults(func=_cmd_plan) + recover = sub.add_parser("recover", help="preserve an interrupted planning journal as interrupted") + recover.add_argument("--run-dir", required=True) + recover.add_argument("--catalog", help="optional normalized-input check before recovery") + recover.set_defaults(func=_cmd_recover) + args = parser.parse_args(argv) + try: + return args.func(args) + except Refused as exc: + print(f"version rehearsal refused: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From 6cb5d161fd0b7fe89efd9c0415a42cadca7312a1 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:01:20 -0500 Subject: [PATCH 05/22] tools: bind rehearsal plans to matching native versions --- .../exiftool-tables/test_version_rehearsal.py | 19 +++++- tools/exiftool-tables/version_rehearsal.py | 59 +++++++++++++++++-- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal.py b/tools/exiftool-tables/test_version_rehearsal.py index 5f490bdfe..b6e0923dd 100644 --- a/tools/exiftool-tables/test_version_rehearsal.py +++ b/tools/exiftool-tables/test_version_rehearsal.py @@ -62,8 +62,14 @@ def test_deterministic_replay_and_ordered_distinct_pairs(self): old, new = first["pairs"][0]["old"]["release"], first["pairs"][0]["new"]["release"] self.assertLess(vr.release_key(old), vr.release_key(new)) self.assertNotEqual(old, new) - self.assertEqual(first["execution"]["read"], "unrun") - self.assertEqual(first["execution"]["write"], "unrun") + self.assertEqual(first["execution"]["per_version_read_vs_native"], "unrun") + self.assertEqual(first["execution"]["per_version_write_vs_native"], "unrun") + self.assertEqual(first["execution"]["native_old_to_native_new_delta"], "unrun") + pair = first["pairs"][0] + self.assertEqual(pair["native_oracles"]["old"]["native_release_identity"], pair["old"]) + self.assertEqual(pair["native_oracles"]["new"]["native_release_identity"], pair["new"]) + self.assertEqual(pair["comparison_contract"]["cross_version_output_equality"], "not_required") + self.assertTrue(pair["comparison_contract"]["newer_native_supersedes_older_native"]) def test_same_version_and_ambiguous_identity_are_refused(self): catalog = self.normalized() @@ -87,6 +93,15 @@ def test_changed_catalog_identity_refuses_plan_reuse(self): with self.assertRaisesRegex(vr.Refused, "catalog identity differs"): vr.verify_plan(plan, changed) + def test_swapped_or_same_native_oracle_is_refused_after_rehash(self): + plan = self.plan() + pair = plan["pairs"][0] + pair["native_oracles"]["old"]["native_release_identity"] = copy.deepcopy(pair["new"]) + payload = {k: v for k, v in plan.items() if k != "plan_sha256"} + plan["plan_sha256"] = vr.sha256_json(payload) + with self.assertRaisesRegex(vr.Refused, "old native oracle binding"): + vr.verify_plan(plan, self.normalized()) + def test_unselected_eligible_release_is_explicitly_untested(self): raw = catalog_entries() raw["entries"].append(release("13.60", OID_D, SHA_B)) diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py index 8a5798245..2c4baf644 100755 --- a/tools/exiftool-tables/version_rehearsal.py +++ b/tools/exiftool-tables/version_rehearsal.py @@ -193,6 +193,16 @@ def _release_identity(entry: dict[str, Any]) -> dict[str, Any]: } +def _oracle_binding(variant: str, release: dict[str, Any]) -> dict[str, Any]: + """Bind one OxiDex variant to the native source of the same release.""" + return { + "variant": variant, + "native_release_identity": _release_identity(release), + "read_vs_native": "unrun", + "write_vs_native": "unrun", + } + + def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: int, repository_commit: str) -> dict[str, Any]: verify_catalog(catalog) if not isinstance(repository_commit, str) or not GIT_OID_RE.fullmatch(repository_commit): @@ -214,7 +224,27 @@ def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: "sample_index": sample_index, "pair_count": pair_count, "pairs": [ - {"pair_index": index, "old": _release_identity(old), "new": _release_identity(new)} + { + "pair_index": index, + "old": _release_identity(old), + "new": _release_identity(new), + "native_oracles": { + "old": _oracle_binding("old", old), + "new": _oracle_binding("new", new), + }, + "comparison_contract": { + "schema": "per-version-native-v1", + "required": [ + "oxidex_old_vs_native_old", + "oxidex_new_vs_native_new", + "native_old_to_native_new_delta", + ], + "cross_version_output_equality": "not_required", + "newer_native_supersedes_older_native": True, + "accepted_native_delta_examples": ["upstream_bug_fix", "new_tag", "type_change", "format_change"], + "unsupported_new_semantics": "explicit_gap_never_old_fallback", + }, + } for index, (old, new) in enumerate(selected) ], "untested_eligible_releases": sorted(untested, key=lambda row: release_key(row["release"])), @@ -225,9 +255,10 @@ def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: ], "execution": { "state": "plan_only", - "read": "unrun", - "write": "unrun", - "explicit_limit": "selection is not native read/write proof or upgrade success", + "per_version_read_vs_native": "unrun", + "per_version_write_vs_native": "unrun", + "native_old_to_native_new_delta": "unrun", + "explicit_limit": "selection is not native read/write proof, native-delta classification, or upgrade success", }, } return {**payload, "plan_sha256": sha256_json(payload)} @@ -268,6 +299,26 @@ def verify_plan(plan: dict[str, Any], catalog: dict[str, Any] | None = None) -> raise Refused("pair release identity differs from catalog") if release_key(old["release"]) >= release_key(new["release"]): raise Refused("pair must have distinct old/new releases in numeric order") + oracles = pair.get("native_oracles") + if not isinstance(oracles, dict): + raise Refused("pair lacks native oracle bindings") + for label, release in (("old", old), ("new", new)): + binding = oracles.get(label) + if not isinstance(binding, dict) or binding.get("variant") != label: + raise Refused(f"pair lacks {label} native oracle binding") + if binding.get("native_release_identity") != release: + raise Refused(f"{label} native oracle binding does not match its selected release") + if binding.get("read_vs_native") != "unrun" or binding.get("write_vs_native") != "unrun": + raise Refused("planning plan cannot claim native comparison success") + contract = pair.get("comparison_contract") + if not isinstance(contract, dict) or contract.get("schema") != "per-version-native-v1": + raise Refused("pair lacks per-version comparison contract") + if contract.get("required") != ["oxidex_old_vs_native_old", "oxidex_new_vs_native_new", "native_old_to_native_new_delta"]: + raise Refused("pair comparison contract is incomplete") + if contract.get("cross_version_output_equality") != "not_required" or contract.get("newer_native_supersedes_older_native") is not True: + raise Refused("pair comparison contract incorrectly freezes cross-version output") + if contract.get("unsupported_new_semantics") != "explicit_gap_never_old_fallback": + raise Refused("pair comparison contract permits unsupported old fallback") pair_key = f"{old['release']}->{new['release']}" if pair_key in seen: raise Refused("duplicate pair in plan") From 21eed548ffe7d35e50c4ea075b24d9eca72d9d3f Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:14:22 -0500 Subject: [PATCH 06/22] tools: verify deterministic rehearsal plans --- .../exiftool-tables/test_version_rehearsal.py | 91 ++++++++-- tools/exiftool-tables/version_rehearsal.py | 163 ++++++++++-------- 2 files changed, 172 insertions(+), 82 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal.py b/tools/exiftool-tables/test_version_rehearsal.py index b6e0923dd..c4f99f07d 100644 --- a/tools/exiftool-tables/test_version_rehearsal.py +++ b/tools/exiftool-tables/test_version_rehearsal.py @@ -37,6 +37,7 @@ def catalog_entries(): {"name": "v13.58", "tag_object": OID_B}, release("13.57", OID_A, SHA_A), {"name": "13.58", "tag_object": OID_B, "peeled_commit": OID_B, "archive": {"url": "https://github.com/exiftool/exiftool/archive/refs/tags/13.58.tar.gz"}}, + release("13.60", OID_D, SHA_B), ], } @@ -48,12 +49,20 @@ def normalized(self): def plan(self, seed=7, sample_index=0, pair_count=1): return vr.make_plan(self.normalized(), seed, sample_index, pair_count, "e" * 40) + @staticmethod + def rehash_plan(plan): + plan["plan_sha256"] = vr.sha256_json({key: value for key, value in plan.items() if key != "plan_sha256"}) + + @staticmethod + def rehash_catalog(catalog): + catalog["catalog_sha256"] = vr.sha256_json({key: value for key, value in catalog.items() if key != "catalog_sha256"}) + def test_catalog_preserves_unclassified_and_excluded_entries(self): catalog = self.normalized() - self.assertEqual(len(catalog["entries"]), 4) + self.assertEqual(len(catalog["entries"]), 5) self.assertEqual(catalog["entries"][1]["classification"], {"state": "excluded", "reason": "tag_name_not_numeric_release"}) self.assertEqual(catalog["entries"][3]["classification"], {"state": "unclassified", "reason": "missing_or_invalid_archive_sha256"}) - self.assertEqual([row["name"] for row in vr.eligible_releases(catalog)], ["13.57", "13.59"]) + self.assertEqual([row["name"] for row in vr.eligible_releases(catalog)], ["13.57", "13.59", "13.60"]) def test_deterministic_replay_and_ordered_distinct_pairs(self): first = self.plan(seed=99, sample_index=4) @@ -70,11 +79,12 @@ def test_deterministic_replay_and_ordered_distinct_pairs(self): self.assertEqual(pair["native_oracles"]["new"]["native_release_identity"], pair["new"]) self.assertEqual(pair["comparison_contract"]["cross_version_output_equality"], "not_required") self.assertTrue(pair["comparison_contract"]["newer_native_supersedes_older_native"]) + self.assertEqual(len(self.plan(seed=99, sample_index=4, pair_count=2)["pairs"]), 2) def test_same_version_and_ambiguous_identity_are_refused(self): catalog = self.normalized() duplicate = copy.deepcopy(catalog_entries()) - duplicate["entries"].append(release("13.59", OID_D, SHA_B)) + duplicate["entries"].extend([release("13.59", OID_D, SHA_B), release("13.60", OID_D, SHA_B)]) duplicate_normalized = vr.normalize_catalog(duplicate) with self.assertRaisesRegex(vr.Refused, "at least two"): vr.eligible_releases(duplicate_normalized) @@ -82,7 +92,7 @@ def test_same_version_and_ambiguous_identity_are_refused(self): bad_plan["pairs"][0]["new"]["release"] = bad_plan["pairs"][0]["old"]["release"] # The immutable checksum catches mutation before the semantic check. with self.assertRaisesRegex(vr.Refused, "identity changed"): - vr.verify_plan(bad_plan) + vr.verify_plan(bad_plan, self.normalized()) def test_changed_catalog_identity_refuses_plan_reuse(self): catalog = self.normalized() @@ -97,15 +107,12 @@ def test_swapped_or_same_native_oracle_is_refused_after_rehash(self): plan = self.plan() pair = plan["pairs"][0] pair["native_oracles"]["old"]["native_release_identity"] = copy.deepcopy(pair["new"]) - payload = {k: v for k, v in plan.items() if k != "plan_sha256"} - plan["plan_sha256"] = vr.sha256_json(payload) - with self.assertRaisesRegex(vr.Refused, "old native oracle binding"): + self.rehash_plan(plan) + with self.assertRaisesRegex(vr.Refused, "deterministic catalog selection"): vr.verify_plan(plan, self.normalized()) def test_unselected_eligible_release_is_explicitly_untested(self): - raw = catalog_entries() - raw["entries"].append(release("13.60", OID_D, SHA_B)) - catalog = vr.normalize_catalog(raw) + catalog = self.normalized() plan = vr.make_plan(catalog, 12, 0, 1, "e" * 40) selected = {side["release"] for pair in plan["pairs"] for side in (pair["old"], pair["new"])} untested = {row["release"]: row["reason"] for row in plan["untested_eligible_releases"]} @@ -129,15 +136,73 @@ def test_interrupted_recovery_preserves_unrun_and_failure_is_accounted(self): run_dir = Path(tmp) / "run" vr.create_run(plan, self.normalized(), run_dir) vr.start_pair(run_dir, 0) - recovered = vr.recover_interrupted(run_dir, self.normalized()) - self.assertEqual(recovered["phase"], "interrupted") - self.assertTrue(all(v["read"] == v["write"] == "unrun" for v in recovered["releases"].values())) selected = plan["pairs"][0]["old"]["release"] failed = vr.record_failure(run_dir, 0, selected, "write", "native validation failed") self.assertEqual(failed["phase"], "failed") self.assertEqual(failed["releases"][selected]["write"], "failed") self.assertEqual(failed["pairs"][0]["failure"]["detail"], "native validation failed") + def test_interrupted_recovery_preserves_unrun(self): + plan = self.plan() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + vr.start_pair(run_dir, 0) + recovered = vr.recover_interrupted(run_dir, self.normalized()) + self.assertEqual(recovered["phase"], "interrupted") + self.assertTrue(all(v["read"] == v["write"] == "unrun" for v in recovered["releases"].values())) + + def test_rehashed_selector_or_scope_mutations_are_refused(self): + for mutation in ( + lambda plan: plan.__setitem__("seed", plan["seed"] + 1), + lambda plan: plan.__setitem__("pair_count", 2), + lambda plan: plan.__setitem__("pairs", []), + lambda plan: plan.__setitem__("untested_eligible_releases", []), + ): + with self.subTest(mutation=mutation): + plan = self.plan() + mutation(plan) + self.rehash_plan(plan) + with self.assertRaisesRegex(vr.Refused, "deterministic catalog selection"): + vr.verify_plan(plan, self.normalized()) + + def test_rehashed_catalog_classification_mutation_is_refused(self): + catalog = self.normalized() + catalog["entries"][0]["classification"] = {"state": "excluded", "reason": "invented"} + self.rehash_catalog(catalog) + with self.assertRaisesRegex(vr.Refused, "catalog identity changed"): + vr.verify_catalog(catalog) + + def test_journal_refuses_missing_extra_or_duplicate_pairs_releases_and_scope(self): + plan = self.plan(pair_count=2) + for mutate in ( + lambda journal: journal["pairs"].pop(), + lambda journal: journal["pairs"].append(copy.deepcopy(journal["pairs"][0])), + lambda journal: journal["pairs"].__setitem__(0, {**journal["pairs"][0], "old_release": "not-selected"}), + lambda journal: journal["releases"].pop(next(iter(journal["releases"]))), + lambda journal: journal["releases"].__setitem__("not-selected", {"read": "unrun", "write": "unrun", "state": "unrun"}), + lambda journal: journal.__setitem__("untested_eligible_releases", [{"release": "not-selected", "reason": "invented"}]), + ): + with self.subTest(mutate=mutate), tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + status_path = run_dir / "status.json" + journal = json.loads(status_path.read_text()) + mutate(journal) + status_path.write_text(json.dumps(journal)) + with self.assertRaisesRegex(vr.Refused, "journal (pairs|releases|untested scope)"): + vr.load_verified_run(run_dir) + + def test_failure_must_belong_to_active_pair(self): + plan = self.plan(pair_count=2) + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + vr.create_run(plan, self.normalized(), run_dir) + vr.start_pair(run_dir, 0) + other = plan["pairs"][1] + with self.assertRaisesRegex(vr.Refused, "active pair"): + vr.record_failure(run_dir, other["pair_index"], other["old"]["release"], "read", "wrong active pair") + def test_mutated_plan_and_journal_identity_are_refused(self): plan = self.plan() with tempfile.TemporaryDirectory() as tmp: diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py index 2c4baf644..af593ca1e 100755 --- a/tools/exiftool-tables/version_rehearsal.py +++ b/tools/exiftool-tables/version_rehearsal.py @@ -141,9 +141,13 @@ def normalize_catalog(raw: dict[str, Any]) -> dict[str, Any]: def verify_catalog(catalog: dict[str, Any]) -> None: if catalog.get("schema") != SCHEMA or not isinstance(catalog.get("entries"), list): raise Refused("unsupported normalized catalog") - expected = catalog.get("catalog_sha256") - payload = {k: v for k, v in catalog.items() if k != "catalog_sha256"} - if not isinstance(expected, str) or expected != sha256_json(payload): + raw = { + "catalog_source": catalog.get("catalog_source"), + "captured_at": catalog.get("captured_at"), + "entries": [entry.get("raw") if isinstance(entry, dict) else entry for entry in catalog["entries"]], + } + expected = normalize_catalog(raw) + if catalog != expected: raise Refused("catalog identity changed or is malformed") @@ -203,8 +207,12 @@ def _oracle_binding(variant: str, release: dict[str, Any]) -> dict[str, Any]: } -def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: int, repository_commit: str) -> dict[str, Any]: +def _plan_payload(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: int, repository_commit: str) -> dict[str, Any]: verify_catalog(catalog) + if (not isinstance(seed, int) or isinstance(seed, bool) or not 0 <= seed < 2**64 + or not isinstance(sample_index, int) or isinstance(sample_index, bool) or sample_index < 0 + or not isinstance(pair_count, int) or isinstance(pair_count, bool) or pair_count < 1): + raise Refused("plan selector inputs are malformed") if not isinstance(repository_commit, str) or not GIT_OID_RE.fullmatch(repository_commit): raise Refused("repository commit must be a full git object id") selected = select_pairs(catalog, seed, sample_index, pair_count) @@ -261,76 +269,40 @@ def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: "explicit_limit": "selection is not native read/write proof, native-delta classification, or upgrade success", }, } + return payload + + +def make_plan(catalog: dict[str, Any], seed: int, sample_index: int, pair_count: int, repository_commit: str) -> dict[str, Any]: + payload = _plan_payload(catalog, seed, sample_index, pair_count, repository_commit) return {**payload, "plan_sha256": sha256_json(payload)} def verify_plan(plan: dict[str, Any], catalog: dict[str, Any] | None = None) -> None: if plan.get("schema") != SCHEMA or plan.get("kind") != "oxidex_exiftool_version_rehearsal_plan": raise Refused("unsupported run plan") + if catalog is None: + raise Refused("catalog is required to verify a deterministic plan") + verify_catalog(catalog) + if plan.get("catalog_sha256") != catalog["catalog_sha256"]: + raise Refused("catalog identity differs from recorded plan") expected = plan.get("plan_sha256") payload = {k: v for k, v in plan.items() if k != "plan_sha256"} if not isinstance(expected, str) or expected != sha256_json(payload): raise Refused("plan identity changed or is malformed") - eligible_by_release: dict[str, dict[str, Any]] | None = None - if catalog is not None: - verify_catalog(catalog) - if plan.get("catalog_sha256") != catalog["catalog_sha256"]: - raise Refused("catalog identity differs from recorded plan") - eligible_by_release = {entry["raw"]["name"]: entry["raw"] for entry in catalog["entries"] - if entry["classification"]["state"] == "eligible"} - seen: set[str] = set() - for pair in plan.get("pairs", []): - if not isinstance(pair, dict): - raise Refused("malformed pair") - old, new = pair.get("old"), pair.get("new") - if not isinstance(old, dict) or not isinstance(new, dict): - raise Refused("pair lacks release identities") - for release in (old, new): - # Reapply the same exact requirements before future execution. - reason = _identity_reason({"name": release.get("release"), **release}) - if not isinstance(release.get("release"), str) or not RELEASE_RE.fullmatch(release["release"]) or reason: - raise Refused("pair has missing or ambiguous release identity") - if eligible_by_release is not None: - actual = eligible_by_release.get(release["release"]) - expected = {"release": actual["name"], "tag_object": actual["tag_object"], - "peeled_commit": actual["peeled_commit"], - "archive": {"url": actual["archive"]["url"], "sha256": actual["archive"]["sha256"]}} if actual else None - if expected != release: - raise Refused("pair release identity differs from catalog") - if release_key(old["release"]) >= release_key(new["release"]): - raise Refused("pair must have distinct old/new releases in numeric order") - oracles = pair.get("native_oracles") - if not isinstance(oracles, dict): - raise Refused("pair lacks native oracle bindings") - for label, release in (("old", old), ("new", new)): - binding = oracles.get(label) - if not isinstance(binding, dict) or binding.get("variant") != label: - raise Refused(f"pair lacks {label} native oracle binding") - if binding.get("native_release_identity") != release: - raise Refused(f"{label} native oracle binding does not match its selected release") - if binding.get("read_vs_native") != "unrun" or binding.get("write_vs_native") != "unrun": - raise Refused("planning plan cannot claim native comparison success") - contract = pair.get("comparison_contract") - if not isinstance(contract, dict) or contract.get("schema") != "per-version-native-v1": - raise Refused("pair lacks per-version comparison contract") - if contract.get("required") != ["oxidex_old_vs_native_old", "oxidex_new_vs_native_new", "native_old_to_native_new_delta"]: - raise Refused("pair comparison contract is incomplete") - if contract.get("cross_version_output_equality") != "not_required" or contract.get("newer_native_supersedes_older_native") is not True: - raise Refused("pair comparison contract incorrectly freezes cross-version output") - if contract.get("unsupported_new_semantics") != "explicit_gap_never_old_fallback": - raise Refused("pair comparison contract permits unsupported old fallback") - pair_key = f"{old['release']}->{new['release']}" - if pair_key in seen: - raise Refused("duplicate pair in plan") - seen.add(pair_key) + try: + canonical = _plan_payload(catalog, plan["seed"], plan["sample_index"], plan["pair_count"], plan["repository_commit"]) + except (KeyError, TypeError) as exc: + raise Refused("plan lacks deterministic selector inputs") from exc + if payload != canonical: + raise Refused("plan differs from deterministic catalog selection") def _run_id(plan: dict[str, Any]) -> str: return f"rehearsal-{plan['plan_sha256'][:16]}" -def initial_journal(plan: dict[str, Any]) -> dict[str, Any]: - verify_plan(plan) +def initial_journal(plan: dict[str, Any], catalog: dict[str, Any]) -> dict[str, Any]: + verify_plan(plan, catalog) variants: dict[str, dict[str, str]] = {} for pair in plan["pairs"]: for release in (pair["old"], pair["new"]): @@ -343,7 +315,16 @@ def initial_journal(plan: dict[str, Any]) -> dict[str, Any]: "phase": "planned", "active": None, "events": [{"event": "plan_created", "at_unix": time.time()}], - "pairs": [{"pair_index": pair["pair_index"], "state": "unrun", "failure": None} for pair in plan["pairs"]], + "pairs": [ + { + "pair_index": pair["pair_index"], + "old_release": pair["old"]["release"], + "new_release": pair["new"]["release"], + "state": "unrun", + "failure": None, + } + for pair in plan["pairs"] + ], "releases": variants, "untested_eligible_releases": plan["untested_eligible_releases"], "execution_limit": "all read/write states are unrun; planning is not proof", @@ -361,7 +342,7 @@ def create_run(plan: dict[str, Any], catalog: dict[str, Any], run_dir: Path) -> journal_path = run_dir / "status.json" atomic_json(catalog_path, catalog) atomic_json(plan_path, plan) - atomic_json(journal_path, initial_journal(plan)) + atomic_json(journal_path, initial_journal(plan, catalog)) return plan_path, journal_path @@ -375,9 +356,18 @@ def load_verified_run(run_dir: Path, catalog: dict[str, Any] | None = None) -> t raise Refused("unsupported journal") if journal.get("plan_sha256") != plan["plan_sha256"] or journal.get("run_id") != _run_id(plan): raise Refused("journal does not belong to immutable plan") - if journal.get("phase") not in {"planned", "running", "interrupted", "failed"}: + phase = journal.get("phase") + if phase not in {"planned", "running", "interrupted", "failed"}: raise Refused("journal has unsupported phase") - for release, state in journal.get("releases", {}).items(): + expected_releases = { + side["release"] + for pair in plan["pairs"] + for side in (pair["old"], pair["new"]) + } + releases = journal.get("releases") + if not isinstance(releases, dict) or set(releases) != expected_releases: + raise Refused("journal releases differ from selected plan releases") + for release, state in releases.items(): if not isinstance(release, str) or not isinstance(state, dict): raise Refused("journal has malformed release state") if state.get("read") not in {"unrun", "failed"} or state.get("write") not in {"unrun", "failed"}: @@ -385,9 +375,42 @@ def load_verified_run(run_dir: Path, catalog: dict[str, Any] | None = None) -> t expected_state = "failed" if "failed" in {state.get("read"), state.get("write")} else "unrun" if state.get("state") != expected_state: raise Refused("journal release aggregate does not match read/write states") - for pair in journal.get("pairs", []): - if not isinstance(pair, dict) or pair.get("state") not in {"unrun", "failed"}: + expected_pairs = [ + { + "pair_index": pair["pair_index"], + "old_release": pair["old"]["release"], + "new_release": pair["new"]["release"], + } + for pair in plan["pairs"] + ] + pairs = journal.get("pairs") + if not isinstance(pairs, list) or len(pairs) != len(expected_pairs): + raise Refused("journal pairs differ from selected plan pairs") + for pair, expected_pair in zip(pairs, expected_pairs, strict=True): + if not isinstance(pair, dict) or any(pair.get(key) != value for key, value in expected_pair.items()): + raise Refused("journal pairs differ from selected plan pairs") + if pair.get("state") not in {"unrun", "failed"}: raise Refused("planning journal cannot claim pair success") + failure = pair.get("failure") + if pair["state"] == "unrun" and failure is not None: + raise Refused("unrun journal pair cannot record a failure") + if pair["state"] == "failed": + if (not isinstance(failure, dict) or failure.get("release") not in {pair["old_release"], pair["new_release"]} + or failure.get("operation") not in {"read", "write"} or not isinstance(failure.get("detail"), str) + or not failure["detail"]): + raise Refused("journal failure does not belong to selected pair") + if releases[failure["release"]].get(failure["operation"]) != "failed": + raise Refused("journal pair failure disagrees with release state") + if journal.get("untested_eligible_releases") != plan["untested_eligible_releases"]: + raise Refused("journal untested scope differs from immutable plan") + active = journal.get("active") + if phase == "running": + if not isinstance(active, dict) or not isinstance(active.get("stage"), str): + raise Refused("running journal lacks an active pair") + if active.get("pair_index") not in {pair["pair_index"] for pair in expected_pairs}: + raise Refused("active journal pair is not selected") + elif active is not None: + raise Refused("non-running journal cannot retain an active pair") return plan, journal @@ -409,16 +432,18 @@ def record_failure(run_dir: Path, pair_index: int, release: str, operation: str, """Record an observed future-run failure without allowing a synthetic pass.""" if operation not in {"read", "write"}: raise Refused("failure operation must be read or write") + if not isinstance(detail, str) or not detail: + raise Refused("failure detail must be nonempty text") plan, journal = load_verified_run(run_dir) - if journal["phase"] not in {"running", "interrupted"}: - raise Refused("only a running or recovered pair can record a failure") + if journal["phase"] != "running" or journal.get("active", {}).get("pair_index") != pair_index: + raise Refused("failure must belong to the active pair") + selected_pair = next((pair for pair in plan["pairs"] if pair["pair_index"] == pair_index), None) + if selected_pair is None or release not in {selected_pair["old"]["release"], selected_pair["new"]["release"]}: + raise Refused("release does not belong to selected pair") if release not in journal["releases"]: raise Refused("release is not selected by this run") journal["releases"][release][operation] = "failed" journal["releases"][release]["state"] = "failed" - selected_pair = next((pair for pair in plan["pairs"] if pair["pair_index"] == pair_index), None) - if selected_pair is None or release not in {selected_pair["old"]["release"], selected_pair["new"]["release"]}: - raise Refused("release does not belong to selected pair") for pair in journal["pairs"]: if pair["pair_index"] == pair_index: pair.update(state="failed", failure={"release": release, "operation": operation, "detail": detail}) From c4e3bb5ad339dd00a01162398ceaec7682f80459 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:20:47 -0500 Subject: [PATCH 07/22] docs: adopt newer native behavior on ExifTool upgrades --- docs/AUTOGENERATION-PLAN.md | 16 +++- docs/AUTOGENERATION-PROGRESS.md | 104 ++++++++++++---------- docs/reference/afinfo2-production-plan.md | 25 ++++-- docs/reference/read-write-version-plan.md | 41 ++++++++- 4 files changed, 130 insertions(+), 56 deletions(-) diff --git a/docs/AUTOGENERATION-PLAN.md b/docs/AUTOGENERATION-PLAN.md index 0cf91ff34..2122ced89 100644 --- a/docs/AUTOGENERATION-PLAN.md +++ b/docs/AUTOGENERATION-PLAN.md @@ -99,6 +99,15 @@ binaries against the newer oracle; add a separate non-promoting rehearsal that regenerates both releases and checks each against its own native read/write behavior. Selection or successful generation alone is not conformance. +The selected newer ExifTool is the authority after upgrading. Its parsing bug +fixes, added or renamed tags, type changes, formatting and write semantics +must replace older behavior. Keep three comparisons: old OxiDex versus old +native, new OxiDex versus new native, and the native old-to-new delta. A change +in native output is an expected upstream change when the new generated build +matches it. Cross-version output equality is not required. A new unsupported +rule is a visible gap requiring shared compiler/runtime work; silently using +the old tag rule is not an acceptable upgrade. + Once that runner passes its own tests, exercise a randomly selected distinct release pair after three relevant merged batches or one week, whichever comes first. The existing hourly continuation records this cadence without launching @@ -114,10 +123,11 @@ been established. | --- | --- | --- | | Shared table compiler and reader | Already exist; some families use them | We have a foundation to extend instead of building a new interpreter for every camera brand. | | Sony focus-table pilot | Merged in PR #746 at `04eaf6e1`, including removal of 17 duplicate entries. Final CI is green at `a1626cb6`; the 4,238-file pair records ten raw-ID fixes and no other per-file changes. | The shared route now replaces the duplicate Sony producer. Remaining source-inventory work is broader than this pilot. | -| Shared binary strings | Merged in PR #747 at `8f0fdaf4`. It preserves raw bytes in saved state, distinguishes a one-byte default from a remainder string, and carries the bounded CameraInfo repair. | This is a shared capability, not a Canon migration. CameraInfo remains a legacy text-domain adapter, and no Canon manual reader has been removed. The prior full-pair supervisor-status limitation remains recorded. | +| Shared binary strings | Merged in PR #747 at `8f0fdaf4`. It preserves raw bytes in saved state, distinguishes a one-byte default from a remainder string, and carries the bounded CameraInfo repair. | This is a shared capability, not a Canon migration. CameraInfo remains a legacy text-domain adapter, and that batch removed no Canon manual reader. The subsequent AFInfo2/AFInfo3 retirement is listed separately. The prior full-pair supervisor-status limitation remains recorded. | | Keyed-directory schema and compiler | Merged in PR #748 at `ebbe1ece`; all final hosted checks passed at `a422e8de`. | Native parent facts and expression declarations are checked. Shared reporting policy merged in #750; the inactive reader merged in #752 at `634e5616`. No production route is active. | -| Shared word-directory processor | Merged in PR #754 at `1138a880`; nine tables, 132 rows, 698 Python tests and all five hosted jobs pass. | Four of five unsupported child processors now have generated descriptors. Canon production routing and manual-reader retirement remain unfinished. | -| Shared serial processor | Foundation merged in #757; #759 at `8887e5d9` expands the eight-table total to 122 emitted alternatives and 10 omissions, with all five hosted checks passing. Canon AFInfo 14/14 and AFInfo2 16/16 are generated and natively replayed. | Real AudioV4 is a validated production caller. Canon's next delivery is complete parent routing and manual AFInfo2 reader retirement; definition readiness alone does not count as that migration. | +| Shared word-directory processor | Merged in PR #754 at `1138a880`; nine tables, 132 rows, 698 Python tests and all five hosted jobs pass. | Four of five unsupported child processors now have generated descriptors. The separate Canon AFInfo2/AFInfo3 serial migration merged in #760; other word-directory callers remain separate work. | +| Shared serial processor | Foundation merged in #757; #759 at `8887e5d9` expands the eight-table total to 122 emitted alternatives and 10 omissions, with all five hosted checks passing. Canon AFInfo 14/14 and AFInfo2 16/16 are generated and natively replayed. | Real AudioV4 is a validated production caller. Canon AFInfo2/AFInfo3 now uses the generated route after #760. Old AFInfo geometry and CanonRaw omissions remain unfinished. | +| Canon AFInfo2/AFInfo3 retirement | Merged in #760 at `4a3eb26c`; all five required hosted checks passed on `2630ded8`. One shared manual arm, eight offsets, two parent IDs and a 20-value enum removed. | Exact native and bounded/full corpus evidence passes. A supported native name change reaches output through regeneration. This does not activate generated writing or establish a new project-wide percentage. | | Real AudioV4 retirement | Merged in #758 at `19cb7650`; one manual reader and its 31-slot sequence removed. Full corpus: one Copyright correction, 4,237 other files unchanged. All five hosted checks pass. | Supported source-name changes reach actual output after regeneration with no tag-specific Python/Rust edit. Native occurrence groups, warning output and V3/V5 remain explicit residuals. | | Recorded source inventory | Merged in PR #749 at `18a8ef17`; all final hosted checks passed at `72e8e664`. The report accounts for 1,512 table identities and retains 119 tables with no named rows. | This establishes the captured source population. Classifying which rules are generated, manual, unsupported or unclassified remains open; source shape is not automation. | | Sony plain generator recovery | PR #745 merged; six tables and 193 rows reproduced | These tables can be rebuilt. This alone does not prove that their behavior is fully automatic. | diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index 7d35e6094..7ea69165e 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -4,47 +4,61 @@ This is the working scoreboard for [the plan](AUTOGENERATION-PLAN.md). ## Current checkpoint — September 13 -The current Canon AFInfo2/AFInfo3 reader retirement is committed and published -at `ff1e364c`, but not merged. It deletes the shared manual reader arm, eight +Canon AFInfo2/AFInfo3 reader retirement **merged in PR #760** as `4a3eb26c` +at 16:11:54 UTC (11:11:54 CDT). All five required hosted checks passed on +`2630ded8`, including 777 canonical Python tests with zero failures/skips +in 643.787 seconds. The merged change deletes the shared manual reader arm, eight private sequence offsets, two parent IDs and the private 20-value mode enum. -The generated reader now owns these two parent routes. Independent source -review, exact public-fixture native replay, the 67-file bounded pair, the full +Generated tables now supply those two parent routes. Independent source +review, complete-carrier native replay, the 67-file bounded pair, the full 4,238-file pair and all 32-artifact regeneration checks pass. Full corpus output is unchanged; the bounded pair adds two correct rows and removes 40 -extras. The 140 focused Python tests execute the actual artifact mutation -checks with no skips. A copied native field rename also reaches actual output after official -regeneration with no handwritten tag-rule edit. Final hosted acceptance remains. See the -[production plan](reference/afinfo2-production-plan.md) for failed attempts, -exact evidence, and remaining old-AFInfo/CanonRaw scope. - -These results concern reading. The plan now explicitly shares source definitions -between reading and writing, while tracking write execution and preservation -separately. Two independent source audits are identifying existing writer rules -and missing shared-schema facts. No generated-writing percentage or full -read/write delivery estimate exists yet. The earlier 97.3% reading conformance -and route-disabled measurements say nothing about write support. - -The ready retirement PR is #760. Its first hosted test run found an old -synthetic AFInfo2 record with a zero size field and positive output assertions. -Pinned native complete-carrier replay proved those assertions invalid. -The repair at `78fb7921` keeps a rejection case and supplies the correct size -for the positive case. The full local Cargo unit/integration/doc invocation -passes: 5,993 passes, zero failures, 124 ignored, 150.641 seconds. The required -hosted checks are being rerun; do not count this as a merged retirement yet. - -The write audits now identify the first vertical pilot as Exif::Main -HostComputer `0x013c`, whose read behavior has no omitted conversion. They -also identify lost placement/inverse/delete/create source facts and unresolved -writer/checker implementation binding as prerequisites. A dedicated native -write-fact capture is in implementation; no generated writer has been enabled. -The existing bump tool is a promotion comparison against the newer oracle, -not per-version native proof. Seeded release-plan/journal primitives and an -independent native write contract are being prepared in parallel. The -[version plan](reference/read-write-version-plan.md) accounts for both -read/write execution and all requested release scope without promoting random -samples to exhaustive evidence. - -### Latest merged checkpoint +extras. A copied native field rename reaches actual output after official +regeneration with no handwritten tag-rule edit. See the +[production record](reference/afinfo2-production-plan.md) for exact evidence +and remaining old-AFInfo/CanonRaw scope. No project-wide autogenerated +percentage has been remeasured. + +The first hosted run failed an old synthetic test that declared zero bytes +while asserting valid output. Native complete-carrier replay confirmed that +fixture was invalid. The repair retains the rejection test and corrects the +positive fixture. Full local Cargo validation passed 5,993 tests with zero +failures and 124 ignored, in 150.641 seconds. The failed attempt is preserved; +all five required hosted checks then passed before merge. + +The next delivery is a complete generated read/write route for the Exif::Main +scalar class, beginning with HostComputer `0x013c`. Source capture and upgrade +planning are integrated work-branch checkpoints, not activated writing. Review +found flattened scalar references and a loader recognizer that accepted changed +executable behavior. Repairs at `04a694e3` passed independent mutation review +and are integrated as `468a114b`; combined regeneration remains to be checked. +Full canonical capture proves +that the serialized read projection for all 153 modules is unchanged; this is +not proof about every live binding after writer loading. + +The native writer contract covers II/MM JPEG and TIFF copies, insertion, +replacement, growth, shrinkage, deletion and payload preservation. A defined +empty API value creates a present NUL ASCII entry; CLI unset deletes instead. +An independent baseline of the existing OxiDex writer found that +`EXIF:HostComputer=` reports success but leaves the tag in all four carriers. +`IFD0:HostComputer=` deletes on JPEG and explicitly refuses on TIFF. The new +generated route must account for these operation differences; successful +set/update calls alone cannot certify it. + +Upgrades must adopt the selected newer ExifTool's parsing fixes, new tags, +types, formatting and writable behavior. Old OxiDex is checked against old +native, new OxiDex against new native, and their native release delta identifies +intentional upstream changes. Unsupported new semantics remain explicit gaps; +preserving old semantics silently is not a successful upgrade. The published +seeded planner records this contract but has no executable rehearsal stages +yet. Review found plan/journal validation gaps; the repair at `44cf6135` +passed 17 focused tests and five independent altered-plan checks, and is +integrated as `21eed548`. Full source identity capture and execution remain +unfinished. The +[version plan](reference/read-write-version-plan.md) separates these unfinished +stages and never treats random samples as proof of all releases. + +### Previous merged definitions checkpoint Latest combined Canon definitions: **AFInfo 14/14 and AFInfo2 16/16**, both independently verified after formatting. Across all eight serial tables, @@ -56,10 +70,10 @@ native/Rust replays. Official regeneration and local shared-reader checks also pass. The definitions are merged; a Canon production carrier is not yet enabled by that merge. The [Canon plan](reference/serial-afinfo-plan.md) records the failed pipeline -attempt, its formatting repair and the remaining acceptance work. The active -[production migration](reference/afinfo2-production-plan.md) connects the -authenticated parent edges, verifies complete output and removes the AFInfo2 -manual reader in one accepted batch. A parent connection alone retires zero +attempt, its formatting repair and the remaining acceptance work. The subsequent +[production migration](reference/afinfo2-production-plan.md) merged in #760, +connecting authenticated parent edges and removing the manual AFInfo2 reader +after complete-output verification. A parent connection alone retires zero manual output code and does not meet that delivery's goal. The following entries retain the completed milestones and their exact evidence. @@ -74,9 +88,9 @@ This is definition coverage; production routing remains inactive. Four of the parent's five unsupported child processors now have generated descriptors. One dynamic-length processor and four omitted parent rows remain. -No Canon manual reader has been retired and no project-wide percentage has -been remeasured. The next measurable results are the remaining processor, -real-carrier verification and removal of duplicate readers. +At the #754 definition checkpoint, no Canon manual reader had been retired. +The subsequent #760 production migration is recorded above. No project-wide +percentage has been remeasured. The combined source at `f613820d` passes 25 Rust reader tests and the explicit native/Rust comparison passes all seven cases with the real generated tables. diff --git a/docs/reference/afinfo2-production-plan.md b/docs/reference/afinfo2-production-plan.md index af9b16d38..34210c117 100644 --- a/docs/reference/afinfo2-production-plan.md +++ b/docs/reference/afinfo2-production-plan.md @@ -1,6 +1,7 @@ # Replace Canon's manual AFInfo2 reader -Updated September 13, 2026. Base: `8887e5d9` (merged PR #759). +Updated September 13, 2026. Merged in PR #760 as `4a3eb26c` at 16:11:54 UTC. +Base: `8887e5d9` (PR #759); all five required hosted checks passed on `2630ded8`. ## Goal and finish conditions @@ -70,8 +71,8 @@ the complete migration and its gates. ## Current state -The source checkpoint is published on -`codex/afinfo2-production-integration-20260913`. Integration removes the manual +The source from `codex/afinfo2-production-integration-20260913` is merged +in PR #760. Integration removes the manual AFInfo2/AFInfo3 arm, eight private sequence offsets, two parent-ID constants and the private 20-value AFAreaMode enum. The shared reader is the sole replacement. An independent source review accepts the ownership and rollback design. @@ -96,8 +97,8 @@ incorrectly emitted as executable. The generator now leaves that distinct edge explicitly unwalked. The processor-oracle repair is integrated at `62535db7`; independent whole-batch source review accepts `ff1e364c`. Exact exported-fixture checks, the pair after retirement and full corpus are accepted -below. Final hosted acceptance remains required before merge. -No Canon reader is counted as merged retirement yet; no project-wide percentage +below. All five final hosted checks passed on `2630ded8` before squash merge. +The AFInfo2/AFInfo3 reader retirement is merged; no project-wide percentage follows from this bounded work. The exact seven TIFF byte vectors used by the public-reader tests are now @@ -174,3 +175,17 @@ passes in 150.641 seconds: 5,993 passes across unit/integration/doc summaries, zero failures and 124 ignored tests. The attempted local nextest invocation could not run because that executable is not installed; its failed attempt is retained. Hosted nextest and all required checks must pass on the final head. + + +## Final publication and merge + +PR #760 squash-merged as `4a3eb26c7bb1a066bfbf22e41b0404ab63e6e351` +at 16:11:54 UTC on September 13. The final tested head was +`2630ded84a96c2973d3130cad2a26ce3629f01c7`. Lint & Audit, Build & Test, +Release Build, Verify Generated Tables and clean-checkout docs all passed. +The canonical Python stage ran 777 tests with zero failures/skips in 643.787 +seconds. +The guarded waiter verified exact head/base and clean tracked source before +merging. BATCH's `landing-retry-state.json`, `merge-verified.json` and +`hosted-logs-retry/` retain the result and full job logs. Earlier failures remain +separate evidence; old AFInfo geometry and CanonRaw scope remain open. diff --git a/docs/reference/read-write-version-plan.md b/docs/reference/read-write-version-plan.md index c2b1132bd..26102674c 100644 --- a/docs/reference/read-write-version-plan.md +++ b/docs/reference/read-write-version-plan.md @@ -17,11 +17,27 @@ catalog and per-version results. Missing source, unsupported semantics, unexercised behaviors and failed versions remain visible. Native read-only fields are explicitly ineligible for writes, not failed writer implementations. +## Newer native behavior takes precedence + +An upgrade adopts the newer native release's parsing corrections, new tags, +type and formatting changes, and writable behavior. Do not preserve an old +result merely because it once matched an older oracle. Test three relationships: + +- Old generated OxiDex against the old native ExifTool. +- New generated OxiDex against the new native ExifTool. +- Native old versus native new, to identify intentional upstream changes. + +Cross-version equality is not a passing requirement. The new build must match +the new oracle. If the generator cannot represent a changed rule, record that +unsupported behavior explicitly and extend the shared machinery. Do not hide it +by retaining the old tag-specific implementation. A sampled pair passing does +not certify other releases or behaviors. + ## Work in order, with independent tasks in parallel -1. Finish Canon AFInfo2/AFInfo3 reader retirement through PR #760. Preserve the - failed legacy synthetic test, correct its invalid size field with native - evidence, run full tests and merge only after the required checks pass. +1. **Done:** Canon AFInfo2/AFInfo3 reader retirement merged in PR #760 at + `4a3eb26c`, with all five required hosted checks passing on `2630ded8`. + Preserve the failed legacy fixture, native correction and acceptance record. 2. Capture one complete source model for both directions. Retain permissions, actual writer/checker functions, placement, forward and inverse conversions, validation, insertion/deletion and ordering rules. Keep unknown property @@ -32,6 +48,11 @@ fields are explicitly ineligible for writes, not failed writer implementations. placement and write type; use the existing JPEG/TIFF surgical mechanisms. Verify insert, update, growth, shrinkage and deletion against native ExifTool in both byte orders, preserving unrelated metadata and image/file payload. + Keep defined-empty values distinct from deletion. Both EXIF family names + and physical IFD names must address the same generated identity; the + pre-migration EXIF-qualified deletion silently succeeds without deleting, + while TIFF deletion is explicitly unsupported. Preserve these as baseline + failures until the complete generated operation is implemented. A copied native name/type/placement change must propagate without another hand-written rule. Retire the replaced manual lookup after proof. 4. Expand shared capabilities and migrate eligible read/write families through @@ -76,3 +97,17 @@ Report the complete catalog population, tested versions/pairs, failures, untested releases and unexercised read/write behaviors. Keep corpus attribution, read conformance, write conformance and source-rule automation as separate measurements. No exact completion date follows from the current partial data. + + +## Implementation checkpoint after the reader merge + +The native write-fact sidecar and offline seeded planner are integrated on +`codex/read-write-upgrade-integration-20260913`. Independent review accepted the +complete loader-token grammar after rejecting two earlier bypasses. The full +153-module read projection is unchanged. Planner repair checks exact catalog +selection, matching-version oracle bindings, selected journal membership and +untested scope; 17 focused tests and five independent altered-plan checks pass. +These are source and planning foundations: writer activation, official live +catalog/source capture, both-version regeneration/builds and real read/write +comparisons are still required. Combined official regeneration is the next +integration check. No successful release upgrade is claimed by a saved plan. From cb18fb91362ddd4f930acae1583313e795b0052d Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:49:04 -0500 Subject: [PATCH 08/22] Record native dump identity after writer fact capture --- tools/exiftool-tables/expr_oracle_ledger.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/exiftool-tables/expr_oracle_ledger.json b/tools/exiftool-tables/expr_oracle_ledger.json index 6199ab8a0..a95e538f0 100644 --- a/tools/exiftool-tables/expr_oracle_ledger.json +++ b/tools/exiftool-tables/expr_oracle_ledger.json @@ -21,7 +21,7 @@ "skip": 14 }, "schema": 2, - "tables_sha256": "8a3bc83b98faf4382a4e91ea0c406e8883ab3cc938e3a18de25cc4012c0a3ae2", + "tables_sha256": "86d12adec98133e5299bb5ac16a49dc24b20a658e7b669335a67bf05064fcb0c", "use_counts": { "total": 7377, "verified": 5511 From b6d364ca77bc39367766d246589258db6c27e76c Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:33:48 -0500 Subject: [PATCH 09/22] feat(tables): stage inactive write descriptors --- tools/exiftool-tables/WRITE_DESCRIPTOR_API.md | 31 ++ tools/exiftool-tables/codegen.py | 26 + .../exiftool-tables/test_write_descriptors.py | 253 ++++++++++ tools/exiftool-tables/write_descriptors.py | 462 ++++++++++++++++++ 4 files changed, 772 insertions(+) create mode 100644 tools/exiftool-tables/WRITE_DESCRIPTOR_API.md create mode 100644 tools/exiftool-tables/test_write_descriptors.py create mode 100644 tools/exiftool-tables/write_descriptors.py diff --git a/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md b/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md new file mode 100644 index 000000000..32e58ea55 --- /dev/null +++ b/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md @@ -0,0 +1,31 @@ +# Inactive write descriptor API + +`write_descriptors.py` consumes `native_write_tables` and may be selected with +`codegen.py --write-out`. Its generated Rust is not re-exported by OxiDex and +creates no writer route. It is an input candidate for a later, independently +verified writer-mechanism contract. + +The first closed class is a plain `Exif::Main` scalar row with native +`Writable => 'string'` and a literal effective `WriteGroup` in `IFD0`, `ExifIFD`, +or `GPS`. Each +candidate carries only source-derived `raw_id`, `name`, `WritePhysicalGroup`, +and `WriteValueType::Ascii`, together with actual loaded autoload/`WRITE_PROC`/ +`CHECK_PROC` provenance: name, relative source file, source SHA-256, B::Deparse +SHA-256, and captured direct dependencies. + +Those procedure facts are **not** a writer admission and are not interpreted +as an implementation of `WriteExif` or `CheckExif`. A later writer must require +its own closed native-mechanism contract before it can use a candidate. + +Every source table and every source row alternative not in the initial class is +recorded in `OMITTED_WRITE_NATIVE_TABLES` or `OMITTED_WRITE_NATIVE_ROWS` with +named reasons. Reader omission flags are never used for this accounting. The +sidecars preserve source identity, including zero-row tables and array +alternative order. + +The descriptor faithfully carries ordinary native physical `IFD0`, `ExifIFD`, +and `GPS` group strings. A future runtime may stage those writer primitives +separately; their presence here is not an activation claim. A changed name or +compatible added string row produces a changed descriptor; an unmodeled +physical group, type, write control, unknown property, or unresolved procedure +provenance is a refusal rather than a guessed writer operation. diff --git a/tools/exiftool-tables/codegen.py b/tools/exiftool-tables/codegen.py index aff995d86..6eabcc30b 100644 --- a/tools/exiftool-tables/codegen.py +++ b/tools/exiftool-tables/codegen.py @@ -4087,6 +4087,10 @@ def main(): "--keyed-out", help="write source facts for native keyed directories (schema/inventory only; no reader activation)", ) + ap.add_argument( + "--write-out", + help="write inactive source-derived write candidates; this does not activate a writer route", + ) args = ap.parse_args() with open(args.tables_json, encoding="utf-8") as fh: @@ -4198,6 +4202,15 @@ def main(): + "\n];\n" ) + # Compile the inactive write artifact before writing any requested output. + # A malformed write sidecar must not leave a partially refreshed binary + # artifact behind when --write-out was explicitly requested. + write_src = None + write_report = None + if args.write_out: + import write_descriptors + write_src, write_report = write_descriptors.generate(doc, names) + # Stamp the release these tables came from. ExifTool renames fields and # inserts enum values between releases, so verifying against a different # one reports hundreds of differences that read as generator bugs rather @@ -4235,6 +4248,19 @@ def main(): fh.write(ifd_index) print(f"wrote IFD tables {args.ifd_out}") + if args.write_out: + # This artifact is deliberately optional and inactive. The source was + # compiled before any output write, so a refusal cannot half-refresh + # the ordinary binary/IFD/keyed artifacts. + with open(args.write_out, "w", encoding="utf-8") as fh: + fh.write(write_src) + print(f"wrote inactive write descriptors {args.write_out}") + print( + " write candidates: " + f"tables={write_report.emitted_tables}, rows={write_report.emitted_rows}; " + f"omitted tables={write_report.omitted_tables}, rows={write_report.omitted_rows}" + ) + if args.keyed_out: # Normal regeneration always requests this output; focused generator # callers may omit it. It carries source facts and omissions, not a diff --git a/tools/exiftool-tables/test_write_descriptors.py b/tools/exiftool-tables/test_write_descriptors.py new file mode 100644 index 000000000..94e14b470 --- /dev/null +++ b/tools/exiftool-tables/test_write_descriptors.py @@ -0,0 +1,253 @@ +"""Focused source-only checks for inactive write candidates.""" + +import copy +import unittest + +import write_descriptors + + +CONTROLS = ( + "CanCreate", "DelValue", "Deletable", "Mandatory", "PrintConvInv", + "RawConvInv", "Validate", "ValueConvInv", "Writable", "WriteAlso", + "WriteCheck", "WriteCondition", "WriteGroup", "WriteHook", "WriteLast", + "WritePseudo", +) + + +def absent_controls(): + return {key: {"present": False} for key in CONTROLS} + + +def code_fact(name, body="sub body", *, resolved=True, dependencies=None): + result = { + "__perl": "CODE", "__name": name, "resolved": resolved, + "__deparse": body if resolved else None, + "source_file": "Image/ExifTool/WriteExif.pl" if resolved else None, + "source_sha256": "a" * 64 if resolved else None, + } + if not resolved: + result["reason"] = "code_ref_unavailable" + if dependencies is not None: + result["dependencies"] = dependencies + return result + + +def procedures(*, resolved=True, dependency=False): + dependencies = ( + {"Image::ExifTool::Exif::Helper": code_fact("Image::ExifTool::Exif::Helper", "helper")} + if dependency else {} + ) + return { + "effective_write_proc": { + "present": True, + "effective": code_fact("Image::ExifTool::Exif::WriteExif", "write-body", resolved=resolved, + dependencies=dependencies), + }, + "effective_check_proc": { + "present": True, + "effective": code_fact("Image::ExifTool::Exif::CheckExif", "check-body", resolved=resolved, + dependencies=dependencies), + }, + } + + +def row(name="SourceName", writable="string", group="IFD0", *, raw_controls=None, extra=None, unknown=None): + controls = absent_controls() + controls["Writable"] = {"present": True, "value": writable} + controls["WriteGroup"] = {"present": True, "value": group} + controls.update(raw_controls or {}) + properties = { + "Name": {"present": True, "value": name}, + "Writable": {"present": True, "value": writable}, + "WriteGroup": {"present": True, "value": group}, + } + properties.update(extra or {}) + return { + "entry_kind": "HASH", "properties": properties, + "write_controls": controls, "unknown_properties": unknown or {}, + } + + +def table(rows=None, *, module="Exif", table_name="Main", resolved=True, unknown_table=None, dependency=False): + table_controls = { + "WRITABLE": {"present": False}, + "WRITE_GROUP": {"present": True, "value": "ExifIFD"}, + "WRITE_PROC": {"present": True, "value": {"__perl": "CODE"}}, + "CHECK_PROC": {"present": True, "value": {"__perl": "CODE"}}, + } + result = { + "module": module, "table": table_name, "full_name": f"Image::ExifTool::{module}::{table_name}", + "table_properties": { + "GROUPS": {"present": True, "value": {"0": "EXIF"}}, + "SET_GROUP1": {"present": True, "value": "IFD0"}, + "WRITE_GROUP": table_controls["WRITE_GROUP"], + "WRITE_PROC": table_controls["WRITE_PROC"], + "CHECK_PROC": table_controls["CHECK_PROC"], + }, + "write_controls": table_controls, + "unknown_table_properties": unknown_table or {}, + "rows": rows if rows is not None else {"316": row()}, + } + result.update(procedures(resolved=resolved, dependency=dependency)) + return result + + +def document(tables=None, *, router_supported=True): + return { + "native_write_autoload": { + "supported": router_supported, + "router": code_fact("Image::ExifTool::DoAutoLoad", "router-body", dependencies={}), + }, + "native_write_tables": tables if tables is not None else {"Exif": {"Main": table()}}, + } + + +def population(doc): + # The renderer is intentionally a pure source compiler. These local + # reverse checks inspect its unambiguous emitted literals rather than + # reconstructing a Rust artifact by hand. + source, report = write_descriptors.generate(doc) + return source, report + + +class InactiveWriteDescriptorTests(unittest.TestCase): + def test_compatible_source_row_emits_dynamic_name_id_placement_and_provenance(self): + source, report = population(document({"Exif": {"Main": table(dependency=True)}})) + self.assertEqual(report.emitted_tables, 1) + self.assertEqual(report.emitted_rows, 1) + self.assertIn('raw_id: 0x013c, name: "SourceName", physical_group: WritePhysicalGroup::IFD0', source) + self.assertIn('name: "Image::ExifTool::Exif::WriteExif"', source) + self.assertIn('name: "Image::ExifTool::Exif::Helper"', source) + self.assertIn('INACTIVE_WRITE_RUNTIME_STATUS: &str = "inactive_source_candidates_no_writer_route"', source) + self.assertIn('do not activate a writer or authenticate the', source) + + def test_name_and_compatible_new_row_follow_source_without_tag_allowlist(self): + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["rows"] = { + "317": row(name="UpgradeName"), + "400": row(name="AdditionalSourceRow"), + } + source, report = population(changed) + self.assertEqual(report.emitted_rows, 2) + self.assertIn('raw_id: 0x013d, name: "UpgradeName"', source) + self.assertIn('raw_id: 0x0190, name: "AdditionalSourceRow"', source) + self.assertNotIn("HostComputer", source) + + def test_type_and_group_changes_are_explicit_refusals(self): + changed = document() + rows = changed["native_write_tables"]["Exif"]["Main"]["rows"] + rows["316"] = row(writable="int16u") + rows["317"] = row(name="Elsewhere", group="UnmodeledGroup") + source, report = population(changed) + self.assertEqual(report.emitted_rows, 0) + self.assertEqual(report.omitted_rows, 2) + self.assertIn('"write_value_type"', source) + self.assertIn('"write_physical_group_unmodeled"', source) + self.assertIn('"write_table_has_no_admitted_rows"', source) + + def test_ordinary_native_group_is_data_not_a_runtime_admission(self): + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["rows"]["316"] = row(group="ExifIFD") + source, report = population(changed) + self.assertEqual(report.emitted_rows, 1) + self.assertIn('physical_group: WritePhysicalGroup::ExifIFD', source) + + def test_table_write_group_default_is_source_placement(self): + changed = document() + candidate = changed["native_write_tables"]["Exif"]["Main"]["rows"]["316"] + candidate["properties"].pop("WriteGroup") + candidate["write_controls"]["WriteGroup"] = {"present": False} + group_default = {"present": True, "value": "GPS"} + changed["native_write_tables"]["Exif"]["Main"]["write_controls"]["WRITE_GROUP"] = group_default + changed["native_write_tables"]["Exif"]["Main"]["table_properties"]["WRITE_GROUP"] = group_default + source, report = population(changed) + self.assertEqual(report.emitted_rows, 1) + self.assertIn('physical_group: WritePhysicalGroup::GPS', source) + + def test_mismatched_source_projections_cannot_select_the_convenient_copy(self): + changed = document() + candidate = changed["native_write_tables"]["Exif"]["Main"]["rows"]["316"] + candidate["properties"]["Writable"] = {"present": True, "value": "int16u"} + source, report = population(changed) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_row_control_projection_mismatch"', source) + + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["table_properties"]["WRITE_GROUP"] = { + "present": True, "value": "GPS" + } + source, report = population(changed) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_table_control_projection_mismatch"', source) + + def test_reference_shaped_group_is_not_collapsed_to_literal(self): + changed = document() + candidate = changed["native_write_tables"]["Exif"]["Main"]["rows"]["316"] + candidate["properties"]["WriteGroup"] = {"present": True, "value": {"__ref": "SCALAR", "value": "IFD0"}} + candidate["write_controls"]["WriteGroup"] = {"present": True, "value": {"__ref": "SCALAR", "value": "IFD0"}} + source, report = population(changed) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_physical_group_missing_or_nonliteral"', source) + + def test_unknown_conversion_and_writer_controls_are_withheld(self): + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["rows"]["316"] = row( + extra={"RawConvInv": {"present": True, "value": "$val"}}, + raw_controls={"RawConvInv": {"present": True, "value": "$val"}}, + unknown={"FutureSwitch": {"present": True, "value": "1"}}, + ) + source, report = population(changed) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_row_property_RawConvInv"', source) + self.assertIn('"write_row_control_RawConvInv"', source) + self.assertIn('"write_row_unknown_property_FutureSwitch"', source) + + def test_variants_and_zero_row_tables_are_conserved_in_omissions(self): + first = row(name="First") + second = row(name="Second", writable="int16u") + main = table(rows={"320": {"entry_kind": "ARRAY", "alternatives": [first, second]}}) + zero = table(rows={}, module="Exif", table_name="Zero") + foreign = table(rows={"1": row(name="Foreign")}, module="Other", table_name="Main") + doc = document({"Exif": {"Main": main, "Zero": zero}, "Other": {"Main": foreign}}) + source, report = population(doc) + self.assertEqual(report.emitted_rows, 1) + self.assertIn('raw_id: "320", variant: true, alternative: 1, name: Some("Second")', source) + self.assertIn('OmittedWriteNativeTable { module: "Exif", table: "Zero"', source) + self.assertIn('OmittedWriteNativeTable { module: "Other", table: "Main"', source) + self.assertIn('"write_source_class_unimplemented"', source) + + def test_unresolved_provenance_and_router_are_named_refusals(self): + unresolved = document({"Exif": {"Main": table(resolved=False)}}) + source, report = population(unresolved) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_provenance_unresolved"', source) + router = document(router_supported=False) + source, report = population(router) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_provenance_unresolved"', source) + + def test_procedure_body_change_changes_emitted_provenance(self): + before, _ = population(document()) + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["effective_write_proc"]["effective"]["__deparse"] = "changed-write-body" + after, _ = population(changed) + self.assertNotEqual(before, after) + + def test_symbolic_write_alias_is_a_named_row_omission_not_a_dump_abort(self): + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["rows"]["Alias"] = row(name="SourceAlias") + source, report = population(changed) + self.assertEqual(report.emitted_rows, 1) + self.assertEqual(report.omitted_rows, 1) + self.assertIn('raw_id: "Alias"', source) + self.assertIn('"write_raw_id"', source) + + def test_malformed_native_facts_fail_loudly_not_as_empty_candidates(self): + malformed = document() + del malformed["native_write_tables"]["Exif"]["Main"]["rows"]["316"]["write_controls"]["Writable"]["present"] + with self.assertRaisesRegex(write_descriptors.WriteDescriptorError, "Writable.present"): + population(malformed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/write_descriptors.py b/tools/exiftool-tables/write_descriptors.py new file mode 100644 index 000000000..25246e979 --- /dev/null +++ b/tools/exiftool-tables/write_descriptors.py @@ -0,0 +1,462 @@ +"""Inactive, source-derived descriptors for a deliberately tiny write class. + +This module consumes ``native_write_tables`` from :mod:`dump_tables.pl`. It +creates no writer route and it does not claim that a captured native writer +body is understood. The initial descriptor class is intentionally closed: +plain scalar ASCII rows in ``Exif::Main`` with an effective physical +``WriteGroup`` represented by a literal native string. Everything outside +that class remains in a named omission sidecar. + +The compiler has no tag-name or raw-id allowlist. A compatible added or +renamed source row is represented; a changed type, placement, conversion, or +other write control is refused with a source-derived reason. The emitted +procedure fingerprints are provenance for a later writer-mechanism contract, +not an admission to execute those procedures. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +import hashlib +import json +import re +from typing import Any, Mapping + + +DESCRIPTOR_VERSION = 1 +RUNTIME_STATUS = "inactive_source_candidates_no_writer_route" +_SOURCE_MODULE = "Exif" +_SOURCE_TABLE = "Main" +_SUPPORTED_WRITABLE = "string" +_SUPPORTED_VALUE_TYPE = "Ascii" + +# These are source fact keys, not an assertion that every other property is +# harmless. Any new property is deliberately named and withheld. +_ALLOWED_TABLE_PROPERTIES = frozenset( + {"GROUPS", "SET_GROUP1", "WRITE_GROUP", "WRITE_PROC", "CHECK_PROC"} +) +_ALLOWED_ROW_PROPERTIES = frozenset({"Name", "Writable", "WriteGroup"}) +_ALLOWED_ROW_CONTROLS = frozenset({"Writable", "WriteGroup"}) +_ID_RE = re.compile(r"^(?:0x[0-9A-Fa-f]+|[0-9]+)$") + + +class WriteDescriptorError(ValueError): + """The captured write sidecar is malformed rather than merely unsupported.""" + + +@dataclass(frozen=True) +class WriteDescriptorReport: + tables_seen: int + candidate_tables: int + emitted_tables: int + emitted_rows: int + omitted_tables: int + omitted_rows: int + omissions_by_reason: dict[str, int] + + +def _mapping(value: Any, context: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise WriteDescriptorError(f"{context} is not an object") + return value + + +def _bool(value: Any, context: str) -> bool: + if not isinstance(value, bool): + raise WriteDescriptorError(f"{context} is not a boolean") + return value + + +def _fact_value(fact: Any, context: str) -> tuple[bool, Any]: + fact = _mapping(fact, context) + present = _bool(fact.get("present"), f"{context}.present") + if present: + if "value" not in fact: + raise WriteDescriptorError(f"{context} is present without a value") + return True, fact["value"] + if "value" in fact: + raise WriteDescriptorError(f"{context} is absent but has a value") + return False, None + + +def _plain_string(value: Any) -> str | None: + # A SCALAR ref such as ``\\'IFD0'`` deliberately does not collapse to its + # inner string: native write behavior may distinguish the reference. + return value if isinstance(value, str) else None + + +def _raw_u16(raw_id: str) -> int: + if not isinstance(raw_id, str) or _ID_RE.fullmatch(raw_id) is None: + raise WriteDescriptorError(f"write row id {raw_id!r} is not a literal integer") + value = int(raw_id, 0) + if not 0 <= value <= 0xFFFF: + raise WriteDescriptorError(f"write row id {raw_id!r} is outside u16") + return value + + +def _row_sort_key(raw_id: str) -> tuple[int, int | str]: + # Native write tables also contain symbolic lookup aliases. They are + # legitimate source rows but not physical u16 entries, so retain them in + # the deterministic omission sidecar rather than aborting the whole dump. + try: + return 0, _raw_u16(raw_id) + except WriteDescriptorError: + return 1, str(raw_id) + + +def _body_sha256(body: Any, context: str) -> str: + if not isinstance(body, str): + raise WriteDescriptorError(f"{context} has no captured B::Deparse body") + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + +def _procedure_provenance(fact: Any, context: str) -> dict[str, Any]: + """Return facts that a later writer contract must authenticate again. + + We retain the source file, source digest and deparse digest. We do not + attempt to recognize the procedure body here: a procedure name/hash is not + a description of native writer behavior. + """ + fact = _mapping(fact, context) + if not _bool(fact.get("resolved"), f"{context}.resolved"): + raise WriteDescriptorError(f"{context} is unresolved") + if fact.get("__perl") != "CODE": + raise WriteDescriptorError(f"{context} is not a CODE fact") + result = {} + for key in ("__name", "source_file", "source_sha256"): + value = fact.get(key) + if not isinstance(value, str) or not value: + raise WriteDescriptorError(f"{context}.{key} is missing") + result[{"__name": "name"}.get(key, key)] = value + result["body_sha256"] = _body_sha256(fact.get("__deparse"), context) + + dependencies = fact.get("dependencies", {}) + dependencies = _mapping(dependencies, f"{context}.dependencies") + result["dependencies"] = [ + _procedure_provenance(dep, f"{context}.dependencies[{name!r}]") + for name, dep in sorted(dependencies.items()) + ] + return result + + +def _router_provenance(doc: Mapping[str, Any]) -> dict[str, Any]: + router = _mapping(doc.get("native_write_autoload"), "native_write_autoload") + if not _bool(router.get("supported"), "native_write_autoload.supported"): + raise WriteDescriptorError("native_write_autoload router is unsupported") + return _procedure_provenance(router.get("router"), "native_write_autoload.router") + + +def _table_provenance(doc: Mapping[str, Any], table: Mapping[str, Any]) -> dict[str, Any]: + router = _router_provenance(doc) + procedures: dict[str, dict[str, Any]] = {} + for kind in ("write", "check"): + proc = _mapping(table.get(f"effective_{kind}_proc"), f"effective_{kind}_proc") + present = _bool(proc.get("present"), f"effective_{kind}_proc.present") + if not present: + raise WriteDescriptorError(f"effective_{kind}_proc is absent") + procedures[kind] = _procedure_provenance(proc.get("effective"), f"effective_{kind}_proc.effective") + return {"autoload_router": router, "write_proc": procedures["write"], "check_proc": procedures["check"]} + + +def _entry_alternatives(entry: Mapping[str, Any], context: str) -> list[tuple[bool, int, Mapping[str, Any]]]: + kind = entry.get("entry_kind") + if kind == "HASH": + return [(False, 0, entry)] + if kind == "ARRAY": + alternatives = entry.get("alternatives") + if not isinstance(alternatives, list): + raise WriteDescriptorError(f"{context}.alternatives is not a list") + result = [] + for index, alternative in enumerate(alternatives): + alternative = _mapping(alternative, f"{context}.alternatives[{index}]") + if alternative.get("entry_kind", "HASH") not in ("HASH", None): + raise WriteDescriptorError(f"{context}.alternatives[{index}] is not a hash alternative") + result.append((True, index, alternative)) + return result + # Scalar and otherwise unrecognized source row kinds are still represented + # in the omission sidecar instead of disappearing. + return [(False, 0, entry)] + + +def _name_hint(entry: Mapping[str, Any]) -> str | None: + properties = entry.get("properties") + if not isinstance(properties, Mapping): + return None + fact = properties.get("Name") + if not isinstance(fact, Mapping): + return None + try: + present, value = _fact_value(fact, "row.properties.Name") + except WriteDescriptorError: + return None + return _plain_string(value) if present else None + + +def _effective_group(entry: Mapping[str, Any], table: Mapping[str, Any]) -> tuple[bool, Any]: + controls = _mapping(entry.get("write_controls"), "row.write_controls") + row_present, row_value = _fact_value(controls.get("WriteGroup"), "row.write_controls.WriteGroup") + if row_present: + return True, row_value + controls = _mapping(table.get("write_controls"), "table.write_controls") + return _fact_value(controls.get("WRITE_GROUP"), "table.write_controls.WRITE_GROUP") + + +def _row_reasons(entry: Mapping[str, Any], table: Mapping[str, Any]) -> tuple[list[str], dict[str, Any] | None]: + reasons: list[str] = [] + if entry.get("entry_kind", "HASH") != "HASH": + return ["write_row_shape"], None + properties = _mapping(entry.get("properties"), "row.properties") + controls = _mapping(entry.get("write_controls"), "row.write_controls") + unknown = _mapping(entry.get("unknown_properties"), "row.unknown_properties") + if unknown: + reasons.extend(f"write_row_unknown_property_{key}" for key in sorted(unknown)) + # The sidecar deliberately presents writer controls twice: in the complete + # native property map and in the convenient write-controls projection. A + # stale or hand-mutated projection may not choose whichever copy happens + # to admit the row. + for key in _ALLOWED_ROW_CONTROLS: + property_fact = properties.get(key, {"present": False}) + control_fact = controls.get(key) + if property_fact != control_fact: + reasons.append("write_row_control_projection_mismatch") + for key in sorted(set(properties) - _ALLOWED_ROW_PROPERTIES): + reasons.append(f"write_row_property_{key}") + for key in sorted(set(controls) - _ALLOWED_ROW_CONTROLS): + present, _ = _fact_value(controls[key], f"row.write_controls.{key}") + if present: + reasons.append(f"write_row_control_{key}") + + name_present, name_value = _fact_value(properties.get("Name"), "row.properties.Name") + name = _plain_string(name_value) if name_present else None + if not name: + reasons.append("write_name") + writable_present, writable_value = _fact_value(controls.get("Writable"), "row.write_controls.Writable") + if not writable_present or writable_value != _SUPPORTED_WRITABLE: + reasons.append("write_value_type") + group_present, group_value = _effective_group(entry, table) + group = _plain_string(group_value) if group_present else None + if group is None: + reasons.append("write_physical_group_missing_or_nonliteral") + elif group not in {"IFD0", "ExifIFD", "GPS"}: + reasons.append("write_physical_group_unmodeled") + # The inactive descriptor faithfully carries ordinary native physical + # groups. A future runtime may stage IFD0/ExifIFD/GPS independently, but + # source capture must not erase a real placement merely because its writer + # primitive has not landed yet. + + if reasons: + return sorted(set(reasons)), None + return [], { + "name": name, + "value_type": _SUPPORTED_VALUE_TYPE, + "physical_group": group, + } + + +def _table_reasons(doc: Mapping[str, Any], module: str, name: str, table: Mapping[str, Any]) -> tuple[list[str], dict[str, Any] | None]: + if (module, name) != (_SOURCE_MODULE, _SOURCE_TABLE): + return ["write_source_class_unimplemented"], None + unknown = _mapping(table.get("unknown_table_properties"), "table.unknown_table_properties") + reasons = [f"write_table_unknown_property_{key}" for key in sorted(unknown)] + props = _mapping(table.get("table_properties"), "table.table_properties") + controls = _mapping(table.get("write_controls"), "table.write_controls") + reasons.extend(f"write_table_property_{key}" for key in sorted(set(props) - _ALLOWED_TABLE_PROPERTIES)) + for property_key, control_key in (("WRITE_GROUP", "WRITE_GROUP"), ("WRITE_PROC", "WRITE_PROC"), ("CHECK_PROC", "CHECK_PROC")): + if props.get(property_key) != controls.get(control_key): + reasons.append("write_table_control_projection_mismatch") + try: + provenance = _table_provenance(doc, table) + except WriteDescriptorError as error: + reasons.append("write_provenance_unresolved") + provenance = None + return sorted(set(reasons)), provenance + + +def _rust_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _rust_provenance(fact: Mapping[str, Any]) -> str: + deps = ", ".join(_rust_provenance(dep) for dep in fact["dependencies"]) + return ( + "NativeWriteProcedureProvenance { " + f"name: {_rust_string(fact['name'])}, source_file: {_rust_string(fact['source_file'])}, " + f"source_sha256: {_rust_string(fact['source_sha256'])}, body_sha256: {_rust_string(fact['body_sha256'])}, " + f"dependencies: &[{deps}] }}" + ) + + +def _ident(module: str, table: str) -> str: + return re.sub(r"[^A-Za-z0-9]", "_", f"WRITE_{module}_{table}").upper() + + +def rust_source(population: Mapping[str, Any]) -> str: + """Render inert Rust facts. No existing module re-exports this source.""" + tables = population["tables"] + row_omissions = population["omitted_rows"] + table_omissions = population["omitted_tables"] + chunks = [f'''// @generated by tools/exiftool-tables/write_descriptors.py. +//! Inactive native write candidates. These are source provenance and a closed +//! scalar-ASCII shape only; they do not activate a writer or authenticate the +//! behavior of WriteExif/CheckExif. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WriteValueType {{ Ascii }} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WritePhysicalGroup {{ IFD0, ExifIFD, GPS }} +pub struct NativeWriteProcedureProvenance {{ + pub name: &'static str, + pub source_file: &'static str, + pub source_sha256: &'static str, + pub body_sha256: &'static str, + pub dependencies: &'static [NativeWriteProcedureProvenance], +}} +pub struct InactiveWriteScalarString {{ + pub raw_id: u16, + pub name: &'static str, + pub physical_group: WritePhysicalGroup, + pub value_type: WriteValueType, +}} +pub struct InactiveWriteTable {{ + pub module: &'static str, + pub table: &'static str, + pub autoload_router: NativeWriteProcedureProvenance, + pub write_proc: NativeWriteProcedureProvenance, + pub check_proc: NativeWriteProcedureProvenance, + pub tags: &'static [InactiveWriteScalarString], +}} +pub struct OmittedWriteNativeRow {{ + pub module: &'static str, + pub table: &'static str, + pub raw_id: &'static str, + pub variant: bool, + pub alternative: usize, + pub name: Option<&'static str>, + pub reasons: &'static [&'static str], +}} +pub struct OmittedWriteNativeTable {{ + pub module: &'static str, + pub table: &'static str, + pub reasons: &'static [&'static str], +}} +pub const INACTIVE_WRITE_DESCRIPTOR_VERSION: u32 = {DESCRIPTOR_VERSION}; +pub const INACTIVE_WRITE_RUNTIME_STATUS: &str = {_rust_string(RUNTIME_STATUS)}; +'''] + refs = [] + for table in tables: + symbol = _ident(table["module"], table["table"]) + refs.append(f" &{symbol},") + tags = ",\n".join( + " InactiveWriteScalarString { " + f"raw_id: 0x{tag['raw_id']:04x}, name: {_rust_string(tag['name'])}, " + f"physical_group: WritePhysicalGroup::{tag['physical_group']}, value_type: WriteValueType::Ascii }}" + for tag in table["tags"] + ) + prov = table["provenance"] + chunks.append( + f"\npub static {symbol}: InactiveWriteTable = InactiveWriteTable {{\n" + f" module: {_rust_string(table['module'])}, table: {_rust_string(table['table'])},\n" + f" autoload_router: {_rust_provenance(prov['autoload_router'])},\n" + f" write_proc: {_rust_provenance(prov['write_proc'])},\n" + f" check_proc: {_rust_provenance(prov['check_proc'])},\n" + f" tags: &[\n{tags}\n ],\n}};\n" + ) + chunks.append("\npub static ALL_INACTIVE_WRITE_TABLES: &[&InactiveWriteTable] = &[\n" + "\n".join(refs) + "\n];\n") + rows = [] + for row in row_omissions: + name = "None" if row["name"] is None else f"Some({_rust_string(row['name'])})" + reasons = ", ".join(_rust_string(reason) for reason in row["reasons"]) + rows.append( + " OmittedWriteNativeRow { " + f"module: {_rust_string(row['module'])}, table: {_rust_string(row['table'])}, raw_id: {_rust_string(row['raw_id'])}, " + f"variant: {str(row['variant']).lower()}, alternative: {row['alternative']}, name: {name}, reasons: &[{reasons}] }}," + ) + chunks.append("\npub static OMITTED_WRITE_NATIVE_ROWS: &[OmittedWriteNativeRow] = &[\n" + "\n".join(rows) + "\n];\n") + omissions = [] + for table in table_omissions: + reasons = ", ".join(_rust_string(reason) for reason in table["reasons"]) + omissions.append( + " OmittedWriteNativeTable { " + f"module: {_rust_string(table['module'])}, table: {_rust_string(table['table'])}, reasons: &[{reasons}] }}," + ) + chunks.append("\npub static OMITTED_WRITE_NATIVE_TABLES: &[OmittedWriteNativeTable] = &[\n" + "\n".join(omissions) + "\n];\n") + return "".join(chunks) + + +def generate(doc: Mapping[str, Any], modules: list[str] | None = None) -> tuple[str, WriteDescriptorReport]: + """Compile the sidecar into inactive candidates and explicit omissions.""" + write_tables = _mapping(doc.get("native_write_tables"), "native_write_tables") + requested = set(modules) if modules else None + population = {"tables": [], "omitted_rows": [], "omitted_tables": []} + counts: Counter[str] = Counter() + + for module in sorted(write_tables): + if requested is not None and module not in requested: + continue + table_map = _mapping(write_tables[module], f"native_write_tables[{module!r}]") + for table_name in sorted(table_map): + table = _mapping(table_map[table_name], f"native_write_tables[{module!r}][{table_name!r}]") + counts["tables_seen"] += 1 + rows = _mapping(table.get("rows"), f"{module}::{table_name}.rows") + table_reasons, provenance = _table_reasons(doc, module, table_name, table) + if not table_reasons: + counts["candidate_tables"] += 1 + admitted = [] + table_rows = [] + for raw_id in sorted(rows, key=_row_sort_key): + source_entry = _mapping(rows[raw_id], f"{module}::{table_name} row {raw_id}") + for variant, alternative, entry in _entry_alternatives(source_entry, f"{module}::{table_name} row {raw_id}"): + name = _name_hint(entry) + reasons = list(table_reasons) + compiled = None + try: + parsed_raw_id = _raw_u16(raw_id) + except WriteDescriptorError: + parsed_raw_id = None + reasons.append("write_raw_id") + if not reasons: + reasons, compiled = _row_reasons(entry, table) + if reasons: + record = { + "module": module, "table": table_name, "raw_id": raw_id, + "variant": variant, "alternative": alternative, "name": name, + "reasons": sorted(set(reasons)), + } + population["omitted_rows"].append(record) + counts["omitted_rows"] += 1 + counts.update(record["reasons"]) + else: + if parsed_raw_id is None: + raise AssertionError("non-u16 write row reached descriptor") + admitted.append({"raw_id": parsed_raw_id, **compiled}) + table_rows.append(raw_id) + counts["emitted_rows"] += 1 + if admitted: + if table_reasons or provenance is None: + raise AssertionError("admitted write row without table provenance") + population["tables"].append({ + "module": module, "table": table_name, "provenance": provenance, + "tags": sorted(admitted, key=lambda tag: tag["raw_id"]), + }) + counts["emitted_tables"] += 1 + else: + reasons = table_reasons or ["write_table_has_no_admitted_rows"] + record = {"module": module, "table": table_name, "reasons": sorted(set(reasons))} + population["omitted_tables"].append(record) + counts["omitted_tables"] += 1 + counts.update(record["reasons"]) + + population["tables"].sort(key=lambda value: (value["module"], value["table"])) + population["omitted_rows"].sort(key=lambda value: (value["module"], value["table"], _row_sort_key(value["raw_id"]), value["alternative"])) + population["omitted_tables"].sort(key=lambda value: (value["module"], value["table"])) + report = WriteDescriptorReport( + tables_seen=counts["tables_seen"], + candidate_tables=counts["candidate_tables"], + emitted_tables=counts["emitted_tables"], + emitted_rows=counts["emitted_rows"], + omitted_tables=counts["omitted_tables"], + omitted_rows=counts["omitted_rows"], + omissions_by_reason=dict(sorted((key, value) for key, value in counts.items() if key.startswith("write_"))), + ) + return rust_source(population), report From 10c89560ac0ec2be08d224194cbc1fbf7a25a613 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:45:05 -0500 Subject: [PATCH 10/22] fix(tables): bind inactive write candidate source facts --- tools/exiftool-tables/WRITE_DESCRIPTOR_API.md | 15 ++- .../exiftool-tables/test_write_descriptors.py | 80 +++++++++++ tools/exiftool-tables/write_descriptors.py | 125 ++++++++++++++++-- 3 files changed, 205 insertions(+), 15 deletions(-) diff --git a/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md b/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md index 32e58ea55..cad8a39f5 100644 --- a/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md +++ b/tools/exiftool-tables/WRITE_DESCRIPTOR_API.md @@ -9,8 +9,13 @@ The first closed class is a plain `Exif::Main` scalar row with native `Writable => 'string'` and a literal effective `WriteGroup` in `IFD0`, `ExifIFD`, or `GPS`. Each candidate carries only source-derived `raw_id`, `name`, `WritePhysicalGroup`, -and `WriteValueType::Ascii`, together with actual loaded autoload/`WRITE_PROC`/ -`CHECK_PROC` provenance: name, relative source file, source SHA-256, B::Deparse +and `WriteValueType::Ascii`, together with its exact fully-qualified native +table identity and effective native table groups. Group values apply +`GetTagTable`'s false-value defaults and retain the group-0 context a later +`CharsetEXIF` encoding contract needs. They are source facts, not a routing or +encoding decision. The candidate also carries actual loaded autoload/ +`WRITE_PROC`/`CHECK_PROC` provenance: fully-qualified callable name, normalized +library-relative source file, lowercase SHA-256 source digest, B::Deparse SHA-256, and captured direct dependencies. Those procedure facts are **not** a writer admission and are not interpreted @@ -29,3 +34,9 @@ separately; their presence here is not an activation claim. A changed name or compatible added string row produces a changed descriptor; an unmodeled physical group, type, write control, unknown property, or unresolved procedure provenance is a refusal rather than a guessed writer operation. + +The compiler binds the outer sidecar map key to the inner `module`, `table`, +and `full_name` before applying this source class. It rejects malformed or +non-relative provenance paths, non-SHA-256 digests, and unrepresentable group +maps. `--write-out` is optional: requesting this inactive artifact does not +change the ordinary generated binary artifact. diff --git a/tools/exiftool-tables/test_write_descriptors.py b/tools/exiftool-tables/test_write_descriptors.py index 94e14b470..155723e12 100644 --- a/tools/exiftool-tables/test_write_descriptors.py +++ b/tools/exiftool-tables/test_write_descriptors.py @@ -1,6 +1,11 @@ """Focused source-only checks for inactive write candidates.""" import copy +import json +from pathlib import Path +import subprocess +import sys +from tempfile import TemporaryDirectory import unittest import write_descriptors @@ -233,6 +238,81 @@ def test_procedure_body_change_changes_emitted_provenance(self): after, _ = population(changed) self.assertNotEqual(before, after) + def test_malformed_procedure_path_or_digest_is_a_named_refusal(self): + for key, value in ( + ("source_file", "../../untracked.pm"), + ("source_file", "/absolute/Image/ExifTool/WriteExif.pl"), + ("source_file", "untracked.pm"), + ("source_sha256", "not-a-sha256"), + ): + with self.subTest(key=key, value=value): + changed = document() + changed["native_write_tables"]["Exif"]["Main"]["effective_write_proc"]["effective"][key] = value + source, report = population(changed) + self.assertEqual(report.emitted_tables, 0) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_provenance_unresolved"', source) + self.assertNotIn(value, source) + + def test_outer_and_inner_table_identity_must_match_before_source_class_selection(self): + # A stale/mutated inner record cannot borrow Exif::Main eligibility + # merely by claiming that identity while living under another map key. + cases = { + "forged_inner_claim": document({"Other": {"Elsewhere": table(module="Exif", table_name="Main")}}), + "mutated_saved_exif_main": document(), + } + mutated = cases["mutated_saved_exif_main"]["native_write_tables"]["Exif"]["Main"] + mutated.update({ + "module": "Other", "table": "Elsewhere", "full_name": "Image::ExifTool::Other::Elsewhere", + }) + for label, changed in cases.items(): + with self.subTest(label=label): + source, report = population(changed) + self.assertEqual(report.emitted_tables, 0) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_table_identity_mismatch"', source) + self.assertNotIn('pub static WRITE_EXIF_MAIN:', source) + + def test_effective_groups_are_emitted_and_unrepresentable_groups_refuse(self): + before, _ = population(document()) + changed = document() + groups = changed["native_write_tables"]["Exif"]["Main"]["table_properties"]["GROUPS"] + groups["value"] = {"0": "ChangedExif", "1": "ChangedIFD", "2": "ChangedImage"} + after, report = population(changed) + self.assertEqual(report.emitted_rows, 1) + self.assertIn('group0: "ChangedExif", group1: "ChangedIFD", group2: "ChangedImage"', after) + self.assertNotEqual(before, after) + + changed["native_write_tables"]["Exif"]["Main"]["table_properties"]["GROUPS"] = { + "present": True, "value": {"3": "Unexpected"}, + } + source, report = population(changed) + self.assertEqual(report.emitted_rows, 0) + self.assertIn('"write_table_groups_unrepresented"', source) + + def test_default_optional_write_output_does_not_change_primary_artifact(self): + # The optional, inactive sidecar must not perturb ordinary codegen + # output. This invokes the public CLI with the same minimal input + # twice rather than treating this module's renderer as a substitute. + payload = document() + payload.update({"exiftool_version": "13.59", "modules": {}}) + with TemporaryDirectory() as directory: + root = Path(directory) + tables = root / "tables.json" + tables.write_text(json.dumps(payload), encoding="utf-8") + ordinary = root / "ordinary.rs" + with_sidecar = root / "with-sidecar.rs" + sidecar = root / "inactive_write.rs" + common = [sys.executable, str(Path(__file__).with_name("codegen.py")), str(tables)] + subprocess.run([*common, "-o", str(ordinary)], check=True, text=True, capture_output=True) + subprocess.run( + [*common, "-o", str(with_sidecar), "--write-out", str(sidecar)], + check=True, text=True, capture_output=True, + ) + self.assertEqual(ordinary.read_bytes(), with_sidecar.read_bytes()) + self.assertTrue(sidecar.is_file()) + self.assertIn("INACTIVE_WRITE_DESCRIPTOR_VERSION", sidecar.read_text(encoding="utf-8")) + def test_symbolic_write_alias_is_a_named_row_omission_not_a_dump_abort(self): changed = document() changed["native_write_tables"]["Exif"]["Main"]["rows"]["Alias"] = row(name="SourceAlias") diff --git a/tools/exiftool-tables/write_descriptors.py b/tools/exiftool-tables/write_descriptors.py index 25246e979..334f39474 100644 --- a/tools/exiftool-tables/write_descriptors.py +++ b/tools/exiftool-tables/write_descriptors.py @@ -20,6 +20,7 @@ from dataclasses import dataclass import hashlib import json +from pathlib import PurePosixPath import re from typing import Any, Mapping @@ -39,6 +40,8 @@ _ALLOWED_ROW_PROPERTIES = frozenset({"Name", "Writable", "WriteGroup"}) _ALLOWED_ROW_CONTROLS = frozenset({"Writable", "WriteGroup"}) _ID_RE = re.compile(r"^(?:0x[0-9A-Fa-f]+|[0-9]+)$") +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_CODE_NAME_RE = re.compile(r"^[A-Za-z_]\w*(?:::[A-Za-z_]\w*)+$") class WriteDescriptorError(ValueError): @@ -111,6 +114,34 @@ def _body_sha256(body: Any, context: str) -> str: return hashlib.sha256(body.encode("utf-8")).hexdigest() +def _relative_source_file(value: Any, context: str) -> str: + """Require the dumper's canonical library-relative source spelling. + + A nonempty string is not provenance. In particular, accepting ``..`` or + an absolute path would make a hand-mutated sidecar appear to bind a source + file outside the selected ExifTool library. + """ + if not isinstance(value, str) or not value or "\\" in value: + raise WriteDescriptorError(f"{context} is not a normalized relative path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or value in (".", "..") + or any(part in ("", ".", "..") for part in path.parts) + or path.as_posix() != value + or not value.startswith("Image/") + or value == "Image/" + ): + raise WriteDescriptorError(f"{context} is not a normalized relative path") + return value + + +def _sha256(value: Any, context: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise WriteDescriptorError(f"{context} is not a SHA-256 digest") + return value + + def _procedure_provenance(fact: Any, context: str) -> dict[str, Any]: """Return facts that a later writer contract must authenticate again. @@ -123,12 +154,14 @@ def _procedure_provenance(fact: Any, context: str) -> dict[str, Any]: raise WriteDescriptorError(f"{context} is unresolved") if fact.get("__perl") != "CODE": raise WriteDescriptorError(f"{context} is not a CODE fact") - result = {} - for key in ("__name", "source_file", "source_sha256"): - value = fact.get(key) - if not isinstance(value, str) or not value: - raise WriteDescriptorError(f"{context}.{key} is missing") - result[{"__name": "name"}.get(key, key)] = value + name = fact.get("__name") + if not isinstance(name, str) or _CODE_NAME_RE.fullmatch(name) is None: + raise WriteDescriptorError(f"{context} has no fully-qualified callable name") + result = { + "name": name, + "source_file": _relative_source_file(fact.get("source_file"), f"{context}.source_file"), + "source_sha256": _sha256(fact.get("source_sha256"), f"{context}.source_sha256"), + } result["body_sha256"] = _body_sha256(fact.get("__deparse"), context) dependencies = fact.get("dependencies", {}) @@ -159,6 +192,48 @@ def _table_provenance(doc: Mapping[str, Any], table: Mapping[str, Any]) -> dict[ return {"autoload_router": router, "write_proc": procedures["write"], "check_proc": procedures["check"]} +def _table_identity_matches(table: Mapping[str, Any], module: str, name: str) -> bool: + """Bind the sidecar map key to the table's own captured identity.""" + return ( + table.get("module") == module + and table.get("table") == name + and table.get("full_name") == f"Image::ExifTool::{module}::{name}" + ) + + +def _effective_groups(table: Mapping[str, Any], module: str) -> dict[str, str]: + """Resolve native ``GetTagTable`` group defaults from the captured table. + + The native loader fills false groups 0 and 1 with the table-owning module + and false group 2 with ``Other``. Keeping those effective values in the + candidate is required for later encoding/CharsetEXIF authentication; it + does not enable a writer. + """ + props = _mapping(table.get("table_properties"), "table.table_properties") + present, raw_groups = _fact_value(props.get("GROUPS", {"present": False}), "table.table_properties.GROUPS") + if not present: + raw_groups = {} + if not isinstance(raw_groups, Mapping): + raise WriteDescriptorError("table GROUPS are not an object") + if any(not isinstance(key, str) or key not in {"0", "1", "2"} for key in raw_groups): + raise WriteDescriptorError("table GROUPS contain an unrepresented family") + if any(not isinstance(value, str) for value in raw_groups.values()): + raise WriteDescriptorError("table GROUPS are not literal strings") + default_module = module.split("::", 1)[0] + if not default_module: + raise WriteDescriptorError("table module has no native group default") + result = {} + for family in range(3): + value = raw_groups.get(str(family)) + # Perl's boolean false values for the source representation are the + # empty string and "0"; native GetTagTable applies these defaults. + if value not in (None, "", "0"): + result[f"group{family}"] = value + else: + result[f"group{family}"] = default_module if family in (0, 1) else "Other" + return result + + def _entry_alternatives(entry: Mapping[str, Any], context: str) -> list[tuple[bool, int, Mapping[str, Any]]]: kind = entry.get("entry_kind") if kind == "HASH": @@ -254,9 +329,14 @@ def _row_reasons(entry: Mapping[str, Any], table: Mapping[str, Any]) -> tuple[li } -def _table_reasons(doc: Mapping[str, Any], module: str, name: str, table: Mapping[str, Any]) -> tuple[list[str], dict[str, Any] | None]: +def _table_reasons( + doc: Mapping[str, Any], module: str, name: str, table: Mapping[str, Any] +) -> tuple[list[str], dict[str, Any] | None, dict[str, str] | None]: + identity_matches = _table_identity_matches(table, module, name) + if not identity_matches: + return ["write_table_identity_mismatch"], None, None if (module, name) != (_SOURCE_MODULE, _SOURCE_TABLE): - return ["write_source_class_unimplemented"], None + return ["write_source_class_unimplemented"], None, None unknown = _mapping(table.get("unknown_table_properties"), "table.unknown_table_properties") reasons = [f"write_table_unknown_property_{key}" for key in sorted(unknown)] props = _mapping(table.get("table_properties"), "table.table_properties") @@ -270,7 +350,12 @@ def _table_reasons(doc: Mapping[str, Any], module: str, name: str, table: Mappin except WriteDescriptorError as error: reasons.append("write_provenance_unresolved") provenance = None - return sorted(set(reasons)), provenance + try: + groups = _effective_groups(table, module) + except WriteDescriptorError: + reasons.append("write_table_groups_unrepresented") + groups = None + return sorted(set(reasons)), provenance, groups def _rust_string(value: str) -> str: @@ -305,6 +390,11 @@ def rust_source(population: Mapping[str, Any]) -> str: pub enum WriteValueType {{ Ascii }} #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum WritePhysicalGroup {{ IFD0, ExifIFD, GPS }} +pub struct NativeWriteTableGroups {{ + pub group0: &'static str, + pub group1: &'static str, + pub group2: &'static str, +}} pub struct NativeWriteProcedureProvenance {{ pub name: &'static str, pub source_file: &'static str, @@ -321,6 +411,11 @@ def rust_source(population: Mapping[str, Any]) -> str: pub struct InactiveWriteTable {{ pub module: &'static str, pub table: &'static str, + pub full_name: &'static str, + /// Effective native groups after GetTagTable's false-value defaults. + /// These facts are needed by a later encoding contract; they do not route + /// a writer or authenticate WriteExif/CheckExif behavior. + pub groups: NativeWriteTableGroups, pub autoload_router: NativeWriteProcedureProvenance, pub write_proc: NativeWriteProcedureProvenance, pub check_proc: NativeWriteProcedureProvenance, @@ -356,7 +451,10 @@ def rust_source(population: Mapping[str, Any]) -> str: prov = table["provenance"] chunks.append( f"\npub static {symbol}: InactiveWriteTable = InactiveWriteTable {{\n" - f" module: {_rust_string(table['module'])}, table: {_rust_string(table['table'])},\n" + f" module: {_rust_string(table['module'])}, table: {_rust_string(table['table'])}, full_name: {_rust_string(table['full_name'])},\n" + " groups: NativeWriteTableGroups { " + f"group0: {_rust_string(table['groups']['group0'])}, group1: {_rust_string(table['groups']['group1'])}, " + f"group2: {_rust_string(table['groups']['group2'])} }},\n" f" autoload_router: {_rust_provenance(prov['autoload_router'])},\n" f" write_proc: {_rust_provenance(prov['write_proc'])},\n" f" check_proc: {_rust_provenance(prov['check_proc'])},\n" @@ -399,7 +497,7 @@ def generate(doc: Mapping[str, Any], modules: list[str] | None = None) -> tuple[ table = _mapping(table_map[table_name], f"native_write_tables[{module!r}][{table_name!r}]") counts["tables_seen"] += 1 rows = _mapping(table.get("rows"), f"{module}::{table_name}.rows") - table_reasons, provenance = _table_reasons(doc, module, table_name, table) + table_reasons, provenance, groups = _table_reasons(doc, module, table_name, table) if not table_reasons: counts["candidate_tables"] += 1 admitted = [] @@ -433,10 +531,11 @@ def generate(doc: Mapping[str, Any], modules: list[str] | None = None) -> tuple[ table_rows.append(raw_id) counts["emitted_rows"] += 1 if admitted: - if table_reasons or provenance is None: + if table_reasons or provenance is None or groups is None: raise AssertionError("admitted write row without table provenance") population["tables"].append({ - "module": module, "table": table_name, "provenance": provenance, + "module": module, "table": table_name, + "full_name": table["full_name"], "groups": groups, "provenance": provenance, "tags": sorted(admitted, key=lambda tag: tag["raw_id"]), }) counts["emitted_tables"] += 1 From 059f56d1c87b4831da6c9468251845d4ed032876 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:47:53 -0500 Subject: [PATCH 11/22] Add inactive scoped TIFF entry editing primitive --- src/writers/tiff_surgical.rs | 263 ++++++++++++++++++++ src/writers/tiff_surgical/entry_edits.rs | 303 +++++++++++++++++++++++ 2 files changed, 566 insertions(+) create mode 100644 src/writers/tiff_surgical/entry_edits.rs diff --git a/src/writers/tiff_surgical.rs b/src/writers/tiff_surgical.rs index 5df3d5a63..8daa24f47 100644 --- a/src/writers/tiff_surgical.rs +++ b/src/writers/tiff_surgical.rs @@ -51,6 +51,11 @@ use crate::writers::exif_surgical::{ validate_changed, }; +// Inactive until source-derived writer semantics and public identity routing +// are accepted. This primitive contains carrier mechanics, not tag knowledge. +#[allow(dead_code)] +pub(crate) mod entry_edits; + /// IFD0 tag pointing to the ExifIFD const EXIF_IFD_POINTER: u16 = 0x8769; /// IFD0 tag pointing to the GPS IFD @@ -1020,4 +1025,262 @@ mod tests { let err = rewrite_tiff_file(&file, &original, &desired).unwrap_err(); assert!(err.to_string().contains("denominator"), "got: {}", err); } + + fn raw_entry_value(file: &[u8], group: IfdKind, id: u16) -> Option<(u16, u32, Vec)> { + let scan = scan_tiff(file).unwrap(); + let entry = scan + .entries + .iter() + .find(|entry| entry.ifd == group && entry.tag_id == id)?; + let at = entry.record_offset; + let count = read_u32(&file[at + 4..at + 8], scan.byte_order); + let size = count as usize + * crate::parsers::common::exif_types::ExifType::from_u16(entry.field_type) + .unwrap() + .size_in_bytes(); + let value_at = if size <= 4 { + at + 8 + } else { + read_u32(&file[at + 8..at + 12], scan.byte_order) as usize + }; + Some(( + entry.field_type, + count, + file[value_at..value_at + size].to_vec(), + )) + } + + fn raw_string_edit(group: IfdKind, id: u16, bytes: &[u8]) -> entry_edits::ScopedEntryEdit { + entry_edits::ScopedEntryEdit { + ifd: group, + tag_id: id, + mutation: entry_edits::EntryMutation::Set { + field_type: 2, + count: bytes.len() as u32, + bytes: bytes.to_vec(), + }, + } + } + + #[test] + fn raw_scoped_strings_preserve_payload_across_set_and_delete() { + use entry_edits::{EntryMutation, ScopedEntryEdit, apply_entry_edits}; + for bo in [ByteOrder::LittleEndian, ByteOrder::BigEndian] { + let mut file = build_tiff(bo); + let long = [vec![b'G'; 96], vec![0]].concat(); + for bytes in [ + b"Alpha\0".as_slice(), + long.as_slice(), + b"Z\0", + b"\0", + "é\0".as_bytes(), + b"A\0B\0", + ] { + let edit = raw_string_edit(IfdKind::Ifd0, 0x013c, bytes); + let out = apply_entry_edits(&file, std::slice::from_ref(&edit)).unwrap(); + assert_eq!(&out[..4], &file[..4]); + assert_eq!(&out[8..file.len()], &file[8..]); + assert_eq!(&out[80..88], &[0xAA; 8]); + assert_eq!( + raw_entry_value(&out, IfdKind::Ifd0, 0x013c), + Some((2, bytes.len() as u32, bytes.to_vec())) + ); + assert_eq!( + apply_entry_edits(&out, &[edit]).unwrap(), + out, + "same value is byte-identical" + ); + file = out; + } + let delete = ScopedEntryEdit { + ifd: IfdKind::Ifd0, + tag_id: 0x013c, + mutation: EntryMutation::Delete, + }; + let out = apply_entry_edits(&file, std::slice::from_ref(&delete)).unwrap(); + assert_eq!(raw_entry_value(&out, IfdKind::Ifd0, 0x013c), None); + assert_eq!(&out[8..file.len()], &file[8..]); + assert_eq!(apply_entry_edits(&out, &[delete]).unwrap(), out); + } + } + + #[test] + fn raw_scoped_edits_create_child_ifds_and_keep_next_ifd_chain() { + use entry_edits::{EntryMutation, ScopedEntryEdit, apply_entry_edits}; + for bo in [ByteOrder::LittleEndian, ByteOrder::BigEndian] { + let mut file = build_tiff(bo); + let next_at = append_aligned(&mut file, &[0; 6]); + put_u32(&mut file[46..50], next_at as u32, bo); + let edits = [ + raw_string_edit(IfdKind::Ifd0, 0x013c, b"root\0"), + raw_string_edit(IfdKind::ExifIfd, 0x013c, b"child\0"), + raw_string_edit(IfdKind::Gps, 0x001d, b"date\0"), + ScopedEntryEdit { + ifd: IfdKind::Ifd0, + tag_id: 0x0112, + mutation: EntryMutation::Delete, + }, + ]; + let out = apply_entry_edits(&file, &edits).unwrap(); + for (group, id, value) in [ + (IfdKind::Ifd0, 0x013c, b"root\0".as_slice()), + (IfdKind::ExifIfd, 0x013c, b"child\0"), + (IfdKind::Gps, 0x001d, b"date\0"), + ] { + assert_eq!( + raw_entry_value(&out, group, id), + Some((2, value.len() as u32, value.to_vec())) + ); + } + assert_eq!(raw_entry_value(&out, IfdKind::Ifd0, 0x0112), None); + let scan = scan_tiff(&out).unwrap(); + let count = read_u16(&out[scan.ifd0_offset..scan.ifd0_offset + 2], bo) as usize; + let next = scan.ifd0_offset + 2 + 12 * count; + assert_eq!(read_u32(&out[next..next + 4], bo), next_at as u32); + assert_eq!(&out[8..file.len()], &file[8..]); + let deleted = apply_entry_edits( + &out, + &[ScopedEntryEdit { + ifd: IfdKind::ExifIfd, + tag_id: 0x013c, + mutation: EntryMutation::Delete, + }], + ) + .unwrap(); + assert_eq!(raw_entry_value(&deleted, IfdKind::ExifIfd, 0x013c), None); + assert_eq!( + raw_entry_value(&deleted, IfdKind::Ifd0, 0x013c), + Some((2, 5, b"root\0".to_vec())) + ); + } + } + + #[test] + fn raw_scoped_edits_reject_invalid_directory_graphs_before_output() { + use entry_edits::apply_entry_edits; + for bo in [ByteOrder::LittleEndian, ByteOrder::BigEndian] { + // Two distinct, empty child directories. The requested edit is + // in IFD0: malformed untouched children must also be detected. + let mut valid = vec![0; 80]; + valid[..2].copy_from_slice(match bo { + ByteOrder::LittleEndian => b"II", + ByteOrder::BigEndian => b"MM", + }); + put_u16(&mut valid[2..4], 42, bo); + put_u32(&mut valid[4..8], 8, bo); + put_u16(&mut valid[8..10], 2, bo); + for (record, tag, offset) in [(10, EXIF_IFD_POINTER, 40), (22, GPS_IFD_POINTER, 56)] { + put_u16(&mut valid[record..record + 2], tag, bo); + put_u16(&mut valid[record + 2..record + 4], LONG_TYPE, bo); + put_u32(&mut valid[record + 4..record + 8], 1, bo); + put_u32(&mut valid[record + 8..record + 12], offset, bo); + } + let edit = raw_string_edit(IfdKind::Ifd0, 0x013c, b"new\0"); + assert!(apply_entry_edits(&valid, std::slice::from_ref(&edit)).is_ok()); + let mut cases = Vec::new(); + for offset in 0..8 { + let mut file = valid.clone(); + put_u32(&mut file[4..8], offset, bo); + cases.push(("root points into header", file)); + } + for record in [10, 22] { + for (label, offset) in [ + ("zero child", 0), + ("header child", 4), + ("child aliases root", 8), + ("child overlaps root next pointer", 34), + ("truncated child", 78), + ] { + let mut file = valid.clone(); + put_u32(&mut file[record + 8..record + 12], offset, bo); + cases.push((label, file)); + } + let mut wrong_type = valid.clone(); + put_u16(&mut wrong_type[record + 2..record + 4], 3, bo); + cases.push(("non-LONG child pointer", wrong_type)); + let mut wrong_count = valid.clone(); + put_u32(&mut wrong_count[record + 4..record + 8], 2, bo); + cases.push(("non-scalar child pointer", wrong_count)); + } + for (label, offset) in [("aliased children", 40), ("overlapping children", 44)] { + let mut file = valid.clone(); + put_u32(&mut file[30..34], offset, bo); + cases.push((label, file)); + } + let mut duplicate = valid.clone(); + put_u16(&mut duplicate[22..24], EXIF_IFD_POINTER, bo); + cases.push(("duplicate directory pointer", duplicate)); + for (label, file) in cases { + let original = file.clone(); + assert!( + apply_entry_edits(&file, std::slice::from_ref(&edit)).is_err(), + "must reject {label} in {bo:?}" + ); + assert_eq!(file, original, "failed {label} must preserve input"); + } + } + } + + #[test] + fn raw_scoped_edits_refuse_ambiguous_or_malformed_requests() { + use entry_edits::{EntryMutation, ScopedEntryEdit, apply_entry_edits}; + let file = build_tiff(ByteOrder::LittleEndian); + let valid = raw_string_edit(IfdKind::Ifd0, 0x013c, b"ok\0"); + assert!(apply_entry_edits(&file, &[valid.clone(), valid]).is_err()); + for edit in [ + raw_string_edit(IfdKind::Ifd1, 0x013c, b"no\0"), + raw_string_edit(IfdKind::Ifd0, EXIF_IFD_POINTER, b"no\0"), + ScopedEntryEdit { + ifd: IfdKind::Ifd0, + tag_id: 0x013c, + mutation: EntryMutation::Set { + field_type: 3, + count: 2, + bytes: vec![0], + }, + }, + ] { + assert!(apply_entry_edits(&file, &[edit]).is_err()); + } + assert!( + apply_entry_edits( + &file[..48], + &[raw_string_edit(IfdKind::Ifd0, 0x013c, b"ok\0")] + ) + .is_err() + ); + let mut duplicate = file.clone(); + let at = grow_ifd( + &mut duplicate, + 8, + &[NewRecord { + tag_id: 0x010f, + field_type: 2, + count: 2, + inline_or_offset: [b'X', 0, 0, 0], + }], + ByteOrder::LittleEndian, + ) + .unwrap(); + put_u32(&mut duplicate[4..8], at as u32, ByteOrder::LittleEndian); + assert!( + apply_entry_edits( + &duplicate, + &[raw_string_edit(IfdKind::Ifd0, 0x010f, b"new\0")] + ) + .is_err() + ); + assert_eq!( + apply_entry_edits( + &file, + &[ScopedEntryEdit { + ifd: IfdKind::ExifIfd, + tag_id: 0x013c, + mutation: EntryMutation::Delete + }] + ) + .unwrap(), + file + ); + } } diff --git a/src/writers/tiff_surgical/entry_edits.rs b/src/writers/tiff_surgical/entry_edits.rs new file mode 100644 index 000000000..b64c8f469 --- /dev/null +++ b/src/writers/tiff_surgical/entry_edits.rs @@ -0,0 +1,303 @@ +//! Scoped raw IFD edits for a generated writer's resolved operations. +//! +//! The caller must supply source-authorized tag identity, type and already +//! encoded bytes in the file's byte order. This layer never consults tag names +//! or the tag registry. It copies changed directories to the end of the file, +//! retaining every untouched record, value offset and next-directory pointer. +//! Production writers do not call this primitive yet. + +use super::*; +use crate::parsers::common::exif_types::ExifType; + +#[derive(Debug, Clone)] +pub(crate) enum EntryMutation { + Set { + field_type: u16, + count: u32, + bytes: Vec, + }, + Delete, +} + +#[derive(Debug, Clone)] +pub(crate) struct ScopedEntryEdit { + pub ifd: IfdKind, + pub tag_id: u16, + pub mutation: EntryMutation, +} + +fn invalid(message: &str) -> ExifToolError { + ExifToolError::parse_error(message) +} + +/// Apply one resolved edit per physical directory/tag identity. Errors return +/// no output and never mutate the input. Set(empty encoded ASCII) must still +/// include its native terminal NUL; deletion is a distinct operation. +pub(crate) fn apply_entry_edits(file: &[u8], edits: &[ScopedEntryEdit]) -> Result> { + let scan = scan_tiff(file)?; + let bo = scan.byte_order; + validate_directory_layout(file, &scan)?; + checked_append_bounds(file.len(), 0)?; + for (index, edit) in edits.iter().enumerate() { + if !matches!(edit.ifd, IfdKind::Ifd0 | IfdKind::ExifIfd | IfdKind::Gps) { + return Err(invalid("Raw edits require IFD0, ExifIFD or GPS")); + } + if matches!(edit.tag_id, EXIF_IFD_POINTER | GPS_IFD_POINTER) { + return Err(invalid("Directory links are managed by the carrier")); + } + if edits[..index] + .iter() + .any(|prior| prior.ifd == edit.ifd && prior.tag_id == edit.tag_id) + { + return Err(invalid("Multiple edits address the same raw IFD entry")); + } + if let EntryMutation::Set { + field_type, + count, + bytes, + } = &edit.mutation + { + let width = ExifType::from_u16(*field_type) + .ok_or_else(|| invalid("Unsupported encoded TIFF field type"))? + .size_in_bytes(); + if (*count as usize).checked_mul(width) != Some(bytes.len()) { + return Err(invalid("Encoded bytes do not match TIFF type and count")); + } + } + } + + let mut out = file.to_vec(); + let mut root_edits: Vec<_> = edits + .iter() + .filter(|edit| edit.ifd == IfdKind::Ifd0) + .cloned() + .collect(); + for (group, offset, pointer_tag) in [ + (IfdKind::ExifIfd, scan.exif_ifd_offset, EXIF_IFD_POINTER), + (IfdKind::Gps, scan.gps_ifd_offset, GPS_IFD_POINTER), + ] { + let changes: Vec<_> = edits + .iter() + .filter(|edit| edit.ifd == group) + .cloned() + .collect(); + if changes.is_empty() { + continue; + } + if let Some(new_at) = rewrite_directory(&mut out, offset, &changes, bo)? { + let new_at = u32::try_from(new_at).map_err(too_big)?; + let mut encoded = vec![0; 4]; + put_u32(&mut encoded, new_at, bo); + root_edits.push(ScopedEntryEdit { + ifd: IfdKind::Ifd0, + tag_id: pointer_tag, + mutation: EntryMutation::Set { + field_type: LONG_TYPE, + count: 1, + bytes: encoded, + }, + }); + } + } + if let Some(new_at) = rewrite_directory(&mut out, Some(scan.ifd0_offset), &root_edits, bo)? { + let new_at = u32::try_from(new_at).map_err(too_big)?; + put_u32(&mut out[4..8], new_at, bo); + } + Ok(out) +} + +/// Validate every directory this primitive may rewrite, including children +/// without edits. A present link with offset zero is malformed, not absent. +fn validate_directory_layout(file: &[u8], scan: &TiffScan) -> Result<()> { + let bo = scan.byte_order; + let mut spans = vec![directory_span(file, scan.ifd0_offset, bo)?]; + let (records, _) = directory_records(file, scan.ifd0_offset, bo)?; + for pointer_tag in [EXIF_IFD_POINTER, GPS_IFD_POINTER] { + let mut links = records + .iter() + .filter(|record| read_u16(&record[..2], bo) == pointer_tag); + let Some(record) = links.next() else { + continue; + }; + if links.next().is_some() + || read_u16(&record[2..4], bo) != LONG_TYPE + || read_u32(&record[4..8], bo) != 1 + { + return Err(invalid("Ambiguous or malformed subdirectory link")); + } + let span = directory_span(file, read_u32(&record[8..12], bo) as usize, bo)?; + if spans + .iter() + .any(|prior| span.start < prior.end && prior.start < span.end) + { + return Err(invalid("IFD record spans alias or overlap")); + } + spans.push(span); + } + Ok(()) +} + +fn directory_span(file: &[u8], at: usize, bo: ByteOrder) -> Result> { + if at < 8 { + return Err(invalid("IFD offset points inside the TIFF header")); + } + let start = at + .checked_add(2) + .filter(|end| *end <= file.len()) + .ok_or_else(|| invalid("IFD count is outside the file"))?; + let count = read_u16(&file[at..start], bo) as usize; + let end = count + .checked_mul(12) + .and_then(|size| start.checked_add(size)) + .filter(|end| end.checked_add(4).is_some_and(|after| after <= file.len())) + .ok_or_else(|| invalid("Truncated IFD records or next-directory pointer"))?; + Ok(at..end + 4) +} + +fn directory_records(file: &[u8], at: usize, bo: ByteOrder) -> Result<(Vec<[u8; 12]>, [u8; 4])> { + let span = directory_span(file, at, bo)?; + let start = at + 2; + let end = span.end - 4; + let records = file[start..end] + .chunks_exact(12) + .map(|record| record.try_into().expect("exact record width")) + .collect(); + let next = file[end..end + 4] + .try_into() + .expect("checked pointer width"); + Ok((records, next)) +} + +/// This classic-TIFF carrier caps the complete output at u32::MAX bytes. +/// Check alignment and the full payload before any append allocation. +fn checked_append_bounds(len: usize, size: usize) -> Result<(usize, usize)> { + let start = len + .checked_add(len & 1) + .ok_or_else(|| invalid("TIFF append alignment overflows"))?; + let end = start + .checked_add(size) + .filter(|end| *end <= u32::MAX as usize) + .ok_or_else(|| invalid("TIFF output exceeds the 32-bit carrier limit"))?; + Ok((start, end)) +} + +fn append_checked(out: &mut Vec, bytes: &[u8]) -> Result { + let (start, _) = checked_append_bounds(out.len(), bytes.len())?; + out.resize(start, 0); + out.extend_from_slice(bytes); + Ok(start) +} + +/// Return a new offset only when a directory actually changes. Unchanged and +/// absent deletions preserve the entire original byte sequence. +fn rewrite_directory( + out: &mut Vec, + at: Option, + changes: &[ScopedEntryEdit], + bo: ByteOrder, +) -> Result> { + if changes.is_empty() { + return Ok(None); + } + let (mut records, next) = match at { + Some(at) => directory_records(out, at, bo)?, + None => (Vec::new(), [0; 4]), + }; + let mut changed = false; + for edit in changes { + let matching: Vec<_> = records + .iter() + .enumerate() + .filter(|(_, record)| read_u16(&record[..2], bo) == edit.tag_id) + .map(|(index, _)| index) + .collect(); + if matching.len() > 1 { + return Err(invalid("Ambiguous duplicate target entry")); + } + let existing = matching.first().copied(); + match &edit.mutation { + EntryMutation::Delete => { + if let Some(index) = existing { + records.remove(index); + changed = true; + } + } + EntryMutation::Set { + field_type, + count, + bytes, + } => { + if let Some(index) = existing { + let old = &records[index]; + let value_at = if bytes.len() <= 4 { + None + } else { + Some(read_u32(&old[8..12], bo) as usize) + }; + let old_bytes = match value_at { + None => Some(&old[8..8 + bytes.len()]), + Some(start) => start + .checked_add(bytes.len()) + .and_then(|end| out.get(start..end)), + }; + if read_u16(&old[2..4], bo) == *field_type + && read_u32(&old[4..8], bo) == *count + && old_bytes == Some(bytes.as_slice()) + { + continue; + } + } + let mut record = [0; 12]; + put_u16(&mut record[..2], edit.tag_id, bo); + put_u16(&mut record[2..4], *field_type, bo); + put_u32(&mut record[4..8], *count, bo); + if bytes.len() <= 4 { + record[8..8 + bytes.len()].copy_from_slice(bytes); + } else { + let offset = u32::try_from(append_checked(out, bytes)?).map_err(too_big)?; + put_u32(&mut record[8..12], offset, bo); + } + if let Some(index) = existing { + records[index] = record; + } else { + records.push(record); + } + changed = true; + } + } + } + if !changed { + return Ok(None); + } + let count = + u16::try_from(records.len()).map_err(|_| invalid("IFD exceeds its entry-count limit"))?; + checked_append_bounds(out.len(), 2 + records.len() * 12 + 4)?; + records.sort_by_key(|record| read_u16(&record[..2], bo)); + let mut table = vec![0; 2]; + put_u16(&mut table, count, bo); + for record in records { + table.extend_from_slice(&record); + } + table.extend_from_slice(&next); + let offset = append_checked(out, &table)?; + Ok(Some(offset)) +} + +#[cfg(test)] +mod tests { + use super::checked_append_bounds; + + #[test] + fn raw_scoped_append_bounds_reject_overflow_without_allocating() { + let limit = u32::MAX as usize; + assert_eq!(checked_append_bounds(9, 6).unwrap(), (10, 16)); + assert_eq!( + checked_append_bounds(limit - 1, 1).unwrap(), + (limit - 1, limit) + ); + assert!(checked_append_bounds(limit - 1, 2).is_err()); + assert!(checked_append_bounds(limit, 0).is_err()); + assert!(checked_append_bounds(usize::MAX, 0).is_err()); + assert!(checked_append_bounds(usize::MAX - 1, 2).is_err()); + } +} From abaae4b8e60e8c1bfbcbcf1994cb6d3f505a143c Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:28:23 -0500 Subject: [PATCH 12/22] tools: capture rehearsal release identities --- .../exiftool-tables/test_version_rehearsal.py | 18 +- .../test_version_rehearsal_catalog.py | 151 +++++ tools/exiftool-tables/version_rehearsal.py | 38 +- .../version_rehearsal_catalog.py | 579 ++++++++++++++++++ 4 files changed, 770 insertions(+), 16 deletions(-) create mode 100755 tools/exiftool-tables/test_version_rehearsal_catalog.py create mode 100755 tools/exiftool-tables/version_rehearsal_catalog.py diff --git a/tools/exiftool-tables/test_version_rehearsal.py b/tools/exiftool-tables/test_version_rehearsal.py index c4f99f07d..cc6ca1c13 100644 --- a/tools/exiftool-tables/test_version_rehearsal.py +++ b/tools/exiftool-tables/test_version_rehearsal.py @@ -61,8 +61,8 @@ def test_catalog_preserves_unclassified_and_excluded_entries(self): catalog = self.normalized() self.assertEqual(len(catalog["entries"]), 5) self.assertEqual(catalog["entries"][1]["classification"], {"state": "excluded", "reason": "tag_name_not_numeric_release"}) - self.assertEqual(catalog["entries"][3]["classification"], {"state": "unclassified", "reason": "missing_or_invalid_archive_sha256"}) - self.assertEqual([row["name"] for row in vr.eligible_releases(catalog)], ["13.57", "13.59", "13.60"]) + self.assertEqual(catalog["entries"][3]["classification"], {"state": "eligible"}) + self.assertEqual([row["name"] for row in vr.eligible_releases(catalog)], ["13.57", "13.58", "13.59", "13.60"]) def test_deterministic_replay_and_ordered_distinct_pairs(self): first = self.plan(seed=99, sample_index=4) @@ -84,7 +84,11 @@ def test_deterministic_replay_and_ordered_distinct_pairs(self): def test_same_version_and_ambiguous_identity_are_refused(self): catalog = self.normalized() duplicate = copy.deepcopy(catalog_entries()) - duplicate["entries"].extend([release("13.59", OID_D, SHA_B), release("13.60", OID_D, SHA_B)]) + duplicate["entries"].extend([ + release("13.58", OID_D, SHA_B), + release("13.59", OID_D, SHA_B), + release("13.60", OID_D, SHA_B), + ]) duplicate_normalized = vr.normalize_catalog(duplicate) with self.assertRaisesRegex(vr.Refused, "at least two"): vr.eligible_releases(duplicate_normalized) @@ -116,9 +120,15 @@ def test_unselected_eligible_release_is_explicitly_untested(self): plan = vr.make_plan(catalog, 12, 0, 1, "e" * 40) selected = {side["release"] for pair in plan["pairs"] for side in (pair["old"], pair["new"])} untested = {row["release"]: row["reason"] for row in plan["untested_eligible_releases"]} - self.assertEqual(selected | set(untested), {"13.57", "13.59", "13.60"}) + self.assertEqual(selected | set(untested), {"13.57", "13.58", "13.59", "13.60"}) self.assertTrue(all(reason == "not_selected_by_seeded_pair_plan" for reason in untested.values())) + def test_archive_presence_does_not_filter_pair_selection(self): + catalog = self.normalized() + plan = self.plan(seed=12) + self.assertTrue(all(pair["source_resolution"]["state"] == "unresolved" for pair in plan["pairs"])) + self.assertIn("13.58", {entry["name"] for entry in vr.eligible_releases(catalog)}) + def test_duplicate_run_outputs_are_refused_and_initial_status_is_all_unrun(self): plan = self.plan() with tempfile.TemporaryDirectory() as tmp: diff --git a/tools/exiftool-tables/test_version_rehearsal_catalog.py b/tools/exiftool-tables/test_version_rehearsal_catalog.py new file mode 100755 index 000000000..459f9d966 --- /dev/null +++ b/tools/exiftool-tables/test_version_rehearsal_catalog.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Offline tests for catalog capture and selected-source resolution.""" +from __future__ import annotations + +import gzip +import importlib.util +import io +import json +import sys +import tarfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("version_rehearsal_catalog", HERE / "version_rehearsal_catalog.py") +assert spec and spec.loader +catalog_stage = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = catalog_stage +spec.loader.exec_module(catalog_stage) +rehearsal = catalog_stage.rehearsal + +OID_A = "a" * 40 +OID_B = "b" * 40 +OID_C = "c" * 40 +OID_D = "d" * 40 + + +def response(value, headers=None, status=200): + body = value if isinstance(value, bytes) else json.dumps(value, separators=(",", ":")).encode() + return catalog_stage.Response(status=status, headers=headers or {}, body=body) + + +def tag(name, commit): + return {"name": name, "commit": {"sha": commit, "url": f"https://api.github.com/repos/exiftool/exiftool/commits/{commit}"}} + + +def tar_gz(label): + output = io.BytesIO() + with gzip.GzipFile(fileobj=output, mode="wb") as zipped: + with tarfile.open(fileobj=zipped, mode="w") as archive: + data = label.encode() + info = tarfile.TarInfo(f"exiftool-{label}/README") + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + return output.getvalue() + + +class FixtureGet: + def __init__(self, responses): + self.responses = responses + self.calls = [] + + def __call__(self, url): + self.calls.append(url) + value = self.responses.get(url) + if isinstance(value, Exception): + raise value + if value is None: + raise catalog_stage.Refused(f"unexpected URL {url}") + return value + + +def complete_responses(): + second = "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=2" + return { + catalog_stage.TAG_PAGE_URL: response( + [tag("13.59", OID_C), tag("v13.58", OID_B)], + {"Link": f'<{second}>; rel="next", ; rel="last"'}, + ), + second: response([tag("13.58", OID_B), tag("13.57", OID_A)]), + catalog_stage._ref_url("13.59"): response({"object": {"type": "tag", "sha": OID_D}}), + catalog_stage._tag_object_url(OID_D): response({"object": {"type": "commit", "sha": OID_C}}), + catalog_stage._ref_url("13.58"): response({"object": {"type": "commit", "sha": OID_B}}), + catalog_stage._ref_url("13.57"): response({"object": {"type": "commit", "sha": OID_A}}), + } + + +class CatalogCaptureTests(unittest.TestCase): + def capture(self): + return catalog_stage.capture_tag_catalog(FixtureGet(complete_responses()), "2026-09-13T12:00:00Z") + + def normalized(self): + return rehearsal.normalize_catalog(catalog_stage.raw_catalog_from_capture(self.capture())) + + def test_pagination_raw_pages_and_annotated_tag_identity_are_preserved(self): + capture = self.capture() + catalog_stage.verify_capture(capture) + self.assertTrue(capture["complete"]) + self.assertEqual(len(capture["pages"]), 2) + self.assertEqual([item["listed"]["name"] for item in capture["entries"]], ["13.59", "v13.58", "13.58", "13.57"]) + annotated = capture["entries"][0]["identity"] + self.assertEqual((annotated["tag_object"], annotated["peeled_commit"]), (OID_D, OID_C)) + self.assertEqual(capture["entries"][1]["identity"], {"state": "not_requested", "reason": "tag_name_not_numeric_release"}) + catalog = self.normalized() + self.assertEqual([row["name"] for row in rehearsal.eligible_releases(catalog)], ["13.57", "13.58", "13.59"]) + + def test_incomplete_pagination_is_preserved_but_cannot_define_selection_population(self): + responses = complete_responses() + second = "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=2" + responses[second] = catalog_stage.Refused("bounded timeout") + capture = catalog_stage.capture_tag_catalog(FixtureGet(responses), "2026-09-13T12:00:00Z") + catalog_stage.verify_capture(capture) + self.assertFalse(capture["complete"]) + self.assertEqual(capture["failures"][0]["kind"], "page_request_failed") + with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): + catalog_stage.raw_catalog_from_capture(capture) + + def test_moved_ref_is_preserved_and_blocks_population(self): + responses = complete_responses() + responses[catalog_stage._ref_url("13.58")] = response({"object": {"type": "commit", "sha": OID_A}}) + capture = catalog_stage.capture_tag_catalog(FixtureGet(responses), "2026-09-13T12:00:00Z") + identity = next(item["identity"] for item in capture["entries"] if item["listed"].get("name") == "13.58") + self.assertEqual(identity["reason"], "listed_commit_differs_from_resolved_ref") + with self.assertRaisesRegex(catalog_stage.Refused, "lacks an immutable resolved identity"): + catalog_stage.raw_catalog_from_capture(capture) + + def test_duplicate_numeric_tags_survive_capture_and_are_not_silently_selected(self): + responses = complete_responses() + second = "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=2" + responses[second] = response([tag("13.58", OID_B), tag("13.58", OID_B), tag("13.57", OID_A)]) + capture = catalog_stage.capture_tag_catalog(FixtureGet(responses), "2026-09-13T12:00:00Z") + catalog = rehearsal.normalize_catalog(catalog_stage.raw_catalog_from_capture(capture)) + duplicate_states = [entry["classification"] for entry in catalog["entries"] if entry["raw"].get("name") == "13.58"] + self.assertEqual(duplicate_states, [{"state": "unclassified", "reason": "ambiguous_duplicate_release"}] * 2) + self.assertEqual([row["name"] for row in rehearsal.eligible_releases(catalog)], ["13.57", "13.59"]) + + def test_selected_archives_are_resolved_by_immutable_commit_only_after_selection(self): + catalog = self.normalized() + plan = rehearsal.make_plan(catalog, 3, 0, 1, "e" * 40) + selected = {side["release"]: side for pair in plan["pairs"] for side in (pair["old"], pair["new"])} + responses = {catalog_stage.immutable_archive_url(release, side["peeled_commit"]): response(tar_gz(release)) for release, side in selected.items()} + get = FixtureGet(responses) + resolved = catalog_stage.resolve_selected_archives(plan, catalog, get) + catalog_stage.verify_source_resolution(resolved, plan, catalog) + self.assertEqual({row["release"] for row in resolved["selected_releases"]}, set(selected)) + self.assertEqual(set(get.calls), set(responses)) + self.assertEqual(resolved["execution"]["native_read"], "unrun") + self.assertEqual(resolved["execution"]["native_write"], "unrun") + + def test_corrupt_selected_archive_is_refused_without_resolving_unselected_releases(self): + catalog = self.normalized() + plan = rehearsal.make_plan(catalog, 3, 0, 1, "e" * 40) + selected = next(side for pair in plan["pairs"] for side in (pair["old"], pair["new"])) + get = FixtureGet({catalog_stage.immutable_archive_url(selected["release"], selected["peeled_commit"]): response(b"not-a-tar")}) + with self.assertRaisesRegex(catalog_stage.Refused, "readable tar.gz"): + catalog_stage.resolve_selected_archives(plan, catalog, get) + self.assertEqual(len(get.calls), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py index af593ca1e..dd03b6bf0 100755 --- a/tools/exiftool-tables/version_rehearsal.py +++ b/tools/exiftool-tables/version_rehearsal.py @@ -22,7 +22,10 @@ from pathlib import Path from typing import Any -SCHEMA = 1 +# Schema 2 intentionally separates all-release pair selection from archive +# retrieval. Schema-1 raw catalog inputs remain normalizable, but a saved +# normalized schema-1 catalog/plan is not silently reused under new semantics. +SCHEMA = 2 SELECTOR = "python-random-mt19937-v1" RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+$") GIT_OID_RE = re.compile(r"^[0-9a-f]{40,64}$") @@ -72,24 +75,29 @@ def release_key(tag: str) -> tuple[int, int]: def _identity_reason(raw: dict[str, Any]) -> str | None: - name = raw.get("name") + """Validate the identity needed to select a release, not its archive. + + Archives are resolved only after seeded selection. Requiring a cached + archive digest here would shrink the candidate population toward releases + somebody happened to fetch previously. + """ tag_object = raw.get("tag_object") peeled_commit = raw.get("peeled_commit") - archive = raw.get("archive") if not isinstance(tag_object, str) or not GIT_OID_RE.fullmatch(tag_object): return "missing_or_invalid_tag_object" if not isinstance(peeled_commit, str) or not GIT_OID_RE.fullmatch(peeled_commit): return "missing_or_invalid_peeled_commit" - if not isinstance(archive, dict): - return "missing_archive_identity" - expected_url = f"https://github.com/exiftool/exiftool/archive/refs/tags/{name}.tar.gz" - if archive.get("url") != expected_url: - return "missing_or_invalid_archive_url" - if not isinstance(archive.get("sha256"), str) or not SHA256_RE.fullmatch(archive["sha256"]): - return "missing_or_invalid_archive_sha256" return None +def archive_url(release: str, peeled_commit: str) -> str: + if not isinstance(release, str) or not RELEASE_RE.fullmatch(release): + raise Refused("archive URL requires a numeric ExifTool release") + if not isinstance(peeled_commit, str) or not GIT_OID_RE.fullmatch(peeled_commit): + raise Refused("archive URL requires the selected immutable commit") + return f"https://github.com/exiftool/exiftool/archive/{peeled_commit}.tar.gz" + + def normalize_catalog(raw: dict[str, Any]) -> dict[str, Any]: """Normalize an offline official-tag catalog without silently dropping tags. @@ -188,12 +196,10 @@ def select_pairs(catalog: dict[str, Any], seed: int, sample_index: int, pair_cou def _release_identity(entry: dict[str, Any]) -> dict[str, Any]: - archive = entry["archive"] return { "release": entry["name"], "tag_object": entry["tag_object"], "peeled_commit": entry["peeled_commit"], - "archive": {"url": archive["url"], "sha256": archive["sha256"]}, } @@ -240,6 +246,14 @@ def _plan_payload(catalog: dict[str, Any], seed: int, sample_index: int, pair_co "old": _oracle_binding("old", old), "new": _oracle_binding("new", new), }, + "source_resolution": { + "state": "unresolved", + "required_archive_urls": { + "old": archive_url(old["name"], old["peeled_commit"]), + "new": archive_url(new["name"], new["peeled_commit"]), + }, + "limit": "archive bytes and digests are resolved only after seeded selection", + }, "comparison_contract": { "schema": "per-version-native-v1", "required": [ diff --git a/tools/exiftool-tables/version_rehearsal_catalog.py b/tools/exiftool-tables/version_rehearsal_catalog.py new file mode 100755 index 000000000..ceb3eb015 --- /dev/null +++ b/tools/exiftool-tables/version_rehearsal_catalog.py @@ -0,0 +1,579 @@ +#!/usr/bin/env python3 +"""Capture and resolve immutable source identities for version rehearsals. + +The capture stage reads only GitHub's official ExifTool tag APIs. It preserves +exact page bodies and digest records, then resolves each numeric tag through +its ref (and annotated-tag chain when needed) to an immutable commit. Pair +selection uses that complete tag/commit population. Archive bytes are fetched +and hashed only after a plan selects releases, so pre-existing archive cache +contents cannot bias selection. + +Neither command changes OxiDex, its ExifTool pin, generated files, builds, +oracles, or promotion state. A resolved source identity is still not native +read/write conformance evidence. +""" +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import re +import sys +import tarfile +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +import version_rehearsal as rehearsal + +CAPTURE_SCHEMA = 1 +RESOLUTION_SCHEMA = 1 +REPOSITORY = "exiftool/exiftool" +API_ROOT = "https://api.github.com" +TAG_PAGE_URL = f"{API_ROOT}/repos/{REPOSITORY}/tags?per_page=100&page=1" +MAX_TAG_DEPTH = 8 +MAX_ARCHIVE_BYTES = 128 * 1024 * 1024 +LINK_NEXT_RE = re.compile(r'<([^>]+)>;\s*rel="?next"?') + + +@dataclass(frozen=True) +class Response: + status: int + headers: dict[str, str] + body: bytes + + +class Refused(ValueError): + """The captured source cannot support an attributable rehearsal.""" + + +def canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_json(value: Any) -> str: + return sha256_bytes(canonical_json(value)) + + +def atomic_json(path: Path, value: dict[str, Any]) -> None: + rehearsal.atomic_json(path, value) + + +def _json_body(response: Response, context: str) -> Any: + try: + return json.loads(response.body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise Refused(f"{context} did not return UTF-8 JSON") from exc + + +def _body_text(body: bytes) -> str: + try: + return body.decode("utf-8") + except UnicodeDecodeError as exc: + raise Refused("official API response was not UTF-8") from exc + + +def _official_api_url(url: str) -> bool: + parsed = urllib.parse.urlparse(url) + return (parsed.scheme == "https" and parsed.netloc == "api.github.com" + and parsed.path.startswith(f"/repos/{REPOSITORY}/")) + + +def _next_url(headers: dict[str, str]) -> str | None: + link = next((value for key, value in headers.items() if key.lower() == "link"), "") + match = LINK_NEXT_RE.search(link) + if match is None: + return None + url = match.group(1) + if not _official_api_url(url): + raise Refused("pagination next link is not the official ExifTool API") + return url + + +def http_get(url: str, timeout: float, max_bytes: int | None = None) -> Response: + """Read one official URL with a caller-bounded timeout and no retries.""" + headers = {"Accept": "application/vnd.github+json", "User-Agent": "oxidex-version-rehearsal"} + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 -- URL is authenticated below + if max_bytes is not None: + content_length = response.headers.get("Content-Length") + if content_length is not None: + try: + declared_length = int(content_length) + except ValueError as exc: + raise Refused("response has an invalid content length") from exc + if declared_length > max_bytes: + raise Refused("response exceeds configured byte limit") + body = response.read(max_bytes + 1) + if len(body) > max_bytes: + raise Refused("response exceeds configured byte limit") + else: + body = response.read() + return Response(response.status, dict(response.headers.items()), body) + except urllib.error.HTTPError as exc: + return Response(exc.code, dict(exc.headers.items()) if exc.headers else {}, exc.read()) + except urllib.error.URLError as exc: + raise Refused(f"request failed for {url}: {exc.reason}") from exc + + +def _response_record(url: str, response: Response) -> dict[str, Any]: + link = next((value for key, value in response.headers.items() if key.lower() == "link"), "") + return { + "url": url, + "status": response.status, + "body_utf8": _body_text(response.body), + "body_sha256": sha256_bytes(response.body), + "link_header": link, + } + + +def _ref_url(tag_name: str) -> str: + return f"{API_ROOT}/repos/{REPOSITORY}/git/ref/tags/{urllib.parse.quote(tag_name, safe='')}" + + +def _tag_object_url(oid: str) -> str: + return f"{API_ROOT}/repos/{REPOSITORY}/git/tags/{oid}" + + +def _list_commit(listed: dict[str, Any]) -> str | None: + commit = listed.get("commit") + sha = commit.get("sha") if isinstance(commit, dict) else None + return sha if isinstance(sha, str) and rehearsal.GIT_OID_RE.fullmatch(sha) else None + + +def _resolve_numeric_tag(name: str, listed: dict[str, Any], get: Callable[[str], Response]) -> dict[str, Any]: + """Resolve a listed tag to its ref object and terminal commit. + + Every response is retained. Ref/type failures are records rather than + discarded tags, so a partial capture cannot masquerade as a complete + candidate population. + """ + trace: list[dict[str, Any]] = [] + try: + url = _ref_url(name) + response = get(url) + trace.append(_response_record(url, response)) + if response.status != 200: + return {"state": "failed", "reason": "tag_ref_http_status", "trace": trace} + ref = _json_body(response, "tag ref") + obj = ref.get("object") if isinstance(ref, dict) else None + kind = obj.get("type") if isinstance(obj, dict) else None + oid = obj.get("sha") if isinstance(obj, dict) else None + if kind not in {"commit", "tag"} or not isinstance(oid, str) or not rehearsal.GIT_OID_RE.fullmatch(oid): + return {"state": "failed", "reason": "tag_ref_missing_object", "trace": trace} + tag_object = oid + depth = 0 + while kind == "tag": + depth += 1 + if depth > MAX_TAG_DEPTH: + return {"state": "failed", "reason": "annotated_tag_depth_exceeded", "trace": trace} + url = _tag_object_url(oid) + response = get(url) + trace.append(_response_record(url, response)) + if response.status != 200: + return {"state": "failed", "reason": "annotated_tag_http_status", "trace": trace} + tag = _json_body(response, "annotated tag") + obj = tag.get("object") if isinstance(tag, dict) else None + kind = obj.get("type") if isinstance(obj, dict) else None + oid = obj.get("sha") if isinstance(obj, dict) else None + if kind not in {"commit", "tag"} or not isinstance(oid, str) or not rehearsal.GIT_OID_RE.fullmatch(oid): + return {"state": "failed", "reason": "annotated_tag_missing_object", "trace": trace} + listed_commit = _list_commit(listed) + if listed_commit is not None and listed_commit != oid: + return { + "state": "failed", + "reason": "listed_commit_differs_from_resolved_ref", + "trace": trace, + "tag_object": tag_object, + "peeled_commit": oid, + "listed_commit": listed_commit, + } + return { + "state": "resolved", + "tag_object": tag_object, + "peeled_commit": oid, + "listed_commit": listed_commit, + "trace": trace, + } + except Refused as exc: + return {"state": "failed", "reason": "identity_request_failed", "detail": str(exc), "trace": trace} + + +def capture_tag_catalog(get: Callable[[str], Response], captured_at: str | None = None) -> dict[str, Any]: + """Capture every official tag page and resolve each numeric release tag.""" + pages: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] + failures: list[dict[str, Any]] = [] + current = TAG_PAGE_URL + seen: set[str] = set() + complete = True + while current is not None: + if current in seen: + complete = False + failures.append({"kind": "pagination_cycle", "url": current}) + break + seen.add(current) + try: + response = get(current) + except Refused as exc: + complete = False + failures.append({"kind": "page_request_failed", "url": current, "detail": str(exc)}) + break + record = _response_record(current, response) + if response.status != 200: + complete = False + failures.append({"kind": "page_http_status", "url": current, "status": response.status, "body_sha256": record["body_sha256"]}) + break + try: + listed = _json_body(response, "tag page") + if not isinstance(listed, list): + raise Refused("tag page was not a JSON list") + next_url = _next_url(response.headers) + except Refused as exc: + complete = False + failures.append({"kind": "page_malformed", "url": current, "detail": str(exc), "body_sha256": record["body_sha256"]}) + break + page_index = len(pages) + pages.append({**record, "next_url": next_url}) + for entry_index, listed_entry in enumerate(listed): + name = listed_entry.get("name") if isinstance(listed_entry, dict) else None + item: dict[str, Any] = { + "source_page": page_index, + "source_index": entry_index, + "listed": listed_entry, + } + if not isinstance(name, str) or not rehearsal.RELEASE_RE.fullmatch(name): + item["identity"] = {"state": "not_requested", "reason": "tag_name_not_numeric_release"} + else: + item["identity"] = _resolve_numeric_tag(name, listed_entry, get) + entries.append(item) + current = next_url + payload = { + "schema": CAPTURE_SCHEMA, + "kind": "oxidex_exiftool_official_tag_capture", + "repository": REPOSITORY, + "tag_page_start_url": TAG_PAGE_URL, + "captured_at": captured_at or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "complete": complete, + "pages": pages, + "entries": entries, + "failures": failures, + "execution": {"state": "source_identity_only", "native_read": "unrun", "native_write": "unrun"}, + } + return {**payload, "capture_sha256": sha256_json(payload)} + + +def verify_capture(capture: dict[str, Any]) -> None: + if (capture.get("schema") != CAPTURE_SCHEMA or capture.get("kind") != "oxidex_exiftool_official_tag_capture" + or capture.get("repository") != REPOSITORY or capture.get("tag_page_start_url") != TAG_PAGE_URL + or not isinstance(capture.get("captured_at"), str) or not isinstance(capture.get("complete"), bool) + or not isinstance(capture.get("pages"), list) or not isinstance(capture.get("entries"), list) + or not isinstance(capture.get("failures"), list)): + raise Refused("unsupported capture manifest") + payload = {key: value for key, value in capture.items() if key != "capture_sha256"} + if capture.get("capture_sha256") != sha256_json(payload): + raise Refused("capture manifest identity changed") + source_rows: list[Any] = [] + for page_index, page in enumerate(capture["pages"]): + if not isinstance(page, dict) or not _official_api_url(page.get("url", "")) or page.get("status") != 200: + raise Refused("capture page identity is malformed") + text = page.get("body_utf8") + if not isinstance(text, str) or page.get("body_sha256") != sha256_bytes(text.encode("utf-8")): + raise Refused("capture page body digest differs") + if not isinstance(page.get("link_header"), str): + raise Refused("capture page link header is malformed") + if page.get("next_url") != _next_url({"Link": page["link_header"]}): + raise Refused("capture pagination link differs from raw header") + if page_index + 1 < len(capture["pages"]) and page["next_url"] != capture["pages"][page_index + 1].get("url"): + raise Refused("capture page sequence differs from pagination links") + if page_index + 1 == len(capture["pages"]) and capture["complete"] and page["next_url"] is not None: + raise Refused("complete capture ended before pagination ended") + try: + listed = json.loads(text) + except json.JSONDecodeError as exc: + raise Refused("capture page body is not JSON") from exc + if not isinstance(listed, list): + raise Refused("capture page body is not a tag list") + for index, row in enumerate(listed): + source_rows.append((page_index, index, row)) + if len(source_rows) != len(capture["entries"]): + raise Refused("capture entries do not cover raw pages") + for item, (page_index, index, listed) in zip(capture["entries"], source_rows, strict=True): + if not isinstance(item, dict) or item.get("source_page") != page_index or item.get("source_index") != index or item.get("listed") != listed: + raise Refused("capture entry differs from raw page") + identity = item.get("identity") + if not isinstance(identity, dict): + raise Refused("capture entry lacks identity record") + name = listed.get("name") if isinstance(listed, dict) else None + if not isinstance(name, str) or not rehearsal.RELEASE_RE.fullmatch(name): + if identity != {"state": "not_requested", "reason": "tag_name_not_numeric_release"}: + raise Refused("nonnumeric tag identity was not preserved") + elif identity.get("state") == "resolved": + _verify_resolved_identity(name, listed, identity) + elif identity.get("state") != "failed" or not isinstance(identity.get("reason"), str): + raise Refused("numeric tag identity state is malformed") + + +def _trace_json(record: dict[str, Any], expected_url: str, context: str) -> Any: + if (not isinstance(record, dict) or record.get("url") != expected_url or record.get("status") != 200 + or not isinstance(record.get("body_utf8"), str) or record.get("body_sha256") != sha256_bytes(record["body_utf8"].encode("utf-8"))): + raise Refused(f"{context} trace differs from saved response") + try: + return json.loads(record["body_utf8"]) + except json.JSONDecodeError as exc: + raise Refused(f"{context} trace is not JSON") from exc + + +def _verify_resolved_identity(name: str, listed: dict[str, Any], identity: dict[str, Any]) -> None: + trace = identity.get("trace") + if not isinstance(trace, list) or not trace: + raise Refused("resolved numeric tag lacks immutable trace") + ref = _trace_json(trace[0], _ref_url(name), "tag ref") + obj = ref.get("object") if isinstance(ref, dict) else None + kind = obj.get("type") if isinstance(obj, dict) else None + oid = obj.get("sha") if isinstance(obj, dict) else None + if kind not in {"commit", "tag"} or not isinstance(oid, str) or not rehearsal.GIT_OID_RE.fullmatch(oid): + raise Refused("tag ref trace has no resolvable object") + tag_object = oid + trace_index = 1 + while kind == "tag": + if trace_index >= len(trace) or trace_index > MAX_TAG_DEPTH: + raise Refused("annotated tag trace is incomplete") + tag = _trace_json(trace[trace_index], _tag_object_url(oid), "annotated tag") + trace_index += 1 + obj = tag.get("object") if isinstance(tag, dict) else None + kind = obj.get("type") if isinstance(obj, dict) else None + oid = obj.get("sha") if isinstance(obj, dict) else None + if kind not in {"commit", "tag"} or not isinstance(oid, str) or not rehearsal.GIT_OID_RE.fullmatch(oid): + raise Refused("annotated tag trace has no resolvable object") + if trace_index != len(trace): + raise Refused("resolved tag trace has unused responses") + listed_commit = _list_commit(listed) + if listed_commit is not None and listed_commit != oid: + raise Refused("resolved tag trace disagrees with listed commit") + if identity != { + "state": "resolved", + "tag_object": tag_object, + "peeled_commit": oid, + "listed_commit": listed_commit, + "trace": trace, + }: + raise Refused("resolved tag identity differs from raw trace") + + +def _identity_fields(identity: dict[str, Any]) -> tuple[str, str] | None: + tag_object, commit = identity.get("tag_object"), identity.get("peeled_commit") + if not isinstance(tag_object, str) or not rehearsal.GIT_OID_RE.fullmatch(tag_object): + return None + if not isinstance(commit, str) or not rehearsal.GIT_OID_RE.fullmatch(commit): + return None + return tag_object, commit + + +def raw_catalog_from_capture(capture: dict[str, Any]) -> dict[str, Any]: + """Make the planner input only when every numeric tag identity is known.""" + verify_capture(capture) + if not capture["complete"]: + raise Refused("incomplete pagination cannot define a release population") + entries: list[dict[str, Any]] = [] + for item in capture["entries"]: + listed = item["listed"] + name = listed.get("name") if isinstance(listed, dict) else None + identity = item["identity"] + if isinstance(name, str) and rehearsal.RELEASE_RE.fullmatch(name): + fields = _identity_fields(identity) + if identity.get("state") != "resolved" or fields is None: + raise Refused(f"numeric release {name!r} lacks an immutable resolved identity") + tag_object, peeled_commit = fields + entries.append({ + "name": name, + "tag_object": tag_object, + "peeled_commit": peeled_commit, + "capture_provenance": {"source_page": item["source_page"], "source_index": item["source_index"], "identity_sha256": sha256_json(identity)}, + }) + else: + entries.append({"name": name, "capture_provenance": {"source_page": item["source_page"], "source_index": item["source_index"]}}) + return { + "catalog_source": { + "kind": "official_exiftool_tag_catalog", + "repository": REPOSITORY, + "capture_schema": CAPTURE_SCHEMA, + "capture_sha256": capture["capture_sha256"], + "pages": [{"url": page["url"], "sha256": page["body_sha256"]} for page in capture["pages"]], + }, + "captured_at": capture["captured_at"], + "entries": entries, + } + + +def immutable_archive_url(release: str, peeled_commit: str) -> str: + if not isinstance(peeled_commit, str) or not rehearsal.GIT_OID_RE.fullmatch(peeled_commit): + raise Refused("immutable archive URL requires a commit object id") + return rehearsal.archive_url(release, peeled_commit) + + +def _read_limited(response: Response, limit: int) -> bytes: + if len(response.body) > limit: + raise Refused("archive exceeds configured byte limit") + return response.body + + +def _verify_tar_gz(body: bytes) -> None: + try: + with gzip.GzipFile(fileobj=io.BytesIO(body)) as stream: + with tarfile.open(fileobj=stream, mode="r:") as archive: + if not archive.getmembers(): + raise Refused("archive has no members") + except (OSError, tarfile.TarError, EOFError) as exc: + raise Refused("archive bytes are not a readable tar.gz") from exc + + +def resolve_selected_archives(plan: dict[str, Any], catalog: dict[str, Any], get: Callable[[str], Response], max_archive_bytes: int = MAX_ARCHIVE_BYTES) -> dict[str, Any]: + """Fetch/hash exactly the unique releases selected by an immutable plan.""" + rehearsal.verify_plan(plan, catalog) + if not isinstance(max_archive_bytes, int) or isinstance(max_archive_bytes, bool) or max_archive_bytes < 1: + raise Refused("archive byte limit must be positive") + selected = { + side["release"]: side + for pair in plan["pairs"] + for side in (pair["old"], pair["new"]) + } + required_urls: dict[str, str] = {} + for pair in plan["pairs"]: + requirement = pair.get("source_resolution") + if (not isinstance(requirement, dict) or requirement.get("state") != "unresolved" + or not isinstance(requirement.get("required_archive_urls"), dict)): + raise Refused("selected pair lacks unresolved archive requirement") + for label in ("old", "new"): + identity = pair[label] + expected_url = immutable_archive_url(identity["release"], identity["peeled_commit"]) + if requirement["required_archive_urls"].get(label) != expected_url: + raise Refused("selected pair archive requirement is not commit-pinned") + required_urls[identity["release"]] = expected_url + resolved: list[dict[str, Any]] = [] + for release, identity in sorted(selected.items(), key=lambda item: rehearsal.release_key(item[0])): + url = required_urls[release] + try: + response = get(url) + if response.status != 200: + raise Refused(f"archive HTTP status {response.status}") + body = _read_limited(response, max_archive_bytes) + _verify_tar_gz(body) + except Refused as exc: + raise Refused(f"selected archive {release} cannot be resolved: {exc}") from exc + resolved.append({ + **identity, + "archive": {"url": url, "sha256": sha256_bytes(body), "bytes": len(body), "format": "tar.gz"}, + }) + payload = { + "schema": RESOLUTION_SCHEMA, + "kind": "oxidex_exiftool_selected_source_resolution", + "plan_sha256": plan["plan_sha256"], + "catalog_sha256": catalog["catalog_sha256"], + "selected_releases": resolved, + "execution": {"state": "source_identity_resolved_only", "native_read": "unrun", "native_write": "unrun"}, + } + return {**payload, "resolution_sha256": sha256_json(payload)} + + +def verify_source_resolution(resolution: dict[str, Any], plan: dict[str, Any], catalog: dict[str, Any]) -> None: + rehearsal.verify_plan(plan, catalog) + payload = {key: value for key, value in resolution.items() if key != "resolution_sha256"} + if (resolution.get("schema") != RESOLUTION_SCHEMA or resolution.get("kind") != "oxidex_exiftool_selected_source_resolution" + or resolution.get("plan_sha256") != plan["plan_sha256"] or resolution.get("catalog_sha256") != catalog["catalog_sha256"] + or resolution.get("resolution_sha256") != sha256_json(payload) or not isinstance(resolution.get("selected_releases"), list)): + raise Refused("source resolution identity changed or is malformed") + expected = { + side["release"]: side + for pair in plan["pairs"] + for side in (pair["old"], pair["new"]) + } + if len(resolution["selected_releases"]) != len(expected) or {row.get("release") for row in resolution["selected_releases"] if isinstance(row, dict)} != set(expected): + raise Refused("source resolution has missing or extra selected releases") + for row in resolution["selected_releases"]: + if not isinstance(row, dict) or row.get("release") not in expected: + raise Refused("source resolution has unselected release") + identity = expected[row["release"]] + if any(row.get(key) != identity[key] for key in ("release", "tag_object", "peeled_commit")): + raise Refused("source resolution identity differs from selected plan") + archive = row.get("archive") + if (not isinstance(archive, dict) or archive.get("url") != immutable_archive_url(row["release"], identity["peeled_commit"]) + or not isinstance(archive.get("sha256"), str) or not rehearsal.SHA256_RE.fullmatch(archive["sha256"]) + or not isinstance(archive.get("bytes"), int) or archive["bytes"] < 1 or archive.get("format") != "tar.gz"): + raise Refused("source resolution archive identity is malformed") + + +def _cmd_capture(args: argparse.Namespace) -> int: + output = Path(args.output) + if output.exists(): + raise Refused(f"capture output already exists: {output}") + capture = capture_tag_catalog(lambda url: http_get(url, args.timeout)) + atomic_json(output, capture) + try: + raw_catalog = raw_catalog_from_capture(capture) + except Refused as exc: + print(json.dumps({"capture": str(output), "selection_ready": False, "reason": str(exc)}, sort_keys=True)) + return 2 + if args.catalog_output: + catalog_output = Path(args.catalog_output) + if catalog_output.exists(): + raise Refused(f"catalog output already exists: {catalog_output}") + atomic_json(catalog_output, raw_catalog) + print(json.dumps({"capture": str(output), "capture_sha256": capture["capture_sha256"], "selection_ready": True, "native_read": "unrun", "native_write": "unrun"}, sort_keys=True)) + return 0 + + +def _cmd_resolve(args: argparse.Namespace) -> int: + output = Path(args.output) + if output.exists(): + raise Refused(f"resolution output already exists: {output}") + catalog = rehearsal.normalize_catalog(rehearsal.read_json(Path(args.catalog))) + plan = rehearsal.read_json(Path(args.plan)) + resolution = resolve_selected_archives(plan, catalog, lambda url: http_get(url, args.timeout, args.max_archive_bytes), args.max_archive_bytes) + atomic_json(output, resolution) + print(json.dumps({"resolution": str(output), "resolution_sha256": resolution["resolution_sha256"], "native_read": "unrun", "native_write": "unrun"}, sort_keys=True)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + capture = sub.add_parser("capture", help="capture official tag pages and tag-to-commit identities only") + capture.add_argument("--output", required=True) + capture.add_argument("--catalog-output", help="optional raw planner input; written only after complete identity capture") + capture.add_argument("--timeout", type=float, default=20.0) + capture.set_defaults(func=_cmd_capture) + resolve = sub.add_parser("resolve-selected", help="fetch/hash immutable commit archives for already selected releases") + resolve.add_argument("--catalog", required=True, help="raw capture-derived catalog input") + resolve.add_argument("--plan", required=True) + resolve.add_argument("--output", required=True) + resolve.add_argument("--timeout", type=float, default=30.0) + resolve.add_argument("--max-archive-bytes", type=int, default=MAX_ARCHIVE_BYTES) + resolve.set_defaults(func=_cmd_resolve) + args = parser.parse_args(argv) + try: + if args.timeout <= 0: + raise Refused("timeout must be positive") + return args.func(args) + except Refused as exc: + print(f"version rehearsal catalog refused: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From b5bef0be1b1ac59644c66a37d00adf196886ddb0 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:29:39 -0500 Subject: [PATCH 13/22] tools: accept GitHub repository-id page links --- tools/exiftool-tables/test_version_rehearsal_catalog.py | 8 ++++---- tools/exiftool-tables/version_rehearsal.py | 6 +++++- tools/exiftool-tables/version_rehearsal_catalog.py | 7 +++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal_catalog.py b/tools/exiftool-tables/test_version_rehearsal_catalog.py index 459f9d966..62e92fcf3 100755 --- a/tools/exiftool-tables/test_version_rehearsal_catalog.py +++ b/tools/exiftool-tables/test_version_rehearsal_catalog.py @@ -61,11 +61,11 @@ def __call__(self, url): def complete_responses(): - second = "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=2" + second = "https://api.github.com/repositories/132751855/tags?per_page=100&page=2" return { catalog_stage.TAG_PAGE_URL: response( [tag("13.59", OID_C), tag("v13.58", OID_B)], - {"Link": f'<{second}>; rel="next", ; rel="last"'}, + {"Link": f'<{second}>; rel="next", ; rel="last"'}, ), second: response([tag("13.58", OID_B), tag("13.57", OID_A)]), catalog_stage._ref_url("13.59"): response({"object": {"type": "tag", "sha": OID_D}}), @@ -96,7 +96,7 @@ def test_pagination_raw_pages_and_annotated_tag_identity_are_preserved(self): def test_incomplete_pagination_is_preserved_but_cannot_define_selection_population(self): responses = complete_responses() - second = "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=2" + second = "https://api.github.com/repositories/132751855/tags?per_page=100&page=2" responses[second] = catalog_stage.Refused("bounded timeout") capture = catalog_stage.capture_tag_catalog(FixtureGet(responses), "2026-09-13T12:00:00Z") catalog_stage.verify_capture(capture) @@ -116,7 +116,7 @@ def test_moved_ref_is_preserved_and_blocks_population(self): def test_duplicate_numeric_tags_survive_capture_and_are_not_silently_selected(self): responses = complete_responses() - second = "https://api.github.com/repos/exiftool/exiftool/tags?per_page=100&page=2" + second = "https://api.github.com/repositories/132751855/tags?per_page=100&page=2" responses[second] = response([tag("13.58", OID_B), tag("13.58", OID_B), tag("13.57", OID_A)]) capture = catalog_stage.capture_tag_catalog(FixtureGet(responses), "2026-09-13T12:00:00Z") catalog = rehearsal.normalize_catalog(catalog_stage.raw_catalog_from_capture(capture)) diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py index dd03b6bf0..512bd1e3b 100755 --- a/tools/exiftool-tables/version_rehearsal.py +++ b/tools/exiftool-tables/version_rehearsal.py @@ -30,6 +30,10 @@ RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+$") GIT_OID_RE = re.compile(r"^[0-9a-f]{40,64}$") SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +OFFICIAL_TAG_PAGE_PREFIXES = ( + "https://api.github.com/repos/exiftool/exiftool/tags", + "https://api.github.com/repositories/132751855/tags", +) class Refused(ValueError): @@ -111,7 +115,7 @@ def normalize_catalog(raw: dict[str, Any]) -> dict[str, Any]: if (not isinstance(source, dict) or source.get("kind") != "official_exiftool_tag_catalog" or not isinstance(pages, list) or not pages or any(not isinstance(page, dict) or not isinstance(page.get("url"), str) - or not page["url"].startswith("https://api.github.com/repos/exiftool/exiftool/tags") + or not page["url"].startswith(OFFICIAL_TAG_PAGE_PREFIXES) or not isinstance(page.get("sha256"), str) or not SHA256_RE.fullmatch(page["sha256"]) for page in pages) or not isinstance(raw.get("captured_at"), str)): diff --git a/tools/exiftool-tables/version_rehearsal_catalog.py b/tools/exiftool-tables/version_rehearsal_catalog.py index ceb3eb015..388b82eb8 100755 --- a/tools/exiftool-tables/version_rehearsal_catalog.py +++ b/tools/exiftool-tables/version_rehearsal_catalog.py @@ -36,6 +36,7 @@ CAPTURE_SCHEMA = 1 RESOLUTION_SCHEMA = 1 REPOSITORY = "exiftool/exiftool" +REPOSITORY_ID = "132751855" API_ROOT = "https://api.github.com" TAG_PAGE_URL = f"{API_ROOT}/repos/{REPOSITORY}/tags?per_page=100&page=1" MAX_TAG_DEPTH = 8 @@ -86,8 +87,10 @@ def _body_text(body: bytes) -> str: def _official_api_url(url: str) -> bool: parsed = urllib.parse.urlparse(url) - return (parsed.scheme == "https" and parsed.netloc == "api.github.com" - and parsed.path.startswith(f"/repos/{REPOSITORY}/")) + if parsed.scheme != "https" or parsed.netloc != "api.github.com": + return False + return (parsed.path.startswith(f"/repos/{REPOSITORY}/") + or parsed.path.startswith(f"/repositories/{REPOSITORY_ID}/tags")) def _next_url(headers: dict[str, str]) -> str | None: From cd08dcf21d93fe14fcae16503308183ebf46c509 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:33:14 -0500 Subject: [PATCH 14/22] tools: bind rehearsal archive resolution to capture --- .../test_version_rehearsal_catalog.py | 15 +++++++++++--- .../version_rehearsal_catalog.py | 20 ++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal_catalog.py b/tools/exiftool-tables/test_version_rehearsal_catalog.py index 62e92fcf3..70d608fb1 100755 --- a/tools/exiftool-tables/test_version_rehearsal_catalog.py +++ b/tools/exiftool-tables/test_version_rehearsal_catalog.py @@ -130,8 +130,9 @@ def test_selected_archives_are_resolved_by_immutable_commit_only_after_selection selected = {side["release"]: side for pair in plan["pairs"] for side in (pair["old"], pair["new"])} responses = {catalog_stage.immutable_archive_url(release, side["peeled_commit"]): response(tar_gz(release)) for release, side in selected.items()} get = FixtureGet(responses) - resolved = catalog_stage.resolve_selected_archives(plan, catalog, get) - catalog_stage.verify_source_resolution(resolved, plan, catalog) + capture = self.capture() + resolved = catalog_stage.resolve_selected_archives(plan, catalog, capture, get) + catalog_stage.verify_source_resolution(resolved, plan, catalog, capture) self.assertEqual({row["release"] for row in resolved["selected_releases"]}, set(selected)) self.assertEqual(set(get.calls), set(responses)) self.assertEqual(resolved["execution"]["native_read"], "unrun") @@ -143,9 +144,17 @@ def test_corrupt_selected_archive_is_refused_without_resolving_unselected_releas selected = next(side for pair in plan["pairs"] for side in (pair["old"], pair["new"])) get = FixtureGet({catalog_stage.immutable_archive_url(selected["release"], selected["peeled_commit"]): response(b"not-a-tar")}) with self.assertRaisesRegex(catalog_stage.Refused, "readable tar.gz"): - catalog_stage.resolve_selected_archives(plan, catalog, get) + catalog_stage.resolve_selected_archives(plan, catalog, self.capture(), get) self.assertEqual(len(get.calls), 1) + def test_archive_resolution_refuses_catalog_not_derived_from_its_capture(self): + catalog = self.normalized() + catalog["entries"][0]["raw"]["peeled_commit"] = OID_D + catalog["catalog_sha256"] = rehearsal.sha256_json({key: value for key, value in catalog.items() if key != "catalog_sha256"}) + plan = rehearsal.make_plan(self.normalized(), 3, 0, 1, "e" * 40) + with self.assertRaisesRegex(catalog_stage.Refused, "differs from its saved source capture"): + catalog_stage.resolve_selected_archives(plan, catalog, self.capture(), FixtureGet({})) + if __name__ == "__main__": unittest.main() diff --git a/tools/exiftool-tables/version_rehearsal_catalog.py b/tools/exiftool-tables/version_rehearsal_catalog.py index 388b82eb8..d27591398 100755 --- a/tools/exiftool-tables/version_rehearsal_catalog.py +++ b/tools/exiftool-tables/version_rehearsal_catalog.py @@ -424,6 +424,14 @@ def raw_catalog_from_capture(capture: dict[str, Any]) -> dict[str, Any]: } +def verify_capture_binding(capture: dict[str, Any], catalog: dict[str, Any]) -> None: + """Require a planner catalog to be exactly derived from saved raw pages.""" + verify_capture(capture) + expected = rehearsal.normalize_catalog(raw_catalog_from_capture(capture)) + if catalog != expected: + raise Refused("planner catalog differs from its saved source capture") + + def immutable_archive_url(release: str, peeled_commit: str) -> str: if not isinstance(peeled_commit, str) or not rehearsal.GIT_OID_RE.fullmatch(peeled_commit): raise Refused("immutable archive URL requires a commit object id") @@ -446,8 +454,9 @@ def _verify_tar_gz(body: bytes) -> None: raise Refused("archive bytes are not a readable tar.gz") from exc -def resolve_selected_archives(plan: dict[str, Any], catalog: dict[str, Any], get: Callable[[str], Response], max_archive_bytes: int = MAX_ARCHIVE_BYTES) -> dict[str, Any]: +def resolve_selected_archives(plan: dict[str, Any], catalog: dict[str, Any], capture: dict[str, Any], get: Callable[[str], Response], max_archive_bytes: int = MAX_ARCHIVE_BYTES) -> dict[str, Any]: """Fetch/hash exactly the unique releases selected by an immutable plan.""" + verify_capture_binding(capture, catalog) rehearsal.verify_plan(plan, catalog) if not isinstance(max_archive_bytes, int) or isinstance(max_archive_bytes, bool) or max_archive_bytes < 1: raise Refused("archive byte limit must be positive") @@ -488,17 +497,20 @@ def resolve_selected_archives(plan: dict[str, Any], catalog: dict[str, Any], get "kind": "oxidex_exiftool_selected_source_resolution", "plan_sha256": plan["plan_sha256"], "catalog_sha256": catalog["catalog_sha256"], + "capture_sha256": capture["capture_sha256"], "selected_releases": resolved, "execution": {"state": "source_identity_resolved_only", "native_read": "unrun", "native_write": "unrun"}, } return {**payload, "resolution_sha256": sha256_json(payload)} -def verify_source_resolution(resolution: dict[str, Any], plan: dict[str, Any], catalog: dict[str, Any]) -> None: +def verify_source_resolution(resolution: dict[str, Any], plan: dict[str, Any], catalog: dict[str, Any], capture: dict[str, Any]) -> None: + verify_capture_binding(capture, catalog) rehearsal.verify_plan(plan, catalog) payload = {key: value for key, value in resolution.items() if key != "resolution_sha256"} if (resolution.get("schema") != RESOLUTION_SCHEMA or resolution.get("kind") != "oxidex_exiftool_selected_source_resolution" or resolution.get("plan_sha256") != plan["plan_sha256"] or resolution.get("catalog_sha256") != catalog["catalog_sha256"] + or resolution.get("capture_sha256") != capture["capture_sha256"] or resolution.get("resolution_sha256") != sha256_json(payload) or not isinstance(resolution.get("selected_releases"), list)): raise Refused("source resolution identity changed or is malformed") expected = { @@ -545,9 +557,10 @@ def _cmd_resolve(args: argparse.Namespace) -> int: output = Path(args.output) if output.exists(): raise Refused(f"resolution output already exists: {output}") + capture = rehearsal.read_json(Path(args.capture)) catalog = rehearsal.normalize_catalog(rehearsal.read_json(Path(args.catalog))) plan = rehearsal.read_json(Path(args.plan)) - resolution = resolve_selected_archives(plan, catalog, lambda url: http_get(url, args.timeout, args.max_archive_bytes), args.max_archive_bytes) + resolution = resolve_selected_archives(plan, catalog, capture, lambda url: http_get(url, args.timeout, args.max_archive_bytes), args.max_archive_bytes) atomic_json(output, resolution) print(json.dumps({"resolution": str(output), "resolution_sha256": resolution["resolution_sha256"], "native_read": "unrun", "native_write": "unrun"}, sort_keys=True)) return 0 @@ -562,6 +575,7 @@ def main(argv: list[str] | None = None) -> int: capture.add_argument("--timeout", type=float, default=20.0) capture.set_defaults(func=_cmd_capture) resolve = sub.add_parser("resolve-selected", help="fetch/hash immutable commit archives for already selected releases") + resolve.add_argument("--capture", required=True, help="raw official capture bound to the planner catalog") resolve.add_argument("--catalog", required=True, help="raw capture-derived catalog input") resolve.add_argument("--plan", required=True) resolve.add_argument("--output", required=True) From 8f3194074d03ac105520ceb181af884c06c770f5 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:34:41 -0500 Subject: [PATCH 15/22] tools: constrain rehearsal catalog page origins --- .../test_version_rehearsal_catalog.py | 12 ++++++++++++ tools/exiftool-tables/version_rehearsal.py | 7 +++---- tools/exiftool-tables/version_rehearsal_catalog.py | 3 +-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal_catalog.py b/tools/exiftool-tables/test_version_rehearsal_catalog.py index 70d608fb1..4c8ec33a3 100755 --- a/tools/exiftool-tables/test_version_rehearsal_catalog.py +++ b/tools/exiftool-tables/test_version_rehearsal_catalog.py @@ -105,6 +105,18 @@ def test_incomplete_pagination_is_preserved_but_cannot_define_selection_populati with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): catalog_stage.raw_catalog_from_capture(capture) + def test_non_tag_pagination_target_is_refused_before_any_population_is_accepted(self): + responses = complete_responses() + responses[catalog_stage.TAG_PAGE_URL] = response( + [tag("13.59", OID_C)], + {"Link": '; rel="next"'}, + ) + capture = catalog_stage.capture_tag_catalog(FixtureGet(responses), "2026-09-13T12:00:00Z") + self.assertFalse(capture["complete"]) + self.assertEqual(capture["failures"][0]["kind"], "page_malformed") + with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): + catalog_stage.raw_catalog_from_capture(capture) + def test_moved_ref_is_preserved_and_blocks_population(self): responses = complete_responses() responses[catalog_stage._ref_url("13.58")] = response({"object": {"type": "commit", "sha": OID_A}}) diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py index 512bd1e3b..bca971b2b 100755 --- a/tools/exiftool-tables/version_rehearsal.py +++ b/tools/exiftool-tables/version_rehearsal.py @@ -30,9 +30,8 @@ RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+$") GIT_OID_RE = re.compile(r"^[0-9a-f]{40,64}$") SHA256_RE = re.compile(r"^[0-9a-f]{64}$") -OFFICIAL_TAG_PAGE_PREFIXES = ( - "https://api.github.com/repos/exiftool/exiftool/tags", - "https://api.github.com/repositories/132751855/tags", +OFFICIAL_TAG_PAGE_RE = re.compile( + r"^https://api\.github\.com/(?:repos/exiftool/exiftool|repositories/132751855)/tags(?:\?.*)?$" ) @@ -115,7 +114,7 @@ def normalize_catalog(raw: dict[str, Any]) -> dict[str, Any]: if (not isinstance(source, dict) or source.get("kind") != "official_exiftool_tag_catalog" or not isinstance(pages, list) or not pages or any(not isinstance(page, dict) or not isinstance(page.get("url"), str) - or not page["url"].startswith(OFFICIAL_TAG_PAGE_PREFIXES) + or OFFICIAL_TAG_PAGE_RE.fullmatch(page["url"]) is None or not isinstance(page.get("sha256"), str) or not SHA256_RE.fullmatch(page["sha256"]) for page in pages) or not isinstance(raw.get("captured_at"), str)): diff --git a/tools/exiftool-tables/version_rehearsal_catalog.py b/tools/exiftool-tables/version_rehearsal_catalog.py index d27591398..166fa5967 100755 --- a/tools/exiftool-tables/version_rehearsal_catalog.py +++ b/tools/exiftool-tables/version_rehearsal_catalog.py @@ -89,8 +89,7 @@ def _official_api_url(url: str) -> bool: parsed = urllib.parse.urlparse(url) if parsed.scheme != "https" or parsed.netloc != "api.github.com": return False - return (parsed.path.startswith(f"/repos/{REPOSITORY}/") - or parsed.path.startswith(f"/repositories/{REPOSITORY_ID}/tags")) + return parsed.path in {f"/repos/{REPOSITORY}/tags", f"/repositories/{REPOSITORY_ID}/tags"} def _next_url(headers: dict[str, str]) -> str | None: From 3131ee70e8625014f5dbef4737c4c17444003ae8 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:42:53 -0500 Subject: [PATCH 16/22] tools: require contiguous rehearsal catalog pages --- .../test_version_rehearsal_catalog.py | 32 ++++++++++ .../version_rehearsal_catalog.py | 60 ++++++++++++++++++- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal_catalog.py b/tools/exiftool-tables/test_version_rehearsal_catalog.py index 4c8ec33a3..3ec073531 100755 --- a/tools/exiftool-tables/test_version_rehearsal_catalog.py +++ b/tools/exiftool-tables/test_version_rehearsal_catalog.py @@ -117,6 +117,38 @@ def test_non_tag_pagination_target_is_refused_before_any_population_is_accepted( with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): catalog_stage.raw_catalog_from_capture(capture) + def test_capture_refuses_skipped_or_ambiguous_pagination_before_fetching_it(self): + for next_url in ( + "https://api.github.com/repositories/132751855/tags?per_page=100&page=4", + "https://api.github.com/repositories/132751855/tags?per_page=50&page=2", + "https://api.github.com/repositories/132751855/tags?per_page=100&page=2&page=3", + "https://api.github.com/repositories/132751855/tags?per_page=100&page=02", + ): + with self.subTest(next_url=next_url): + responses = complete_responses() + responses[catalog_stage.TAG_PAGE_URL] = response( + [tag("13.59", OID_C)], {"Link": f"<{next_url}>; rel=\"next\""} + ) + get = FixtureGet(responses) + capture = catalog_stage.capture_tag_catalog(get, "2026-09-13T12:00:00Z") + self.assertFalse(capture["complete"]) + self.assertEqual(capture["failures"][0]["kind"], "page_malformed") + self.assertEqual(get.calls, [catalog_stage.TAG_PAGE_URL]) + with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): + catalog_stage.raw_catalog_from_capture(capture) + + def test_rehashed_capture_cannot_skip_a_page_in_replay_validation(self): + capture = self.capture() + skipped = "https://api.github.com/repositories/132751855/tags?per_page=100&page=4" + capture["pages"][0]["link_header"] = f"<{skipped}>; rel=\"next\"" + capture["pages"][0]["next_url"] = skipped + capture["pages"][1]["url"] = skipped + capture["capture_sha256"] = catalog_stage.sha256_json( + {key: value for key, value in capture.items() if key != "capture_sha256"} + ) + with self.assertRaisesRegex(catalog_stage.Refused, "does not advance exactly one page"): + catalog_stage.verify_capture(capture) + def test_moved_ref_is_preserved_and_blocks_population(self): responses = complete_responses() responses[catalog_stage._ref_url("13.58")] = response({"object": {"type": "commit", "sha": OID_A}}) diff --git a/tools/exiftool-tables/version_rehearsal_catalog.py b/tools/exiftool-tables/version_rehearsal_catalog.py index 166fa5967..985eca4bb 100755 --- a/tools/exiftool-tables/version_rehearsal_catalog.py +++ b/tools/exiftool-tables/version_rehearsal_catalog.py @@ -92,14 +92,45 @@ def _official_api_url(url: str) -> bool: return parsed.path in {f"/repos/{REPOSITORY}/tags", f"/repositories/{REPOSITORY_ID}/tags"} +def _page_coordinates(url: str) -> tuple[int, int]: + """Return the canonical official tag-page coordinates. + + GitHub may switch the repository path in a Link header, but the page + sequence itself is part of the captured population. Accepting arbitrary + query strings would let a saved page 1 point directly at page 4 and still + appear complete. Parse query pairs rather than using a dict so duplicate + keys cannot make the effective page ambiguous. + """ + if not _official_api_url(url): + raise Refused("pagination link is not the official ExifTool tag API") + parsed = urllib.parse.urlparse(url) + pairs = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) + values: dict[str, str] = {} + for key, value in pairs: + if key in values: + raise Refused("pagination link has a duplicate query key") + values[key] = value + if set(values) != {"per_page", "page"}: + raise Refused("pagination link has unsupported or missing query keys") + try: + per_page = int(values["per_page"]) + page = int(values["page"]) + except ValueError as exc: + raise Refused("pagination link has a nonnumeric page coordinate") from exc + if not 1 <= per_page <= 100 or page < 1: + raise Refused("pagination link has an out-of-range page coordinate") + if values["per_page"] != str(per_page) or values["page"] != str(page): + raise Refused("pagination link has a noncanonical page coordinate") + return per_page, page + + def _next_url(headers: dict[str, str]) -> str | None: link = next((value for key, value in headers.items() if key.lower() == "link"), "") match = LINK_NEXT_RE.search(link) if match is None: return None url = match.group(1) - if not _official_api_url(url): - raise Refused("pagination next link is not the official ExifTool API") + _page_coordinates(url) return url @@ -223,8 +254,17 @@ def capture_tag_catalog(get: Callable[[str], Response], captured_at: str | None failures: list[dict[str, Any]] = [] current = TAG_PAGE_URL seen: set[str] = set() + expected_per_page, expected_page = _page_coordinates(current) complete = True while current is not None: + try: + per_page, page = _page_coordinates(current) + if per_page != expected_per_page or page != expected_page: + raise Refused("pagination sequence is not contiguous") + except Refused as exc: + complete = False + failures.append({"kind": "page_malformed", "url": current, "detail": str(exc)}) + break if current in seen: complete = False failures.append({"kind": "pagination_cycle", "url": current}) @@ -246,6 +286,10 @@ def capture_tag_catalog(get: Callable[[str], Response], captured_at: str | None if not isinstance(listed, list): raise Refused("tag page was not a JSON list") next_url = _next_url(response.headers) + if next_url is not None: + next_per_page, next_page = _page_coordinates(next_url) + if next_per_page != expected_per_page or next_page != expected_page + 1: + raise Refused("pagination next link does not advance exactly one page") except Refused as exc: complete = False failures.append({"kind": "page_malformed", "url": current, "detail": str(exc), "body_sha256": record["body_sha256"]}) @@ -265,6 +309,7 @@ def capture_tag_catalog(get: Callable[[str], Response], captured_at: str | None item["identity"] = _resolve_numeric_tag(name, listed_entry, get) entries.append(item) current = next_url + expected_page += 1 payload = { "schema": CAPTURE_SCHEMA, "kind": "oxidex_exiftool_official_tag_capture", @@ -290,10 +335,16 @@ def verify_capture(capture: dict[str, Any]) -> None: payload = {key: value for key, value in capture.items() if key != "capture_sha256"} if capture.get("capture_sha256") != sha256_json(payload): raise Refused("capture manifest identity changed") + expected_per_page, expected_page = _page_coordinates(TAG_PAGE_URL) + if capture["complete"] and not capture["pages"]: + raise Refused("complete capture has no tag pages") source_rows: list[Any] = [] for page_index, page in enumerate(capture["pages"]): if not isinstance(page, dict) or not _official_api_url(page.get("url", "")) or page.get("status") != 200: raise Refused("capture page identity is malformed") + per_page, page_number = _page_coordinates(page["url"]) + if per_page != expected_per_page or page_number != expected_page: + raise Refused("capture page sequence is not contiguous") text = page.get("body_utf8") if not isinstance(text, str) or page.get("body_sha256") != sha256_bytes(text.encode("utf-8")): raise Refused("capture page body digest differs") @@ -301,6 +352,10 @@ def verify_capture(capture: dict[str, Any]) -> None: raise Refused("capture page link header is malformed") if page.get("next_url") != _next_url({"Link": page["link_header"]}): raise Refused("capture pagination link differs from raw header") + if page["next_url"] is not None: + next_per_page, next_page = _page_coordinates(page["next_url"]) + if next_per_page != expected_per_page or next_page != expected_page + 1: + raise Refused("capture pagination next link does not advance exactly one page") if page_index + 1 < len(capture["pages"]) and page["next_url"] != capture["pages"][page_index + 1].get("url"): raise Refused("capture page sequence differs from pagination links") if page_index + 1 == len(capture["pages"]) and capture["complete"] and page["next_url"] is not None: @@ -313,6 +368,7 @@ def verify_capture(capture: dict[str, Any]) -> None: raise Refused("capture page body is not a tag list") for index, row in enumerate(listed): source_rows.append((page_index, index, row)) + expected_page += 1 if len(source_rows) != len(capture["entries"]): raise Refused("capture entries do not cover raw pages") for item, (page_index, index, listed) in zip(capture["entries"], source_rows, strict=True): From 4c584dd8b4ff32a701e0892fdf3cc31a29c7dec9 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:48:59 -0500 Subject: [PATCH 17/22] tools: reject ambiguous catalog next links --- .../test_version_rehearsal_catalog.py | 28 +++++++++++++++++++ .../version_rehearsal_catalog.py | 8 ++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal_catalog.py b/tools/exiftool-tables/test_version_rehearsal_catalog.py index 3ec073531..06d195d1f 100755 --- a/tools/exiftool-tables/test_version_rehearsal_catalog.py +++ b/tools/exiftool-tables/test_version_rehearsal_catalog.py @@ -137,6 +137,22 @@ def test_capture_refuses_skipped_or_ambiguous_pagination_before_fetching_it(self with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): catalog_stage.raw_catalog_from_capture(capture) + def test_capture_refuses_multiple_next_relations_before_fetching_any_target(self): + page_two = "https://api.github.com/repositories/132751855/tags?per_page=100&page=2" + page_four = "https://api.github.com/repositories/132751855/tags?per_page=100&page=4" + responses = complete_responses() + responses[catalog_stage.TAG_PAGE_URL] = response( + [tag("13.59", OID_C)], + {"Link": f'<{page_two}>; rel="next", <{page_four}>; rel="next"'}, + ) + get = FixtureGet(responses) + capture = catalog_stage.capture_tag_catalog(get, "2026-09-13T12:00:00Z") + self.assertFalse(capture["complete"]) + self.assertEqual(capture["failures"][0]["kind"], "page_malformed") + self.assertEqual(get.calls, [catalog_stage.TAG_PAGE_URL]) + with self.assertRaisesRegex(catalog_stage.Refused, "incomplete pagination"): + catalog_stage.raw_catalog_from_capture(capture) + def test_rehashed_capture_cannot_skip_a_page_in_replay_validation(self): capture = self.capture() skipped = "https://api.github.com/repositories/132751855/tags?per_page=100&page=4" @@ -149,6 +165,18 @@ def test_rehashed_capture_cannot_skip_a_page_in_replay_validation(self): with self.assertRaisesRegex(catalog_stage.Refused, "does not advance exactly one page"): catalog_stage.verify_capture(capture) + def test_rehashed_capture_cannot_hide_a_second_next_relation(self): + capture = self.capture() + page_two = "https://api.github.com/repositories/132751855/tags?per_page=100&page=2" + page_four = "https://api.github.com/repositories/132751855/tags?per_page=100&page=4" + capture["pages"][0]["link_header"] = f'<{page_two}>; rel="next", <{page_four}>; rel="next"' + capture["pages"][0]["next_url"] = page_two + capture["capture_sha256"] = catalog_stage.sha256_json( + {key: value for key, value in capture.items() if key != "capture_sha256"} + ) + with self.assertRaisesRegex(catalog_stage.Refused, "multiple rel=next"): + catalog_stage.verify_capture(capture) + def test_moved_ref_is_preserved_and_blocks_population(self): responses = complete_responses() responses[catalog_stage._ref_url("13.58")] = response({"object": {"type": "commit", "sha": OID_A}}) diff --git a/tools/exiftool-tables/version_rehearsal_catalog.py b/tools/exiftool-tables/version_rehearsal_catalog.py index 985eca4bb..134ed6999 100755 --- a/tools/exiftool-tables/version_rehearsal_catalog.py +++ b/tools/exiftool-tables/version_rehearsal_catalog.py @@ -126,10 +126,12 @@ def _page_coordinates(url: str) -> tuple[int, int]: def _next_url(headers: dict[str, str]) -> str | None: link = next((value for key, value in headers.items() if key.lower() == "link"), "") - match = LINK_NEXT_RE.search(link) - if match is None: + matches = LINK_NEXT_RE.findall(link) + if not matches: return None - url = match.group(1) + if len(matches) != 1: + raise Refused("pagination link has multiple rel=next relations") + url = matches[0] _page_coordinates(url) return url From 16432502683756c07f018ca8086e4be38b43d218 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 11:54:09 -0500 Subject: [PATCH 18/22] Record source candidates and version capture with execution gaps --- docs/AUTOGENERATION-PLAN.md | 2 +- docs/AUTOGENERATION-PROGRESS.md | 17 ++++++++++- docs/reference/read-write-version-plan.md | 35 +++++++++++++++++++---- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/docs/AUTOGENERATION-PLAN.md b/docs/AUTOGENERATION-PLAN.md index 2122ced89..ee5bdf808 100644 --- a/docs/AUTOGENERATION-PLAN.md +++ b/docs/AUTOGENERATION-PLAN.md @@ -149,7 +149,7 @@ order and independent native validation. | **2. Prove one complete migration.** | Move Sony's 17-entry focus-point table, `Tag202a`, through the shared machinery. Reuse the existing ability to save a value and evaluate conditions on it; extend it for binary tables. Preserve native selection rules and list any remaining parent-routing work. | All 17 entries and their conditions are accounted for. Real-file and synthetic boundary comparisons agree with pinned ExifTool. Change a name, enum, offset and supported new row in a copied native source: regeneration must reflect each change without a new handwritten tag rule. Stale output must fail verification. | | **3. Remove what the pilot replaces.** | Switch the validated family to the shared path, then remove its duplicate custom declarations and handling. Preserve shared file-reading/decryption mechanisms and explicitly list any remaining tag-specific routing or helper rules. | A merged change identifies the exact manual rules removed, shows the shared path actually executed, and has zero unexplained per-file regressions. Keeping the old path as the real producer does not pass this step. | | **4. Expand by shared capability.** | Choose the next unsupported behavior that serves several families. Candidates include saved-value effects, suppression rules, lookup conversions and verified handling of encrypted blocks. Generate other families through the same machinery; use one difficult neighboring family to test that the design generalizes. | Each batch lists the native families unlocked, manual rules retired, remaining unsupported rules, actual output gain and validation scope. Add no new vendor-specific copy of an already supported expression. The first additional family must reuse the new capability without adding another interpreter. | -| **5. Prove an upgrade needs less intervention.** | Run the existing isolated upgrade tool against another pinned release. Record every manual edit and its cause. Turn repeated causes into shared capabilities, then rerun. | Supported native changes regenerate with zero tag-specific Python/Rust edits. Unknown semantics fail visibly and preserve working output. Report elapsed time, build time, manual interventions and corpus changes. A failed or unexercised change stays open. | +| **5. Prove an upgrade needs less intervention.** | Run the existing isolated upgrade tool against another pinned release. Record every manual edit and its cause. Turn repeated causes into shared capabilities, then rerun. | Supported native changes regenerate with zero tag-specific Python/Rust edits. Unknown new semantics refuse promotion and remain a visible gap; they must not silently retain old rules in a new-version build. Report elapsed time, build time, manual interventions and corpus changes. A failed or unexercised change stays open. | | **6. Close the full remaining inventory.** | Repeat the capability/migration/retirement cycle until no native rule in scope depends on manual tag knowledge. Broaden fixtures for unexercised formats and keep testing later release changes. | Zero manually maintained tag-specific rules, zero unclassified rules, and zero required unsupported behaviors remain in scope. All required parity and upgrade checks pass. A corpus percentage alone cannot certify this finish line. | Step 1 measurement and Step 2 implementation can proceed in parallel. We will diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index 7ea69165e..be70492fe 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -53,11 +53,26 @@ preserving old semantics silently is not a successful upgrade. The published seeded planner records this contract but has no executable rehearsal stages yet. Review found plan/journal validation gaps; the repair at `44cf6135` passed 17 focused tests and five independent altered-plan checks, and is -integrated as `21eed548`. Full source identity capture and execution remain +integrated as `21eed548`. Official catalog capture now records four pages and 388 numeric release tags +with immutable commit identities. Archive materialization and execution remain unfinished. The [version plan](reference/read-write-version-plan.md) separates these unfinished stages and never treats random samples as proof of all releases. +The integrated inactive compiler emits **22 string candidates in one table** +from the recorded native dump. This is source classification, with **zero new +production writer routes**. Independent mutation checks cover malformed source +provenance, table identities and group defaults. The shared raw TIFF editing +primitive passes **16 focused tests** and full Clippy after independent review; +it contains no tag-name lookup and remains inactive. Complete JPEG/TIFF native +proof, actual helper translation, public identity resolution and exact-value +input are the next writer requirements. + +The combined regeneration attempt completed tier-1 oracle checks and then +failed its workspace guard because the untracked handoff was edited. Preserve +the failed result; it is not a passing full regeneration. The retry must freeze +all workspace files and complete tier 2 plus the full Python suite. + ### Previous merged definitions checkpoint Latest combined Canon definitions: **AFInfo 14/14 and AFInfo2 16/16**, both diff --git a/docs/reference/read-write-version-plan.md b/docs/reference/read-write-version-plan.md index 26102674c..a45a6c476 100644 --- a/docs/reference/read-write-version-plan.md +++ b/docs/reference/read-write-version-plan.md @@ -48,7 +48,8 @@ not certify other releases or behaviors. placement and write type; use the existing JPEG/TIFF surgical mechanisms. Verify insert, update, growth, shrinkage and deletion against native ExifTool in both byte orders, preserving unrelated metadata and image/file payload. - Keep defined-empty values distinct from deletion. Both EXIF family names + Include default UTF-8 text and embedded NUL through a generic exact-value + input, and keep defined-empty values distinct from deletion. Both EXIF family names and physical IFD names must address the same generated identity; the pre-migration EXIF-qualified deletion silently succeeds without deleting, while TIFF deletion is explicitly unsupported. Preserve these as baseline @@ -107,7 +108,31 @@ complete loader-token grammar after rejecting two earlier bypasses. The full 153-module read projection is unchanged. Planner repair checks exact catalog selection, matching-version oracle bindings, selected journal membership and untested scope; 17 focused tests and five independent altered-plan checks pass. -These are source and planning foundations: writer activation, official live -catalog/source capture, both-version regeneration/builds and real read/write -comparisons are still required. Combined official regeneration is the next -integration check. No successful release upgrade is claimed by a saved plan. +The inactive writer compiler is integrated through `10c89560`. On the recorded +native dump it emits one table with 22 string candidates; those rows have no +production write route. Captured identities, procedure provenance and effective +native groups are preserved. Independent mutations reject malformed identities +and change the candidate when supported source facts change. Procedure hashes +are evidence of origin, not proof that the procedure is translated. + +The official release capture is integrated through `4c584dd8`. The saved capture +contains four pages and 388 numeric release tags with resolved commit identities. +Review found skipped-page and duplicate-next-link acceptance; both repairs and +failed probes are retained. Selection and immutable source identity are now +implemented. Archive materialization, both-version regeneration/builds, native +old-to-new comparison and real read/write comparisons remain unfinished. + +The inactive raw TIFF editing primitive is integrated through `059f56d1`. +It accepts resolved directory/tag/type/value operations and deletion without a +manual tag-name lookup. Sixteen focused tests and full Clippy pass; independent +review accepted directory-graph and size-bound fixes. Complete-carrier proof, +source helper translation and public generated identity routing remain separate +requirements. No production writing behavior is changed by this checkpoint. + +The first combined official regeneration stopped at its workspace guard because +an untracked handoff changed during the run. Tier-1 oracle checks completed; +tier 2 and the full Python suite did not run. Preserve that failed attempt, and +freeze all workspace files, including handoff notes, for the retry. The recorded +expression ledger changed only its native dump identity after write-fact capture. +Combined validation remains pending. No successful release upgrade is claimed +by a saved plan or by the 388-entry catalog. From a80452da8dee96408ec21acd0f1d3dee03789106 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 12:22:37 -0500 Subject: [PATCH 19/22] docs: record validated foundation and first selected release sources --- docs/AUTOGENERATION-PLAN.md | 2 +- docs/AUTOGENERATION-PROGRESS.md | 17 ++++++++++++----- docs/reference/read-write-version-plan.md | 22 ++++++++++++++++++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/AUTOGENERATION-PLAN.md b/docs/AUTOGENERATION-PLAN.md index ee5bdf808..dea1f9ea8 100644 --- a/docs/AUTOGENERATION-PLAN.md +++ b/docs/AUTOGENERATION-PLAN.md @@ -100,7 +100,7 @@ regenerates both releases and checks each against its own native read/write behavior. Selection or successful generation alone is not conformance. The selected newer ExifTool is the authority after upgrading. Its parsing bug -fixes, added or renamed tags, type changes, formatting and write semantics +fixes, added/removed/renamed tags, type changes, formatting and write semantics must replace older behavior. Keep three comparisons: old OxiDex versus old native, new OxiDex versus new native, and the native old-to-new delta. A change in native output is an expected upstream change when the new generated build diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index be70492fe..28fa960c3 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -31,7 +31,7 @@ scalar class, beginning with HostComputer `0x013c`. Source capture and upgrade planning are integrated work-branch checkpoints, not activated writing. Review found flattened scalar references and a loader recognizer that accepted changed executable behavior. Repairs at `04a694e3` passed independent mutation review -and are integrated as `468a114b`; combined regeneration remains to be checked. +and are integrated as `468a114b`; combined regeneration now passes at `16432502`. Full canonical capture proves that the serialized read projection for all 153 modules is unchanged; this is not proof about every live binding after writer loading. @@ -54,8 +54,11 @@ seeded planner records this contract but has no executable rehearsal stages yet. Review found plan/journal validation gaps; the repair at `44cf6135` passed 17 focused tests and five independent altered-plan checks, and is integrated as `21eed548`. Official catalog capture now records four pages and 388 numeric release tags -with immutable commit identities. Archive materialization and execution remain -unfinished. The +with immutable commit identities. The separately reviewed materializer +`05729f1b` downloaded and verified the first persisted random pair, **11.78 to +12.64**. Their native version checks pass; native capability checks, generated +builds and read/write comparisons remain unfinished. Source identity is not +upgrade conformance. The [version plan](reference/read-write-version-plan.md) separates these unfinished stages and never treats random samples as proof of all releases. @@ -70,8 +73,12 @@ input are the next writer requirements. The combined regeneration attempt completed tier-1 oracle checks and then failed its workspace guard because the untracked handoff was edited. Preserve -the failed result; it is not a passing full regeneration. The retry must freeze -all workspace files and complete tier 2 plus the full Python suite. +the failed result. The frozen retry at `16432502` passed all 32-artifact +regeneration with zero changes (236.627 seconds), native processor checks, +831 Python tests with zero skips (528.143 seconds), and 6,022 Rust tests with +zero failures and 124 ignored (136.323 seconds). Formatting, full Clippy and +diff checks also passed. The ignored tests are unexercised. This foundation +awaits hosted review and landing; no new production writer is enabled. ### Previous merged definitions checkpoint diff --git a/docs/reference/read-write-version-plan.md b/docs/reference/read-write-version-plan.md index a45a6c476..eef05acb6 100644 --- a/docs/reference/read-write-version-plan.md +++ b/docs/reference/read-write-version-plan.md @@ -119,8 +119,13 @@ The official release capture is integrated through `4c584dd8`. The saved capture contains four pages and 388 numeric release tags with resolved commit identities. Review found skipped-page and duplicate-next-link acceptance; both repairs and failed probes are retained. Selection and immutable source identity are now -implemented. Archive materialization, both-version regeneration/builds, native -old-to-new comparison and real read/write comparisons remain unfinished. +implemented in this branch. A separately reviewed materializer at `05729f1b` +has downloaded, extracted and verified both selected source archives. The first +persisted random pair is **11.78 to 12.64**, selected once from the complete +388-entry catalog. Both report the expected version under canonical Perl 5.38.2. +This proves source identity only. Both-version regeneration/builds, native +capability checks, old-to-new comparison and real read/write comparisons remain +unfinished. The materializer is not integrated in this foundation branch. The inactive raw TIFF editing primitive is integrated through `059f56d1`. It accepts resolved directory/tag/type/value operations and deletion without a @@ -134,5 +139,14 @@ an untracked handoff changed during the run. Tier-1 oracle checks completed; tier 2 and the full Python suite did not run. Preserve that failed attempt, and freeze all workspace files, including handoff notes, for the retry. The recorded expression ledger changed only its native dump identity after write-fact capture. -Combined validation remains pending. No successful release upgrade is claimed -by a saved plan or by the 388-entry catalog. +The frozen retry at `16432502` passed on September 13 at 17:12:18 UTC: +all 32 artifacts regenerated with zero changes (236.627 seconds), the native +processor oracle passed, all 831 Python tests passed with zero skips +(528.143 seconds), and `cargo test --workspace --all-features` passed 6,022 +tests with 124 ignored (136.323 seconds). Formatting, full Clippy and diff +checks also passed. The 124 ignored tests are not counted as exercised. +Evidence relative to the continuation evidence root is +`shared-pilot/write-upgrade-integration-20260913/retry-16432502/validation-state.json` +and its stage logs. This is local foundation validation; hosted review and +landing remain pending. No successful release upgrade is claimed by a saved +plan, a verified archive, or the 388-entry catalog. From 66f28379ab9f1f0b20710b4bd474f3b701583567 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 12:27:20 -0500 Subject: [PATCH 20/22] fix(tables): preserve writer bindings and source-distinct rehearsals --- .../exiftool-tables/test_version_rehearsal.py | 23 +++++++++++++++ .../exiftool-tables/test_write_descriptors.py | 21 ++++++++++++++ tools/exiftool-tables/version_rehearsal.py | 25 ++++++++++++++-- tools/exiftool-tables/write_descriptors.py | 29 +++++++++++++++---- 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/tools/exiftool-tables/test_version_rehearsal.py b/tools/exiftool-tables/test_version_rehearsal.py index cc6ca1c13..d4f1448b5 100644 --- a/tools/exiftool-tables/test_version_rehearsal.py +++ b/tools/exiftool-tables/test_version_rehearsal.py @@ -98,6 +98,29 @@ def test_same_version_and_ambiguous_identity_are_refused(self): with self.assertRaisesRegex(vr.Refused, "identity changed"): vr.verify_plan(bad_plan, self.normalized()) + def test_same_peeled_commit_pairs_refuse_on_creation_and_verification(self): + same_source = { + "catalog_source": catalog_entries()["catalog_source"], + "captured_at": "2026-09-13T00:00:00Z", + "entries": [ + release("13.58", OID_A, SHA_A), + release("13.59", OID_A, SHA_B), + ], + } + with self.assertRaisesRegex(vr.Refused, "distinct-source pairs"): + vr.make_plan(vr.normalize_catalog(same_source), 7, 0, 1, "e" * 40) + + forged = self.plan() + pair = forged["pairs"][0] + pair["new"]["peeled_commit"] = pair["old"]["peeled_commit"] + pair["native_oracles"]["new"]["native_release_identity"]["peeled_commit"] = pair["old"]["peeled_commit"] + pair["source_resolution"]["required_archive_urls"]["new"] = vr.archive_url( + pair["new"]["release"], pair["old"]["peeled_commit"] + ) + self.rehash_plan(forged) + with self.assertRaisesRegex(vr.Refused, "same peeled source commit"): + vr.verify_plan(forged, self.normalized()) + def test_changed_catalog_identity_refuses_plan_reuse(self): catalog = self.normalized() plan = vr.make_plan(catalog, 3, 0, 1, "e" * 40) diff --git a/tools/exiftool-tables/test_write_descriptors.py b/tools/exiftool-tables/test_write_descriptors.py index 155723e12..1ba854680 100644 --- a/tools/exiftool-tables/test_write_descriptors.py +++ b/tools/exiftool-tables/test_write_descriptors.py @@ -126,6 +126,27 @@ def test_compatible_source_row_emits_dynamic_name_id_placement_and_provenance(se self.assertIn('INACTIVE_WRITE_RUNTIME_STATUS: &str = "inactive_source_candidates_no_writer_route"', source) self.assertIn('do not activate a writer or authenticate the', source) + def test_dependency_binding_survives_anonymous_final_callable(self): + changed = document() + deps = changed["native_write_tables"]["Exif"]["Main"]["effective_write_proc"]["effective"]["dependencies"] + deps["Image::ExifTool::Exif::WriteHelper"] = code_fact( + "Image::ExifTool::__ANON__", "anonymous-helper-body" + ) + source, report = population(changed) + self.assertEqual(report.emitted_rows, 1) + self.assertIn( + 'binding: "Image::ExifTool::Exif::WriteHelper", fact: NativeWriteProcedureProvenance { name: "Image::ExifTool::__ANON__"', + source, + ) + + def test_malformed_dependency_binding_refuses_candidate(self): + changed = document() + deps = changed["native_write_tables"]["Exif"]["Main"]["effective_write_proc"]["effective"]["dependencies"] + deps["not a callable binding"] = code_fact("Image::ExifTool::Exif::WriteHelper", "helper") + source, report = population(changed) + self.assertEqual((report.emitted_tables, report.emitted_rows), (0, 0)) + self.assertIn('"write_provenance_unresolved"', source) + def test_name_and_compatible_new_row_follow_source_without_tag_allowlist(self): changed = document() changed["native_write_tables"]["Exif"]["Main"]["rows"] = { diff --git a/tools/exiftool-tables/version_rehearsal.py b/tools/exiftool-tables/version_rehearsal.py index bca971b2b..5090fc77c 100755 --- a/tools/exiftool-tables/version_rehearsal.py +++ b/tools/exiftool-tables/version_rehearsal.py @@ -185,9 +185,16 @@ def select_pairs(catalog: dict[str, Any], seed: int, sample_index: int, pair_cou if sample_index < 0 or pair_count < 1: raise Refused("sample index must be nonnegative and pair count positive") releases = eligible_releases(catalog) - pairs = list(itertools.combinations(releases, 2)) + # A release name alone is not a source-version boundary: two distinct tags + # can legally peel to one commit. A rehearsal must compare two immutable + # sources, so exclude those duplicate-source pairs before seeded sampling. + pairs = [ + (old, new) + for old, new in itertools.combinations(releases, 2) + if old["peeled_commit"] != new["peeled_commit"] + ] if pair_count > len(pairs): - raise Refused(f"pair count {pair_count} exceeds {len(pairs)} available unordered pairs") + raise Refused(f"pair count {pair_count} exceeds {len(pairs)} available distinct-source pairs") rng = random.Random(f"{SELECTOR}:{seed}:{sample_index}") selected = rng.sample(pairs, pair_count) # itertools combinations has each pair in increasing release order; verify @@ -304,6 +311,20 @@ def verify_plan(plan: dict[str, Any], catalog: dict[str, Any] | None = None) -> raise Refused("catalog identity differs from recorded plan") expected = plan.get("plan_sha256") payload = {k: v for k, v in plan.items() if k != "plan_sha256"} + # Do not let a rehashed historical or hand-authored plan reuse one native + # source under two release labels. Creation filters these pairs; this + # direct check keeps verification fail-closed if a stored plan is altered. + pairs = plan.get("pairs") + if isinstance(pairs, list): + for pair in pairs: + if not isinstance(pair, dict): + continue + old = pair.get("old") + new = pair.get("new") + if (isinstance(old, dict) and isinstance(new, dict) + and isinstance(old.get("peeled_commit"), str) + and old.get("peeled_commit") == new.get("peeled_commit")): + raise Refused("plan selects the same peeled source commit twice") if not isinstance(expected, str) or expected != sha256_json(payload): raise Refused("plan identity changed or is malformed") try: diff --git a/tools/exiftool-tables/write_descriptors.py b/tools/exiftool-tables/write_descriptors.py index 334f39474..c043eb38f 100644 --- a/tools/exiftool-tables/write_descriptors.py +++ b/tools/exiftool-tables/write_descriptors.py @@ -166,10 +166,18 @@ def _procedure_provenance(fact: Any, context: str) -> dict[str, Any]: dependencies = fact.get("dependencies", {}) dependencies = _mapping(dependencies, f"{context}.dependencies") - result["dependencies"] = [ - _procedure_provenance(dep, f"{context}.dependencies[{name!r}]") - for name, dep in sorted(dependencies.items()) - ] + result["dependencies"] = [] + for binding, dependency in sorted(dependencies.items()): + # The source fact is keyed by the exact glob that the owning procedure + # calls. Keeping only the final CV identity would lose an anonymous + # or rebound glob, making a later mechanism authenticate the wrong + # callable. This is provenance only, never execution admission. + if not isinstance(binding, str) or _CODE_NAME_RE.fullmatch(binding) is None: + raise WriteDescriptorError(f"{context}.dependencies has a malformed binding") + result["dependencies"].append({ + "binding": binding, + "fact": _procedure_provenance(dependency, f"{context}.dependencies[{binding!r}]"), + }) return result @@ -363,7 +371,11 @@ def _rust_string(value: str) -> str: def _rust_provenance(fact: Mapping[str, Any]) -> str: - deps = ", ".join(_rust_provenance(dep) for dep in fact["dependencies"]) + deps = ", ".join( + "NativeWriteProcedureDependency { " + f"binding: {_rust_string(dependency['binding'])}, fact: {_rust_provenance(dependency['fact'])} }}" + for dependency in fact["dependencies"] + ) return ( "NativeWriteProcedureProvenance { " f"name: {_rust_string(fact['name'])}, source_file: {_rust_string(fact['source_file'])}, " @@ -400,7 +412,12 @@ def rust_source(population: Mapping[str, Any]) -> str: pub source_file: &'static str, pub source_sha256: &'static str, pub body_sha256: &'static str, - pub dependencies: &'static [NativeWriteProcedureProvenance], + /// Each dependency retains the exact source glob binding and final CV fact. + pub dependencies: &'static [NativeWriteProcedureDependency], +}} +pub struct NativeWriteProcedureDependency {{ + pub binding: &'static str, + pub fact: NativeWriteProcedureProvenance, }} pub struct InactiveWriteScalarString {{ pub raw_id: u16, From 149387d2e35b05f53d1fb85744d40d7157fafa2d Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 12:30:15 -0500 Subject: [PATCH 21/22] docs: record foundation review fixes and remaining upgrade proof --- docs/AUTOGENERATION-PROGRESS.md | 5 ++++- docs/reference/read-write-version-plan.md | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index 28fa960c3..1f80c6443 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -78,7 +78,10 @@ regeneration with zero changes (236.627 seconds), native processor checks, 831 Python tests with zero skips (528.143 seconds), and 6,022 Rust tests with zero failures and 124 ignored (136.323 seconds). Formatting, full Clippy and diff checks also passed. The ignored tests are unexercised. This foundation -awaits hosted review and landing; no new production writer is enabled. +is in ready PR #761. Two independent review fixes preserve dependency binding +names and reject same-source upgrade pairs; all 57 affected Python tests pass, +and a binding-only mutation changes compilable generated candidate Rust. +Hosted checks and landing remain pending; no new production writer is enabled. ### Previous merged definitions checkpoint diff --git a/docs/reference/read-write-version-plan.md b/docs/reference/read-write-version-plan.md index eef05acb6..45cb8d0c2 100644 --- a/docs/reference/read-write-version-plan.md +++ b/docs/reference/read-write-version-plan.md @@ -150,3 +150,13 @@ Evidence relative to the continuation evidence root is and its stage logs. This is local foundation validation; hosted review and landing remain pending. No successful release upgrade is claimed by a saved plan, a verified archive, or the 388-entry catalog. + +Independent foundation review then found two source-identity gaps. Repair +`66f28379` retains each requested dependency binding alongside its final +callable provenance, including anonymous callables, and rejects release pairs +that resolve to the same source commit during selection and verification. +All 57 affected Python tests pass. An independent binding-only mutation changes +the rendered candidate, and that rendered Rust compiles. The saved 11.78/12.64 +plan still verifies without reselection. No runtime Rust or canonical generated +artifact changed after the full gate above. Ready PR #761 is awaiting hosted +checks on the final source head. From 675a736e16d3dfdefe31db42629d3fdce9781a3e Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sun, 13 Sep 2026 19:26:17 -0500 Subject: [PATCH 22/22] ci: allow complete native table verification within a bounded job --- .github/workflows/ci.yml | 14 +++++++------- docs/AUTOGENERATION-PROGRESS.md | 6 +++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01106d147..5f161e42c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -442,14 +442,14 @@ jobs: # throwaway harness to execute the committed Start/Base forms. verify-tables: name: Verify Generated Tables - # 2x on purpose: this is a tarball fetch plus a perl/python pass over one - # file, ~30s and entirely I/O-bound. The sizing measurements above apply - # to compile-bound jobs and buy nothing here -- 2x is for the Warp queue - # (seconds, vs minutes on the GitHub-hosted pool), not for the cores. + # This job now verifies generated artifacts and runs the complete native + # mutation/replay suite. Keep the existing runner size; allow the serial + # Perl/Python work to finish instead of treating it as a short I/O check. runs-on: warp-ubuntu-latest-x64-2x - # Native source replay adds a measured 399-second local suite to the - # previous six-minute verification job. Keep room for cold hosted runs. - timeout-minutes: 20 + # PR #761 at 149387d2 used 5m47s for setup/tier verification, then hit the + # former 20m job cap during native inventory mutations (679 tests reached). + # No check is removed; retain a finite bound with room for the full suite. + timeout-minutes: 35 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # The release comes from .exiftool-version, the repo-wide pin. This job diff --git a/docs/AUTOGENERATION-PROGRESS.md b/docs/AUTOGENERATION-PROGRESS.md index 1f80c6443..f2a4e53d8 100644 --- a/docs/AUTOGENERATION-PROGRESS.md +++ b/docs/AUTOGENERATION-PROGRESS.md @@ -81,7 +81,11 @@ diff checks also passed. The ignored tests are unexercised. This foundation is in ready PR #761. Two independent review fixes preserve dependency binding names and reject same-source upgrade pairs; all 57 affected Python tests pass, and a binding-only mutation changes compilable generated candidate Rust. -Hosted checks and landing remain pending; no new production writer is enabled. +Four required hosted checks passed on `149387d2`; generated-table verification +was cancelled at its 20-minute job limit during the native inventory mutation +test (679 Python cases reached). This is incomplete verification, not a pass. +The job allowance is now 35 minutes with all checks retained; its new hosted +run and landing remain pending. No new production writer is enabled. ### Previous merged definitions checkpoint