From 09a23fb932615affa6f784aecb94fcb22691d122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:51:06 +0300 Subject: [PATCH 01/19] research: bind canonical repeat stability artifact --- .../repeat/v1/repeat-dataset-manifest.json | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 research/repeat/v1/repeat-dataset-manifest.json diff --git a/research/repeat/v1/repeat-dataset-manifest.json b/research/repeat/v1/repeat-dataset-manifest.json new file mode 100644 index 0000000..5e4b99c --- /dev/null +++ b/research/repeat/v1/repeat-dataset-manifest.json @@ -0,0 +1,75 @@ +{ + "schema_version": "bpfcompat.research.repeat-dataset-manifest.v1", + "dataset_version": "v1", + "status": "canonical-repeat-stability-collection", + "canonical_pilot_run": { + "id": "35393833464", + "head_sha": "5de550e5cc8c659872a90fc262eb0250c229bf23" + }, + "repeat_workflow_run": { + "id": "35445834557", + "head_sha": "d2a78e05178ea6dd82ead9684f5eedf49066903f", + "branch": "main", + "conclusion": "success" + }, + "actions_artifact": { + "id": "10584409793", + "name": "research-repeat-v1-35445834557", + "sha256": "sha256:895fd41dae147c592d5cdec91bbd99150f9dd14c35afadf72bc14954a192a0e5", + "size_bytes": 455756, + "retention_note": "Actions artifact is staging evidence; normalized evidence and checksums are bound here and the full artifact is intended for the later DOI archive." + }, + "collection": { + "planned_tuples": 7, + "repeats_per_tuple": 3, + "planned_attempts": 21, + "observed_attempts": 21, + "collection_complete": true, + "stable_same_environment": 21, + "environment_drift": 0, + "verdict_instability": 0 + }, + "committed_evidence": { + "repeat_provenance": { + "path": "data/repeat-provenance.json", + "sha256": "sha256:7fc550f7fab1590bef5f284e4fc72f331e9ea78ef20af9b0bfd66f1844dbef52" + }, + "stability_summary": { + "path": "data/stability-summary.json", + "sha256": "sha256:add7441dd16b74d31845f3b35b30edd65e1f102799417dbc068c6c3d105c1a9c" + }, + "results": { + "path": "data/RESULTS.md", + "sha256": "sha256:51cdea803353736c77a2b944554079f7e6a230ed5614be30badf425c1677c3b0" + }, + "raw_report_checksums": { + "path": "data/raw-report-checksums.json", + "sha256": "sha256:304f7521eb310b607a4a3cc36d45614e329b83461ce97d838244b1202e229dfd" + }, + "repeat_executions_artifact_sha256": "sha256:844ff0b27d8bcd7ecdb0e3368147b7486305c9cc0f3710c0fc95dc4d25aed11c" + }, + "projection_metadata": [ + { + "path": "data/projection-metadata/inconclusive-oracle-environment.json", + "sha256": "sha256:77783e88eb13923d343245eebe007fbc735a8a81623aabd576c156cf5d3ff42b" + }, + { + "path": "data/projection-metadata/negative-ringbuf-boundary.json", + "sha256": "sha256:e563729f5204fa8bf9e84c2bdf577272f5d41e4c9f4dc0515de5b2bed4a2b6cf" + }, + { + "path": "data/projection-metadata/positive-controlled-modern.json", + "sha256": "sha256:3b3eedf25d470206b24a7cda9244d40d305f562a2e1ab94ce510121c8926cbe4" + }, + { + "path": "data/projection-metadata/positive-ringbuf-backport.json", + "sha256": "sha256:3369a03fc3daa362b86e6ac96fb9e49e3aae85640ab099fc780199626a7c8e74" + } + ], + "notes": [ + "The first manual repeat run 35443831085 failed before 12 libbpf-backed attempts because singleton repeat matrices conflicted with frozen manifests listing all study profiles; it is diagnostic only and is not part of this dataset.", + "The successful repeat runner verifies the frozen manifest Git blob before creating a repeat-only required_profiles singleton projection; program, attach, and validation semantics are unchanged.", + "Raw repeat reports and repeat-executions.jsonl remain bound by SHA-256 in the successful workflow artifact rather than being duplicated into Git history.", + "This is a purposeful post-collection stability sample and must not be interpreted as a population-wide nondeterminism estimate." + ] +} From d4036b2289f682430918dfb6125163beaaa423f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:51:08 +0300 Subject: [PATCH 02/19] research: publish repeat provenance --- research/repeat/v1/data/repeat-provenance.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 research/repeat/v1/data/repeat-provenance.json diff --git a/research/repeat/v1/data/repeat-provenance.json b/research/repeat/v1/data/repeat-provenance.json new file mode 100644 index 0000000..ffa43d3 --- /dev/null +++ b/research/repeat/v1/data/repeat-provenance.json @@ -0,0 +1,16 @@ +{ + "schema_version": "bpfcompat.research.repeat-provenance.v1", + "corpus_version": "v1", + "workflow_source_commit": "d2a78e05178ea6dd82ead9684f5eedf49066903f", + "sample_sha256": "sha256:daf3f51ec5179897d9788c5eed89bbc453690383085043e03eb38034ed54560f", + "materialization_sha256": "sha256:a8c3f1745f27037d2cf8cd083176e021497f5b98fe0cd5a9220a10bb2379bab4", + "study_plan_sha256": "sha256:90f7562fa331d8f38832e155f2f267e0b35e244417c226245f4d92b523e61408", + "profile_lock_sha256": "sha256:702a8beb2859871d69d84f56c37e7ff4b1d1144fc0ccee8788aaee45cc84f63a", + "manifest_projection_script_sha256": "sha256:527b041ccc7a2c53282a7f365b2b833a0091600644ba2e33c62d1d72736562e9", + "bpfcompat_cli_sha256": "sha256:1365337098474dc0a42484271ee6383d46cb1cd20da09d248f9a0063c5147f0e", + "validator_sha256": "sha256:4ae1d5b838be07e6e7c304d753389a239c19eb92f6ba3bd77657e5c9583b9d04", + "loaders": { + "cilium-ebpf-v022-loader": "sha256:2e0517a0a068cf169ebf6a562cd2354ff9f18da708de7e54af52027a893598a4", + "falco-modern-bpf-scap-open": "sha256:736c307379603a8320434217440e346ce03260d31a5e7d58326f5526e2c2f2cd" + } +} From 27c96dc7442271ca28cd2a2abf396bedc1680cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:51:10 +0300 Subject: [PATCH 03/19] research: publish repeat stability result --- research/repeat/v1/data/RESULTS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 research/repeat/v1/data/RESULTS.md diff --git a/research/repeat/v1/data/RESULTS.md b/research/repeat/v1/data/RESULTS.md new file mode 100644 index 0000000..c72448f --- /dev/null +++ b/research/repeat/v1/data/RESULTS.md @@ -0,0 +1,11 @@ +# Pilot v1 repeat-run stability + +This is a purposeful post-collection stability sample, not a random population sample. + +- Planned attempts: 21 +- Observed attempts: 21 +- Stable on the same exact environment: 21 +- Environment drift: 0 +- Verdict instability on the same environment: 0 + +Environment drift is kept separate from verdict instability. From e0676cbc4a1e0102360ddb8b32f0306124a2354d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:51:35 +0300 Subject: [PATCH 04/19] research: publish canonical repeat stability summary --- .../repeat/v1/data/stability-summary.json | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 research/repeat/v1/data/stability-summary.json diff --git a/research/repeat/v1/data/stability-summary.json b/research/repeat/v1/data/stability-summary.json new file mode 100644 index 0000000..616ab8a --- /dev/null +++ b/research/repeat/v1/data/stability-summary.json @@ -0,0 +1,97 @@ +{ + "canonical_run_id": "35393833464", + "canonical_run_sha": "5de550e5cc8c659872a90fc262eb0250c229bf23", + "collection_complete": true, + "corpus_version": "v1", + "environment_drift": 0, + "interpretation": "Verdict instability is counted only when the exact environment ID matches the canonical run. Environment drift is reported separately and is not treated as nondeterministic compatibility behavior.", + "missing": [], + "observed_attempts": 21, + "planned_attempts": 21, + "planned_tuples": 7, + "repeat_provenance_sha256": "sha256:7fc550f7fab1590bef5f284e4fc72f331e9ea78ef20af9b0bfd66f1844dbef52", + "repeats_per_tuple": 3, + "sampling_design": "purposeful_stratified_post_collection", + "schema_version": "bpfcompat.research.repeat-summary.v1", + "stable_same_environment": 21, + "tuple_summaries": [ + { + "case_id": "simple-pass-libbpf", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "ubuntu-22.04-5.15", + "stable_same_environment": 3, + "stratum": "compatible_controlled", + "tuple_id": "positive-controlled-modern", + "verdict_instability": 0 + }, + { + "case_id": "ringbuf-modern-libbpf", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "ubuntu-20.04-5.4", + "stable_same_environment": 3, + "stratum": "incompatible_controlled", + "tuple_id": "negative-ringbuf-boundary", + "verdict_instability": 0 + }, + { + "case_id": "ringbuf-modern-libbpf", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "almalinux-8-4.18", + "stable_same_environment": 3, + "stratum": "compatible_vendor_backport", + "tuple_id": "positive-ringbuf-backport", + "verdict_instability": 0 + }, + { + "case_id": "falco-modern-bpf-scap-open", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "amazon-linux-2-4.14", + "stable_same_environment": 3, + "stratum": "incompatible_real_loader", + "tuple_id": "negative-falco-real-loader", + "verdict_instability": 0 + }, + { + "case_id": "falco-modern-bpf-scap-open", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "ubuntu-22.04-5.15", + "stable_same_environment": 3, + "stratum": "compatible_real_loader", + "tuple_id": "positive-falco-real-loader", + "verdict_instability": 0 + }, + { + "case_id": "cilium-tracepoint-ebpf-go", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "ubuntu-22.04-5.15", + "stable_same_environment": 3, + "stratum": "compatible_project_loader", + "tuple_id": "positive-cilium-project-loader", + "verdict_instability": 0 + }, + { + "case_id": "simple-pass-libbpf", + "environment_drift": 0, + "observed_repeats": 3, + "planned_repeats": 3, + "profile_id": "oracle-linux-9-uek7-5.15", + "stable_same_environment": 3, + "stratum": "inconclusive_environment_mismatch", + "tuple_id": "inconclusive-oracle-environment", + "verdict_instability": 0 + } + ], + "verdict_instability": 0 +} From 42b0e6bf2c52bc8a252c0f4c2530718db7603b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:51:38 +0300 Subject: [PATCH 05/19] research: freeze repeat raw report checksums --- .../repeat/v1/data/raw-report-checksums.json | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 research/repeat/v1/data/raw-report-checksums.json diff --git a/research/repeat/v1/data/raw-report-checksums.json b/research/repeat/v1/data/raw-report-checksums.json new file mode 100644 index 0000000..a251fb1 --- /dev/null +++ b/research/repeat/v1/data/raw-report-checksums.json @@ -0,0 +1,27 @@ +{ + "schema_version": "bpfcompat.research.repeat-raw-checksums.v1", + "count": 21, + "reports": [ + {"path":"raw/inconclusive-oracle-environment/repeat-1.json","sha256":"sha256:7c2e440a586ce1c35221cbc402556709b972e753d91e8227164d69b5331207cb","size_bytes":3680}, + {"path":"raw/inconclusive-oracle-environment/repeat-2.json","sha256":"sha256:0ba66a0687db2914bc295d30000f102ade1269fb8c736751c90bb626cde906ac","size_bytes":3680}, + {"path":"raw/inconclusive-oracle-environment/repeat-3.json","sha256":"sha256:b2a51f2f0bc9d1b036f52307895f6d581cafd8c27acf2fa927e5f3fc1750ec2f","size_bytes":3680}, + {"path":"raw/negative-falco-real-loader/repeat-1.json","sha256":"sha256:2a2a802239a5159ef54a5ff1840567c5c1b7efc5301a6b6bd68e0b8964a99a30","size_bytes":4386}, + {"path":"raw/negative-falco-real-loader/repeat-2.json","sha256":"sha256:3c0e3764a010c7fabe6ad56bd80a2b2df6d9851874611121fc60a7b9655c25a6","size_bytes":4386}, + {"path":"raw/negative-falco-real-loader/repeat-3.json","sha256":"sha256:f43e2673a1ae4df1571ad94bead9d4acfaaf331646a509e70af4c249f3d8a55d","size_bytes":4386}, + {"path":"raw/negative-ringbuf-boundary/repeat-1.json","sha256":"sha256:6c57fe27727ef6111e9d4a9f5ce9acfacbf76513f22b36554a5439a67a5ed591","size_bytes":4402}, + {"path":"raw/negative-ringbuf-boundary/repeat-2.json","sha256":"sha256:de7ce80dc46feb42eaa6319f2932ba3af96763290fc40c19a558947538d89b04","size_bytes":4402}, + {"path":"raw/negative-ringbuf-boundary/repeat-3.json","sha256":"sha256:026d53ede2f95d484068cf9f003e347c6fd60d81fa33669c44e7c4c20844c048","size_bytes":4402}, + {"path":"raw/positive-cilium-project-loader/repeat-1.json","sha256":"sha256:4199ea28caf34e335313a53ea45be87669e82ed132c74540c54545c97e6fa7a7","size_bytes":3286}, + {"path":"raw/positive-cilium-project-loader/repeat-2.json","sha256":"sha256:53bc690811e97bea1df88a55e764a65413c41146415369f36285149ea8a892ba","size_bytes":3286}, + {"path":"raw/positive-cilium-project-loader/repeat-3.json","sha256":"sha256:44c513a3d6386e7f19a1b0a15cbe71dc9aaaa2456735055ad98683c9d9171ffd","size_bytes":3286}, + {"path":"raw/positive-controlled-modern/repeat-1.json","sha256":"sha256:cedfb535ec3204fff8df2b6b8f94eb23498ea9d5787015d05df10a36b3fd819d","size_bytes":3423}, + {"path":"raw/positive-controlled-modern/repeat-2.json","sha256":"sha256:1e59171367ad93cc2bb87233b1bee7bbd8f9ea8370ec552c47ca845cf37b3fc4","size_bytes":3423}, + {"path":"raw/positive-controlled-modern/repeat-3.json","sha256":"sha256:ebc116612557332cb9b46eb60720fb98e75d6d1a51afe3f49eb660c74234da7c","size_bytes":3423}, + {"path":"raw/positive-falco-real-loader/repeat-1.json","sha256":"sha256:79541b0f295adb81e5fcec66a30bf890367fb435f4d285c962f3a07a31722000","size_bytes":7284}, + {"path":"raw/positive-falco-real-loader/repeat-2.json","sha256":"sha256:7835ebc6c595972329fb3c7200c877f9b229f078a399d79cceafca5d944d6e91","size_bytes":7284}, + {"path":"raw/positive-falco-real-loader/repeat-3.json","sha256":"sha256:bfaecccd536fbb70a608247997f39724d0626c61dd6a127b8d0d3bd478abd633","size_bytes":7284}, + {"path":"raw/positive-ringbuf-backport/repeat-1.json","sha256":"sha256:1f13f4e7d123610f89bab66a6fea0bbddb6b817712e1940633adae368d788146","size_bytes":3552}, + {"path":"raw/positive-ringbuf-backport/repeat-2.json","sha256":"sha256:e52668535177618250ac18b05e98956d48d48b203ef1ddaeb44f852349713132","size_bytes":3552}, + {"path":"raw/positive-ringbuf-backport/repeat-3.json","sha256":"sha256:8347013949bf085c8cb32a4e3fe9fe45f56df10749a9ac95f97f90e895ae8d27","size_bytes":3552} + ] +} From aa1f7a447503758f3cc3faaf0126c3d337d71f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:52:05 +0300 Subject: [PATCH 06/19] research: publish repeat projection metadata --- .../inconclusive-oracle-environment.json | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 research/repeat/v1/data/projection-metadata/inconclusive-oracle-environment.json diff --git a/research/repeat/v1/data/projection-metadata/inconclusive-oracle-environment.json b/research/repeat/v1/data/projection-metadata/inconclusive-oracle-environment.json new file mode 100644 index 0000000..db89d84 --- /dev/null +++ b/research/repeat/v1/data/projection-metadata/inconclusive-oracle-environment.json @@ -0,0 +1,25 @@ +{ + "profile_id": "oracle-linux-9-uek7-5.15", + "projected_path": "reports/research-repeat-v1/manifests/inconclusive-oracle-environment.yaml", + "projected_required_profiles": [ + "oracle-linux-9-uek7-5.15" + ], + "projected_sha256": "sha256:72c544536b5e081353b50d45842969b111240e4b13ed1d0dc5ed59b5ba5ba301", + "projection": "required_profiles_singleton", + "schema_version": "bpfcompat.research.repeat-manifest-projection.v1", + "source_git_blob": "f813d866b40469815699577ab15ee9b4881e3d3c", + "source_path": "research/corpus/v1/manifests/simple-pass.yaml", + "source_required_profiles": [ + "ubuntu-20.04-5.4", + "ubuntu-20.10-5.8", + "ubuntu-22.04-5.15", + "debian-12-6.1", + "ubuntu-24.04-6.8", + "almalinux-8-4.18", + "amazon-linux-2-4.14", + "amazon-linux-2-5.10", + "oracle-linux-9-uek7-5.15", + "opensuse-leap-15.6-6.4" + ], + "source_sha256": "sha256:c2c210941988588e7299bf8e69f1b7c0b643057225a9fe7532a46578622f5dcb" +} From 98d6a046c33c744dc1770597d63dba4e38a8c594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:52:07 +0300 Subject: [PATCH 07/19] research: publish repeat projection metadata --- .../negative-ringbuf-boundary.json | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 research/repeat/v1/data/projection-metadata/negative-ringbuf-boundary.json diff --git a/research/repeat/v1/data/projection-metadata/negative-ringbuf-boundary.json b/research/repeat/v1/data/projection-metadata/negative-ringbuf-boundary.json new file mode 100644 index 0000000..0fe31f5 --- /dev/null +++ b/research/repeat/v1/data/projection-metadata/negative-ringbuf-boundary.json @@ -0,0 +1,25 @@ +{ + "profile_id": "ubuntu-20.04-5.4", + "projected_path": "reports/research-repeat-v1/manifests/negative-ringbuf-boundary.yaml", + "projected_required_profiles": [ + "ubuntu-20.04-5.4" + ], + "projected_sha256": "sha256:7134def39ea8204916fb24ae4690d759e21f6b97c4f3b0a23a233c573a7b85de", + "projection": "required_profiles_singleton", + "schema_version": "bpfcompat.research.repeat-manifest-projection.v1", + "source_git_blob": "e639703c69bc49130f1959a095f5fe8befcec0bd", + "source_path": "research/corpus/v1/manifests/ringbuf-modern.yaml", + "source_required_profiles": [ + "ubuntu-20.04-5.4", + "ubuntu-20.10-5.8", + "ubuntu-22.04-5.15", + "debian-12-6.1", + "ubuntu-24.04-6.8", + "almalinux-8-4.18", + "amazon-linux-2-4.14", + "amazon-linux-2-5.10", + "oracle-linux-9-uek7-5.15", + "opensuse-leap-15.6-6.4" + ], + "source_sha256": "sha256:1a39b0164114981cb336cc864caedbd69302b3bf8397d964a77757fae59a48a0" +} From da6dc87b9664e5836903bd5cd7ae0bc2568c6fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:52:09 +0300 Subject: [PATCH 08/19] research: publish repeat projection metadata --- .../positive-controlled-modern.json | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 research/repeat/v1/data/projection-metadata/positive-controlled-modern.json diff --git a/research/repeat/v1/data/projection-metadata/positive-controlled-modern.json b/research/repeat/v1/data/projection-metadata/positive-controlled-modern.json new file mode 100644 index 0000000..78e553f --- /dev/null +++ b/research/repeat/v1/data/projection-metadata/positive-controlled-modern.json @@ -0,0 +1,25 @@ +{ + "profile_id": "ubuntu-22.04-5.15", + "projected_path": "reports/research-repeat-v1/manifests/positive-controlled-modern.yaml", + "projected_required_profiles": [ + "ubuntu-22.04-5.15" + ], + "projected_sha256": "sha256:b786af1a52de3fb51d97581591166e94b788ee1795bbad5fff9ba4bd1f29fc88", + "projection": "required_profiles_singleton", + "schema_version": "bpfcompat.research.repeat-manifest-projection.v1", + "source_git_blob": "f813d866b40469815699577ab15ee9b4881e3d3c", + "source_path": "research/corpus/v1/manifests/simple-pass.yaml", + "source_required_profiles": [ + "ubuntu-20.04-5.4", + "ubuntu-20.10-5.8", + "ubuntu-22.04-5.15", + "debian-12-6.1", + "ubuntu-24.04-6.8", + "almalinux-8-4.18", + "amazon-linux-2-4.14", + "amazon-linux-2-5.10", + "oracle-linux-9-uek7-5.15", + "opensuse-leap-15.6-6.4" + ], + "source_sha256": "sha256:c2c210941988588e7299bf8e69f1b7c0b643057225a9fe7532a46578622f5dcb" +} From 88c597dba265489667b98a55797d7735e3d13f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:52:11 +0300 Subject: [PATCH 09/19] research: publish repeat projection metadata --- .../positive-ringbuf-backport.json | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 research/repeat/v1/data/projection-metadata/positive-ringbuf-backport.json diff --git a/research/repeat/v1/data/projection-metadata/positive-ringbuf-backport.json b/research/repeat/v1/data/projection-metadata/positive-ringbuf-backport.json new file mode 100644 index 0000000..0b2c5d0 --- /dev/null +++ b/research/repeat/v1/data/projection-metadata/positive-ringbuf-backport.json @@ -0,0 +1,25 @@ +{ + "profile_id": "almalinux-8-4.18", + "projected_path": "reports/research-repeat-v1/manifests/positive-ringbuf-backport.yaml", + "projected_required_profiles": [ + "almalinux-8-4.18" + ], + "projected_sha256": "sha256:dcb56ff7c7f7c5473b955d007f04d2f36fa96da3dfa8052b5678cbc93a496621", + "projection": "required_profiles_singleton", + "schema_version": "bpfcompat.research.repeat-manifest-projection.v1", + "source_git_blob": "e639703c69bc49130f1959a095f5fe8befcec0bd", + "source_path": "research/corpus/v1/manifests/ringbuf-modern.yaml", + "source_required_profiles": [ + "ubuntu-20.04-5.4", + "ubuntu-20.10-5.8", + "ubuntu-22.04-5.15", + "debian-12-6.1", + "ubuntu-24.04-6.8", + "almalinux-8-4.18", + "amazon-linux-2-4.14", + "amazon-linux-2-5.10", + "oracle-linux-9-uek7-5.15", + "opensuse-leap-15.6-6.4" + ], + "source_sha256": "sha256:1a39b0164114981cb336cc864caedbd69302b3bf8397d964a77757fae59a48a0" +} From d360f1b870964a75fff3ee5c3f99a76a303a61f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:52:48 +0300 Subject: [PATCH 10/19] research: make repeat snapshot verification byte-driven --- .../repeat/v1/repeat-dataset-manifest.json | 44 ++++--------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/research/repeat/v1/repeat-dataset-manifest.json b/research/repeat/v1/repeat-dataset-manifest.json index 5e4b99c..eca2f90 100644 --- a/research/repeat/v1/repeat-dataset-manifest.json +++ b/research/repeat/v1/repeat-dataset-manifest.json @@ -30,42 +30,16 @@ "verdict_instability": 0 }, "committed_evidence": { - "repeat_provenance": { - "path": "data/repeat-provenance.json", - "sha256": "sha256:7fc550f7fab1590bef5f284e4fc72f331e9ea78ef20af9b0bfd66f1844dbef52" - }, - "stability_summary": { - "path": "data/stability-summary.json", - "sha256": "sha256:add7441dd16b74d31845f3b35b30edd65e1f102799417dbc068c6c3d105c1a9c" - }, - "results": { - "path": "data/RESULTS.md", - "sha256": "sha256:51cdea803353736c77a2b944554079f7e6a230ed5614be30badf425c1677c3b0" - }, - "raw_report_checksums": { - "path": "data/raw-report-checksums.json", - "sha256": "sha256:304f7521eb310b607a4a3cc36d45614e329b83461ce97d838244b1202e229dfd" - }, - "repeat_executions_artifact_sha256": "sha256:844ff0b27d8bcd7ecdb0e3368147b7486305c9cc0f3710c0fc95dc4d25aed11c" + "repeat_provenance": "data/repeat-provenance.json", + "stability_summary": "data/stability-summary.json", + "results": "data/RESULTS.md", + "raw_report_checksums": "data/raw-report-checksums.json", + "projection_metadata_dir": "data/projection-metadata/" + }, + "artifact_evidence": { + "repeat_executions_path": "normalized/repeat-executions.jsonl", + "repeat_executions_sha256": "sha256:844ff0b27d8bcd7ecdb0e3368147b7486305c9cc0f3710c0fc95dc4d25aed11c" }, - "projection_metadata": [ - { - "path": "data/projection-metadata/inconclusive-oracle-environment.json", - "sha256": "sha256:77783e88eb13923d343245eebe007fbc735a8a81623aabd576c156cf5d3ff42b" - }, - { - "path": "data/projection-metadata/negative-ringbuf-boundary.json", - "sha256": "sha256:e563729f5204fa8bf9e84c2bdf577272f5d41e4c9f4dc0515de5b2bed4a2b6cf" - }, - { - "path": "data/projection-metadata/positive-controlled-modern.json", - "sha256": "sha256:3b3eedf25d470206b24a7cda9244d40d305f562a2e1ab94ce510121c8926cbe4" - }, - { - "path": "data/projection-metadata/positive-ringbuf-backport.json", - "sha256": "sha256:3369a03fc3daa362b86e6ac96fb9e49e3aae85640ab099fc780199626a7c8e74" - } - ], "notes": [ "The first manual repeat run 35443831085 failed before 12 libbpf-backed attempts because singleton repeat matrices conflicted with frozen manifests listing all study profiles; it is diagnostic only and is not part of this dataset.", "The successful repeat runner verifies the frozen manifest Git blob before creating a repeat-only required_profiles singleton projection; program, attach, and validation semantics are unchanged.", From f47613e8a9daefdf5dd78134e148c6e97433dab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:53:32 +0300 Subject: [PATCH 11/19] research: add repeat snapshot integrity verifier --- scripts/research/verify-repeat-snapshot-v1.py | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 scripts/research/verify-repeat-snapshot-v1.py diff --git a/scripts/research/verify-repeat-snapshot-v1.py b/scripts/research/verify-repeat-snapshot-v1.py new file mode 100644 index 0000000..d51faee --- /dev/null +++ b/scripts/research/verify-repeat-snapshot-v1.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Verify the committed pilot-v1 repeat-stability snapshot.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +REPEAT = ROOT / "research/repeat/v1" +DATA = REPEAT / "data" + + +def fail(message: str) -> None: + raise SystemExit(f"[verify-repeat-snapshot-v1] {message}") + + +def load_json(path: Path): + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + fail(f"cannot read {path.relative_to(ROOT)}: {exc}") + + +def sha256_file(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def git_blob_sha1(path: Path) -> str: + data = path.read_bytes() + header = f"blob {len(data)}\0".encode() + return hashlib.sha1(header + data).hexdigest() + + +def main() -> int: + manifest = load_json(REPEAT / "repeat-dataset-manifest.json") + summary = load_json(DATA / "stability-summary.json") + provenance = load_json(DATA / "repeat-provenance.json") + raw = load_json(DATA / "raw-report-checksums.json") + sample_path = REPEAT / "stability-sample.json" + sample = load_json(sample_path) + pilot_manifest = load_json(ROOT / "research/data/v1/dataset-manifest.json") + study_plan_path = ROOT / "research/corpus/v1/study-plan.json" + study_plan = load_json(study_plan_path) + profile_lock_path = ROOT / "research/corpus/v1/profile-identities.json" + identity_lock = load_json(ROOT / "research/corpus/v1/materialized-identities.json") + projection_script = ROOT / "scripts/research/project-repeat-manifest-v1.py" + + if manifest.get("schema_version") != "bpfcompat.research.repeat-dataset-manifest.v1": + fail("unexpected repeat dataset manifest schema") + if summary.get("schema_version") != "bpfcompat.research.repeat-summary.v1": + fail("unexpected stability summary schema") + if provenance.get("schema_version") != "bpfcompat.research.repeat-provenance.v1": + fail("unexpected repeat provenance schema") + if raw.get("schema_version") != "bpfcompat.research.repeat-raw-checksums.v1": + fail("unexpected raw checksum schema") + + canonical = pilot_manifest.get("canonical_workflow_run") or {} + expected_canonical = manifest.get("canonical_pilot_run") or {} + if str(canonical.get("id")) != str(expected_canonical.get("id")): + fail("canonical pilot run id drift") + if canonical.get("head_sha") != expected_canonical.get("head_sha"): + fail("canonical pilot run SHA drift") + if str(summary.get("canonical_run_id")) != str(expected_canonical.get("id")): + fail("summary canonical run id drift") + if summary.get("canonical_run_sha") != expected_canonical.get("head_sha"): + fail("summary canonical run SHA drift") + + repeat_run = manifest.get("repeat_workflow_run") or {} + if repeat_run != { + "id": "35445834557", + "head_sha": "d2a78e05178ea6dd82ead9684f5eedf49066903f", + "branch": "main", + "conclusion": "success", + }: + fail("canonical repeat workflow identity drift") + + artifact = manifest.get("actions_artifact") or {} + if artifact.get("id") != "10584409793": + fail("canonical repeat artifact id drift") + if artifact.get("sha256") != "sha256:895fd41dae147c592d5cdec91bbd99150f9dd14c35afadf72bc14954a192a0e5": + fail("canonical repeat artifact SHA-256 drift") + + expected_collection = { + "planned_tuples": 7, + "repeats_per_tuple": 3, + "planned_attempts": 21, + "observed_attempts": 21, + "collection_complete": True, + "stable_same_environment": 21, + "environment_drift": 0, + "verdict_instability": 0, + } + if manifest.get("collection") != expected_collection: + fail("repeat dataset collection summary drift") + + for key, value in expected_collection.items(): + if key in summary and summary.get(key) != value: + fail(f"stability summary drift: {key}") + if summary.get("missing") != []: + fail("repeat summary has missing observations") + if summary.get("sampling_design") != "purposeful_stratified_post_collection": + fail("repeat sampling design drift") + + provenance_digest = sha256_file(DATA / "repeat-provenance.json") + if summary.get("repeat_provenance_sha256") != provenance_digest: + fail("summary does not bind committed repeat provenance") + + if provenance.get("workflow_source_commit") != repeat_run["head_sha"]: + fail("repeat provenance source commit drift") + if provenance.get("sample_sha256") != sha256_file(sample_path): + fail("repeat sample SHA-256 drift") + if provenance.get("study_plan_sha256") != sha256_file(study_plan_path): + fail("study plan SHA-256 drift") + if provenance.get("profile_lock_sha256") != sha256_file(profile_lock_path): + fail("profile lock SHA-256 drift") + if provenance.get("manifest_projection_script_sha256") != sha256_file(projection_script): + fail("repeat projection script SHA-256 drift") + + identities = { + row["id"]: "sha256:" + str(row["sha256"]).removeprefix("sha256:") + for row in identity_lock.get("artifacts") or [] + } + if provenance.get("bpfcompat_cli_sha256") != identities.get("bpfcompat-v037-cli"): + fail("BPFCompat CLI identity drift") + if provenance.get("validator_sha256") != identities.get("bpfcompat-v037-validator"): + fail("validator identity drift") + loaders = provenance.get("loaders") or {} + if loaders.get("cilium-ebpf-v022-loader") != identities.get("cilium-ebpf-v022-loader"): + fail("cilium loader identity drift") + if loaders.get("falco-modern-bpf-scap-open") != identities.get("falco-modern-bpf-scap-open"): + fail("Falco loader identity drift") + + tuples = sample.get("tuples") or [] + repeats = int(sample.get("repeats_per_tuple") or 0) + expected_raw_paths = { + f"raw/{row['id']}/repeat-{repeat}.json" + for row in tuples + for repeat in range(1, repeats + 1) + } + reports = raw.get("reports") or [] + observed_raw_paths = {row.get("path") for row in reports} + if raw.get("count") != 21 or len(reports) != 21: + fail("raw checksum manifest must contain exactly 21 reports") + if observed_raw_paths != expected_raw_paths: + fail("raw checksum paths do not match the frozen repeat sample") + if len({row.get("sha256") for row in reports}) != 21: + fail("raw report SHA-256 values must be unique") + for row in reports: + digest = str(row.get("sha256") or "") + if not digest.startswith("sha256:") or len(digest) != 71: + fail(f"invalid raw report SHA-256: {row!r}") + if int(row.get("size_bytes") or 0) <= 0: + fail(f"invalid raw report size: {row!r}") + + tuple_summaries = summary.get("tuple_summaries") or [] + by_id = {row.get("tuple_id"): row for row in tuple_summaries} + if set(by_id) != {row["id"] for row in tuples}: + fail("tuple summary coverage drift") + for row in tuples: + got = by_id[row["id"]] + if got.get("case_id") != row["case_id"] or got.get("profile_id") != row["profile_id"]: + fail(f"tuple binding drift: {row['id']}") + if got.get("stratum") != row["stratum"]: + fail(f"tuple stratum drift: {row['id']}") + if ( + got.get("planned_repeats") != 3 + or got.get("observed_repeats") != 3 + or got.get("stable_same_environment") != 3 + or got.get("environment_drift") != 0 + or got.get("verdict_instability") != 0 + ): + fail(f"tuple stability drift: {row['id']}") + + cases = {row["id"]: row for row in study_plan.get("cases") or []} + libbpf_tuples = [ + row for row in tuples + if cases[row["case_id"]].get("mode") in {"load_only", "load_attach"} + ] + projection_dir = DATA / "projection-metadata" + projection_files = sorted(projection_dir.glob("*.json")) + if len(projection_files) != len(libbpf_tuples): + fail("projection metadata count drift") + + for row in libbpf_tuples: + case = cases[row["case_id"]] + meta_path = projection_dir / f"{row['id']}.json" + if not meta_path.is_file(): + fail(f"missing projection metadata: {row['id']}") + meta = load_json(meta_path) + source_path = ROOT / case["manifest"] + if meta.get("schema_version") != "bpfcompat.research.repeat-manifest-projection.v1": + fail(f"projection schema drift: {row['id']}") + if meta.get("projection") != "required_profiles_singleton": + fail(f"projection rule drift: {row['id']}") + if meta.get("profile_id") != row["profile_id"]: + fail(f"projection profile drift: {row['id']}") + if meta.get("source_path") != case["manifest"]: + fail(f"projection source path drift: {row['id']}") + if meta.get("source_git_blob") != case["manifest_git_blob"]: + fail(f"projection source blob binding drift: {row['id']}") + if git_blob_sha1(source_path) != case["manifest_git_blob"]: + fail(f"frozen source manifest Git blob drift: {row['id']}") + if meta.get("source_sha256") != sha256_file(source_path): + fail(f"projection source SHA-256 drift: {row['id']}") + if meta.get("projected_required_profiles") != [row["profile_id"]]: + fail(f"projection singleton profile drift: {row['id']}") + if row["profile_id"] not in (meta.get("source_required_profiles") or []): + fail(f"sampled profile absent from frozen manifest metadata: {row['id']}") + + with tempfile.TemporaryDirectory() as tmp: + projected = Path(tmp) / "manifest.yaml" + generated_meta = Path(tmp) / "meta.json" + subprocess.run( + [ + sys.executable, + str(projection_script), + "--source", + case["manifest"], + "--profile-id", + row["profile_id"], + "--expected-git-blob", + case["manifest_git_blob"], + "--out", + str(projected), + "--metadata-out", + str(generated_meta), + ], + cwd=ROOT, + check=True, + stdout=subprocess.DEVNULL, + ) + regenerated = load_json(generated_meta) + if regenerated.get("projected_sha256") != meta.get("projected_sha256"): + fail(f"projection is not reproducible: {row['id']}") + + artifact_evidence = manifest.get("artifact_evidence") or {} + if artifact_evidence.get("repeat_executions_sha256") != "sha256:844ff0b27d8bcd7ecdb0e3368147b7486305c9cc0f3710c0fc95dc4d25aed11c": + fail("repeat-executions artifact binding drift") + + print("[verify-repeat-snapshot-v1] PASS: canonical 21/21 repeat snapshot is internally consistent") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5e387d0b702121d9f72a5b63834a806fd73a09f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:54:06 +0300 Subject: [PATCH 12/19] research: verify committed repeat snapshot in preflight --- .github/workflows/research-repeat-v1.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/research-repeat-v1.yml b/.github/workflows/research-repeat-v1.yml index 05ddca2..7500613 100644 --- a/.github/workflows/research-repeat-v1.yml +++ b/.github/workflows/research-repeat-v1.yml @@ -11,6 +11,7 @@ on: - "scripts/research/run-repeat-v1.sh" - "scripts/research/analyze-repeat-v1.py" - "scripts/research/project-repeat-manifest-v1.py" + - "scripts/research/verify-repeat-snapshot-v1.py" - "scripts/research/materialize-v1.sh" - "scripts/research/verify-materialization-v1.sh" - "scripts/research/verify-profile-lock-v1.sh" @@ -39,6 +40,7 @@ jobs: bash -n scripts/research/run-repeat-v1.sh python3 -m py_compile scripts/research/analyze-repeat-v1.py python3 -m py_compile scripts/research/project-repeat-manifest-v1.py + python3 -m py_compile scripts/research/verify-repeat-snapshot-v1.py python3 scripts/research/analyze-repeat-v1.py --self-test python3 scripts/research/project-repeat-manifest-v1.py --self-test jq -e . research/repeat/v1/stability-sample.json >/dev/null @@ -84,6 +86,10 @@ jobs: print("[research-repeat-v1] sample bindings PASS") PY + - name: Verify committed repeat snapshot + if: hashFiles('research/repeat/v1/repeat-dataset-manifest.json') != '' + run: python3 scripts/research/verify-repeat-snapshot-v1.py + execute: name: Execute 21-attempt repeat stability sample if: github.event_name == 'workflow_dispatch' From 1e3997afe6357e385cf5ce88a68440d02b83c755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:54:09 +0300 Subject: [PATCH 13/19] research: document canonical repeat collection --- research/repeat/v1/README.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/research/repeat/v1/README.md b/research/repeat/v1/README.md index 7e279a7..6176acb 100644 --- a/research/repeat/v1/README.md +++ b/research/repeat/v1/README.md @@ -42,15 +42,32 @@ the exact environment ID and normalized verdict match. The workflow is manual-only because it boots real vendor VMs and is intended as a bounded research validation, not a routine CI gate. -## Run +## Canonical repeat collection -After this protocol lands on `main`: +The canonical successful repeat collection is GitHub Actions run +`35445834557` at commit +`d2a78e05178ea6dd82ead9684f5eedf49066903f`. Its Actions artifact is +`10584409793` with SHA-256 +`895fd41dae147c592d5cdec91bbd99150f9dd14c35afadf72bc14954a192a0e5`. -```bash -gh workflow run research-repeat-v1.yml --repo Kernel-Guard/bpfcompat --ref main -``` +The run completed all **21/21** planned attempts across the seven frozen sample +tuples. All 21 observations reproduced the canonical verdict on the same exact +environment. Environment drift was **0** and same-environment verdict +instability was **0**. -The workflow publishes raw repeat reports, logs, repeat-only manifest -projections and their provenance metadata, a repeat provenance record, -`repeat-executions.jsonl`, `stability-summary.json`, and generated -`RESULTS.md` as a staging Actions artifact. +Repository evidence: + +- `repeat-dataset-manifest.json` binds the canonical run and Actions artifact; +- `data/stability-summary.json` preserves the generated tuple-level result; +- `data/repeat-provenance.json` binds the runner inputs and binary identities; +- `data/raw-report-checksums.json` freezes all 21 raw report paths and hashes; +- `data/projection-metadata/` preserves the repeat-only manifest projections; +- `scripts/research/verify-repeat-snapshot-v1.py` verifies the committed hash + and provenance chain. + +The row-level `repeat-executions.jsonl`, raw reports, VM logs, and generated +projection YAMLs remain in the content-addressed workflow artifact for later DOI +archival rather than being duplicated into Git history. + +The earlier manual run `35443831085` is diagnostic only: it exposed the +singleton-matrix/manifest scheduling bug and is not part of the repeat dataset. From 9f5ce142e0dea90a89f7019acbc2133631d85873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:54:23 +0300 Subject: [PATCH 14/19] research: record completed repeat stability gate --- research/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/research/README.md b/research/README.md index 485f7c2..8378c53 100644 --- a/research/README.md +++ b/research/README.md @@ -30,8 +30,9 @@ Repository snapshot and integrity metadata: raw-report checksums, and dataset manifest; - [`analysis/v1/`](analysis/v1/) — deterministic descriptive RQ1–RQ4 tables and machine-readable analysis summary; -- [`repeat/v1/`](repeat/v1/) — merged post-collection stability protocol; the - manual 21-attempt execution is still pending; +- [`repeat/v1/`](repeat/v1/) — canonical post-collection stability snapshot: + 21/21 attempts stable on the same exact environment, with 0 environment drift + and 0 same-environment verdict instability; - `scripts/research/analyze-study-v1.py` — analysis generator; - `scripts/research/test-analysis-v1.sh` — checksum and reproduction gate. @@ -47,8 +48,8 @@ Publication-facing material is kept explicit and evidence-bounded: third-party compiled-loader policy; - [paper/PREPRINT.md](paper/PREPRINT.md) — working manuscript scaffold whose numerical claims are constrained to generated v1 analysis; -- [ROADMAP.md](ROADMAP.md) — remaining repeat-run, figure, archival, DOI, and - external academic-use gates. +- [ROADMAP.md](ROADMAP.md) — remaining figure, archival, DOI, and external + academic-use gates. No university, lab, course, or paper should be listed as an academic reference until a public independent source can be verified. A resource-page link is a From 1c1a13ff18afdf5f05f2488f88ccb5ebcae1493c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:54:25 +0300 Subject: [PATCH 15/19] research: close repeat stability roadmap gate --- research/ROADMAP.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/research/ROADMAP.md b/research/ROADMAP.md index bd08083..c6214b0 100644 --- a/research/ROADMAP.md +++ b/research/ROADMAP.md @@ -32,9 +32,11 @@ - [x] Generate pilot v1 descriptive RQ1–RQ4 tables from normalized data. - [ ] Generate every final paper table and figure from normalized data. - [x] Define a bounded representative repeat-run protocol (PR #152). -- [ ] Execute the 21-attempt repeat-run workflow from `main`, archive its - provenance/results, and evaluate environment drift separately from - same-environment verdict instability. +- [x] Execute the 21-attempt repeat-run workflow from `main`, bind its + provenance/results to the repository snapshot, and evaluate environment + drift separately from same-environment verdict instability. Canonical run + `35445834557`: 21/21 stable same-environment observations, 0 drift, + 0 verdict instability. - [x] Document the v1 execution boundary, KVM/VM prerequisites, exact-environment identity, and collection completeness rules. - [x] Publish raw/processed dataset checksums and bind the repository snapshot to the canonical successful workflow artifact. From b80acb88d144cfe18a39025e3185ca4b905b2198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:54:28 +0300 Subject: [PATCH 16/19] research: bind canonical repeat evidence for archival --- research/ARCHIVAL.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/research/ARCHIVAL.md b/research/ARCHIVAL.md index 0a216f6..5a25e9e 100644 --- a/research/ARCHIVAL.md +++ b/research/ARCHIVAL.md @@ -13,8 +13,14 @@ research archive. It is an engineering reproducibility policy, not legal advice. - planned/observed executions: **70/70** - repository dataset: `research/data/v1/` - deterministic analysis: `research/analysis/v1/` -- repeat-run protocol: merged in PR #152; the manual 21-attempt execution is - still required before the research archive is frozen. +- canonical repeat workflow run: `35445834557` +- repeat source commit: `d2a78e05178ea6dd82ead9684f5eedf49066903f` +- repeat Actions artifact: `10584409793` +- repeat artifact SHA-256: + `895fd41dae147c592d5cdec91bbd99150f9dd14c35afadf72bc14954a192a0e5` +- repeat stability: **21/21** stable on the same exact environment, + **0** environment drift, **0** same-environment verdict instability +- repeat repository snapshot: `research/repeat/v1/` ## Archive layers From 4f7471050b3447d65456b5ebdb7171e056b35db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 16:54:40 +0300 Subject: [PATCH 17/19] research: add canonical repeat result to preprint --- research/paper/PREPRINT.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/research/paper/PREPRINT.md b/research/paper/PREPRINT.md index 2c636e8..6b15db1 100644 --- a/research/paper/PREPRINT.md +++ b/research/paper/PREPRINT.md @@ -27,7 +27,11 @@ booted 6.12 UEK. Excluding the calibration case, the pilot contains 50 compatibl 8 of 9 conclusive observations; AlmaLinux 8's observed 4.18 vendor kernel was the below-threshold compatible exception. The dataset also contains two cross-vendor version inversions in which an older-numbered kernel passed while a -newer-numbered kernel failed. +newer-numbered kernel failed. A purposefully stratified post-collection repeat +sample then reran seven canonical tuples three times each. All 21/21 repeats +matched the canonical verdict on the same exact environment; no environment +drift or same-environment verdict instability was observed in that bounded +sample. These results are descriptive evidence from a selected pilot, not population estimates for Linux deployments. The study publishes normalized evidence, @@ -113,8 +117,20 @@ release, image SHA-256, exact environment ID, and evaluability. ### Table 4 — repeat-run stability -Populate only after the merged PR #152 manual workflow is executed. Separate -environment drift from same-environment verdict instability. +Canonical repeat run `35445834557` executed seven purposefully selected tuples +three times each: + +| Measure | Result | +| --- | ---: | +| Planned attempts | 21 | +| Observed attempts | 21 | +| Stable on the same exact environment | 21 | +| Environment drift | 0 | +| Same-environment verdict instability | 0 | + +This table is a bounded stability check, not an estimate of a population-wide +nondeterminism rate. The source of truth is +`research/repeat/v1/data/stability-summary.json`. ## Limitations @@ -131,7 +147,7 @@ environment drift from same-environment verdict instability. Do not convert this working draft into a submission-ready manuscript until: -1. the 21-attempt repeat-run stability sample is executed and archived; +1. the committed 21-attempt repeat-run stability snapshot remains verifiable; 2. final figures/tables are generated from normalized data; 3. the v1 archival/redistribution manifest passes; 4. a research-tagged immutable release is created; From a23a066aabe2b5e7bfad6ddc1de6ebe6647332d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 17:48:00 +0300 Subject: [PATCH 18/19] research: bind repeat snapshot checks to canonical artifact --- .github/workflows/research-repeat-v1.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/research-repeat-v1.yml b/.github/workflows/research-repeat-v1.yml index 7500613..b0c7571 100644 --- a/.github/workflows/research-repeat-v1.yml +++ b/.github/workflows/research-repeat-v1.yml @@ -18,6 +18,7 @@ on: permissions: contents: read + actions: read env: GO_VERSION: "1.25.14" @@ -86,9 +87,22 @@ jobs: print("[research-repeat-v1] sample bindings PASS") PY + - name: Download canonical repeat artifact + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + artifact_id="$(jq -r '.actions_artifact.id' research/repeat/v1/repeat-dataset-manifest.json)" + [[ "$artifact_id" == "10584409793" ]] || { + echo "::error::unexpected canonical repeat artifact id: $artifact_id" + exit 1 + } + gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > /tmp/research-repeat-v1.zip + - name: Verify committed repeat snapshot - if: hashFiles('research/repeat/v1/repeat-dataset-manifest.json') != '' - run: python3 scripts/research/verify-repeat-snapshot-v1.py + run: >- + python3 scripts/research/verify-repeat-snapshot-v1.py + --artifact-zip /tmp/research-repeat-v1.zip execute: name: Execute 21-attempt repeat stability sample From 0bce98d5eedfafc11b325b72d7227c7a3d501c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eren=20Ar=C4=B1?= Date: Sat, 19 Sep 2026 17:48:49 +0300 Subject: [PATCH 19/19] research: verify repeat evidence against pinned artifact bytes --- scripts/research/verify-repeat-snapshot-v1.py | 235 ++++++++++++++++-- 1 file changed, 214 insertions(+), 21 deletions(-) diff --git a/scripts/research/verify-repeat-snapshot-v1.py b/scripts/research/verify-repeat-snapshot-v1.py index d51faee..3ab7cae 100644 --- a/scripts/research/verify-repeat-snapshot-v1.py +++ b/scripts/research/verify-repeat-snapshot-v1.py @@ -3,41 +3,213 @@ from __future__ import annotations +import argparse import hashlib import json import subprocess import sys import tempfile +import zipfile from pathlib import Path +from typing import Any ROOT = Path(__file__).resolve().parents[2] REPEAT = ROOT / "research/repeat/v1" DATA = REPEAT / "data" +ARTIFACT_PREFIX = "reports/research-repeat-v1/" def fail(message: str) -> None: + """Abort verification with a stable, grep-friendly error prefix.""" raise SystemExit(f"[verify-repeat-snapshot-v1] {message}") -def load_json(path: Path): +def load_json(path: Path) -> Any: + """Load UTF-8 JSON from the repository and fail closed on parse/read errors.""" try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: - fail(f"cannot read {path.relative_to(ROOT)}: {exc}") + try: + display = path.relative_to(ROOT) + except ValueError: + display = path + fail(f"cannot read {display}: {exc}") + + +def sha256_bytes(data: bytes) -> str: + """Return a normalized sha256: digest for in-memory bytes.""" + return "sha256:" + hashlib.sha256(data).hexdigest() def sha256_file(path: Path) -> str: - return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + """Return a normalized sha256: digest for a file.""" + try: + return sha256_bytes(path.read_bytes()) + except OSError as exc: + fail(f"cannot hash {path}: {exc}") def git_blob_sha1(path: Path) -> str: - data = path.read_bytes() + """Return the Git blob SHA-1 for the exact bytes at path.""" + try: + data = path.read_bytes() + except OSError as exc: + fail(f"cannot read Git-blob source {path}: {exc}") header = f"blob {len(data)}\0".encode() return hashlib.sha1(header + data).hexdigest() -def main() -> int: +def require_exact_fields( + mapping: dict[str, Any], + expected: dict[str, Any], + label: str, +) -> None: + """Require every expected field to exist with both matching type and value.""" + for key, value in expected.items(): + if key not in mapping: + fail(f"{label} missing required field: {key}") + got = mapping[key] + if type(got) is not type(value) or got != value: + fail(f"{label} drift: {key}") + + +def verify_artifact_member_matches_file( + archive: zipfile.ZipFile, + member: str, + committed: Path, +) -> None: + """Require an artifact member to be byte-identical to a committed file.""" + try: + data = archive.read(member) + except KeyError: + fail(f"canonical artifact is missing {member}") + try: + committed_data = committed.read_bytes() + except OSError as exc: + fail(f"cannot read committed evidence {committed}: {exc}") + if data != committed_data: + fail(f"artifact/committed evidence mismatch: {member}") + + +def verify_artifact_zip( + artifact_zip: Path, + artifact: dict[str, Any], + reports: list[dict[str, Any]], + expected_raw_paths: set[str], + manifest: dict[str, Any], +) -> None: + """Verify the pinned artifact archive and bind every raw report to its bytes.""" + if not artifact_zip.is_file(): + fail(f"canonical artifact ZIP is missing: {artifact_zip}") + + expected_digest = artifact.get("sha256") + if sha256_file(artifact_zip) != expected_digest: + fail("canonical repeat artifact ZIP SHA-256 mismatch") + + expected_size = artifact.get("size_bytes") + try: + actual_size = artifact_zip.stat().st_size + except OSError as exc: + fail(f"cannot stat canonical artifact ZIP: {exc}") + if type(expected_size) is not int or actual_size != expected_size: + fail("canonical repeat artifact ZIP size mismatch") + + try: + archive = zipfile.ZipFile(artifact_zip) + except (OSError, zipfile.BadZipFile) as exc: + fail(f"cannot open canonical artifact ZIP: {exc}") + + with archive: + bad_member = archive.testzip() + if bad_member is not None: + fail(f"canonical artifact ZIP CRC failure: {bad_member}") + + names = set(archive.namelist()) + expected_members = {ARTIFACT_PREFIX + path for path in expected_raw_paths} + raw_members = { + name + for name in names + if name.startswith(ARTIFACT_PREFIX + "raw/") and name.endswith(".json") + } + if raw_members != expected_members: + fail("canonical artifact raw-report membership drift") + + for row in reports: + path = row["path"] + member = ARTIFACT_PREFIX + path + try: + info = archive.getinfo(member) + data = archive.read(info) + except KeyError: + fail(f"canonical artifact is missing raw report: {path}") + + expected_report_size = row["size_bytes"] + if ( + type(expected_report_size) is not int + or info.file_size != expected_report_size + or len(data) != expected_report_size + ): + fail(f"raw report size mismatch against artifact: {path}") + if sha256_bytes(data) != row["sha256"]: + fail(f"raw report SHA-256 mismatch against artifact: {path}") + + verify_artifact_member_matches_file( + archive, + ARTIFACT_PREFIX + "normalized/stability-summary.json", + DATA / "stability-summary.json", + ) + verify_artifact_member_matches_file( + archive, + ARTIFACT_PREFIX + "repeat-provenance.json", + DATA / "repeat-provenance.json", + ) + verify_artifact_member_matches_file( + archive, + ARTIFACT_PREFIX + "normalized/RESULTS.md", + DATA / "RESULTS.md", + ) + + projection_dir = DATA / "projection-metadata" + for committed in sorted(projection_dir.glob("*.json")): + verify_artifact_member_matches_file( + archive, + ARTIFACT_PREFIX + "manifests/" + committed.name, + committed, + ) + + artifact_evidence = manifest.get("artifact_evidence") or {} + repeat_executions_member = ( + ARTIFACT_PREFIX + + str(artifact_evidence.get("repeat_executions_path") or "") + ) + try: + repeat_executions = archive.read(repeat_executions_member) + except KeyError: + fail("canonical artifact is missing repeat-executions.jsonl") + if ( + sha256_bytes(repeat_executions) + != artifact_evidence.get("repeat_executions_sha256") + ): + fail("repeat-executions artifact binding drift") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse verifier CLI arguments.""" + ap = argparse.ArgumentParser() + ap.add_argument( + "--artifact-zip", + required=True, + help="Downloaded canonical repeat Actions artifact ZIP", + ) + return ap.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Verify committed metadata, provenance, projections, and artifact bytes.""" + args = parse_args(argv) + artifact_zip = Path(args.artifact_zip) + manifest = load_json(REPEAT / "repeat-dataset-manifest.json") summary = load_json(DATA / "stability-summary.json") provenance = load_json(DATA / "repeat-provenance.json") @@ -83,8 +255,13 @@ def main() -> int: artifact = manifest.get("actions_artifact") or {} if artifact.get("id") != "10584409793": fail("canonical repeat artifact id drift") - if artifact.get("sha256") != "sha256:895fd41dae147c592d5cdec91bbd99150f9dd14c35afadf72bc14954a192a0e5": + if ( + artifact.get("sha256") + != "sha256:895fd41dae147c592d5cdec91bbd99150f9dd14c35afadf72bc14954a192a0e5" + ): fail("canonical repeat artifact SHA-256 drift") + if artifact.get("size_bytes") != 455756: + fail("canonical repeat artifact size drift") expected_collection = { "planned_tuples": 7, @@ -98,10 +275,8 @@ def main() -> int: } if manifest.get("collection") != expected_collection: fail("repeat dataset collection summary drift") + require_exact_fields(summary, expected_collection, "stability summary") - for key, value in expected_collection.items(): - if key in summary and summary.get(key) != value: - fail(f"stability summary drift: {key}") if summary.get("missing") != []: fail("repeat summary has missing observations") if summary.get("sampling_design") != "purposeful_stratified_post_collection": @@ -119,7 +294,9 @@ def main() -> int: fail("study plan SHA-256 drift") if provenance.get("profile_lock_sha256") != sha256_file(profile_lock_path): fail("profile lock SHA-256 drift") - if provenance.get("manifest_projection_script_sha256") != sha256_file(projection_script): + if provenance.get("manifest_projection_script_sha256") != sha256_file( + projection_script + ): fail("repeat projection script SHA-256 drift") identities = { @@ -128,12 +305,18 @@ def main() -> int: } if provenance.get("bpfcompat_cli_sha256") != identities.get("bpfcompat-v037-cli"): fail("BPFCompat CLI identity drift") - if provenance.get("validator_sha256") != identities.get("bpfcompat-v037-validator"): + if provenance.get("validator_sha256") != identities.get( + "bpfcompat-v037-validator" + ): fail("validator identity drift") loaders = provenance.get("loaders") or {} - if loaders.get("cilium-ebpf-v022-loader") != identities.get("cilium-ebpf-v022-loader"): + if loaders.get("cilium-ebpf-v022-loader") != identities.get( + "cilium-ebpf-v022-loader" + ): fail("cilium loader identity drift") - if loaders.get("falco-modern-bpf-scap-open") != identities.get("falco-modern-bpf-scap-open"): + if loaders.get("falco-modern-bpf-scap-open") != identities.get( + "falco-modern-bpf-scap-open" + ): fail("Falco loader identity drift") tuples = sample.get("tuples") or [] @@ -155,7 +338,7 @@ def main() -> int: digest = str(row.get("sha256") or "") if not digest.startswith("sha256:") or len(digest) != 71: fail(f"invalid raw report SHA-256: {row!r}") - if int(row.get("size_bytes") or 0) <= 0: + if type(row.get("size_bytes")) is not int or row["size_bytes"] <= 0: fail(f"invalid raw report size: {row!r}") tuple_summaries = summary.get("tuple_summaries") or [] @@ -164,7 +347,9 @@ def main() -> int: fail("tuple summary coverage drift") for row in tuples: got = by_id[row["id"]] - if got.get("case_id") != row["case_id"] or got.get("profile_id") != row["profile_id"]: + if got.get("case_id") != row["case_id"] or got.get("profile_id") != row[ + "profile_id" + ]: fail(f"tuple binding drift: {row['id']}") if got.get("stratum") != row["stratum"]: fail(f"tuple stratum drift: {row['id']}") @@ -179,7 +364,8 @@ def main() -> int: cases = {row["id"]: row for row in study_plan.get("cases") or []} libbpf_tuples = [ - row for row in tuples + row + for row in tuples if cases[row["case_id"]].get("mode") in {"load_only", "load_attach"} ] projection_dir = DATA / "projection-metadata" @@ -239,11 +425,18 @@ def main() -> int: if regenerated.get("projected_sha256") != meta.get("projected_sha256"): fail(f"projection is not reproducible: {row['id']}") - artifact_evidence = manifest.get("artifact_evidence") or {} - if artifact_evidence.get("repeat_executions_sha256") != "sha256:844ff0b27d8bcd7ecdb0e3368147b7486305c9cc0f3710c0fc95dc4d25aed11c": - fail("repeat-executions artifact binding drift") - - print("[verify-repeat-snapshot-v1] PASS: canonical 21/21 repeat snapshot is internally consistent") + verify_artifact_zip( + artifact_zip, + artifact, + reports, + expected_raw_paths, + manifest, + ) + + print( + "[verify-repeat-snapshot-v1] PASS: canonical 21/21 repeat snapshot " + "and pinned artifact bytes are internally consistent" + ) return 0