From fb02634689d99f0365674f880cfa6d8866bbcc36 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 03:59:59 +0200 Subject: [PATCH 01/62] test(mpi): prove exact nonzero balance terms --- .../mpi/test_async_balance_cadence_mpi.py | 98 ++++++++++++++----- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/tests/python/integration/mpi/test_async_balance_cadence_mpi.py b/tests/python/integration/mpi/test_async_balance_cadence_mpi.py index 40ce1a255..0ba380290 100644 --- a/tests/python/integration/mpi/test_async_balance_cadence_mpi.py +++ b/tests/python/integration/mpi/test_async_balance_cadence_mpi.py @@ -5,8 +5,11 @@ ``Case -> Program.cadence -> compile -> mpi_world -> bind -> run`` route. The Program closes one stride-3 window every third accepted macro-step, while async Balance consumers fire every two and three accepted steps. Held windows must therefore publish exact zero ledgers and due windows must -publish native nonzero ledgers. A separate every-step async field series proves that each worker -receives the accepted field image captured on its own tick, never the latest native state. +publish an exact signed five-term ledger built from five real collective Program reductions. The +fixture explicitly authors that accounting split; it proves transport, signs, residual closure and +rank agreement, not automatic extraction of AMR reflux or projection terms. A separate every-step +async field series proves that each worker receives the accepted field image captured on its own +tick, never the latest native state. """ from __future__ import annotations @@ -163,22 +166,35 @@ def _authored_case(*, adaptive: bool) -> tuple[pops.Case, Any]: case.numerics(numerics, block=block) program = pops.Program("async-balance-%s-program" % label) temporal = program.state(evolved) - total = program.sum(temporal.n) - zero = total * 0.0 - ledger = BalanceLedger("accepted-mass") - program.record_balance( - ledger, - storage_change=total, - outward_boundary_flux=zero, - sources=zero, - reflux=zero, - projection=zero, - ) accepted = program.value( "accepted_growth", temporal.n + program.dt * Fraction(1, 2) * temporal.n, at=temporal.next.point, ) + increment = program.value( + "accepted_increment", + accepted - temporal.n, + at=temporal.next.point, + ) + # This is an explicitly authored accounting fixture, not an automatic AMR-term extractor. + # Every term owns a real native Program.sum so the installed mpiexec route enters five + # collectives. The signed split closes the actual accepted storage increment exactly: + # storage + outward - sources - reflux - projection + # = q - q - q - q - (-2q) = 0. + storage_change = program.sum(increment) + outward_boundary_flux = -program.sum(increment) + sources = program.sum(increment) + reflux = program.sum(increment) + projection = -2.0 * program.sum(increment) + ledger = BalanceLedger("accepted-mass") + program.record_balance( + ledger, + storage_change=storage_change, + outward_boundary_flux=outward_boundary_flux, + sources=sources, + reflux=reflux, + projection=projection, + ) program.commit(temporal.next, accepted) program.cadence(stride=3) program.step_strategy(FixedDt(DT)) @@ -310,6 +326,34 @@ def _balance(reopened: Any) -> tuple[float, dict[str, float]]: ) +def _require_exact_signed_balance( + label: str, + step: int, + value: float, + terms: dict[str, float], +) -> None: + q = terms["storage_change"] + expected = { + "storage_change": q, + "outward_boundary_flux": -q, + "sources": q, + "reflux": q, + "projection": -2.0 * q, + } + residual = ( + terms["storage_change"] + + terms["outward_boundary_flux"] + - terms["sources"] + - terms["reflux"] + - terms["projection"] + ) + if q <= 0.0 or terms != expected or residual != 0.0 or value != residual: + raise AssertionError( + "%s due step %d did not preserve its exact signed five-term Balance: " + "value=%r terms=%r" % (label, step, value, terms) + ) + + def _verify(root: Path, *, adaptive: bool) -> None: if RANK != 0: return @@ -354,17 +398,13 @@ def _verify(root: Path, *, adaptive: bool) -> None: for series, steps in ((every_two, (6,)), (every_three, (3, 6))): for step in steps: value, terms = _balance(series[step]) - if set(terms) != expected_terms \ - or value <= 0.0 \ - or terms["storage_change"] <= 0.0 \ - or any( - terms[name] != 0.0 - for name in expected_terms - {"storage_change"} - ): - raise AssertionError( - "%s due step %d did not publish its native nonzero Balance ledger" - % (label, step) - ) + if set(terms) != expected_terms: + raise AssertionError("%s due step %d omitted a Balance term" % (label, step)) + _require_exact_signed_balance(label, step, value, terms) + if _balance(every_two[6]) != _balance(every_three[6]): + raise AssertionError( + "%s independent due consumers disagreed on the accepted step-6 Balance" % label + ) def _run_case(root: Path, *, adaptive: bool) -> None: @@ -405,6 +445,16 @@ def _run_case(root: Path, *, adaptive: bool) -> None: ) if any(row != reports[0] for row in reports[1:]) or reports[0][0] != NSTEPS: raise AssertionError("%s run report differs across ranks: %r" % (label, reports)) + accepted_balance = tuple( + row + for row in runtime.inspect().to_dict()["instance"]["accepted_diagnostics"] + if row["key"]["reduction"] == "discrete_balance" + ) + accepted_by_rank = allgather_value(COMM, accepted_balance) + if not accepted_balance or any(row != accepted_by_rank[0] for row in accepted_by_rank[1:]): + raise AssertionError( + "%s accepted Balance registry differs across ranks: %r" % (label, accepted_by_rank) + ) barrier(COMM) _collective_local(label + " output verification", lambda: _verify(root, adaptive=adaptive)) From 6de9bbf35a2b942113f0fd8018d1e3d87041ee8b Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:08:59 +0200 Subject: [PATCH 02/62] test(output): add mandatory native reopen proofs --- .../integration/io/m4_native_reopen_proof.py | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 tests/python/integration/io/m4_native_reopen_proof.py diff --git a/tests/python/integration/io/m4_native_reopen_proof.py b/tests/python/integration/io/m4_native_reopen_proof.py new file mode 100644 index 000000000..b1f0ecc50 --- /dev/null +++ b/tests/python/integration/io/m4_native_reopen_proof.py @@ -0,0 +1,150 @@ +"""Mandatory format-native reopen proofs executed only by the explicit M4 gate.""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + +from pops.identity import make_identity +from pops.model import Handle, OwnerKind, OwnerPath +from pops.output import ( + ArrayPiece, + FieldKey, + FieldPayload, + HDF5Writer, + LevelGeometry, + NPZWriter, + OutputClock, + OutputProvenance, + OutputRequest, + OutputSnapshot, + ParaViewWriter, + ParallelMode, + read_hdf5, +) + + +def _identity(domain: str, name: str): + return make_identity(domain, {"name": name}) + + +def _snapshot_and_request(): + layout = _identity("layout-plan", "m4-native-reopen") + component = _identity("component-manifest", "m4-native-reopen") + owner = OwnerPath.case("m4-native-reopen").child(OwnerKind.BLOCK, "fluid") + state = Handle("rho", kind="state", owner=owner) + key = FieldKey(state, component, layout, 0, "accepted") + values = np.asarray([[1.0, 2.0], [3.0, 4.0]], dtype=np.float64) + geometry = LevelGeometry( + layout, + "uniform", + 0, + (0.0, 0.0), + (0.5, 0.5), + (2, 2), + ((0, 0, 2, 2),), + np.zeros((2, 2), dtype=np.bool_), + np.full((2, 2), 0.25, dtype=np.float64), + ) + field = FieldPayload( + key, + "cell", + "kg.m-3", + (), + (2, 2), + (ArrayPiece((0, 0), (2, 2), values, 0, 0, False),), + ) + snapshot = OutputSnapshot( + OutputClock.at("macro", 0.25, 4, stage="accepted"), + OutputProvenance( + _identity("resolved-plan", "m4-native-reopen"), + _identity("bind", "m4-native-reopen"), + _identity("run", "m4-native-reopen"), + "accepted-step-transaction", + ), + (geometry,), + (field,), + {"case": "m4-native-reopen"}, + ) + request = OutputRequest("rho-output", (key,), ParallelMode.SERIAL) + return snapshot, request, values + + +def _publish(writer, target): + snapshot, request, expected = _snapshot_and_request() + session = writer.prepare_session(snapshot, request, target) + session.stage() + receipt = session.publish() + session.finalize() + assert receipt.path == target + return receipt.path, request, expected + + +def _field_dataset(manifest: dict, request: OutputRequest) -> str: + key = request.selection[0].identity.token + return manifest["datasets"]["fields"][key] + + +def test_npz_reopens_with_numpy_without_a_pops_reader(tmp_path): + path, request, expected = _publish(NPZWriter(), tmp_path / "native.npz") + + with np.load(path, allow_pickle=False) as archive: + manifest = json.loads(str(archive["pops_output_manifest"])) + dataset = _field_dataset(manifest, request) + np.testing.assert_array_equal(archive[dataset], expected) + assert set(archive.files) == set(manifest["arrays"]) | { + "pops_output_manifest" + } + assert manifest["snapshot"]["clock"]["time"] == float.hex(0.25) + + +def test_hdf5_reopens_with_h5py_without_a_pops_reader(tmp_path): + import h5py + + path, request, expected = _publish(HDF5Writer(), tmp_path / "native.h5") + + with h5py.File(path, "r") as output: + manifest = json.loads(str(output.attrs["pops_output_manifest"])) + dataset = _field_dataset(manifest, request) + np.testing.assert_array_equal(output[dataset][...], expected) + assert set(output.attrs) == {"pops_output_manifest"} + assert manifest["snapshot"]["clock"]["time"] == float.hex(0.25) + + +def test_hdf5_authenticated_reader_rejects_native_dataset_tampering(tmp_path): + import h5py + + path, request, _expected = _publish(HDF5Writer(), tmp_path / "tampered.h5") + with h5py.File(path, "r+") as output: + manifest = json.loads(str(output.attrs["pops_output_manifest"])) + dataset = _field_dataset(manifest, request) + output[dataset][0, 0] = np.float64(99.0) + + with pytest.raises(ValueError, match="content verification"): + read_hdf5(path) + + +def test_paraview_reopens_with_vtk_without_a_pops_reader(tmp_path): + from vtkmodules.vtkIOXML import vtkXMLUnstructuredGridReader + + path, _request, expected = _publish( + ParaViewWriter(), tmp_path / "native.vtu" + ) + + reader = vtkXMLUnstructuredGridReader() + reader.SetFileName(str(path)) + reader.Update() + grid = reader.GetOutput() + assert grid.GetNumberOfCells() == 4 + assert grid.GetNumberOfPoints() == 9 + rho = grid.GetCellData().GetArray("rho") + assert rho is not None + assert [rho.GetTuple1(index) for index in range(4)] == expected.ravel().tolist() + assert grid.GetCellData().GetArray("field_0000") is None + assert [ + grid.GetCellData().GetArray("pops_level").GetTuple1(index) + for index in range(4) + ] == [0.0, 0.0, 0.0, 0.0] + assert grid.GetFieldData().GetArray("TimeValue").GetTuple1(0) == 0.25 From d92459b8148ba1d2946ef0244dd99a79ad450567 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:09:15 +0200 Subject: [PATCH 03/62] test(gate): add exact fail-closed M4 evidence ledger --- scripts/run_m4_gate.py | 728 ++++++++++++++++++ tests/gates/m4_runtime_io.toml | 347 +++++++++ .../architecture/test_m4_runtime_io_gate.py | 454 +++++++++++ 3 files changed, 1529 insertions(+) create mode 100644 scripts/run_m4_gate.py create mode 100644 tests/gates/m4_runtime_io.toml create mode 100644 tests/python/architecture/test_m4_runtime_io_gate.py diff --git a/scripts/run_m4_gate.py b/scripts/run_m4_gate.py new file mode 100644 index 000000000..75ca9480e --- /dev/null +++ b/scripts/run_m4_gate.py @@ -0,0 +1,728 @@ +#!/usr/bin/env python3 +"""Audit and run the fail-closed M4 native-runtime/IO conformance matrix.""" + +from __future__ import annotations + +import argparse +import ast +from collections import Counter, defaultdict +from collections.abc import Iterable +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST = ROOT / "tests/gates/m4_runtime_io.toml" +TEST_MANIFEST = ROOT / "tests/test_manifest.toml" +EXPECTED_ISSUES = tuple("ADC-%d" % number for number in range(679, 688)) +REQUIRED_POLARITIES = { + "component_manifest": {"positive", "refusal"}, + "generated_registry": {"positive", "refusal"}, + "external_package": {"positive", "refusal"}, + "external_flux": {"positive"}, + "external_boundary": {"positive"}, + "external_tagger": {"positive"}, + "external_transfer": {"positive"}, + "external_solver": {"positive"}, + "external_writer": {"positive"}, + "native_interfaces": {"positive", "refusal"}, + "flux_contract": {"positive", "refusal"}, + "platform_execution": {"positive", "refusal"}, + "runtime_instance": {"positive", "refusal"}, + "consumer_graph": {"positive", "refusal"}, + "accepted_publication": {"positive", "refusal"}, + "exact_npz": {"positive", "refusal"}, + "exact_hdf5": {"positive", "refusal"}, + "exact_paraview": {"positive", "refusal"}, + "collective_hdf5": {"positive"}, + "strict_checkpoint": {"positive", "refusal"}, + "diagnostics": {"positive", "refusal"}, + "tamper_capability_abi": {"refusal"}, + "legacy_stepper_retirement": {"positive"}, +} +REQUIREMENT_ISSUES = { + "component_manifest": {"ADC-679"}, + "generated_registry": {"ADC-679"}, + "external_package": {"ADC-680"}, + "external_flux": {"ADC-680"}, + "native_interfaces": {"ADC-681"}, + "external_boundary": {"ADC-681"}, + "external_tagger": {"ADC-681"}, + "flux_contract": {"ADC-682"}, + "platform_execution": {"ADC-683"}, + "runtime_instance": {"ADC-684"}, + "external_transfer": {"ADC-684"}, + "external_writer": {"ADC-685"}, + "consumer_graph": {"ADC-685"}, + "accepted_publication": {"ADC-685"}, + "exact_npz": {"ADC-686"}, + "exact_hdf5": {"ADC-686"}, + "exact_paraview": {"ADC-686"}, + "collective_hdf5": {"ADC-686"}, + "strict_checkpoint": {"ADC-686"}, + "diagnostics": {"ADC-686"}, + "external_solver": {"ADC-687"}, + "legacy_stepper_retirement": {"ADC-687"}, + "tamper_capability_abi": {"ADC-679", "ADC-680", "ADC-683", "ADC-687"}, +} +NATIVE_PYTEST_PREFIXES = ( + "tests/python/integration/amr/", + "tests/python/integration/io/", + "tests/python/integration/mpi/", + "tests/python/integration/native_loader/", + "tests/python/integration/runtime/", +) +_GTEST_DECLARATION = re.compile( + r"\bTEST(?:_F)?\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*,\s*" + r"([A-Za-z_][A-Za-z0-9_]*)\s*\)" +) +_CPP_RAW_STRING_START = re.compile(r'(?:u8|u|U|L)?R"([^\s()\\]{0,16})\(') +_MOCK_FIXTURES = {"monkeypatch", "mocker", "mock", "patch"} +_FORBIDDEN_CALLS = { + "pytest.skip", + "pytest.xfail", + "pytest.importorskip", + "unittest.mock.patch", + "mock.patch", + "require_mpi_or_skip", +} + + +def _dotted_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + prefix = _dotted_name(node.value) + return "%s.%s" % (prefix, node.attr) if prefix else node.attr + if isinstance(node, ast.Call): + return _dotted_name(node.func) + return "" + + +def _forbidden_python_markers(node: ast.AST) -> list[str]: + markers: list[str] = [] + for decorator in getattr(node, "decorator_list", ()): + name = _dotted_name(decorator) + if name.endswith((".skip", ".skipif", ".xfail")) or name in { + "skip", + "skipif", + "xfail", + }: + markers.append(name) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + fixtures = { + argument.arg + for argument in ( + *node.args.posonlyargs, + *node.args.args, + *node.args.kwonlyargs, + ) + } + markers.extend("fixture:%s" % name for name in sorted(fixtures & _MOCK_FIXTURES)) + for child in ast.walk(node): + if isinstance(child, ast.Call): + name = _dotted_name(child.func) + if name in _FORBIDDEN_CALLS or name.endswith( + (".importorskip", ".skip", ".xfail", ".mock", ".patch") + ): + markers.append(name) + elif isinstance(child, (ast.Import, ast.ImportFrom)): + module = child.module if isinstance(child, ast.ImportFrom) else "" + names = [alias.name for alias in child.names] + if module.startswith(("unittest.mock", "pytest_mock")) or any( + name.startswith(("unittest.mock", "pytest_mock")) for name in names + ): + markers.append("mock-import") + elif isinstance(child, ast.Try): + for handler in child.handlers: + caught = _dotted_name(handler.type) if handler.type is not None else "" + if caught in {"ImportError", "ModuleNotFoundError"}: + markers.append("optional-import-fallback") + return markers + + +def _ctest_suites() -> dict[str, dict]: + data = tomllib.loads(TEST_MANIFEST.read_text(encoding="utf-8")) + return {str(row["name"]): row for row in data.get("cpp", {}).get("suite", ())} + + +def _python_suites() -> tuple[dict, ...]: + data = tomllib.loads(TEST_MANIFEST.read_text(encoding="utf-8")) + return tuple(data.get("python", {}).get("suite", ())) + + +def _python_mpi_entrypoints() -> dict[str, int]: + entries: dict[str, int] = {} + for suite in _python_suites(): + for row in suite.get("mpi_entrypoints", ()): + path = str(row.get("path", "")) + nproc = row.get("nproc") + if not path or isinstance(nproc, bool) or not isinstance(nproc, int) or nproc < 1: + raise ValueError("invalid Python MPI entrypoint %r" % row) + if path in entries: + raise ValueError("duplicate Python MPI entrypoint %s" % path) + entries[path] = nproc + return entries + + +def _python_mpi_orchestrators() -> set[str]: + orchestrators: set[str] = set() + for suite in _python_suites(): + for row in suite.get("mpi_orchestrators", ()): + if not isinstance(row, dict) or set(row) != {"path"}: + raise ValueError( + "invalid Python MPI orchestrator %r; expected exactly one path field" + % row + ) + path = row["path"] + if not isinstance(path, str) or not path: + raise ValueError("invalid Python MPI orchestrator path %r" % path) + if path in orchestrators: + raise ValueError("duplicate Python MPI orchestrator %s" % path) + orchestrators.add(path) + return orchestrators + + +def _python_suite_owns(relative: str) -> bool: + path = Path(relative) + return any( + path == Path(str(suite.get("path", ""))) + or Path(str(suite.get("path", ""))) in path.parents + for suite in _python_suites() + ) + + +def _cpp_code_only(source: str) -> str: + """Mask comments and literals while preserving source positions and newlines.""" + code = list(source) + size = len(source) + + def mask(begin: int, end: int) -> None: + for offset in range(begin, end): + if code[offset] != "\n": + code[offset] = " " + + index = 0 + while index < size: + if source.startswith("//", index): + end = source.find("\n", index + 2) + end = size if end < 0 else end + mask(index, end) + index = end + continue + if source.startswith("/*", index): + end = source.find("*/", index + 2) + end = size if end < 0 else end + 2 + mask(index, end) + index = end + continue + raw = _CPP_RAW_STRING_START.match(source, index) + if raw is not None: + terminator = ")" + raw.group(1) + '"' + end = source.find(terminator, raw.end()) + end = size if end < 0 else end + len(terminator) + mask(index, end) + index = end + continue + if source[index] in {'"', "'"}: + quote = source[index] + end = index + 1 + while end < size: + if source[end] == "\\": + end = min(size, end + 2) + continue + end += 1 + if source[end - 1] == quote: + break + mask(index, end) + index = end + continue + index += 1 + return "".join(code) + + +def _registered_gtest_cases(source: str) -> set[str]: + return { + "%s.%s" % declaration + for declaration in _GTEST_DECLARATION.findall(_cpp_code_only(source)) + } + + +def _registered_ctest_cases(target: str, suite: dict) -> set[str]: + cases: set[str] = set() + for relative in suite.get("sources", ()): + source = ROOT / relative + if source.is_file(): + cases.update(_registered_gtest_cases(source.read_text(encoding="utf-8"))) + for field in ("mpi_nproc", "mpi_rank_parity", "mpi_variants"): + cases.update( + "%s_np%d" % (target, nproc) + for nproc in suite.get(field, ()) + if not isinstance(nproc, bool) and isinstance(nproc, int) and nproc > 0 + ) + return cases + + +def _validate_exact_ctest_selector( + selector: object, + target: str, + suite: dict, + where: str, + errors: list[str], +) -> None: + if not isinstance(selector, str) or not selector: + errors.append("%s CTest row requires a non-empty test_regex" % where) + return + exact = { + "^%s$" % re.escape(case) + for case in _registered_ctest_cases(target, suite) + } + if selector not in exact: + errors.append( + "%s CTest selector %r is not one exact source-registered case for target %r" + % (where, selector, target) + ) + + +def _validate_python_nodeid( + nodeid: object, + where: str, + errors: list[str], +) -> str | None: + if not isinstance(nodeid, str) or nodeid.count("::") != 1: + errors.append("%s must contain one exact file::test nodeid" % where) + return None + relative, function_name = nodeid.split("::") + test_path = ROOT / relative + if not test_path.is_file(): + errors.append("%s references missing test file %s" % (where, relative)) + return None + if not _python_suite_owns(relative): + errors.append("%s is not owned by tests/test_manifest.toml" % relative) + tree = ast.parse(test_path.read_text(encoding="utf-8"), filename=str(test_path)) + functions = { + node.name: node + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + function = functions.get(function_name) + if function is None: + errors.append("%s references missing test function %s" % (where, nodeid)) + return None + module_nodes = [ + node + for node in tree.body + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + ] + markers = _forbidden_python_markers(function) + markers.extend( + _forbidden_python_markers(ast.Module(body=module_nodes, type_ignores=[])) + ) + if markers: + errors.append( + "%s is not an unconditional real proof; found %s" + % (nodeid, sorted(set(markers))) + ) + return relative + + +def _validate_deferred(data: dict, errors: list[str]) -> set[str]: + rows = data.get("deferred") + if not isinstance(rows, list): + errors.append("deferred must be an array of explicit gap tables") + return set() + requirements: set[str] = set() + identities = Counter() + for index, row in enumerate(rows, 1): + where = "deferred[%d]" % index + expected = {"issue", "requirement", "reason", "evidence_paths"} + if not isinstance(row, dict) or set(row) != expected: + errors.append("%s must contain issue/requirement/reason/evidence_paths" % where) + continue + issue = row.get("issue") + requirement = row.get("requirement") + reason = row.get("reason") + evidence_paths = row.get("evidence_paths") + if issue not in EXPECTED_ISSUES: + errors.append("%s has unknown issue %r" % (where, issue)) + if requirement not in REQUIRED_POLARITIES: + errors.append("%s has unknown requirement %r" % (where, requirement)) + elif issue not in REQUIREMENT_ISSUES[requirement]: + errors.append( + "%s requirement %r cannot be deferred under %r" + % (where, requirement, issue) + ) + if not isinstance(reason, str) or len(reason.strip()) < 20: + errors.append("%s requires a precise non-empty reason" % where) + if not isinstance(evidence_paths, list) or not evidence_paths: + errors.append("%s requires at least one evidence path" % where) + else: + for relative in evidence_paths: + if not isinstance(relative, str) or not relative: + errors.append("%s has an invalid evidence path %r" % (where, relative)) + elif not (ROOT / relative).exists(): + errors.append( + "%s gap evidence path no longer exists: %s" % (where, relative) + ) + identities[(issue, requirement)] += 1 + requirements.add(str(requirement)) + duplicates = sorted(identity for identity, count in identities.items() if count > 1) + if duplicates: + errors.append("duplicate deferred gaps: %s" % duplicates) + return requirements + + +def audit_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: + """Return source-only structural errors without pretending deferred gaps are closed.""" + errors: list[str] = [] + try: + data = tomllib.loads(path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + return {}, ["cannot read M4 gate manifest %s: %s" % (path, exc)] + + if data.get("schema_version") != 1: + errors.append("schema_version must be exactly 1") + if data.get("gate") != "m4-runtime-io": + errors.append("gate must be exactly 'm4-runtime-io'") + if set(data) != {"schema_version", "gate", "issues", "deferred", "check"}: + errors.append("manifest fields must be schema_version/gate/issues/deferred/check") + if data.get("issues") != list(EXPECTED_ISSUES): + errors.append("issues must list ADC-679..ADC-687 exactly once") + + deferred_requirements = _validate_deferred(data, errors) + checks = data.get("check") + if not isinstance(checks, list) or not checks: + errors.append("manifest must contain [[check]] rows") + checks = [] + + identities = Counter() + issue_coverage: dict[str, set[str]] = defaultdict(set) + requirement_coverage: dict[str, set[str]] = defaultdict(set) + native_positive_issues: set[str] = set() + cpp_suites = _ctest_suites() + try: + mpi_entrypoints = _python_mpi_entrypoints() + mpi_orchestrators = _python_mpi_orchestrators() + except (OSError, tomllib.TOMLDecodeError, ValueError) as exc: + errors.append("cannot read Python MPI ownership: %s" % exc) + mpi_entrypoints = {} + mpi_orchestrators = set() + + for index, row in enumerate(checks, 1): + where = "check[%d]" % index + base = {"issue", "requirement", "polarity", "kind", "target"} + kind = row.get("kind") if isinstance(row, dict) else None + expected = ( + base | {"nodeid", "nproc"} + if kind == "mpi_python" + else base | ({"nodeid"} if kind == "pytest" else {"test_regex"}) + ) + if not isinstance(row, dict) or set(row) != expected: + errors.append("%s has unknown or missing fields: %s" % (where, sorted(row))) + continue + + issue = row.get("issue") + requirement = row.get("requirement") + polarity = row.get("polarity") + target = row.get("target") + if issue not in EXPECTED_ISSUES: + errors.append("%s has unknown issue %r" % (where, issue)) + if requirement not in REQUIRED_POLARITIES: + errors.append("%s has unknown requirement %r" % (where, requirement)) + elif issue not in REQUIREMENT_ISSUES[requirement]: + errors.append( + "%s requirement %r cannot be attributed to %r" + % (where, requirement, issue) + ) + if kind != "ctest" and target != requirement: + errors.append( + "%s target must equal its exact requirement %r" % (where, requirement) + ) + if polarity not in {"positive", "refusal"}: + errors.append("%s polarity must be positive or refusal" % where) + else: + issue_coverage[str(issue)].add(polarity) + requirement_coverage[str(requirement)].add(polarity) + + identity = (kind, row.get("nodeid", row.get("test_regex"))) + identities[identity] += 1 + if kind == "pytest": + relative = _validate_python_nodeid(row.get("nodeid"), where, errors) + if ( + relative is not None + and relative.startswith("tests/python/integration/mpi/") + and relative not in mpi_orchestrators + ): + errors.append( + "%s is not a manifest-owned serial MPI orchestrator" % relative + ) + if ( + polarity == "positive" + and relative is not None + and relative.startswith(NATIVE_PYTEST_PREFIXES) + ): + native_positive_issues.add(str(issue)) + elif kind == "mpi_python": + relative = _validate_python_nodeid(row.get("nodeid"), where, errors) + nproc = row.get("nproc") + if isinstance(nproc, bool) or not isinstance(nproc, int) or nproc < 1: + errors.append("%s MPI Python row requires a positive integer nproc" % where) + elif relative is not None: + expected_nproc = mpi_entrypoints.get(relative) + if expected_nproc is None: + errors.append("%s is not a manifest-owned MPI Python entrypoint" % relative) + elif expected_nproc != nproc: + errors.append( + "%s requires nproc=%d, not %d" + % (relative, expected_nproc, nproc) + ) + if polarity == "positive": + native_positive_issues.add(str(issue)) + elif kind == "ctest": + target_name = row.get("target") + # CTest rows still carry the semantic requirement in target, so the + # build target is encoded as "requirement@ctest-target". + if not isinstance(target_name, str) or "@" not in target_name: + errors.append( + "%s CTest target must be requirement@manifest-suite" % where + ) + continue + semantic, suite_name = target_name.split("@", 1) + if semantic != requirement: + errors.append( + "%s CTest target must start with requirement %r" % (where, requirement) + ) + suite = cpp_suites.get(suite_name) + if suite is None: + errors.append("%s references unknown CTest target %r" % (where, suite_name)) + continue + _validate_exact_ctest_selector( + row.get("test_regex"), suite_name, suite, where, errors + ) + for relative in suite.get("sources", ()): + source = ROOT / relative + if not source.is_file(): + errors.append( + "%s target %r has missing source %s" + % (where, suite_name, relative) + ) + else: + text = source.read_text(encoding="utf-8") + if "DISABLED_" in text: + errors.append( + "%s target %r contains a disabled test" % (where, suite_name) + ) + if polarity == "positive": + native_positive_issues.add(str(issue)) + else: + errors.append("%s kind must be pytest, mpi_python, or ctest" % where) + + duplicates = sorted(identity for identity, count in identities.items() if count > 1) + if duplicates: + errors.append("duplicate executable checks: %s" % duplicates) + for issue in EXPECTED_ISSUES: + missing = {"positive", "refusal"} - issue_coverage[issue] + if missing: + errors.append("%s lacks %s coverage" % (issue, "/".join(sorted(missing)))) + if issue not in native_positive_issues: + errors.append("%s lacks a mandatory native positive proof" % issue) + for requirement, required in sorted(REQUIRED_POLARITIES.items()): + missing = required - requirement_coverage[requirement] + if missing and requirement not in deferred_requirements: + errors.append( + "%s lacks %s coverage" + % (requirement, "/".join(sorted(missing))) + ) + return data, errors + + +def validate_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: + """Fail closed when even one structurally valid M4 requirement is deferred.""" + data, errors = audit_manifest(path) + if errors: + return data, errors + for row in data["deferred"]: + errors.append( + "%s/%s remains deferred: %s" + % (row["issue"], row["requirement"], row["reason"]) + ) + return data, errors + + +def _run(command: list[str], *, env: dict[str, str] | None = None) -> None: + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=ROOT, check=True, env=env) + + +def _required_environment() -> dict[str, str]: + environment = os.environ.copy() + environment["POPS_REQUIRE_MPI_TESTS"] = "1" + environment["POPS_REQUIRE_NATIVE_TESTS"] = "1" + return environment + + +def _mpi_python_command(mpi_exec: str, nproc: int, relative: str) -> list[str]: + if shutil.which(mpi_exec) is None: + raise RuntimeError("required MPI launcher %r is unavailable" % mpi_exec) + return [mpi_exec, "-n", str(nproc), sys.executable, str(ROOT / relative)] + + +def _junit_skip_count(report: Path, producer: str) -> int: + if not report.is_file(): + raise RuntimeError("M4 %s did not produce its mandatory JUnit report" % producer) + try: + root = ET.parse(report).getroot() + except ET.ParseError as exc: + raise RuntimeError("M4 %s produced an invalid JUnit report" % producer) from exc + return len(root.findall(".//skipped")) + + +def _run_required_pytest(nodeids: list[str]) -> None: + environment = _required_environment() + with tempfile.TemporaryDirectory(prefix="pops-m4-gate-") as temporary: + report = Path(temporary) / "pytest.xml" + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "--strict-markers", + "-o", + "xfail_strict=true", + "--junitxml", + str(report), + *nodeids, + ] + print( + "+ POPS_REQUIRE_MPI_TESTS=1 POPS_REQUIRE_NATIVE_TESTS=1", + " ".join(command), + flush=True, + ) + completed = subprocess.run( + command, + cwd=ROOT, + env=environment, + check=False, + ) + skipped = _junit_skip_count(report, "pytest") + if skipped: + raise RuntimeError( + "M4 pytest reported %d skipped/xfail proof(s); every proof is mandatory" + % skipped + ) + if completed.returncode != 0: + raise subprocess.CalledProcessError(completed.returncode, command) + + +def _chunks(values: list[str], size: int) -> Iterable[list[str]]: + for index in range(0, len(values), size): + yield values[index : index + size] + + +def _run_ctest(build_dir: Path, target: str, selector: str) -> None: + listed = subprocess.run( + ["ctest", "--test-dir", str(build_dir), "-N", "-R", selector], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + if "Total Tests: 0" in listed.stdout or "Test #" not in listed.stdout: + raise RuntimeError( + "M4 CTest target %r (%s) is not built in %s" + % (target, selector, build_dir) + ) + with tempfile.TemporaryDirectory(prefix="pops-m4-ctest-") as temporary: + report = Path(temporary) / "ctest.xml" + command = [ + "ctest", + "--test-dir", + str(build_dir), + "--output-on-failure", + "--output-junit", + str(report), + "-R", + selector, + ] + print("+", " ".join(command), flush=True) + completed = subprocess.run(command, cwd=ROOT, check=False) + skipped = _junit_skip_count(report, "CTest") + if skipped: + raise RuntimeError( + "M4 CTest %r reported %d skipped proof(s)" % (selector, skipped) + ) + if completed.returncode != 0: + raise subprocess.CalledProcessError(completed.returncode, command) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "--audit-only", + action="store_true", + help="verify exact evidence and explicit gaps without claiming M4 closure", + ) + mode.add_argument("--check-only", action="store_true") + parser.add_argument("--python-only", action="store_true") + parser.add_argument("--build-dir", type=Path, default=ROOT / "build-mpi") + parser.add_argument("--mpi-exec", default="mpiexec") + args = parser.parse_args(argv) + + if args.audit_only: + data, errors = audit_manifest(args.manifest) + else: + data, errors = validate_manifest(args.manifest) + if errors: + print("M4 gate is incomplete or invalid:", file=sys.stderr) + for error in errors: + print(" -", error, file=sys.stderr) + return 2 + + checks = data["check"] + print( + "M4 gate source matrix: %s (%d executable, %d deferred)" + % ( + "AUDITED OPEN" if args.audit_only else "CLOSED", + len(checks), + len(data["deferred"]), + ) + ) + if args.audit_only or args.check_only: + return 0 + + nodeids = [row["nodeid"] for row in checks if row["kind"] == "pytest"] + for chunk in _chunks(nodeids, 24): + _run_required_pytest(chunk) + mpi_entrypoints = sorted( + { + (row["nodeid"].split("::", 1)[0], row["nproc"]) + for row in checks + if row["kind"] == "mpi_python" + } + ) + for relative, nproc in mpi_entrypoints: + _run( + _mpi_python_command(args.mpi_exec, nproc, relative), + env=_required_environment(), + ) + if not args.python_only: + for row in sorted( + (row for row in checks if row["kind"] == "ctest"), + key=lambda value: (value["target"], value["test_regex"]), + ): + _semantic, target = row["target"].split("@", 1) + _run_ctest(args.build_dir, target, row["test_regex"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml new file mode 100644 index 000000000..ad18818e3 --- /dev/null +++ b/tests/gates/m4_runtime_io.toml @@ -0,0 +1,347 @@ +schema_version = 1 +gate = "m4-runtime-io" +issues = [ + "ADC-679", + "ADC-680", + "ADC-681", + "ADC-682", + "ADC-683", + "ADC-684", + "ADC-685", + "ADC-686", + "ADC-687", +] +deferred = [] + +# This is an exact evidence ledger, not a list of nearby suites. Every row names +# one source-registered proof. The runner rejects mocks, optional imports, +# skip/xfail, non-exact CTest selectors, duplicate proofs, and missing manifest +# ownership before it launches anything. + +[[check]] +issue = "ADC-679" +requirement = "component_manifest" +polarity = "positive" +kind = "pytest" +target = "component_manifest" +nodeid = "tests/python/unit/codegen/test_component_manifest_v2.py::test_native_parser_normalizer_matches_python_canonical_bytes" + +[[check]] +issue = "ADC-679" +requirement = "component_manifest" +polarity = "refusal" +kind = "pytest" +target = "component_manifest" +nodeid = "tests/python/unit/codegen/test_component_manifest_v2.py::test_unknown_semantic_top_level_field_is_a_structured_refusal" + +[[check]] +issue = "ADC-679" +requirement = "component_manifest" +polarity = "refusal" +kind = "pytest" +target = "component_manifest" +nodeid = "tests/python/unit/codegen/test_component_manifest_v2.py::test_interface_bindings_are_exact_closed_and_entry_point_checked" + +[[check]] +issue = "ADC-679" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "pytest" +target = "tamper_capability_abi" +nodeid = "tests/python/unit/codegen/test_component_manifest_v2.py::test_target_capability_refusal_contains_requested_and_supported_evidence" + +[[check]] +issue = "ADC-679" +requirement = "generated_registry" +polarity = "positive" +kind = "ctest" +target = "generated_registry@test_brick_catalog" +test_regex = "^BrickCatalog\\.MirrorsRegistryAndRouteTablesRowForRow$" + +[[check]] +issue = "ADC-679" +requirement = "generated_registry" +polarity = "refusal" +kind = "pytest" +target = "generated_registry" +nodeid = "tests/python/architecture/test_route_registry_parity.py::test_no_unknown_fields_can_hide_in_catalog_rows" + +[[check]] +issue = "ADC-680" +requirement = "external_package" +polarity = "positive" +kind = "pytest" +target = "external_package" +nodeid = "tests/python/unit/codegen/test_component_packages.py::test_source_package_verifies_content_before_authoring_registry" + +[[check]] +issue = "ADC-680" +requirement = "external_package" +polarity = "refusal" +kind = "pytest" +target = "external_package" +nodeid = "tests/python/unit/codegen/test_component_packages.py::test_tampered_manifest_digest_is_rejected" + +[[check]] +issue = "ADC-680" +requirement = "external_flux" +polarity = "positive" +kind = "pytest" +target = "external_flux" +nodeid = "tests/python/integration/native_loader/test_external_component_package.py::test_source_component_executes_through_generic_native_loader_and_flux_consumer" + +[[check]] +issue = "ADC-680" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "pytest" +target = "tamper_capability_abi" +nodeid = "tests/python/unit/codegen/test_component_packages.py::test_fixed_binary_cannot_claim_template_genericity" + +[[check]] +issue = "ADC-681" +requirement = "native_interfaces" +polarity = "positive" +kind = "ctest" +target = "native_interfaces@test_component_interfaces" +test_regex = "^ComponentInterfaces\\.ExactAbiConsumersExecuteEveryClosedScientificFamily$" + +[[check]] +issue = "ADC-681" +requirement = "native_interfaces" +polarity = "refusal" +kind = "pytest" +target = "native_interfaces" +nodeid = "tests/python/unit/codegen/test_component_adapters.py::test_registration_rejects_malformed_interface_and_target_before_mutation" + +[[check]] +issue = "ADC-681" +requirement = "external_boundary" +polarity = "positive" +kind = "ctest" +target = "external_boundary@test_amr_native_loader" +test_regex = "^test_amr_native_loader\\.BoundaryPlanSessionsOwnFreshLaneQualifiedComponentStates$" + +[[check]] +issue = "ADC-681" +requirement = "external_tagger" +polarity = "positive" +kind = "ctest" +target = "external_tagger@test_amr_native_loader" +test_regex = "^test_amr_native_loader\\.PreparedAmrProvidersExecuteExactTablesAndProvenance$" + +[[check]] +issue = "ADC-682" +requirement = "flux_contract" +polarity = "positive" +kind = "ctest" +target = "flux_contract@test_flux_interfaces" +test_regex = "^test_flux_interfaces\\.equal_state_consistency_and_declared_stability$" + +[[check]] +issue = "ADC-682" +requirement = "flux_contract" +polarity = "refusal" +kind = "ctest" +target = "flux_contract@test_flux_interfaces" +test_regex = "^test_flux_interfaces\\.invalid_trace_stability_is_rejected_on_both_orientations$" + +[[check]] +issue = "ADC-683" +requirement = "platform_execution" +polarity = "positive" +kind = "ctest" +target = "platform_execution@test_platform_manifest" +test_regex = "^PlatformManifest\\.GenericTwoDimensionalDoubleRouteLaunches$" + +[[check]] +issue = "ADC-683" +requirement = "platform_execution" +polarity = "refusal" +kind = "ctest" +target = "platform_execution@test_platform_manifest" +test_regex = "^PlatformManifest\\.FieldAndCommunicatorMismatchesRefuseBeforeKernel$" + +[[check]] +issue = "ADC-683" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "pytest" +target = "tamper_capability_abi" +nodeid = "tests/python/unit/runtime/test_platform_manifest.py::test_aot_component_rejects_openmpi_mpich_abi_mix_even_with_same_headers_and_standard" + +[[check]] +issue = "ADC-684" +requirement = "runtime_instance" +polarity = "positive" +kind = "pytest" +target = "runtime_instance" +nodeid = "tests/python/integration/runtime/test_shared_interface_runtime.py::test_runtime_instance_executes_one_two_sided_shared_flux" + +[[check]] +issue = "ADC-684" +requirement = "runtime_instance" +polarity = "refusal" +kind = "pytest" +target = "runtime_instance" +nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_mid_step_child_failure_preserves_root_error_and_rolls_back_composite" + +[[check]] +issue = "ADC-684" +requirement = "external_transfer" +polarity = "positive" +kind = "pytest" +target = "external_transfer" +nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_two_native_layouts_execute_sliced_programs_and_exact_transfer" + +[[check]] +issue = "ADC-685" +requirement = "external_writer" +polarity = "positive" +kind = "pytest" +target = "external_writer" +nodeid = "tests/python/integration/native_loader/test_external_component_package.py::test_qualified_writer_runs_through_uniform_and_amr_runtime_transactions" + +[[check]] +issue = "ADC-685" +requirement = "consumer_graph" +polarity = "positive" +kind = "pytest" +target = "consumer_graph" +nodeid = "tests/python/unit/runtime/test_consumer_transactions.py::test_graph_and_plan_are_semantic_and_insertion_order_independent" + +[[check]] +issue = "ADC-685" +requirement = "consumer_graph" +polarity = "refusal" +kind = "pytest" +target = "consumer_graph" +nodeid = "tests/python/unit/runtime/test_consumer_transactions.py::test_rejected_attempt_discards_temporaries_without_publication_or_cursor_advance" + +[[check]] +issue = "ADC-685" +requirement = "accepted_publication" +polarity = "positive" +kind = "pytest" +target = "accepted_publication" +nodeid = "tests/python/examples/final/test_imex_amr_final_example.py::test_example_runs_and_every_scientific_format_reopens" + +[[check]] +issue = "ADC-685" +requirement = "accepted_publication" +polarity = "refusal" +kind = "pytest" +target = "accepted_publication" +nodeid = "tests/python/unit/runtime/test_consumer_transactions.py::test_stale_field_requires_explicit_policy_and_records_recompute_without_solving" + +[[check]] +issue = "ADC-686" +requirement = "exact_npz" +polarity = "positive" +kind = "pytest" +target = "exact_npz" +nodeid = "tests/python/integration/io/m4_native_reopen_proof.py::test_npz_reopens_with_numpy_without_a_pops_reader" + +[[check]] +issue = "ADC-686" +requirement = "exact_npz" +polarity = "refusal" +kind = "pytest" +target = "exact_npz" +nodeid = "tests/python/unit/output/test_exact_writers.py::test_npz_collision_and_discard_never_publish_partial_content" + +[[check]] +issue = "ADC-686" +requirement = "exact_hdf5" +polarity = "positive" +kind = "pytest" +target = "exact_hdf5" +nodeid = "tests/python/integration/io/m4_native_reopen_proof.py::test_hdf5_reopens_with_h5py_without_a_pops_reader" + +[[check]] +issue = "ADC-686" +requirement = "exact_hdf5" +polarity = "refusal" +kind = "pytest" +target = "exact_hdf5" +nodeid = "tests/python/integration/io/m4_native_reopen_proof.py::test_hdf5_authenticated_reader_rejects_native_dataset_tampering" + +[[check]] +issue = "ADC-686" +requirement = "exact_paraview" +polarity = "positive" +kind = "pytest" +target = "exact_paraview" +nodeid = "tests/python/integration/io/m4_native_reopen_proof.py::test_paraview_reopens_with_vtk_without_a_pops_reader" + +[[check]] +issue = "ADC-686" +requirement = "exact_paraview" +polarity = "refusal" +kind = "pytest" +target = "exact_paraview" +nodeid = "tests/python/unit/output/test_exact_writers.py::test_paraview_rejects_inconsistent_logical_field_family_levels" + +[[check]] +issue = "ADC-686" +requirement = "collective_hdf5" +polarity = "positive" +kind = "ctest" +target = "collective_hdf5@test_mpi_hdf5_collective" +test_regex = "^test_mpi_hdf5_collective_np2$" + +[[check]] +issue = "ADC-686" +requirement = "strict_checkpoint" +polarity = "positive" +kind = "pytest" +target = "strict_checkpoint" +nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count" + +[[check]] +issue = "ADC-686" +requirement = "strict_checkpoint" +polarity = "refusal" +kind = "pytest" +target = "strict_checkpoint" +nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_failed_child_restart_rolls_back_already_restored_layouts" + +[[check]] +issue = "ADC-686" +requirement = "diagnostics" +polarity = "positive" +kind = "ctest" +target = "diagnostics@test_program_context_contract" +test_regex = "^ProgramContextContract\\.AcceptedBalanceEvidenceIsCurrentAttemptExactAndFailClosed$" + +[[check]] +issue = "ADC-686" +requirement = "diagnostics" +polarity = "refusal" +kind = "pytest" +target = "diagnostics" +nodeid = "tests/python/unit/output/test_exact_writers.py::test_composite_integrals_refuses_non_cartesian_cell_measure" + +[[check]] +issue = "ADC-687" +requirement = "external_solver" +polarity = "positive" +kind = "pytest" +target = "external_solver" +nodeid = "tests/python/integration/native_loader/test_external_field_solver_runtime.py::test_external_field_pair_executes_and_reports_materialized_topology" + +[[check]] +issue = "ADC-687" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "ctest" +target = "tamper_capability_abi@test_native_loader_param_overflow" +test_regex = "^test_native_loader_param_overflow\\.Runs$" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_no_schur_header_leak.py::test_native_source_stage_headers_are_retired" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py new file mode 100644 index 000000000..55de744c4 --- /dev/null +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -0,0 +1,454 @@ +"""Source-only integrity checks for the executable M4 runtime/IO gate.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = ROOT / "tests/gates/m4_runtime_io.toml" +RUNNER = ROOT / "scripts/run_m4_gate.py" + + +def _load_runner(): + spec = importlib.util.spec_from_file_location("pops_run_m4_gate", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _mutated_manifest(tmp_path: Path, old: str, new: str) -> Path: + source = MANIFEST.read_text(encoding="utf-8") + assert old in source + path = tmp_path / "m4.toml" + path.write_text(source.replace(old, new, 1), encoding="utf-8") + return path + + +def test_m4_manifest_is_a_closed_exact_mandatory_matrix(): + data, errors = _load_runner().validate_manifest(MANIFEST) + + assert not errors, "M4 gate matrix is incomplete:\n " + "\n ".join(errors) + assert data["deferred"] == [] + assert len(data["check"]) >= 41 + assert data["issues"] == [ + "ADC-679", + "ADC-680", + "ADC-681", + "ADC-682", + "ADC-683", + "ADC-684", + "ADC-685", + "ADC-686", + "ADC-687", + ] + assert {row["issue"] for row in data["check"]} == set(data["issues"]) + + +def test_m4_gate_pins_every_external_component_family(): + data, errors = _load_runner().validate_manifest(MANIFEST) + assert not errors + + executable = { + ( + row["requirement"], + row["polarity"], + row.get("nodeid", row.get("test_regex")), + ) + for row in data["check"] + } + assert { + ( + "external_flux", + "positive", + "tests/python/integration/native_loader/" + "test_external_component_package.py::" + "test_source_component_executes_through_generic_native_loader_and_flux_consumer", + ), + ( + "external_boundary", + "positive", + r"^test_amr_native_loader\." + r"BoundaryPlanSessionsOwnFreshLaneQualifiedComponentStates$", + ), + ( + "external_tagger", + "positive", + r"^test_amr_native_loader\." + r"PreparedAmrProvidersExecuteExactTablesAndProvenance$", + ), + ( + "external_transfer", + "positive", + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_two_native_layouts_execute_sliced_programs_and_exact_transfer", + ), + ( + "external_solver", + "positive", + "tests/python/integration/native_loader/" + "test_external_field_solver_runtime.py::" + "test_external_field_pair_executes_and_reports_materialized_topology", + ), + ( + "external_writer", + "positive", + "tests/python/integration/native_loader/" + "test_external_component_package.py::" + "test_qualified_writer_runs_through_uniform_and_amr_runtime_transactions", + ), + } <= executable + + +def test_m4_gate_pins_runtime_instance_multi_layout_and_strict_checkpoint(): + data, errors = _load_runner().validate_manifest(MANIFEST) + assert not errors + checks = data["check"] + + assert { + "issue": "ADC-684", + "requirement": "runtime_instance", + "polarity": "positive", + "kind": "pytest", + "target": "runtime_instance", + "nodeid": ( + "tests/python/integration/runtime/test_shared_interface_runtime.py::" + "test_runtime_instance_executes_one_two_sided_shared_flux" + ), + } in checks + assert { + "issue": "ADC-684", + "requirement": "external_transfer", + "polarity": "positive", + "kind": "pytest", + "target": "external_transfer", + "nodeid": ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_two_native_layouts_execute_sliced_programs_and_exact_transfer" + ), + } in checks + assert { + "issue": "ADC-685", + "requirement": "external_writer", + "polarity": "positive", + "kind": "pytest", + "target": "external_writer", + "nodeid": ( + "tests/python/integration/native_loader/" + "test_external_component_package.py::" + "test_qualified_writer_runs_through_uniform_and_amr_runtime_transactions" + ), + } in checks + assert { + "issue": "ADC-686", + "requirement": "strict_checkpoint", + "polarity": "positive", + "kind": "pytest", + "target": "strict_checkpoint", + "nodeid": ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count" + ), + } in checks + assert { + "issue": "ADC-686", + "requirement": "strict_checkpoint", + "polarity": "refusal", + "kind": "pytest", + "target": "strict_checkpoint", + "nodeid": ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_failed_child_restart_rolls_back_already_restored_layouts" + ), + } in checks + + +def test_m4_gate_pins_capability_tamper_and_native_abi_refusals(): + data, errors = _load_runner().validate_manifest(MANIFEST) + assert not errors + + refusals = { + row.get("nodeid", row.get("test_regex")) + for row in data["check"] + if row["requirement"] == "tamper_capability_abi" + and row["polarity"] == "refusal" + } + assert { + ( + "tests/python/unit/codegen/test_component_manifest_v2.py::" + "test_target_capability_refusal_contains_requested_and_supported_evidence" + ), + ( + "tests/python/unit/codegen/test_component_packages.py::" + "test_fixed_binary_cannot_claim_template_genericity" + ), + ( + "tests/python/unit/runtime/test_platform_manifest.py::" + "test_aot_component_rejects_openmpi_mpich_abi_mix_even_with_same_headers_and_standard" + ), + r"^test_native_loader_param_overflow\.Runs$", + } <= refusals + + +def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): + data, errors = _load_runner().validate_manifest(MANIFEST) + assert not errors + checks = data["check"] + + native_reopen = { + row["requirement"]: row["nodeid"] + for row in checks + if row["requirement"] in {"exact_npz", "exact_hdf5", "exact_paraview"} + and row["polarity"] == "positive" + } + assert native_reopen == { + "exact_npz": ( + "tests/python/integration/io/m4_native_reopen_proof.py::" + "test_npz_reopens_with_numpy_without_a_pops_reader" + ), + "exact_hdf5": ( + "tests/python/integration/io/m4_native_reopen_proof.py::" + "test_hdf5_reopens_with_h5py_without_a_pops_reader" + ), + "exact_paraview": ( + "tests/python/integration/io/m4_native_reopen_proof.py::" + "test_paraview_reopens_with_vtk_without_a_pops_reader" + ), + } + source = ( + ROOT / "tests/python/integration/io/m4_native_reopen_proof.py" + ).read_text(encoding="utf-8") + assert "pytest.importorskip" not in source + assert "import h5py" in source + assert "from vtkmodules.vtkIOXML import vtkXMLUnstructuredGridReader" in source + assert { + "issue": "ADC-686", + "requirement": "collective_hdf5", + "polarity": "positive", + "kind": "ctest", + "target": "collective_hdf5@test_mpi_hdf5_collective", + "test_regex": "^test_mpi_hdf5_collective_np2$", + } in checks + + +def test_m4_gate_pins_schur_retirement_and_ci_check_only_command(): + data, errors = _load_runner().validate_manifest(MANIFEST) + assert not errors + assert { + "issue": "ADC-687", + "requirement": "legacy_stepper_retirement", + "polarity": "positive", + "kind": "pytest", + "target": "legacy_stepper_retirement", + "nodeid": ( + "tests/python/architecture/test_no_schur_header_leak.py::" + "test_native_source_stage_headers_are_retired" + ), + } in data["check"] + + workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + job = workflow.split("\n gate-python-architecture:\n", 1)[1] + job = job.split("\n gate-python-build:\n", 1)[0] + command = "run: python3 scripts/run_m4_gate.py --check-only" + assert [line.strip() for line in job.splitlines()].count(command) == 1 + + +def test_m4_gate_rejects_fake_nodeid_before_execution(tmp_path): + manifest = _mutated_manifest( + tmp_path, + ( + "tests/python/unit/codegen/test_component_manifest_v2.py::" + "test_native_parser_normalizer_matches_python_canonical_bytes" + ), + ( + "tests/python/unit/codegen/test_component_manifest_v2.py::" + "test_definitely_missing_m4_proof" + ), + ) + + _, errors = _load_runner().validate_manifest(manifest) + + assert any("references missing test function" in error for error in errors) + + +def test_m4_gate_rejects_wildcard_ctest_selector_before_build(tmp_path): + manifest = _mutated_manifest( + tmp_path, + 'test_regex = "^test_mpi_hdf5_collective_np2$"', + 'test_regex = "^test_mpi_hdf5_collective_.*$"', + ) + + _, errors = _load_runner().validate_manifest(manifest) + + assert any( + "is not one exact source-registered case for target " + "'test_mpi_hdf5_collective'" in error + for error in errors + ) + + +def test_m4_gate_rejects_requirement_attributed_to_the_wrong_issue(tmp_path): + manifest = _mutated_manifest( + tmp_path, + ( + 'issue = "ADC-679"\n' + 'requirement = "component_manifest"\n' + 'polarity = "positive"' + ), + ( + 'issue = "ADC-680"\n' + 'requirement = "component_manifest"\n' + 'polarity = "positive"' + ), + ) + + _, errors = _load_runner().validate_manifest(manifest) + + assert any( + "requirement 'component_manifest' cannot be attributed to 'ADC-680'" in error + for error in errors + ) + + +def test_m4_gate_rejects_importorskip_and_mock_proofs(tmp_path): + optional_manifest = _mutated_manifest( + tmp_path, + ( + "tests/python/integration/io/m4_native_reopen_proof.py::" + "test_hdf5_reopens_with_h5py_without_a_pops_reader" + ), + ( + "tests/python/unit/output/test_exact_writers.py::" + "test_hdf5_is_reopened_with_native_reader_and_exact_selection" + ), + ) + _, optional_errors = _load_runner().validate_manifest(optional_manifest) + assert any( + "is not an unconditional real proof" in error + and "pytest.importorskip" in error + for error in optional_errors + ) + + mock_manifest = _mutated_manifest( + tmp_path, + ( + "tests/python/unit/runtime/test_consumer_transactions.py::" + "test_graph_and_plan_are_semantic_and_insertion_order_independent" + ), + ( + "tests/python/architecture/test_m3_amr_multilayout_gate.py::" + "test_m3_mpi_python_proof_is_exact_and_manifest_owned" + ), + ) + _, mock_errors = _load_runner().validate_manifest(mock_manifest) + assert any( + "is not an unconditional real proof" in error + and "fixture:monkeypatch" in error + for error in mock_errors + ) + + +def test_m4_gate_rejects_every_deferred_requirement(tmp_path): + manifest = _mutated_manifest( + tmp_path, + "deferred = []", + ( + "[[deferred]]\n" + 'issue = "ADC-687"\n' + 'requirement = "legacy_stepper_retirement"\n' + 'reason = "The mandatory Schur retirement proof is deliberately deferred."\n' + "evidence_paths = " + '["tests/python/architecture/test_no_schur_header_leak.py"]' + ), + ) + + data, audit_errors = _load_runner().audit_manifest(manifest) + assert not audit_errors + assert len(data["deferred"]) == 1 + + _, errors = _load_runner().validate_manifest(manifest) + assert any("remains deferred" in error for error in errors) + + +def test_m4_required_pytest_execution_rejects_junit_skips(monkeypatch): + runner = _load_runner() + skipped_xml = ( + '' + '' + '' + '' + "" + ) + + def successful_pytest_with_a_skip(command, *, cwd, env, check): + assert cwd == ROOT + assert env["POPS_REQUIRE_MPI_TESTS"] == "1" + assert env["POPS_REQUIRE_NATIVE_TESTS"] == "1" + assert check is False + assert "xfail_strict=true" in command + report = Path(command[command.index("--junitxml") + 1]) + report.write_text(skipped_xml, encoding="utf-8") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(runner.subprocess, "run", successful_pytest_with_a_skip) + with pytest.raises(RuntimeError, match="reported 1 skipped/xfail proof"): + runner._run_required_pytest( + [ + "tests/python/integration/io/m4_native_reopen_proof.py::" + "test_npz_reopens_with_numpy_without_a_pops_reader" + ] + ) + + +def test_m4_required_ctest_execution_rejects_junit_skips(tmp_path, monkeypatch): + runner = _load_runner() + skipped_xml = ( + '' + '' + '' + '' + "" + ) + calls = 0 + + def ctest_with_a_skip(command, **kwargs): + nonlocal calls + calls += 1 + assert kwargs["cwd"] == ROOT + if "-N" in command: + assert kwargs["check"] is True + assert kwargs["capture_output"] is True + return SimpleNamespace( + returncode=0, + stdout="Test #1: ComponentInterfaces.Proof\nTotal Tests: 1\n", + ) + assert kwargs["check"] is False + report = Path(command[command.index("--output-junit") + 1]) + report.write_text(skipped_xml, encoding="utf-8") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(runner.subprocess, "run", ctest_with_a_skip) + with pytest.raises(RuntimeError, match="reported 1 skipped proof"): + runner._run_ctest( + tmp_path / "build", + "test_component_interfaces", + r"^ComponentInterfaces\.Proof$", + ) + assert calls == 2 + + +def test_m4_check_only_never_consults_launcher_or_build(monkeypatch): + runner = _load_runner() + + def forbidden_call(*_args, **_kwargs): + raise AssertionError("--check-only attempted to launch an executable") + + monkeypatch.setattr(runner.shutil, "which", forbidden_call) + monkeypatch.setattr(runner.subprocess, "run", forbidden_call) + + assert runner.main(["--check-only"]) == 0 From 4b43d53ab08ca97e188b5b3e0bbd1f273522f44a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:09:25 +0200 Subject: [PATCH 04/62] ci(docs): register the M4 conformance gate --- .github/workflows/ci.yml | 3 ++ docs/design/m4-conformance-gate.md | 53 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 docs/design/m4-conformance-gate.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32c0131a5..3a1a0ddfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -715,6 +715,9 @@ jobs: - name: M3 AMR and multi-layout gate manifest run: python3 scripts/run_m3_gate.py --check-only + - name: M4 native runtime and scientific I/O gate manifest + run: python3 scripts/run_m4_gate.py --check-only + - name: Generated component catalog env: PYTHONPATH: ${{ github.workspace }}/python diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md new file mode 100644 index 000000000..fb5d5999a --- /dev/null +++ b/docs/design/m4-conformance-gate.md @@ -0,0 +1,53 @@ +# M4 native runtime and scientific I/O conformance gate + +`python scripts/run_m4_gate.py` is the reviewed executable acceptance matrix for +ADC-679 through ADC-687. It pins exact source-registered proofs instead of broad +test directories or nearby suites. The architecture CI runs `--check-only`, so +renaming, deleting, making optional, or removing a selected proof from +`tests/test_manifest.toml` fails before a build is launched. + +The matrix covers: + +- canonical component manifests, generated registries, and external AOT + packages; +- external flux, boundary, tagger, transfer, solver, and writer components; +- native interface, provider-pack, platform, capability, and ABI refusals; +- one `RuntimeInstance` across Uniform, AMR, and multiple mapped layouts; +- transactional `ConsumerGraph` publication and rollback; +- direct format-native reopen of NPZ with NumPy, HDF5 with h5py, and ParaView + VTU with VTK; +- real two-rank collective HDF5 and its exact CTest selector; +- strict multi-layout checkpoint/restart, including atomic restore refusal; +- exact diagnostics and the source-level retirement fence for the old Schur + source steppers. + +`deferred = []` is normative for closure. Every issue needs positive and refusal +coverage and at least one native positive proof. Each scientific family has its +own required polarity. The validator rejects duplicate or wildcard selectors, +missing manifest ownership, pytest skip/xfail and optional imports, mock-based +proofs, disabled CTests, and any explicit deferred gap. + +Use: + +```bash +python scripts/run_m4_gate.py --audit-only +python scripts/run_m4_gate.py --check-only +python scripts/run_m4_gate.py --python-only +python scripts/run_m4_gate.py --build-dir build-mpi +``` + +`--audit-only` validates the ledger while deliberately making no closure claim, +even when there are no deferred rows. `--check-only` additionally requires the +ledger to be closed, but still launches no test, compiler, MPI process, or +native reader. `--python-only` executes every selected Python proof with native +requirements forced on and omits CTest. The last command is the full gate and +requires an MPI-enabled build containing every selected CTest, plus real NumPy, +h5py, and VTK installations. Both pytest and CTest must produce JUnit reports +with zero skipped or xfailed proofs. + +The native-reader tests intentionally use no PoPS reader to interpret the +written payload. PoPS is used only to produce and authenticate the output; +NumPy, h5py, and VTK independently prove that the published formats are usable. +Their proof module is deliberately named `m4_native_reopen_proof.py`, so normal +`test_*.py` shard discovery does not silently turn VTK into a dependency of +every Python shard. The exact nodeids remain mandatory in the explicit M4 gate. From 18191953dca60d22354f82448d3b71ca3a156fe5 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:18:38 +0200 Subject: [PATCH 05/62] test(output): fix mandatory native reopen proofs --- tests/python/integration/io/m4_native_reopen_proof.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/python/integration/io/m4_native_reopen_proof.py b/tests/python/integration/io/m4_native_reopen_proof.py index b1f0ecc50..7f8b0173e 100644 --- a/tests/python/integration/io/m4_native_reopen_proof.py +++ b/tests/python/integration/io/m4_native_reopen_proof.py @@ -84,7 +84,12 @@ def _publish(writer, target): def _field_dataset(manifest: dict, request: OutputRequest) -> str: key = request.selection[0].identity.token - return manifest["datasets"]["fields"][key] + dataset = manifest["datasets"]["fields"][key] + if isinstance(dataset, str): + return dataset + pieces = dataset["pieces"] + assert len(pieces) == 1 + return pieces[0]["name"] def test_npz_reopens_with_numpy_without_a_pops_reader(tmp_path): @@ -122,7 +127,7 @@ def test_hdf5_authenticated_reader_rejects_native_dataset_tampering(tmp_path): dataset = _field_dataset(manifest, request) output[dataset][0, 0] = np.float64(99.0) - with pytest.raises(ValueError, match="content verification"): + with pytest.raises(ValueError, match="parallel piece failed verification"): read_hdf5(path) @@ -130,7 +135,7 @@ def test_paraview_reopens_with_vtk_without_a_pops_reader(tmp_path): from vtkmodules.vtkIOXML import vtkXMLUnstructuredGridReader path, _request, expected = _publish( - ParaViewWriter(), tmp_path / "native.vtu" + ParaViewWriter(collection=False), tmp_path / "native.vtu" ) reader = vtkXMLUnstructuredGridReader() From 7ae0a12a4a9c86a7cb204e9c54709663079c43c1 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:27:43 +0200 Subject: [PATCH 06/62] test(gate): keep incomplete M4 evidence audited open --- .github/workflows/ci.yml | 2 +- scripts/run_m4_gate.py | 78 ++++++-- tests/gates/m4_runtime_io.toml | 139 +++++++++----- .../architecture/test_m4_runtime_io_gate.py | 169 ++++++++++++------ 4 files changed, 274 insertions(+), 114 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a1a0ddfb..3cbeed497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -716,7 +716,7 @@ jobs: run: python3 scripts/run_m3_gate.py --check-only - name: M4 native runtime and scientific I/O gate manifest - run: python3 scripts/run_m4_gate.py --check-only + run: python3 scripts/run_m4_gate.py --audit-only - name: Generated component catalog env: diff --git a/scripts/run_m4_gate.py b/scripts/run_m4_gate.py index 75ca9480e..433cf4b82 100644 --- a/scripts/run_m4_gate.py +++ b/scripts/run_m4_gate.py @@ -46,6 +46,7 @@ "diagnostics": {"positive", "refusal"}, "tamper_capability_abi": {"refusal"}, "legacy_stepper_retirement": {"positive"}, + "gate_execution": {"positive"}, } REQUIREMENT_ISSUES = { "component_manifest": {"ADC-679"}, @@ -70,6 +71,7 @@ "diagnostics": {"ADC-686"}, "external_solver": {"ADC-687"}, "legacy_stepper_retirement": {"ADC-687"}, + "gate_execution": {"ADC-687"}, "tamper_capability_abi": {"ADC-679", "ADC-680", "ADC-683", "ADC-687"}, } NATIVE_PYTEST_PREFIXES = ( @@ -333,21 +335,33 @@ def _validate_python_nodeid( return relative -def _validate_deferred(data: dict, errors: list[str]) -> set[str]: +def _validate_deferred( + data: dict, errors: list[str] +) -> set[tuple[str, str, str]]: rows = data.get("deferred") if not isinstance(rows, list): errors.append("deferred must be an array of explicit gap tables") return set() - requirements: set[str] = set() + gaps: set[tuple[str, str, str]] = set() identities = Counter() for index, row in enumerate(rows, 1): where = "deferred[%d]" % index - expected = {"issue", "requirement", "reason", "evidence_paths"} + expected = { + "issue", + "requirement", + "polarity", + "reason", + "evidence_paths", + } if not isinstance(row, dict) or set(row) != expected: - errors.append("%s must contain issue/requirement/reason/evidence_paths" % where) + errors.append( + "%s must contain issue/requirement/polarity/reason/evidence_paths" + % where + ) continue issue = row.get("issue") requirement = row.get("requirement") + polarity = row.get("polarity") reason = row.get("reason") evidence_paths = row.get("evidence_paths") if issue not in EXPECTED_ISSUES: @@ -359,6 +373,16 @@ def _validate_deferred(data: dict, errors: list[str]) -> set[str]: "%s requirement %r cannot be deferred under %r" % (where, requirement, issue) ) + if polarity not in {"positive", "refusal"}: + errors.append("%s polarity must be positive or refusal" % where) + elif ( + requirement in REQUIRED_POLARITIES + and polarity not in REQUIRED_POLARITIES[requirement] + ): + errors.append( + "%s requirement %r has no %r polarity" + % (where, requirement, polarity) + ) if not isinstance(reason, str) or len(reason.strip()) < 20: errors.append("%s requires a precise non-empty reason" % where) if not isinstance(evidence_paths, list) or not evidence_paths: @@ -371,12 +395,13 @@ def _validate_deferred(data: dict, errors: list[str]) -> set[str]: errors.append( "%s gap evidence path no longer exists: %s" % (where, relative) ) - identities[(issue, requirement)] += 1 - requirements.add(str(requirement)) + identity = (str(issue), str(requirement), str(polarity)) + identities[identity] += 1 + gaps.add(identity) duplicates = sorted(identity for identity, count in identities.items() if count > 1) if duplicates: errors.append("duplicate deferred gaps: %s" % duplicates) - return requirements + return gaps def audit_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: @@ -396,7 +421,7 @@ def audit_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: if data.get("issues") != list(EXPECTED_ISSUES): errors.append("issues must list ADC-679..ADC-687 exactly once") - deferred_requirements = _validate_deferred(data, errors) + deferred_gaps = _validate_deferred(data, errors) checks = data.get("check") if not isinstance(checks, list) or not checks: errors.append("manifest must contain [[check]] rows") @@ -529,16 +554,36 @@ def audit_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: errors.append("duplicate executable checks: %s" % duplicates) for issue in EXPECTED_ISSUES: missing = {"positive", "refusal"} - issue_coverage[issue] - if missing: - errors.append("%s lacks %s coverage" % (issue, "/".join(sorted(missing)))) + unresolved = { + polarity + for polarity in missing + if not any( + deferred_issue == issue and deferred_polarity == polarity + for deferred_issue, _requirement, deferred_polarity in deferred_gaps + ) + } + if unresolved: + errors.append( + "%s lacks %s coverage" + % (issue, "/".join(sorted(unresolved))) + ) if issue not in native_positive_issues: errors.append("%s lacks a mandatory native positive proof" % issue) for requirement, required in sorted(REQUIRED_POLARITIES.items()): missing = required - requirement_coverage[requirement] - if missing and requirement not in deferred_requirements: + unresolved = { + polarity + for polarity in missing + if not any( + deferred_requirement == requirement + and deferred_polarity == polarity + for _issue, deferred_requirement, deferred_polarity in deferred_gaps + ) + } + if unresolved: errors.append( "%s lacks %s coverage" - % (requirement, "/".join(sorted(missing))) + % (requirement, "/".join(sorted(unresolved))) ) return data, errors @@ -550,8 +595,13 @@ def validate_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: return data, errors for row in data["deferred"]: errors.append( - "%s/%s remains deferred: %s" - % (row["issue"], row["requirement"], row["reason"]) + "%s/%s/%s remains deferred: %s" + % ( + row["issue"], + row["requirement"], + row["polarity"], + row["reason"], + ) ) return data, errors diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index ad18818e3..dac8ad254 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -11,12 +11,103 @@ issues = [ "ADC-686", "ADC-687", ] -deferred = [] + +[[deferred]] +issue = "ADC-687" +requirement = "gate_execution" +polarity = "positive" +reason = "CI only audits the source ledger; no required lane installs VTK and executes every selected pytest and CTest proof with zero skips." +evidence_paths = [ + ".github/workflows/ci.yml", + "environment.yml", + "scripts/run_m4_gate.py", +] + +[[deferred]] +issue = "ADC-683" +requirement = "tamper_capability_abi" +polarity = "refusal" +reason = "The matrix lacks a real runtime or backend launch whose unknown capability is refused before any kernel executes; schema-only target validation is insufficient." +evidence_paths = [ + "tests/python/unit/codegen/test_component_manifest_v2.py", + "tests/cpp/unit/runtime/test_platform_manifest.cpp", +] + +[[deferred]] +issue = "ADC-687" +requirement = "tamper_capability_abi" +polarity = "refusal" +reason = "A genuine wrong-ABI shared object refusal exists, but the executable M4 matrix does not yet select and run that DSO proof." +evidence_paths = [ + "tests/cpp/integration/native_loader/test_amr_native_loader.cpp", + "tests/gates/m4_runtime_io.toml", +] + +[[deferred]] +issue = "ADC-684" +requirement = "runtime_instance" +polarity = "positive" +reason = "Uniform, AMR, and multi-layout executions are covered separately and only partially inspect reports; one complete public-contract and report-parity proof is still absent." +evidence_paths = [ + "tests/python/integration/native_loader/test_external_component_package.py", + "tests/python/integration/runtime/test_multi_layout_runtime.py", + "tests/python/integration/runtime/test_shared_interface_runtime.py", +] + +[[deferred]] +issue = "ADC-684" +requirement = "runtime_instance" +polarity = "refusal" +reason = "Composite step rollback is only exercised through an injected FailFirstStep wrapper; a failure from a real prepared runtime component is still required." +evidence_paths = [ + "tests/python/integration/runtime/test_multi_layout_runtime.py", +] + +[[deferred]] +issue = "ADC-685" +requirement = "consumer_graph" +polarity = "refusal" +reason = "The available transaction refusal uses handwritten publisher and prepared-publication fakes; M4 still needs the same refusal through a real writer or consumer." +evidence_paths = [ + "tests/python/unit/runtime/test_consumer_transactions.py", + "tests/python/integration/native_loader/test_external_component_package.py", +] + +[[deferred]] +issue = "ADC-686" +requirement = "strict_checkpoint" +polarity = "refusal" +reason = "Atomic restart rollback is only exercised through an injected FailFirstRestart wrapper; a real checkpoint provider failure must prove restoration rollback." +evidence_paths = [ + "tests/python/integration/runtime/test_multi_layout_runtime.py", +] + +[[deferred]] +issue = "ADC-686" +requirement = "exact_paraview" +polarity = "positive" +reason = "Serial VTU reopens with VTK, but native VTK reopening of the mandatory MPI PVD to PVTU to rank-VTU hierarchy is still optional and therefore unproved." +evidence_paths = [ + "tests/python/integration/io/m4_native_reopen_proof.py", + "tests/python/integration/mpi/test_scientific_output_mpi.py", +] + +[[deferred]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +reason = "The selected Schur-header fence does not by itself prove Program-only execution or the absence of every legacy stepper, concrete central dispatch, and silent fallback." +evidence_paths = [ + "tests/python/architecture/test_no_schur_header_leak.py", + "tests/python/architecture/test_program_only_temporal_facades.py", + "tests/python/architecture/test_no_legacy_runtime_routes.py", + "tests/python/architecture/test_component_interface_dispatch.py", +] # This is an exact evidence ledger, not a list of nearby suites. Every row names -# one source-registered proof. The runner rejects mocks, optional imports, -# skip/xfail, non-exact CTest selectors, duplicate proofs, and missing manifest -# ownership before it launches anything. +# one source-registered proof. The runner rejects mock fixtures/imports, +# optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, +# and missing manifest ownership before it launches anything. [[check]] issue = "ADC-679" @@ -42,14 +133,6 @@ kind = "pytest" target = "component_manifest" nodeid = "tests/python/unit/codegen/test_component_manifest_v2.py::test_interface_bindings_are_exact_closed_and_entry_point_checked" -[[check]] -issue = "ADC-679" -requirement = "tamper_capability_abi" -polarity = "refusal" -kind = "pytest" -target = "tamper_capability_abi" -nodeid = "tests/python/unit/codegen/test_component_manifest_v2.py::test_target_capability_refusal_contains_requested_and_supported_evidence" - [[check]] issue = "ADC-679" requirement = "generated_registry" @@ -162,14 +245,6 @@ kind = "ctest" target = "platform_execution@test_platform_manifest" test_regex = "^PlatformManifest\\.FieldAndCommunicatorMismatchesRefuseBeforeKernel$" -[[check]] -issue = "ADC-683" -requirement = "tamper_capability_abi" -polarity = "refusal" -kind = "pytest" -target = "tamper_capability_abi" -nodeid = "tests/python/unit/runtime/test_platform_manifest.py::test_aot_component_rejects_openmpi_mpich_abi_mix_even_with_same_headers_and_standard" - [[check]] issue = "ADC-684" requirement = "runtime_instance" @@ -178,14 +253,6 @@ kind = "pytest" target = "runtime_instance" nodeid = "tests/python/integration/runtime/test_shared_interface_runtime.py::test_runtime_instance_executes_one_two_sided_shared_flux" -[[check]] -issue = "ADC-684" -requirement = "runtime_instance" -polarity = "refusal" -kind = "pytest" -target = "runtime_instance" -nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_mid_step_child_failure_preserves_root_error_and_rolls_back_composite" - [[check]] issue = "ADC-684" requirement = "external_transfer" @@ -210,14 +277,6 @@ kind = "pytest" target = "consumer_graph" nodeid = "tests/python/unit/runtime/test_consumer_transactions.py::test_graph_and_plan_are_semantic_and_insertion_order_independent" -[[check]] -issue = "ADC-685" -requirement = "consumer_graph" -polarity = "refusal" -kind = "pytest" -target = "consumer_graph" -nodeid = "tests/python/unit/runtime/test_consumer_transactions.py::test_rejected_attempt_discards_temporaries_without_publication_or_cursor_advance" - [[check]] issue = "ADC-685" requirement = "accepted_publication" @@ -298,14 +357,6 @@ kind = "pytest" target = "strict_checkpoint" nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count" -[[check]] -issue = "ADC-686" -requirement = "strict_checkpoint" -polarity = "refusal" -kind = "pytest" -target = "strict_checkpoint" -nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_failed_child_restart_rolls_back_already_restored_layouts" - [[check]] issue = "ADC-686" requirement = "diagnostics" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 55de744c4..2473aa793 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -4,6 +4,8 @@ import importlib.util from pathlib import Path +import subprocess +import sys from types import SimpleNamespace import pytest @@ -30,12 +32,13 @@ def _mutated_manifest(tmp_path: Path, old: str, new: str) -> Path: return path -def test_m4_manifest_is_a_closed_exact_mandatory_matrix(): - data, errors = _load_runner().validate_manifest(MANIFEST) +def test_m4_manifest_is_an_audited_open_exact_matrix(): + runner = _load_runner() + data, errors = runner.audit_manifest(MANIFEST) - assert not errors, "M4 gate matrix is incomplete:\n " + "\n ".join(errors) - assert data["deferred"] == [] - assert len(data["check"]) >= 41 + assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) + assert len(data["deferred"]) == 9 + assert len(data["check"]) >= 36 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -48,10 +51,53 @@ def test_m4_manifest_is_a_closed_exact_mandatory_matrix(): "ADC-687", ] assert {row["issue"] for row in data["check"]} == set(data["issues"]) + assert { + (row["issue"], row["requirement"], row["polarity"]) + for row in data["deferred"] + } == { + ("ADC-683", "tamper_capability_abi", "refusal"), + ("ADC-684", "runtime_instance", "positive"), + ("ADC-684", "runtime_instance", "refusal"), + ("ADC-685", "consumer_graph", "refusal"), + ("ADC-686", "strict_checkpoint", "refusal"), + ("ADC-686", "exact_paraview", "positive"), + ("ADC-687", "gate_execution", "positive"), + ("ADC-687", "tamper_capability_abi", "refusal"), + ("ADC-687", "legacy_stepper_retirement", "positive"), + } + + _, closure_errors = runner.validate_manifest(MANIFEST) + assert len(closure_errors) == len(data["deferred"]) + assert all("remains deferred" in error for error in closure_errors) + + +def test_m4_cli_reports_open_and_check_only_refuses_closure(): + audit = subprocess.run( + [sys.executable, str(RUNNER), "--audit-only"], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + assert audit.returncode == 0 + assert "M4 gate source matrix: AUDITED OPEN" in audit.stdout + + closure = subprocess.run( + [sys.executable, str(RUNNER), "--check-only"], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + assert closure.returncode == 2 + assert "M4 gate is incomplete or invalid" in closure.stdout + assert "remains deferred" in closure.stdout def test_m4_gate_pins_every_external_component_family(): - data, errors = _load_runner().validate_manifest(MANIFEST) + data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors executable = { @@ -105,8 +151,8 @@ def test_m4_gate_pins_every_external_component_family(): } <= executable -def test_m4_gate_pins_runtime_instance_multi_layout_and_strict_checkpoint(): - data, errors = _load_runner().validate_manifest(MANIFEST) +def test_m4_gate_pins_real_runtime_instance_and_positive_checkpoint_proofs(): + data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors checks = data["check"] @@ -155,21 +201,22 @@ def test_m4_gate_pins_runtime_instance_multi_layout_and_strict_checkpoint(): "test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count" ), } in checks - assert { - "issue": "ADC-686", - "requirement": "strict_checkpoint", - "polarity": "refusal", - "kind": "pytest", - "target": "strict_checkpoint", - "nodeid": ( - "tests/python/integration/runtime/test_multi_layout_runtime.py::" - "test_failed_child_restart_rolls_back_already_restored_layouts" - ), - } in checks - - -def test_m4_gate_pins_capability_tamper_and_native_abi_refusals(): - data, errors = _load_runner().validate_manifest(MANIFEST) + selected = { + row.get("nodeid", row.get("test_regex")) + for row in checks + } + assert ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_mid_step_child_failure_preserves_root_error_and_rolls_back_composite" + ) not in selected + assert ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_failed_child_restart_rolls_back_already_restored_layouts" + ) not in selected + + +def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): + data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors refusals = { @@ -179,24 +226,32 @@ def test_m4_gate_pins_capability_tamper_and_native_abi_refusals(): and row["polarity"] == "refusal" } assert { - ( - "tests/python/unit/codegen/test_component_manifest_v2.py::" - "test_target_capability_refusal_contains_requested_and_supported_evidence" - ), ( "tests/python/unit/codegen/test_component_packages.py::" "test_fixed_binary_cannot_claim_template_genericity" ), - ( - "tests/python/unit/runtime/test_platform_manifest.py::" - "test_aot_component_rejects_openmpi_mpich_abi_mix_even_with_same_headers_and_standard" - ), r"^test_native_loader_param_overflow\.Runs$", } <= refusals + assert ( + "tests/python/unit/codegen/test_component_manifest_v2.py::" + "test_target_capability_refusal_contains_requested_and_supported_evidence" + ) not in refusals + assert ( + "tests/python/unit/runtime/test_platform_manifest.py::" + "test_aot_component_rejects_openmpi_mpich_abi_mix_even_with_same_headers_and_standard" + ) not in refusals + assert { + (row["issue"], row["requirement"], row["polarity"]) + for row in data["deferred"] + if row["requirement"] == "tamper_capability_abi" + } == { + ("ADC-683", "tamper_capability_abi", "refusal"), + ("ADC-687", "tamper_capability_abi", "refusal"), + } def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): - data, errors = _load_runner().validate_manifest(MANIFEST) + data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors checks = data["check"] @@ -236,8 +291,8 @@ def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): } in checks -def test_m4_gate_pins_schur_retirement_and_ci_check_only_command(): - data, errors = _load_runner().validate_manifest(MANIFEST) +def test_m4_gate_keeps_schur_evidence_but_ci_only_claims_an_open_audit(): + data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors assert { "issue": "ADC-687", @@ -254,8 +309,15 @@ def test_m4_gate_pins_schur_retirement_and_ci_check_only_command(): workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") job = workflow.split("\n gate-python-architecture:\n", 1)[1] job = job.split("\n gate-python-build:\n", 1)[0] - command = "run: python3 scripts/run_m4_gate.py --check-only" + command = "run: python3 scripts/run_m4_gate.py --audit-only" assert [line.strip() for line in job.splitlines()].count(command) == 1 + assert "run: python3 scripts/run_m4_gate.py --check-only" not in job + + documentation = ( + ROOT / "docs/design/m4-conformance-gate.md" + ).read_text(encoding="utf-8") + assert "current status is **AUDITED OPEN**" in documentation + assert "four serial proofs" in documentation def test_m4_gate_rejects_fake_nodeid_before_execution(tmp_path): @@ -353,26 +415,23 @@ def test_m4_gate_rejects_importorskip_and_mock_proofs(tmp_path): ) -def test_m4_gate_rejects_every_deferred_requirement(tmp_path): - manifest = _mutated_manifest( - tmp_path, - "deferred = []", - ( - "[[deferred]]\n" - 'issue = "ADC-687"\n' - 'requirement = "legacy_stepper_retirement"\n' - 'reason = "The mandatory Schur retirement proof is deliberately deferred."\n' - "evidence_paths = " - '["tests/python/architecture/test_no_schur_header_leak.py"]' - ), - ) - - data, audit_errors = _load_runner().audit_manifest(manifest) +def test_m4_gate_rejects_every_explicit_deferred_gap(): + runner = _load_runner() + data, audit_errors = runner.audit_manifest(MANIFEST) assert not audit_errors - assert len(data["deferred"]) == 1 + assert data["deferred"] - _, errors = _load_runner().validate_manifest(manifest) - assert any("remains deferred" in error for error in errors) + _, errors = runner.validate_manifest(MANIFEST) + expected = { + "%s/%s/%s" % (row["issue"], row["requirement"], row["polarity"]) + for row in data["deferred"] + } + observed = { + error.split(" remains deferred:", 1)[0] + for error in errors + if " remains deferred:" in error + } + assert observed == expected def test_m4_required_pytest_execution_rejects_junit_skips(monkeypatch): @@ -442,7 +501,7 @@ def ctest_with_a_skip(command, **kwargs): assert calls == 2 -def test_m4_check_only_never_consults_launcher_or_build(monkeypatch): +def test_m4_check_only_refuses_open_ledger_before_launcher_or_build(monkeypatch): runner = _load_runner() def forbidden_call(*_args, **_kwargs): @@ -451,4 +510,4 @@ def forbidden_call(*_args, **_kwargs): monkeypatch.setattr(runner.shutil, "which", forbidden_call) monkeypatch.setattr(runner.subprocess, "run", forbidden_call) - assert runner.main(["--check-only"]) == 0 + assert runner.main(["--check-only"]) == 2 From b15515f9c4ac443f3be2261f39491a44598bf9fb Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:27:47 +0200 Subject: [PATCH 07/62] docs(gate): record audited-open M4 limits --- docs/design/m4-conformance-gate.md | 129 +++++++++++++++++++---------- 1 file changed, 84 insertions(+), 45 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index fb5d5999a..8d38274c1 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -1,53 +1,92 @@ # M4 native runtime and scientific I/O conformance gate -`python scripts/run_m4_gate.py` is the reviewed executable acceptance matrix for -ADC-679 through ADC-687. It pins exact source-registered proofs instead of broad -test directories or nearby suites. The architecture CI runs `--check-only`, so -renaming, deleting, making optional, or removing a selected proof from -`tests/test_manifest.toml` fails before a build is launched. - -The matrix covers: - -- canonical component manifests, generated registries, and external AOT - packages; -- external flux, boundary, tagger, transfer, solver, and writer components; -- native interface, provider-pack, platform, capability, and ABI refusals; -- one `RuntimeInstance` across Uniform, AMR, and multiple mapped layouts; -- transactional `ConsumerGraph` publication and rollback; -- direct format-native reopen of NPZ with NumPy, HDF5 with h5py, and ParaView - VTU with VTK; -- real two-rank collective HDF5 and its exact CTest selector; -- strict multi-layout checkpoint/restart, including atomic restore refusal; -- exact diagnostics and the source-level retirement fence for the old Schur - source steppers. - -`deferred = []` is normative for closure. Every issue needs positive and refusal -coverage and at least one native positive proof. Each scientific family has its -own required polarity. The validator rejects duplicate or wildcard selectors, -missing manifest ownership, pytest skip/xfail and optional imports, mock-based -proofs, disabled CTests, and any explicit deferred gap. - -Use: +The current status is **AUDITED OPEN**. The ledger in +`tests/gates/m4_runtime_io.toml` records exact executable evidence for +ADC-679 through ADC-687 and exact deferred gaps. It deliberately does not +claim M4 closure while any `[[deferred]]` row remains. + +The source audit already authenticates real proofs for: + +- external flux, boundary, tagger, transfer, solver, and writer components + that are compiled, loaded, and executed; +- canonical component manifests, generated registries, exact interface + tables, and platform launch checks; +- source, manifest, and installed-binary tamper refusals, provider absence, + and native parameter capacity overflow; +- real Uniform and AMR writer transactions, a real multi-layout transfer, + and a positive multi-layout checkpoint/restart; +- accepted scientific publication, diagnostics, and two-rank collective + HDF5. + +This evidence is intentionally narrower than the final ADC-687 acceptance +contract. The deferred rows name the missing polarity and the nearby source +that must not be mistaken for closure. They currently cover: + +- a CI lane that installs every mandatory dependency, including VTK, and + executes every selected pytest and CTest proof with zero skips; +- an unknown capability refused by a real runtime/backend before execution; +- selection and execution of the existing genuine wrong-ABI DSO refusal; +- ConsumerGraph and composite runtime rollback without handwritten publishers + or injected failure wrappers; +- checkpoint restore rollback caused by a real provider failure; +- complete Uniform, AMR, and multi-layout public-contract/report parity; +- mandatory native VTK reopen of the MPI PVD to PVTU to rank-VTU hierarchy; +- Program-only execution and complete retirement of legacy steppers, central + concrete dispatch, and silent fallback. + +## Serial output evidence + +There are four serial proofs that are real and remain selected: + +1. the final IMEX/AMR example publishes and reopens its serial scientific + formats through the public PoPS readers; +2. NPZ is independently reopened with NumPy and its arrays and physical clock + are checked; +3. HDF5 is independently reopened with h5py and its dataset and physical clock + are checked; +4. a serial VTU is independently reopened with VTK and its mesh, public field + name, AMR level array, and `TimeValue` are checked. + +An additional HDF5 refusal mutates a dataset with h5py and proves that the +authenticated PoPS reader rejects it. These tests contain no optional import +or skip. That makes their dependencies mandatory wherever the executable gate +runs; it does not prove that CI currently provisions those dependencies. + +The ParaView proof is limited to one serial `.vtu`. The MPI test authenticates +the `.pvd`, `.pvtu`, and rank-local `.vtu` hierarchy with PoPS and XML, but its +independent VTK reader is optional today. Consequently the standard parallel +ParaView hierarchy is useful existing evidence, not a closed native-reader +proof. + +## Gate modes + +The architecture CI runs: ```bash python scripts/run_m4_gate.py --audit-only +``` + +`--audit-only` verifies the exact nodeids, CTest selectors, manifest ownership, +deferred-gap schema, and source-level anti-skip rules. It prints +`AUDITED OPEN` and launches no compiler, test, MPI process, or native reader. + +The closure check is intentionally red while the ledger is open: + +```bash python scripts/run_m4_gate.py --check-only -python scripts/run_m4_gate.py --python-only -python scripts/run_m4_gate.py --build-dir build-mpi ``` -`--audit-only` validates the ledger while deliberately making no closure claim, -even when there are no deferred rows. `--check-only` additionally requires the -ledger to be closed, but still launches no test, compiler, MPI process, or -native reader. `--python-only` executes every selected Python proof with native -requirements forced on and omits CTest. The last command is the full gate and -requires an MPI-enabled build containing every selected CTest, plus real NumPy, -h5py, and VTK installations. Both pytest and CTest must produce JUnit reports -with zero skipped or xfailed proofs. - -The native-reader tests intentionally use no PoPS reader to interpret the -written payload. PoPS is used only to produce and authenticate the output; -NumPy, h5py, and VTK independently prove that the published formats are usable. -Their proof module is deliberately named `m4_native_reopen_proof.py`, so normal -`test_*.py` shard discovery does not silently turn VTK into a dependency of -every Python shard. The exact nodeids remain mandatory in the explicit M4 gate. +`--check-only` rejects every remaining deferred row and exits nonzero before +launching anything. Running the script without either audit flag, or with +`--python-only`, is also fail-closed until all deferred gaps are replaced by +real selected proofs. + +Once `deferred = []` is honestly restored, the full command requires an +MPI-enabled build containing every selected CTest and environments with NumPy, +h5py, and VTK. Every pytest and CTest execution must emit a JUnit report with +zero skipped or xfailed proofs. + +Each deferred row contains `issue`, `requirement`, `polarity`, a precise +`reason`, and existing `evidence_paths`. The validator rejects malformed or +duplicate gaps, wildcard selectors, missing manifest ownership, optional +pytest imports, skip/xfail markers, mock fixtures/imports, and disabled CTests. From 54f68a4305c4fa33cad68f6d676a7eac4aa54076 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:30:18 +0200 Subject: [PATCH 08/62] test(gate): select the real wrong-ABI DSO refusal --- docs/design/m4-conformance-gate.md | 3 +-- tests/gates/m4_runtime_io.toml | 18 ++++++++---------- .../architecture/test_m4_runtime_io_gate.py | 11 ++++------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 8d38274c1..739515808 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -12,7 +12,7 @@ The source audit already authenticates real proofs for: - canonical component manifests, generated registries, exact interface tables, and platform launch checks; - source, manifest, and installed-binary tamper refusals, provider absence, - and native parameter capacity overflow; + native parameter capacity overflow, and a genuine wrong-ABI DSO refusal; - real Uniform and AMR writer transactions, a real multi-layout transfer, and a positive multi-layout checkpoint/restart; - accepted scientific publication, diagnostics, and two-rank collective @@ -25,7 +25,6 @@ that must not be mistaken for closure. They currently cover: - a CI lane that installs every mandatory dependency, including VTK, and executes every selected pytest and CTest proof with zero skips; - an unknown capability refused by a real runtime/backend before execution; -- selection and execution of the existing genuine wrong-ABI DSO refusal; - ConsumerGraph and composite runtime rollback without handwritten publishers or injected failure wrappers; - checkpoint restore rollback caused by a real provider failure; diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index dac8ad254..43f9be1bf 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -33,16 +33,6 @@ evidence_paths = [ "tests/cpp/unit/runtime/test_platform_manifest.cpp", ] -[[deferred]] -issue = "ADC-687" -requirement = "tamper_capability_abi" -polarity = "refusal" -reason = "A genuine wrong-ABI shared object refusal exists, but the executable M4 matrix does not yet select and run that DSO proof." -evidence_paths = [ - "tests/cpp/integration/native_loader/test_amr_native_loader.cpp", - "tests/gates/m4_runtime_io.toml", -] - [[deferred]] issue = "ADC-684" requirement = "runtime_instance" @@ -389,6 +379,14 @@ kind = "ctest" target = "tamper_capability_abi@test_native_loader_param_overflow" test_regex = "^test_native_loader_param_overflow\\.Runs$" +[[check]] +issue = "ADC-687" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "ctest" +target = "tamper_capability_abi@test_amr_native_loader" +test_regex = "^test_amr_native_loader\\.RefusesComponentBuiltForAnotherNativeAbi$" + [[check]] issue = "ADC-687" requirement = "legacy_stepper_retirement" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 2473aa793..06ff33cbc 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -37,8 +37,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 9 - assert len(data["check"]) >= 36 + assert len(data["deferred"]) == 8 + assert len(data["check"]) >= 37 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -62,7 +62,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): ("ADC-686", "strict_checkpoint", "refusal"), ("ADC-686", "exact_paraview", "positive"), ("ADC-687", "gate_execution", "positive"), - ("ADC-687", "tamper_capability_abi", "refusal"), ("ADC-687", "legacy_stepper_retirement", "positive"), } @@ -231,6 +230,7 @@ def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): "test_fixed_binary_cannot_claim_template_genericity" ), r"^test_native_loader_param_overflow\.Runs$", + r"^test_amr_native_loader\.RefusesComponentBuiltForAnotherNativeAbi$", } <= refusals assert ( "tests/python/unit/codegen/test_component_manifest_v2.py::" @@ -244,10 +244,7 @@ def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): (row["issue"], row["requirement"], row["polarity"]) for row in data["deferred"] if row["requirement"] == "tamper_capability_abi" - } == { - ("ADC-683", "tamper_capability_abi", "refusal"), - ("ADC-687", "tamper_capability_abi", "refusal"), - } + } == {("ADC-683", "tamper_capability_abi", "refusal")} def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): From 883522a5b0f33427fabf1ed65fb03eebcd394df3 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:35:43 +0200 Subject: [PATCH 09/62] test(gate): prove Program-only runtime retirement --- docs/design/m4-conformance-gate.md | 5 +- tests/gates/m4_runtime_io.toml | 68 +++++++++++++++---- .../architecture/test_m4_runtime_io_gate.py | 41 +++++++---- 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 739515808..b1a271670 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -13,6 +13,9 @@ The source audit already authenticates real proofs for: tables, and platform launch checks; - source, manifest, and installed-binary tamper refusals, provider absence, native parameter capacity overflow, and a genuine wrong-ABI DSO refusal; +- Program-only Uniform/AMR temporal facades, retired native source-stage + headers and schedulers, typed component dispatch, and fail-closed unbound + native interfaces; - real Uniform and AMR writer transactions, a real multi-layout transfer, and a positive multi-layout checkpoint/restart; - accepted scientific publication, diagnostics, and two-rank collective @@ -30,8 +33,6 @@ that must not be mistaken for closure. They currently cover: - checkpoint restore rollback caused by a real provider failure; - complete Uniform, AMR, and multi-layout public-contract/report parity; - mandatory native VTK reopen of the MPI PVD to PVTU to rank-VTU hierarchy; -- Program-only execution and complete retirement of legacy steppers, central - concrete dispatch, and silent fallback. ## Serial output evidence diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 43f9be1bf..bc5924000 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -82,18 +82,6 @@ evidence_paths = [ "tests/python/integration/mpi/test_scientific_output_mpi.py", ] -[[deferred]] -issue = "ADC-687" -requirement = "legacy_stepper_retirement" -polarity = "positive" -reason = "The selected Schur-header fence does not by itself prove Program-only execution or the absence of every legacy stepper, concrete central dispatch, and silent fallback." -evidence_paths = [ - "tests/python/architecture/test_no_schur_header_leak.py", - "tests/python/architecture/test_program_only_temporal_facades.py", - "tests/python/architecture/test_no_legacy_runtime_routes.py", - "tests/python/architecture/test_component_interface_dispatch.py", -] - # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, # optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, @@ -394,3 +382,59 @@ polarity = "positive" kind = "pytest" target = "legacy_stepper_retirement" nodeid = "tests/python/architecture/test_no_schur_header_leak.py::test_native_source_stage_headers_are_retired" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_program_only_temporal_facades.py::test_system_temporal_facades_dispatch_only_through_an_installed_program" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_program_only_temporal_facades.py::test_amr_temporal_facades_use_amr_runtime_only_as_the_spatial_engine" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_program_only_temporal_facades.py::test_historical_block_scheduler_is_not_an_installed_temporal_authority" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_program_only_temporal_facades.py::test_production_has_no_second_amr_time_engine" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_component_interface_dispatch.py::test_component_trust_boundary_never_classifies_the_scientific_component_type" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/architecture/test_component_interface_dispatch.py::test_native_registry_has_no_rtti_or_untyped_capability_escape_hatch" + +[[check]] +issue = "ADC-687" +requirement = "legacy_stepper_retirement" +polarity = "positive" +kind = "pytest" +target = "legacy_stepper_retirement" +nodeid = "tests/python/unit/codegen/test_component_adapters.py::test_native_interface_is_declared_and_unbound_never_falls_back" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 06ff33cbc..96db88a24 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -37,8 +37,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 8 - assert len(data["check"]) >= 37 + assert len(data["deferred"]) == 7 + assert len(data["check"]) >= 44 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -62,7 +62,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): ("ADC-686", "strict_checkpoint", "refusal"), ("ADC-686", "exact_paraview", "positive"), ("ADC-687", "gate_execution", "positive"), - ("ADC-687", "legacy_stepper_retirement", "positive"), } _, closure_errors = runner.validate_manifest(MANIFEST) @@ -288,20 +287,32 @@ def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): } in checks -def test_m4_gate_keeps_schur_evidence_but_ci_only_claims_an_open_audit(): +def test_m4_gate_pins_complete_program_only_dispatch_and_fallback_fences(): data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors - assert { - "issue": "ADC-687", - "requirement": "legacy_stepper_retirement", - "polarity": "positive", - "kind": "pytest", - "target": "legacy_stepper_retirement", - "nodeid": ( - "tests/python/architecture/test_no_schur_header_leak.py::" - "test_native_source_stage_headers_are_retired" - ), - } in data["check"] + selected = { + row["nodeid"] + for row in data["check"] + if row["requirement"] == "legacy_stepper_retirement" + } + assert selected == { + "tests/python/architecture/test_no_schur_header_leak.py::" + "test_native_source_stage_headers_are_retired", + "tests/python/architecture/test_program_only_temporal_facades.py::" + "test_system_temporal_facades_dispatch_only_through_an_installed_program", + "tests/python/architecture/test_program_only_temporal_facades.py::" + "test_amr_temporal_facades_use_amr_runtime_only_as_the_spatial_engine", + "tests/python/architecture/test_program_only_temporal_facades.py::" + "test_historical_block_scheduler_is_not_an_installed_temporal_authority", + "tests/python/architecture/test_program_only_temporal_facades.py::" + "test_production_has_no_second_amr_time_engine", + "tests/python/architecture/test_component_interface_dispatch.py::" + "test_component_trust_boundary_never_classifies_the_scientific_component_type", + "tests/python/architecture/test_component_interface_dispatch.py::" + "test_native_registry_has_no_rtti_or_untyped_capability_escape_hatch", + "tests/python/unit/codegen/test_component_adapters.py::" + "test_native_interface_is_declared_and_unbound_never_falls_back", + } workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") job = workflow.split("\n gate-python-architecture:\n", 1)[1] From b182591c8937f8d691a815b425a30904912eb8c9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:37:30 +0200 Subject: [PATCH 10/62] test(runtime): refuse unknown capabilities before launch --- docs/design/m4-conformance-gate.md | 3 ++- .../unit/runtime/test_platform_manifest.cpp | 14 ++++++++++++++ tests/gates/m4_runtime_io.toml | 18 ++++++++---------- .../architecture/test_m4_runtime_io_gate.py | 8 ++++---- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index b1a271670..f31b1e393 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -13,6 +13,8 @@ The source audit already authenticates real proofs for: tables, and platform launch checks; - source, manifest, and installed-binary tamper refusals, provider absence, native parameter capacity overflow, and a genuine wrong-ABI DSO refusal; +- an unknown device capability refused by the runtime launch validator before + the candidate kernel is invoked; - Program-only Uniform/AMR temporal facades, retired native source-stage headers and schedulers, typed component dispatch, and fail-closed unbound native interfaces; @@ -27,7 +29,6 @@ that must not be mistaken for closure. They currently cover: - a CI lane that installs every mandatory dependency, including VTK, and executes every selected pytest and CTest proof with zero skips; -- an unknown capability refused by a real runtime/backend before execution; - ConsumerGraph and composite runtime rollback without handwritten publishers or injected failure wrappers; - checkpoint restore rollback caused by a real provider failure; diff --git a/tests/cpp/unit/runtime/test_platform_manifest.cpp b/tests/cpp/unit/runtime/test_platform_manifest.cpp index 01afeb06f..c8415277a 100644 --- a/tests/cpp/unit/runtime/test_platform_manifest.cpp +++ b/tests/cpp/unit/runtime/test_platform_manifest.cpp @@ -68,6 +68,20 @@ TEST(PlatformManifest, UnknownIsMissingProofAndThreeDimensionsRemainRepresentabl pops::platform::ContractError); } +TEST(PlatformManifest, UnknownCapabilityRefusesBeforeKernel) { + auto missing = platform(); + missing.device = pops::platform::CapabilityProof::unknown(); + int launches = 0; + EXPECT_THROW(pops::platform::launch_checked(missing, context(), {field()}, + [&](const auto&, const auto&) { + ++launches; + return 0; + }, + {field()}), + pops::platform::ContractError); + EXPECT_EQ(launches, 0); +} + TEST(PlatformManifest, FieldAndCommunicatorMismatchesRefuseBeforeKernel) { int launches = 0; auto kernel = [&](const auto&, const auto&) { return ++launches; }; diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index bc5924000..5063d028e 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -23,16 +23,6 @@ evidence_paths = [ "scripts/run_m4_gate.py", ] -[[deferred]] -issue = "ADC-683" -requirement = "tamper_capability_abi" -polarity = "refusal" -reason = "The matrix lacks a real runtime or backend launch whose unknown capability is refused before any kernel executes; schema-only target validation is insufficient." -evidence_paths = [ - "tests/python/unit/codegen/test_component_manifest_v2.py", - "tests/cpp/unit/runtime/test_platform_manifest.cpp", -] - [[deferred]] issue = "ADC-684" requirement = "runtime_instance" @@ -223,6 +213,14 @@ kind = "ctest" target = "platform_execution@test_platform_manifest" test_regex = "^PlatformManifest\\.FieldAndCommunicatorMismatchesRefuseBeforeKernel$" +[[check]] +issue = "ADC-683" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "ctest" +target = "tamper_capability_abi@test_platform_manifest" +test_regex = "^PlatformManifest\\.UnknownCapabilityRefusesBeforeKernel$" + [[check]] issue = "ADC-684" requirement = "runtime_instance" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 96db88a24..3cffd1acd 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -37,8 +37,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 7 - assert len(data["check"]) >= 44 + assert len(data["deferred"]) == 6 + assert len(data["check"]) >= 45 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -55,7 +55,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): (row["issue"], row["requirement"], row["polarity"]) for row in data["deferred"] } == { - ("ADC-683", "tamper_capability_abi", "refusal"), ("ADC-684", "runtime_instance", "positive"), ("ADC-684", "runtime_instance", "refusal"), ("ADC-685", "consumer_graph", "refusal"), @@ -230,6 +229,7 @@ def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): ), r"^test_native_loader_param_overflow\.Runs$", r"^test_amr_native_loader\.RefusesComponentBuiltForAnotherNativeAbi$", + r"^PlatformManifest\.UnknownCapabilityRefusesBeforeKernel$", } <= refusals assert ( "tests/python/unit/codegen/test_component_manifest_v2.py::" @@ -243,7 +243,7 @@ def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): (row["issue"], row["requirement"], row["polarity"]) for row in data["deferred"] if row["requirement"] == "tamper_capability_abi" - } == {("ADC-683", "tamper_capability_abi", "refusal")} + } == set() def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): From b5759507bbeb324b82daadaeb91659ffcb90b4f9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:48:10 +0200 Subject: [PATCH 11/62] test(m4): require native MPI ParaView hierarchy reopen --- scripts/run_m4_gate.py | 33 ++++++- tests/gates/m4_runtime_io.toml | 19 ++-- .../architecture/test_m4_runtime_io_gate.py | 53 ++++++++++- .../mpi/test_scientific_output_mpi.py | 94 +++++++++++++++---- 4 files changed, 166 insertions(+), 33 deletions(-) diff --git a/scripts/run_m4_gate.py b/scripts/run_m4_gate.py index 433cf4b82..e437aef38 100644 --- a/scripts/run_m4_gate.py +++ b/scripts/run_m4_gate.py @@ -150,6 +150,15 @@ def _forbidden_python_markers(node: ast.AST) -> list[str]: return markers +def _has_authenticated_mpi_guard(module: ast.Module) -> bool: + return any( + isinstance(node, ast.ImportFrom) + and node.module == "tests.python.support.requirements" + and any(alias.name == "require_mpi_or_skip" for alias in node.names) + for node in ast.walk(module) + ) + + def _ctest_suites() -> dict[str, dict]: data = tomllib.loads(TEST_MANIFEST.read_text(encoding="utf-8")) return {str(row["name"]): row for row in data.get("cpp", {}).get("suite", ())} @@ -297,6 +306,8 @@ def _validate_python_nodeid( nodeid: object, where: str, errors: list[str], + *, + mpi_entrypoint: bool = False, ) -> str | None: if not isinstance(nodeid, str) or nodeid.count("::") != 1: errors.append("%s must contain one exact file::test nodeid" % where) @@ -324,9 +335,18 @@ def _validate_python_nodeid( if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) ] markers = _forbidden_python_markers(function) - markers.extend( - _forbidden_python_markers(ast.Module(body=module_nodes, type_ignores=[])) - ) + module = ast.Module(body=module_nodes, type_ignores=[]) + module_markers = _forbidden_python_markers(module) + if mpi_entrypoint and "require_mpi_or_skip" in module_markers: + if not _has_authenticated_mpi_guard(module): + errors.append( + "%s uses an unauthenticated MPI prerequisite guard" % nodeid + ) + module_markers = [ + marker for marker in module_markers + if marker != "require_mpi_or_skip" + ] + markers.extend(module_markers) if markers: errors.append( "%s is not an unconditional real proof; found %s" @@ -495,7 +515,12 @@ def audit_manifest(path: Path = DEFAULT_MANIFEST) -> tuple[dict, list[str]]: ): native_positive_issues.add(str(issue)) elif kind == "mpi_python": - relative = _validate_python_nodeid(row.get("nodeid"), where, errors) + relative = _validate_python_nodeid( + row.get("nodeid"), + where, + errors, + mpi_entrypoint=True, + ) nproc = row.get("nproc") if isinstance(nproc, bool) or not isinstance(nproc, int) or nproc < 1: errors.append("%s MPI Python row requires a positive integer nproc" % where) diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 5063d028e..85f51e703 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -62,16 +62,6 @@ evidence_paths = [ "tests/python/integration/runtime/test_multi_layout_runtime.py", ] -[[deferred]] -issue = "ADC-686" -requirement = "exact_paraview" -polarity = "positive" -reason = "Serial VTU reopens with VTK, but native VTK reopening of the mandatory MPI PVD to PVTU to rank-VTU hierarchy is still optional and therefore unproved." -evidence_paths = [ - "tests/python/integration/io/m4_native_reopen_proof.py", - "tests/python/integration/mpi/test_scientific_output_mpi.py", -] - # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, # optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, @@ -309,6 +299,15 @@ kind = "pytest" target = "exact_paraview" nodeid = "tests/python/integration/io/m4_native_reopen_proof.py::test_paraview_reopens_with_vtk_without_a_pops_reader" +[[check]] +issue = "ADC-686" +requirement = "exact_paraview" +polarity = "positive" +kind = "mpi_python" +target = "exact_paraview" +nodeid = "tests/python/integration/mpi/test_scientific_output_mpi.py::_validate_paraview" +nproc = 2 + [[check]] issue = "ADC-686" requirement = "exact_paraview" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 3cffd1acd..665bf1ef0 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import importlib.util from pathlib import Path import subprocess @@ -37,8 +38,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 6 - assert len(data["check"]) >= 45 + assert len(data["deferred"]) == 5 + assert len(data["check"]) >= 46 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -59,7 +60,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): ("ADC-684", "runtime_instance", "refusal"), ("ADC-685", "consumer_graph", "refusal"), ("ADC-686", "strict_checkpoint", "refusal"), - ("ADC-686", "exact_paraview", "positive"), ("ADC-687", "gate_execution", "positive"), } @@ -256,6 +256,7 @@ def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): for row in checks if row["requirement"] in {"exact_npz", "exact_hdf5", "exact_paraview"} and row["polarity"] == "positive" + and row["kind"] == "pytest" } assert native_reopen == { "exact_npz": ( @@ -277,6 +278,31 @@ def test_m4_gate_pins_mandatory_native_reopen_and_collective_hdf5_np2(): assert "pytest.importorskip" not in source assert "import h5py" in source assert "from vtkmodules.vtkIOXML import vtkXMLUnstructuredGridReader" in source + mpi_native = [ + row + for row in checks + if row["requirement"] == "exact_paraview" + and row["kind"] == "mpi_python" + ] + assert mpi_native == [{ + "issue": "ADC-686", + "requirement": "exact_paraview", + "polarity": "positive", + "kind": "mpi_python", + "target": "exact_paraview", + "nodeid": ( + "tests/python/integration/mpi/test_scientific_output_mpi.py::" + "_validate_paraview" + ), + "nproc": 2, + }] + mpi_source = ( + ROOT / "tests/python/integration/mpi/test_scientific_output_mpi.py" + ).read_text(encoding="utf-8") + assert "vtkXMLPUnstructuredGridReader" in mpi_source + assert "vtkXMLUnstructuredGridReader" in mpi_source + assert "native PVD/PVTU traversal" in mpi_source + assert "except ImportError" not in mpi_source assert { "issue": "ADC-686", "requirement": "collective_hdf5", @@ -423,6 +449,27 @@ def test_m4_gate_rejects_importorskip_and_mock_proofs(tmp_path): ) +def test_m4_mpi_entrypoint_accepts_only_the_required_prerequisite_guard(): + runner = _load_runner() + data, errors = runner.audit_manifest(MANIFEST) + assert not errors + mpi_proof = next( + row for row in data["check"] + if row["kind"] == "mpi_python" + ) + assert mpi_proof["nodeid"] == ( + "tests/python/integration/mpi/test_scientific_output_mpi.py::" + "_validate_paraview" + ) + assert runner._required_environment()["POPS_REQUIRE_MPI_TESTS"] == "1" + trusted = ast.parse( + "from tests.python.support.requirements import require_mpi_or_skip\n" + ) + untrusted = ast.parse("def require_mpi_or_skip(_reason):\n return None\n") + assert runner._has_authenticated_mpi_guard(trusted) + assert not runner._has_authenticated_mpi_guard(untrusted) + + def test_m4_gate_rejects_every_explicit_deferred_gap(): runner = _load_runner() data, audit_errors = runner.audit_manifest(MANIFEST) diff --git a/tests/python/integration/mpi/test_scientific_output_mpi.py b/tests/python/integration/mpi/test_scientific_output_mpi.py index e01111287..5e39b435b 100644 --- a/tests/python/integration/mpi/test_scientific_output_mpi.py +++ b/tests/python/integration/mpi/test_scientific_output_mpi.py @@ -76,6 +76,10 @@ from pops.projection import ConservativeCellAverage from pops.output._writers.hdf5 import _collective_temporary_owner from pops.time import FixedDt, StagePoint, TimePoint, every + from vtkmodules.vtkIOXML import ( + vtkXMLPUnstructuredGridReader, + vtkXMLUnstructuredGridReader, + ) except Exception as exc: # noqa: BLE001 -- optional outside the required MPI lane require_mpi_or_skip("scientific-output MPI/HDF5 runtime import failed: %s" % exc) @@ -691,22 +695,80 @@ def validate() -> None: if tuple(sorted(all_leaf_paths)) != leaves: raise AssertionError("PVTU catalogues do not cover every emitted VTU leaf exactly") - # The exact PoPS reopen above is mandatory and authenticates every component. When the - # independently maintained VTK Python reader is installed in the MPI lane, also prove that - # the standard PVTU is directly consumable without any PoPS-specific adapter. - try: - from vtkmodules.vtkIOXML import vtkXMLPUnstructuredGridReader - except ImportError: - vtkXMLPUnstructuredGridReader = None - if vtkXMLPUnstructuredGridReader is not None: - for pvtu_path in parallel: - reader = vtkXMLPUnstructuredGridReader() - reader.SetFileName(str(pvtu_path)) - reader.Update() - grid = reader.GetOutput() - if grid.GetNumberOfCells() < 1 \ - or grid.GetCellData().GetArray("U") is None: - raise AssertionError("the native VTK reader could not consume the PVTU") + # Traverse from the standard PVD itself, then reopen every referenced PVTU and every + # rank-local VTU with the independently maintained VTK readers. This is mandatory in the + # M4 lane: absence of VTK is a required-test failure, never an optional local success. + catalog_paths = tuple( + (collections[-1].parent / node.attrib["file"]).resolve() + for node in datasets + ) + if catalog_paths != tuple(path.resolve() for path in parallel): + raise AssertionError("native PVD traversal differs from the exact temporal series") + native_leaf_paths = [] + for macro_step, (dataset, pvtu_path) in enumerate( + zip(datasets, catalog_paths, strict=True), start=1): + expected_time = macro_step * DT + if float(dataset.attrib["timestep"]) != expected_time: + raise AssertionError("native PVD traversal lost the physical output time") + + reopened_parallel = read_paraview_parallel(pvtu_path) + native_parallel = vtkXMLPUnstructuredGridReader() + native_parallel.SetFileName(str(pvtu_path)) + native_parallel.Update() + if native_parallel.GetErrorCode() != 0: + raise AssertionError("the native VTK reader rejected the PVTU") + parallel_grid = native_parallel.GetOutput() + + expected_cells = 0 + for leaf_path in reopened_parallel.paths: + native_leaf_paths.append(leaf_path.resolve()) + xml_piece = ET.parse(leaf_path).getroot().find( + "./UnstructuredGrid/Piece") + if xml_piece is None: + raise AssertionError("rank-local VTU has no UnstructuredGrid piece") + leaf_cells = int(xml_piece.attrib["NumberOfCells"]) + leaf_points = int(xml_piece.attrib["NumberOfPoints"]) + expected_cells += leaf_cells + + native_leaf = vtkXMLUnstructuredGridReader() + native_leaf.SetFileName(str(leaf_path)) + native_leaf.Update() + if native_leaf.GetErrorCode() != 0: + raise AssertionError("the native VTK reader rejected a rank-local VTU") + leaf_grid = native_leaf.GetOutput() + if leaf_grid.GetNumberOfCells() != leaf_cells \ + or leaf_grid.GetNumberOfPoints() != leaf_points: + raise AssertionError( + "native VTU geometry differs from the rank-local XML piece") + for name in ("U", "pops_level", "vtkGhostType"): + array = leaf_grid.GetCellData().GetArray(name) + if array is None or array.GetNumberOfTuples() != leaf_cells: + raise AssertionError( + "native VTU reader lost rank-local cell array %s" % name) + time_value = leaf_grid.GetFieldData().GetArray("TimeValue") + if time_value is None \ + or time_value.GetNumberOfTuples() != 1 \ + or time_value.GetTuple1(0) != expected_time: + raise AssertionError( + "native VTU reader lost the rank-local physical output time") + + if parallel_grid.GetNumberOfCells() != expected_cells: + raise AssertionError( + "native PVTU reader did not assemble every rank-local VTU cell") + for name in ("U", "pops_level", "vtkGhostType"): + array = parallel_grid.GetCellData().GetArray(name) + if array is None or array.GetNumberOfTuples() != expected_cells: + raise AssertionError( + "native PVTU reader lost assembled cell array %s" % name) + public_field = parallel_grid.GetCellData().GetArray("U") + if public_field.GetNumberOfComponents() != 1 \ + or public_field.GetComponentName(0) != "rho": + raise AssertionError( + "native PVTU reader lost the user-authored U/rho field name") + + if tuple(sorted(native_leaf_paths)) != tuple(path.resolve() for path in leaves): + raise AssertionError( + "native PVD/PVTU traversal did not reopen every rank-local VTU exactly once") observed_ranks = set() observed_steps = set() From 2119b3f3046709d345c063be68becc75686d3297 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:48:10 +0200 Subject: [PATCH 12/62] docs(m4): record mandatory parallel VTK reopen --- docs/design/m4-conformance-gate.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index f31b1e393..988c86450 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -20,8 +20,8 @@ The source audit already authenticates real proofs for: native interfaces; - real Uniform and AMR writer transactions, a real multi-layout transfer, and a positive multi-layout checkpoint/restart; -- accepted scientific publication, diagnostics, and two-rank collective - HDF5. +- accepted scientific publication, diagnostics, two-rank collective HDF5, + and a two-rank PVD/PVTU/rank-VTU hierarchy reopened by native VTK readers. This evidence is intentionally narrower than the final ADC-687 acceptance contract. The deferred rows name the missing polarity and the nearby source @@ -32,10 +32,9 @@ that must not be mistaken for closure. They currently cover: - ConsumerGraph and composite runtime rollback without handwritten publishers or injected failure wrappers; - checkpoint restore rollback caused by a real provider failure; -- complete Uniform, AMR, and multi-layout public-contract/report parity; -- mandatory native VTK reopen of the MPI PVD to PVTU to rank-VTU hierarchy; +- complete Uniform, AMR, and multi-layout public-contract/report parity. -## Serial output evidence +## Exact output evidence There are four serial proofs that are real and remain selected: @@ -53,11 +52,14 @@ authenticated PoPS reader rejects it. These tests contain no optional import or skip. That makes their dependencies mandatory wherever the executable gate runs; it does not prove that CI currently provisions those dependencies. -The ParaView proof is limited to one serial `.vtu`. The MPI test authenticates -the `.pvd`, `.pvtu`, and rank-local `.vtu` hierarchy with PoPS and XML, but its -independent VTK reader is optional today. Consequently the standard parallel -ParaView hierarchy is useful existing evidence, not a closed native-reader -proof. +The selected two-rank ParaView entrypoint starts from the standard `.pvd` +catalogue, preserves its exact temporal ordering, and requires the native VTK +parallel reader to assemble every referenced `.pvtu`. It also reopens every +rank-local `.vtu` directly with VTK and checks its geometry, public arrays, +component name, and `TimeValue`. VTK imports are unconditional in the required +MPI lane: `POPS_REQUIRE_MPI_TESTS=1` turns an absent reader into a test failure. +The separate `gate_execution` gap remains open until CI provisions VTK and +executes this selected entrypoint rather than auditing only its source. ## Gate modes From 55058ae77397df26a1446c82804817a848891cb1 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 03:48:01 +0200 Subject: [PATCH 13/62] Fix Kokkos concurrency reporting compatibility (cherry picked from commit 0c40994e0d4865624ae1b836ed7d26332d12b1f1) --- include/pops/runtime/runtime_environment.hpp | 3 ++- tests/cpp/integration/runtime/test_runtime_environment.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/pops/runtime/runtime_environment.hpp b/include/pops/runtime/runtime_environment.hpp index 73ee92e84..a0f52b878 100644 --- a/include/pops/runtime/runtime_environment.hpp +++ b/include/pops/runtime/runtime_environment.hpp @@ -136,7 +136,8 @@ inline RuntimeEnvironmentReport runtime_environment_report() { report.kokkos_stream = native_stream_identity(); report.kokkos_stream_synchronous = native_stream_is_synchronous(); if (report.kokkos_initialized) { - report.kokkos_concurrency = Kokkos::DefaultExecutionSpace::concurrency(); + const Kokkos::DefaultExecutionSpace execution_space{}; + report.kokkos_concurrency = execution_space.concurrency(); } if (report.kokkos_initialized_by_pops) { report.kokkos_ownership = "pops-owned-lazy"; diff --git a/tests/cpp/integration/runtime/test_runtime_environment.cpp b/tests/cpp/integration/runtime/test_runtime_environment.cpp index 808b73135..dbdbeb82c 100644 --- a/tests/cpp/integration/runtime/test_runtime_environment.cpp +++ b/tests/cpp/integration/runtime/test_runtime_environment.cpp @@ -29,7 +29,7 @@ TEST(RuntimeEnvironment, ReportsDimensionPrecisionAndBackends) { EXPECT_TRUE(report.has_kokkos) << "has_kokkos"; EXPECT_TRUE(!report.kokkos_backend.empty()) << "kokkos_backend_named"; if (report.kokkos_initialized) { - EXPECT_TRUE(report.kokkos_concurrency == Kokkos::DefaultExecutionSpace::concurrency()) + EXPECT_TRUE(report.kokkos_concurrency == Kokkos::DefaultExecutionSpace{}.concurrency()) << "initialized_kokkos_concurrency"; } else { EXPECT_TRUE(report.kokkos_concurrency == 0) << "inactive_kokkos_concurrency"; @@ -45,7 +45,7 @@ TEST(RuntimeEnvironment, ReportsDimensionPrecisionAndBackends) { const RuntimeEnvironmentReport initialized_report = runtime_environment_report(); EXPECT_TRUE(initialized_report.kokkos_initialized) << "kokkos_initialized_for_exact_probe"; EXPECT_TRUE(initialized_report.kokkos_concurrency == - Kokkos::DefaultExecutionSpace::concurrency()) + Kokkos::DefaultExecutionSpace{}.concurrency()) << "exact_default_execution_space_concurrency"; } #else From d4c3a5750fad455b8187a0838e244e4ba0f045c9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:58:39 +0200 Subject: [PATCH 14/62] test(m4): prove real checkpoint provider rollback --- tests/gates/m4_runtime_io.toml | 17 +++-- .../architecture/test_m4_runtime_io_gate.py | 16 ++++- .../amr/test_amr_regrid_on_restart.py | 68 +++++++++++++++++++ 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 85f51e703..1f3a398f4 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -53,15 +53,6 @@ evidence_paths = [ "tests/python/integration/native_loader/test_external_component_package.py", ] -[[deferred]] -issue = "ADC-686" -requirement = "strict_checkpoint" -polarity = "refusal" -reason = "Atomic restart rollback is only exercised through an injected FailFirstRestart wrapper; a real checkpoint provider failure must prove restoration rollback." -evidence_paths = [ - "tests/python/integration/runtime/test_multi_layout_runtime.py", -] - # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, # optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, @@ -332,6 +323,14 @@ kind = "pytest" target = "strict_checkpoint" nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count" +[[check]] +issue = "ADC-686" +requirement = "strict_checkpoint" +polarity = "refusal" +kind = "pytest" +target = "strict_checkpoint" +nodeid = "tests/python/integration/amr/test_amr_regrid_on_restart.py::test_authenticated_amr_contract_refusal_rolls_back_native_restart_transaction" + [[check]] issue = "ADC-686" requirement = "diagnostics" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 665bf1ef0..6775b2749 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -38,8 +38,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 5 - assert len(data["check"]) >= 46 + assert len(data["deferred"]) == 4 + assert len(data["check"]) >= 47 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -59,7 +59,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): ("ADC-684", "runtime_instance", "positive"), ("ADC-684", "runtime_instance", "refusal"), ("ADC-685", "consumer_graph", "refusal"), - ("ADC-686", "strict_checkpoint", "refusal"), ("ADC-687", "gate_execution", "positive"), } @@ -198,6 +197,17 @@ def test_m4_gate_pins_real_runtime_instance_and_positive_checkpoint_proofs(): "test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count" ), } in checks + assert { + "issue": "ADC-686", + "requirement": "strict_checkpoint", + "polarity": "refusal", + "kind": "pytest", + "target": "strict_checkpoint", + "nodeid": ( + "tests/python/integration/amr/test_amr_regrid_on_restart.py::" + "test_authenticated_amr_contract_refusal_rolls_back_native_restart_transaction" + ), + } in checks selected = { row.get("nodeid", row.get("test_regex")) for row in checks diff --git a/tests/python/integration/amr/test_amr_regrid_on_restart.py b/tests/python/integration/amr/test_amr_regrid_on_restart.py index 0cd022601..ccf829050 100644 --- a/tests/python/integration/amr/test_amr_regrid_on_restart.py +++ b/tests/python/integration/amr/test_amr_regrid_on_restart.py @@ -14,6 +14,7 @@ from __future__ import annotations +import json from pathlib import Path import numpy as np @@ -228,6 +229,73 @@ def _assert_same_accepted_image(runtime, expected): np.testing.assert_array_equal(current, recorded) +def test_authenticated_amr_contract_refusal_rolls_back_native_restart_transaction( + native_cxx, + kokkos_root, + tmp_path, +): + """A real post-apply checkpoint-provider refusal restores the previous accepted image.""" + del kokkos_root + artifact = pops.compile(_resolved(native_cxx)) + source = _bind(artifact) + report = pops.run( + source, + t_end=NSTEPS * DT, + max_steps=NSTEPS, + console=False, + ) + assert report.accepted_steps == NSTEPS + checkpoint = Path(source.checkpoint(tmp_path / "provider-contract-source")) + + # Preserve a fully valid, content-addressed checkpoint envelope while making only its dynamic + # accepted-ledger claim inconsistent with the opaque Program image. Static preflight therefore + # succeeds; the real AMR provider can refuse only after applying the checkpoint inside its + # native restart transaction. + from pops.runtime._checkpoint_manifest import ( + IDENTITY_KEY, + MANIFEST_KEY, + seal_checkpoint_payload, + ) + + with np.load(checkpoint, allow_pickle=False) as stored: + payload = { + name: np.asarray(stored[name]).copy() + for name in stored.files + if name not in {MANIFEST_KEY, IDENTITY_KEY} + } + contract = json.loads(str(payload["amr_accepted_contract"])) + contract["ledger"]["accepted_entries"] = int( + contract["ledger"]["accepted_entries"] + ) + 1 + payload["amr_accepted_contract"] = np.asarray( + json.dumps(contract, sort_keys=True, separators=(",", ":"), allow_nan=False) + ) + seal_checkpoint_payload(source, payload, runtime_kind="amr") + refused_checkpoint = tmp_path / "provider-contract-refusal.npz" + with refused_checkpoint.open("wb") as stream: + np.savez_compressed(stream, **payload) + + restarted = _bind(artifact) + rollback_image = _accepted_image(restarted) + with pytest.raises( + ValueError, + match="restored AMR accepted-state image differs from its authenticated contract", + ): + restarted.restart(refused_checkpoint) + + _assert_same_accepted_image(restarted, rollback_image) + assert restarted._executor.last_restart_regrid_receipt() is None + assert "_checkpoint_restart_python_snapshot" not in restarted._executor.__dict__ + + # The same real provider remains usable after compensation: retrying the unmodified checkpoint + # succeeds and publishes one transformed-hierarchy restart receipt. + restart_identity = restarted.restart(checkpoint) + receipt = restarted._executor.last_restart_regrid_receipt() + assert restart_identity == restarted.last_restart_identity + assert receipt["changed"] is True + assert receipt["before"]["topology_identity"] != receipt["after"]["topology_identity"] + + def test_regrid_on_restart_changes_real_boxes_and_rolls_back_post_regrid_fault( native_cxx, isolated_native_cache, From 2ca31bf62e3dd68e3756b685ca6414e56eb32e03 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:58:39 +0200 Subject: [PATCH 15/62] docs(m4): record provider-backed restart refusal --- docs/design/m4-conformance-gate.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 988c86450..08e8ef237 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -31,7 +31,6 @@ that must not be mistaken for closure. They currently cover: executes every selected pytest and CTest proof with zero skips; - ConsumerGraph and composite runtime rollback without handwritten publishers or injected failure wrappers; -- checkpoint restore rollback caused by a real provider failure; - complete Uniform, AMR, and multi-layout public-contract/report parity. ## Exact output evidence @@ -61,6 +60,14 @@ MPI lane: `POPS_REQUIRE_MPI_TESTS=1` turns an absent reader into a test failure. The separate `gate_execution` gap remains open until CI provisions VTK and executes this selected entrypoint rather than auditing only its source. +The strict-checkpoint refusal is also provider-backed. A correctly sealed AMR +checkpoint with an inconsistent dynamic accepted-ledger claim passes the real +`RestartV3` file reopen and static preflight, then fails only when the native +AMR provider validates the restored Program image. The test proves that the +active restart transaction restores fields, hierarchy, histories, clocks, +counters, run identity, and consumer cursors, and that the same provider can +successfully retry the unmodified checkpoint. + ## Gate modes The architecture CI runs: From 91b756bc150845111a32e35230985bf2bd700f71 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:06:23 +0200 Subject: [PATCH 16/62] test(m4): prove real writer transaction compensation --- .../test_external_component_package.py | 92 ++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) diff --git a/tests/python/integration/native_loader/test_external_component_package.py b/tests/python/integration/native_loader/test_external_component_package.py index fb341a7ce..87f623f3d 100644 --- a/tests/python/integration/native_loader/test_external_component_package.py +++ b/tests/python/integration/native_loader/test_external_component_package.py @@ -29,6 +29,7 @@ from pops.output import ( CoarseOnly, ConsumerGraph, ExternalWriter, ParallelMode, ScientificOutput, ) +from pops.runtime._consumer_transaction import ConsumerPublicationError from pops.runtime._runtime_consumers import RuntimeConsumerPublisher from pops.time import every, on_start @@ -430,10 +431,18 @@ def _load_example(): return module -def _writer_case(example, artifacts, *, adaptive: bool): +def _writer_case( + example, + artifacts, + *, + adaptive: bool, + paired_start_transaction: bool = False, +): from pops.layouts import Uniform from pops.output import SelectedLevels + if adaptive and paired_start_transaction: + raise ValueError("paired Writer transaction is a Uniform-only test route") core = example.build_authoring(output_root="unused") core.numerics.boundaries.add(example.build_transport_boundaries(core)) core.case.numerics(core.numerics, block=core.tracer) @@ -466,7 +475,11 @@ def _writer_case(example, artifacts, *, adaptive: bool): outputs.append(ScientificOutput( format=ExternalWriter( artifacts[-1], extension=".popsbin", mode=output_mode), - schedule=every(1, clock=core.program.clock), + schedule=( + on_start(clock=core.program.clock) + if paired_start_transaction + else every(1, clock=core.program.clock) + ), fields=(core.tracer_state,), levels=SelectedLevels(0, 1) if adaptive else CoarseOnly(), target="amr-writer" if adaptive else "uniform-writer", @@ -507,6 +520,81 @@ def _bind_writer_case(example, core, layout, artifacts, initial_state=None): return simulation +def test_real_writer_collision_compensates_the_complete_consumer_graph_transaction(tmp_path): + example = _load_example() + first = _compile_writer(tmp_path / "transaction-one", "transaction_writer_one") + second = _compile_writer(tmp_path / "transaction-two", "transaction_writer_two") + core, layout, initial_state = _writer_case( + example, + (first, second), + adaptive=False, + paired_start_transaction=True, + ) + runtime = _bind_writer_case( + example, + core, + layout, + (first, second), + initial_state, + ) + output_root = tmp_path / "transaction-output" + runtime._output_root = output_root + + accepted_before = { + "time": runtime.time(), + "macro_step": runtime.macro_step(), + "state": np.asarray( + runtime.state_global("tracer"), dtype=np.float64 + ).copy(), + "cursors": runtime.consumer_cursors.to_data(), + "reports": tuple(runtime._consumer_reports), + } + transactions = runtime._stage_consumers(at_start=True) + assert len(transactions) == 1 + transaction = transactions[0] + prepared = tuple(row[1] for row in transaction._prepared) + assert len(prepared) == 2 + targets = tuple(row.target for row in prepared) + assert all(target is not None for target in targets) + first_target, collision_target = targets + collision_bytes = b"pre-existing user-owned publication" + collision_target.write_bytes(collision_bytes) + + with pytest.raises(ConsumerPublicationError, match="FileExistsError") as failure: + transaction.accept() + + report = failure.value.report + assert report.status == "failed" + assert report.published == () + assert report.cursors.to_data() == accepted_before["cursors"] + assert len(report.staged_effects) == 2 + assert report.rolled_back_effects == tuple(reversed(report.staged_effects)) + assert not first_target.exists() + assert collision_target.read_bytes() == collision_bytes + assert not tuple(output_root.rglob(".*.writer-stage*")) + assert not tuple(output_root.rglob("*.component-published")) + assert runtime.time() == accepted_before["time"] + assert runtime.macro_step() == accepted_before["macro_step"] + assert np.array_equal( + np.asarray(runtime.state_global("tracer"), dtype=np.float64), + accepted_before["state"], + ) + assert runtime.consumer_cursors.to_data() == accepted_before["cursors"] + assert tuple(runtime._consumer_reports) == accepted_before["reports"] + + collision_target.unlink() + accepted_reports = runtime._fire_consumers(at_start=True) + assert len(accepted_reports) == 1 + assert accepted_reports[0].status == "accepted" + assert len(accepted_reports[0].published) == 2 + assert runtime.consumer_cursors.to_data() != accepted_before["cursors"] + published = tuple(sorted(output_root.rglob("*.popsbin"))) + assert len(published) == 2 + assert all("fields=1" in path.read_text(encoding="utf-8") for path in published) + assert not tuple(output_root.rglob(".*.writer-stage*")) + assert not tuple(output_root.rglob("*.component-published")) + + def test_qualified_writer_runs_through_uniform_and_amr_runtime_transactions(tmp_path): example = _load_example() first = _compile_writer(tmp_path / "source-one", "writer_one") From f439e1a5680002e6ba6e93d14900fad227c73073 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:06:23 +0200 Subject: [PATCH 17/62] gate(m4): select real consumer graph refusal --- tests/gates/m4_runtime_io.toml | 18 +++--- .../architecture/test_m4_runtime_io_gate.py | 58 ++++++++++++++++++- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 1f3a398f4..8aec17638 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -43,16 +43,6 @@ evidence_paths = [ "tests/python/integration/runtime/test_multi_layout_runtime.py", ] -[[deferred]] -issue = "ADC-685" -requirement = "consumer_graph" -polarity = "refusal" -reason = "The available transaction refusal uses handwritten publisher and prepared-publication fakes; M4 still needs the same refusal through a real writer or consumer." -evidence_paths = [ - "tests/python/unit/runtime/test_consumer_transactions.py", - "tests/python/integration/native_loader/test_external_component_package.py", -] - # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, # optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, @@ -234,6 +224,14 @@ kind = "pytest" target = "consumer_graph" nodeid = "tests/python/unit/runtime/test_consumer_transactions.py::test_graph_and_plan_are_semantic_and_insertion_order_independent" +[[check]] +issue = "ADC-685" +requirement = "consumer_graph" +polarity = "refusal" +kind = "pytest" +target = "consumer_graph" +nodeid = "tests/python/integration/native_loader/test_external_component_package.py::test_real_writer_collision_compensates_the_complete_consumer_graph_transaction" + [[check]] issue = "ADC-685" requirement = "accepted_publication" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 6775b2749..83a651567 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -38,8 +38,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 4 - assert len(data["check"]) >= 47 + assert len(data["deferred"]) == 3 + assert len(data["check"]) >= 48 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -58,7 +58,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): } == { ("ADC-684", "runtime_instance", "positive"), ("ADC-684", "runtime_instance", "refusal"), - ("ADC-685", "consumer_graph", "refusal"), ("ADC-687", "gate_execution", "positive"), } @@ -222,6 +221,59 @@ def test_m4_gate_pins_real_runtime_instance_and_positive_checkpoint_proofs(): ) not in selected +def test_m4_gate_pins_real_writer_refusal_without_publication_fakes(): + data, errors = _load_runner().audit_manifest(MANIFEST) + assert not errors + expected = { + "issue": "ADC-685", + "requirement": "consumer_graph", + "polarity": "refusal", + "kind": "pytest", + "target": "consumer_graph", + "nodeid": ( + "tests/python/integration/native_loader/" + "test_external_component_package.py::" + "test_real_writer_collision_compensates_the_complete_consumer_graph_transaction" + ), + } + assert expected in data["check"] + + path = ( + ROOT + / "tests/python/integration/native_loader/test_external_component_package.py" + ) + tree = ast.parse(path.read_text(encoding="utf-8")) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name + == "test_real_writer_collision_compensates_the_complete_consumer_graph_transaction" + ) + calls = { + ( + node.func.id + if isinstance(node.func, ast.Name) + else node.func.attr + if isinstance(node.func, ast.Attribute) + else "" + ) + for node in ast.walk(function) + if isinstance(node, ast.Call) + } + assert { + "_compile_writer", + "_bind_writer_case", + "_stage_consumers", + "accept", + "_fire_consumers", + } <= calls + names = {node.id for node in ast.walk(function) if isinstance(node, ast.Name)} + assert names.isdisjoint( + {"_Publisher", "_Prepared", "SimpleNamespace", "Mock", "MagicMock"} + ) + + def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors From cff76414bf438c91a65e89893e32f4a6d7409285 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:06:23 +0200 Subject: [PATCH 18/62] docs(m4): record real consumer graph compensation --- docs/design/m4-conformance-gate.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 08e8ef237..f2f342eeb 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -19,7 +19,8 @@ The source audit already authenticates real proofs for: headers and schedulers, typed component dispatch, and fail-closed unbound native interfaces; - real Uniform and AMR writer transactions, a real multi-layout transfer, - and a positive multi-layout checkpoint/restart; + a real two-writer collision with complete ConsumerGraph compensation, and a + positive multi-layout checkpoint/restart; - accepted scientific publication, diagnostics, two-rank collective HDF5, and a two-rank PVD/PVTU/rank-VTU hierarchy reopened by native VTK readers. @@ -29,8 +30,7 @@ that must not be mistaken for closure. They currently cover: - a CI lane that installs every mandatory dependency, including VTK, and executes every selected pytest and CTest proof with zero skips; -- ConsumerGraph and composite runtime rollback without handwritten publishers - or injected failure wrappers; +- composite runtime rollback without an injected failure wrapper; - complete Uniform, AMR, and multi-layout public-contract/report parity. ## Exact output evidence @@ -68,6 +68,15 @@ active restart transaction restores fields, hierarchy, histories, clocks, counters, run identity, and consumer cursors, and that the same provider can successfully retry the unmodified checkpoint. +The ConsumerGraph refusal is likewise provider-backed. Two separately +qualified native Writer components are compiled and staged in one transaction. +After the first Writer publishes, a pre-existing user-owned target makes the +second Writer fail at the runtime's atomic publication link. The transaction +must compensate the first artifact, preserve the colliding file byte-for-byte, +remove every private staging path, retain the exact accepted numerical state +and consumer cursors, and then publish both Writers on a clean retry. The +selected proof contains no handwritten publisher or prepared-publication fake. + ## Gate modes The architecture CI runs: From 440c9cccc70bcb9345eb4ae722ace38c937ffe30 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:12:23 +0200 Subject: [PATCH 19/62] test(m4): prove prepared runtime component rollback --- .../test_external_field_solver_runtime.py | 83 +++++++++++++++++-- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/tests/python/integration/native_loader/test_external_field_solver_runtime.py b/tests/python/integration/native_loader/test_external_field_solver_runtime.py index 8b44a208e..66c2094d5 100644 --- a/tests/python/integration/native_loader/test_external_field_solver_runtime.py +++ b/tests/python/integration/native_loader/test_external_field_solver_runtime.py @@ -261,10 +261,13 @@ def _solver_source(manifest, *, solution_expression="7.0"): ''' -def _nonfinite_solver_source(manifest): +def _first_nonfinite_solver_source(manifest): return _solver_source( manifest, - solution_expression="std::numeric_limits::quiet_NaN()", + solution_expression=( + "state->solve_count == 1 " + "? std::numeric_limits::quiet_NaN() : 7.0" + ), ) @@ -340,7 +343,7 @@ def test_external_field_pair_executes_and_reports_materialized_topology(tmp_path assert simulation.inspect().to_dict()["instance"]["field_providers"] == providers -def test_external_field_solver_rejects_converged_nonfinite_solution_without_publishing( +def test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retries( tmp_path, ): topology = _component( @@ -348,7 +351,7 @@ def test_external_field_solver_rejects_converged_nonfinite_solution_without_publ source_factory=_topology_source) solver = _component( tmp_path, name="nonfinite-solver", interface=interfaces.FieldSolver, - source_factory=_nonfinite_solver_source, + source_factory=_first_nonfinite_solver_source, manifest_parameters=({"name": "answer", "kind": "runtime"},), instance_parameters={"answer": 7}) provider = ExternalFieldSolver( @@ -365,8 +368,25 @@ def test_external_field_solver_rejects_converged_nonfinite_solution_without_publ initial_state={"material": np.ones((1, 8, 8), dtype=np.float64)}, ) slot, = simulation.field_provider_slots() - before = np.asarray(simulation.field_potential_global(slot)).copy() - assert before.size == 64 and np.all(before == 0.0) + accepted_before = { + "time": simulation.time(), + "macro_step": simulation.macro_step(), + "state": np.asarray( + simulation.state_global("material"), dtype=np.float64 + ).copy(), + "potential": np.asarray( + simulation.field_potential_global(slot), dtype=np.float64 + ).copy(), + "cursors": simulation.consumer_cursors.to_data(), + "reports": tuple(simulation._consumer_reports), + "temporal": json.dumps( + simulation._executor._temporal_restart_state.to_data(), + sort_keys=True, + ), + "providers": simulation.inspect().to_dict()["instance"]["field_providers"], + } + assert accepted_before["potential"].size == 64 + assert np.all(accepted_before["potential"] == 0.0) with pytest.raises( RuntimeError, @@ -374,6 +394,53 @@ def test_external_field_solver_rejects_converged_nonfinite_solution_without_publ ): pops.run(simulation, t_end=1.0e-4, max_steps=1) - after = np.asarray(simulation.field_potential_global(slot)) - np.testing.assert_array_equal(after, before) + np.testing.assert_array_equal( + np.asarray(simulation.state_global("material"), dtype=np.float64), + accepted_before["state"], + ) + after = np.asarray(simulation.field_potential_global(slot), dtype=np.float64) + np.testing.assert_array_equal(after, accepted_before["potential"]) assert np.all(np.isfinite(after)) + assert simulation.time() == accepted_before["time"] + assert simulation.macro_step() == accepted_before["macro_step"] + assert simulation.consumer_cursors.to_data() == accepted_before["cursors"] + assert tuple(simulation._consumer_reports) == accepted_before["reports"] + assert json.dumps( + simulation._executor._temporal_restart_state.to_data(), + sort_keys=True, + ) == accepted_before["temporal"] + assert ( + simulation.inspect().to_dict()["instance"]["field_providers"] + == accepted_before["providers"] + ) + failed = simulation._executor._last_step_transaction_report + assert (failed.status, failed.phase, failed.action) == ( + "failed", + "solve", + "fail_run", + ) + assert failed.committed_effects == () + assert failed.staged_effects + assert failed.rolled_back_effects == failed.staged_effects + + retry = pops.run(simulation, t_end=1.0e-4, max_steps=1) + assert retry.accepted_steps == 1 + assert simulation.time() == 1.0e-4 + assert simulation.macro_step() == 1 + np.testing.assert_array_equal( + np.asarray(simulation.state_global("material"), dtype=np.float64), + accepted_before["state"], + ) + potential = np.asarray(simulation.field_potential_global(slot), dtype=np.float64) + assert potential.size == 64 + assert np.all(np.isfinite(potential)) + assert np.all(potential == 0.0) + accepted = simulation._executor._last_step_transaction_report + assert (accepted.status, accepted.phase, accepted.action) == ( + "accepted", + "commit", + "commit", + ) + assert accepted.staged_effects + assert accepted.committed_effects == accepted.staged_effects + assert accepted.rolled_back_effects == () From b0896990032951a2ba1c1aa34eab8679b870e210 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:12:23 +0200 Subject: [PATCH 20/62] gate(m4): select real runtime refusal --- tests/gates/m4_runtime_io.toml | 17 ++-- .../architecture/test_m4_runtime_io_gate.py | 80 ++++++++++++++++++- 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 8aec17638..83aa60801 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -34,15 +34,6 @@ evidence_paths = [ "tests/python/integration/runtime/test_shared_interface_runtime.py", ] -[[deferred]] -issue = "ADC-684" -requirement = "runtime_instance" -polarity = "refusal" -reason = "Composite step rollback is only exercised through an injected FailFirstStep wrapper; a failure from a real prepared runtime component is still required." -evidence_paths = [ - "tests/python/integration/runtime/test_multi_layout_runtime.py", -] - # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, # optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, @@ -200,6 +191,14 @@ kind = "pytest" target = "runtime_instance" nodeid = "tests/python/integration/runtime/test_shared_interface_runtime.py::test_runtime_instance_executes_one_two_sided_shared_flux" +[[check]] +issue = "ADC-684" +requirement = "runtime_instance" +polarity = "refusal" +kind = "pytest" +target = "runtime_instance" +nodeid = "tests/python/integration/native_loader/test_external_field_solver_runtime.py::test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retries" + [[check]] issue = "ADC-684" requirement = "external_transfer" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 83a651567..7930a0dac 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -38,8 +38,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 3 - assert len(data["check"]) >= 48 + assert len(data["deferred"]) == 2 + assert len(data["check"]) >= 49 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -57,7 +57,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): for row in data["deferred"] } == { ("ADC-684", "runtime_instance", "positive"), - ("ADC-684", "runtime_instance", "refusal"), ("ADC-687", "gate_execution", "positive"), } @@ -162,6 +161,18 @@ def test_m4_gate_pins_real_runtime_instance_and_positive_checkpoint_proofs(): "test_runtime_instance_executes_one_two_sided_shared_flux" ), } in checks + assert { + "issue": "ADC-684", + "requirement": "runtime_instance", + "polarity": "refusal", + "kind": "pytest", + "target": "runtime_instance", + "nodeid": ( + "tests/python/integration/native_loader/" + "test_external_field_solver_runtime.py::" + "test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retries" + ), + } in checks assert { "issue": "ADC-684", "requirement": "external_transfer", @@ -221,6 +232,69 @@ def test_m4_gate_pins_real_runtime_instance_and_positive_checkpoint_proofs(): ) not in selected +def test_m4_runtime_refusal_uses_a_real_prepared_component_without_step_wrapper(): + data, errors = _load_runner().audit_manifest(MANIFEST) + assert not errors + nodeid = ( + "tests/python/integration/native_loader/" + "test_external_field_solver_runtime.py::" + "test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retries" + ) + assert [ + row + for row in data["check"] + if row.get("nodeid") == nodeid + ] == [{ + "issue": "ADC-684", + "requirement": "runtime_instance", + "polarity": "refusal", + "kind": "pytest", + "target": "runtime_instance", + "nodeid": nodeid, + }] + + source_path = ( + ROOT + / "tests/python/integration/native_loader/test_external_field_solver_runtime.py" + ) + tree = ast.parse(source_path.read_text(encoding="utf-8")) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name + == "test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retries" + ) + calls = { + ( + node.func.id + if isinstance(node.func, ast.Name) + else node.func.attr + if isinstance(node.func, ast.Attribute) + else "" + ) + for node in ast.walk(function) + if isinstance(node, ast.Call) + } + assert {"_component", "compile", "bind", "run"} <= calls + names = {node.id for node in ast.walk(function) if isinstance(node, ast.Name)} + assert names.isdisjoint( + { + "FailFirstStep", + "_RankLocalFailureTarget", + "Mock", + "MagicMock", + "SimpleNamespace", + } + ) + attributes = { + node.attr for node in ast.walk(function) if isinstance(node, ast.Attribute) + } + assert "_native_step_target" not in attributes + assert "_engines" not in attributes + assert not any(isinstance(node, ast.ClassDef) for node in ast.walk(function)) + + def test_m4_gate_pins_real_writer_refusal_without_publication_fakes(): data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors From 8c6cf9ce0e9b3cfff4e27bed28b01a1ef928aa8d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:12:24 +0200 Subject: [PATCH 21/62] docs(m4): record prepared component rollback --- docs/design/m4-conformance-gate.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index f2f342eeb..3701edc81 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -21,6 +21,8 @@ The source audit already authenticates real proofs for: - real Uniform and AMR writer transactions, a real multi-layout transfer, a real two-writer collision with complete ConsumerGraph compensation, and a positive multi-layout checkpoint/restart; +- a prepared native FieldSolver whose invalid first result is refused through + RuntimeInstance with exact accepted-state rollback and a successful retry; - accepted scientific publication, diagnostics, two-rank collective HDF5, and a two-rank PVD/PVTU/rank-VTU hierarchy reopened by native VTK readers. @@ -30,7 +32,6 @@ that must not be mistaken for closure. They currently cover: - a CI lane that installs every mandatory dependency, including VTK, and executes every selected pytest and CTest proof with zero skips; -- composite runtime rollback without an injected failure wrapper; - complete Uniform, AMR, and multi-layout public-contract/report parity. ## Exact output evidence @@ -77,6 +78,17 @@ remove every private staging path, retain the exact accepted numerical state and consumer cursors, and then publish both Writers on a clean retry. The selected proof contains no handwritten publisher or prepared-publication fake. +The RuntimeInstance refusal no longer relies on `FailFirstStep`. A qualified +native FieldTopology/FieldSolver pair is packaged, compiled, resolved, bound, +and prepared through the production component ABI. Its first solve reports +convergence while returning non-finite values, so the production field +validation fails inside the native Program step. RuntimeInstance must restore +the conservative state, field potential, accepted clock, macro-step, temporal +authority, consumer cursors, reports, and provider evidence exactly. The same +prepared component then returns a finite result and the unchanged +RuntimeInstance accepts the retry. The selected test defines no step wrapper +and never replaces a native engine or step target. + ## Gate modes The architecture CI runs: From 0351f92904b340fd10340674695929166d6ee008 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:22:07 +0200 Subject: [PATCH 22/62] test(m4): keep prepared refusal provider stateless --- .../test_external_field_solver_runtime.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/python/integration/native_loader/test_external_field_solver_runtime.py b/tests/python/integration/native_loader/test_external_field_solver_runtime.py index 66c2094d5..f1f40d1f0 100644 --- a/tests/python/integration/native_loader/test_external_field_solver_runtime.py +++ b/tests/python/integration/native_loader/test_external_field_solver_runtime.py @@ -147,13 +147,21 @@ def _topology_source(manifest): ''' -def _solver_source(manifest, *, solution_expression="7.0"): +def _solver_source( + manifest, + *, + solution_expression="7.0", + solve_count_statement="++state->solve_count;", + iterations_expression="state->solve_count", + extra_includes="", +): expected_parameters_json = json.dumps( {"answer": 7}, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return f'''#include #include #include #include +{extra_includes} namespace {{ struct State {{ int prepare_count; int solve_count; }}; @@ -199,7 +207,7 @@ def _solver_source(manifest, *, solution_expression="7.0"): !request->boundary_contract_json || std::strstr(request->boundary_contract_json, "identity") == nullptr) return 3; - ++state->solve_count; + {solve_count_statement} for (std::size_t local = 0; local < request->local_patch_count; ++local) {{ const auto& patch = request->local_patches[local]; if (patch.metadata_index >= request->topology.patch_count || @@ -227,7 +235,7 @@ def _solver_source(manifest, *, solution_expression="7.0"): }} report->status = POPS_SOLVE_SOLVED_V2; report->action = POPS_SOLVE_ACTION_NONE_V2; - report->iterations = state->solve_count; + report->iterations = {iterations_expression}; report->relative_residual = 0.0; report->reference_residual_norm = 1.0; report->residual_norm = 0.0; @@ -261,13 +269,17 @@ def _solver_source(manifest, *, solution_expression="7.0"): ''' -def _first_nonfinite_solver_source(manifest): +def _externally_faulted_solver_source(manifest, fault_marker): return _solver_source( manifest, solution_expression=( - "state->solve_count == 1 " + "std::filesystem::exists(%s) " "? std::numeric_limits::quiet_NaN() : 7.0" + % json.dumps(str(fault_marker)) ), + solve_count_statement="", + iterations_expression="1", + extra_includes="#include ", ) @@ -346,12 +358,16 @@ def test_external_field_pair_executes_and_reports_materialized_topology(tmp_path def test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retries( tmp_path, ): + fault_marker = tmp_path / "external-field-solver-fault" + fault_marker.write_text("force a non-finite component result", encoding="utf-8") topology = _component( tmp_path, name="nonfinite-topology", interface=interfaces.FieldTopology, source_factory=_topology_source) solver = _component( tmp_path, name="nonfinite-solver", interface=interfaces.FieldSolver, - source_factory=_first_nonfinite_solver_source, + source_factory=lambda manifest: _externally_faulted_solver_source( + manifest, fault_marker + ), manifest_parameters=({"name": "answer", "kind": "runtime"},), instance_parameters={"answer": 7}) provider = ExternalFieldSolver( @@ -423,6 +439,8 @@ def test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retr assert failed.staged_effects assert failed.rolled_back_effects == failed.staged_effects + assert fault_marker.is_file() + fault_marker.unlink() retry = pops.run(simulation, t_end=1.0e-4, max_steps=1) assert retry.accepted_steps == 1 assert simulation.time() == 1.0e-4 From f83a07d207e557c1ad42fd3ce885b77714dca072 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:22:11 +0200 Subject: [PATCH 23/62] docs(m4): clarify stateless prepared refusal --- docs/design/m4-conformance-gate.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 3701edc81..48669bf10 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -80,14 +80,16 @@ selected proof contains no handwritten publisher or prepared-publication fake. The RuntimeInstance refusal no longer relies on `FailFirstStep`. A qualified native FieldTopology/FieldSolver pair is packaged, compiled, resolved, bound, -and prepared through the production component ABI. Its first solve reports -convergence while returning non-finite values, so the production field -validation fails inside the native Program step. RuntimeInstance must restore +and prepared through the production component ABI. An authenticated external +fault marker makes its solve report convergence while returning non-finite +values, so the production field validation fails inside the native Program +step. RuntimeInstance must restore the conservative state, field potential, accepted clock, macro-step, temporal -authority, consumer cursors, reports, and provider evidence exactly. The same -prepared component then returns a finite result and the unchanged -RuntimeInstance accepts the retry. The selected test defines no step wrapper -and never replaces a native engine or step target. +authority, consumer cursors, reports, and provider evidence exactly. The +component's prepared state is not mutated by this failure. After the external +fault is removed, the same prepared component returns a finite result and the +unchanged RuntimeInstance accepts the retry. The selected test defines no step +wrapper and never replaces a native engine or step target. ## Gate modes From 5076f6295df6394cce2f7d5e89dd8bed5abfbe2f Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:25:01 +0200 Subject: [PATCH 24/62] runtime(m4): unify multi-layout program reports --- python/pops/runtime/_multi_layout_executor.py | 196 ++++++++++++++++++ python/pops/runtime/inspection.py | 10 +- 2 files changed, 204 insertions(+), 2 deletions(-) diff --git a/python/pops/runtime/_multi_layout_executor.py b/python/pops/runtime/_multi_layout_executor.py index 9731e5b45..890e418bf 100644 --- a/python/pops/runtime/_multi_layout_executor.py +++ b/python/pops/runtime/_multi_layout_executor.py @@ -389,6 +389,202 @@ def executor_for_block(self, block: str) -> Any: def block_names(self) -> tuple[str, ...]: return tuple(self._block_layouts) + def _ordered_program_reports(self) -> tuple[tuple[Any, tuple[str, ...], Any], ...]: + """Return one authenticated report for every independently installed Program.""" + from pops.runtime.program_report import ProgramRuntimeReport + + layout_programs = tuple(self._plan.artifact.layout_programs) + layout_ids = tuple(row.layout_id for row in layout_programs) + if layout_ids != tuple(self._engines): + raise RuntimeError( + "multi-layout Program reports differ from the installed layout order" + ) + rows = [] + for layout_program in layout_programs: + engine = self._engines[layout_program.layout_id] + report = engine.program_report() + if type(report) is not ProgramRuntimeReport: + raise TypeError( + "multi-layout child returned a non-canonical ProgramRuntimeReport" + ) + if not report.installed or not isinstance(report.program_hash, str) or not ( + report.program_hash + ): + raise RuntimeError( + "multi-layout child has no authenticated installed Program" + ) + engine_blocks = tuple(engine.block_names()) + if ( + len(engine_blocks) != len(set(engine_blocks)) + or set(engine_blocks) != set(layout_program.block_names) + ): + raise RuntimeError( + "multi-layout child block registry differs from its compiled partition" + ) + local_map = tuple(report.block_map) + if ( + len(local_map) != len(engine_blocks) + or any( + isinstance(index, bool) + or not isinstance(index, int) + or index < 0 + for index in local_map + ) + or tuple(sorted(local_map)) != tuple(range(len(engine_blocks))) + ): + raise RuntimeError( + "multi-layout child Program block map is not an exact local bijection" + ) + parameter_blocks = tuple(row.get("program_block") for row in report.params) + if ( + len(parameter_blocks) != len(local_map) + or any( + isinstance(index, bool) + or not isinstance(index, int) + for index in parameter_blocks + ) + or tuple(sorted(parameter_blocks)) != tuple(range(len(local_map))) + ): + raise RuntimeError( + "multi-layout child Program parameter report is not exact" + ) + rows.append((layout_program, engine_blocks, report)) + return tuple(rows) + + def program_report(self) -> Any: + """Aggregate every real child Program without inventing a single native engine.""" + from pops.identity import make_identity + from pops.runtime.program_report import ProgramRuntimeReport + + children = self._ordered_program_reports() + global_blocks = self.block_names() + global_block_indices = { + name: index for index, name in enumerate(global_blocks) + } + if len(global_block_indices) != len(global_blocks): + raise RuntimeError("multi-layout global block registry contains a duplicate") + + block_map = [] + params = [] + diagnostics = {} + histories = [] + cache = [] + clocks = [] + level_relations = [] + flux_ledger = [] + synchronization = [] + program_offset = 0 + + qualified_row_sets = ( + ("history", histories, "histories"), + ("clock", clocks, "clocks"), + ("level relation", level_relations, "level_relations"), + ("flux ledger", flux_ledger, "flux_ledger"), + ("synchronization", synchronization, "synchronization"), + ) + for layout_program, engine_blocks, report in children: + layout_id = layout_program.layout_id + local_map = tuple(report.block_map) + local_program_blocks = tuple( + engine_blocks[local_system_index] + for local_system_index in local_map + ) + block_map.extend( + global_block_indices[name] for name in local_program_blocks + ) + + for raw in report.params: + row = dict(raw) + local_program_block = row["program_block"] + if "layout_id" in row or "block" in row: + raise RuntimeError( + "multi-layout child parameter report contains reserved qualifiers" + ) + row["program_block"] = program_offset + local_program_block + row["layout_id"] = layout_id + row["block"] = local_program_blocks[local_program_block] + params.append(row) + + for name, value in report.diagnostics.items(): + if not isinstance(name, str) or not name: + raise RuntimeError( + "multi-layout child diagnostic name must be non-empty" + ) + diagnostics["%s::%s" % (layout_id, name)] = value + + for label, destination, attribute in qualified_row_sets: + for raw in getattr(report, attribute): + row = dict(raw) + if "layout_id" in row: + raise RuntimeError( + "multi-layout child %s report contains a reserved qualifier" + % label + ) + row["layout_id"] = layout_id + destination.append(row) + + for raw in report.cache: + row = dict(raw) + if "layout_id" in row or "layout_node_id" in row: + raise RuntimeError( + "multi-layout child cache report contains reserved qualifiers" + ) + local_node_id = row.get("node_id") + if ( + isinstance(local_node_id, bool) + or not isinstance(local_node_id, int) + or local_node_id < 0 + ): + raise RuntimeError( + "multi-layout child cache report has an invalid node identity" + ) + row["layout_id"] = layout_id + row["layout_node_id"] = local_node_id + row["node_id"] = len(cache) + cache.append(row) + program_offset += len(local_map) + + program_hash = make_identity( + "multi-layout-program", + [ + { + "layout_id": layout_program.layout_id, + "layout_program_identity": layout_program.identity.token, + "installed_program_hash": report.program_hash, + } + for layout_program, _engine_blocks, report in children + ], + ).hexdigest + return ProgramRuntimeReport( + installed=True, + program_hash=program_hash, + step_transaction=_common_exact( + (report.step_transaction for _row, _blocks, report in children), + where="multi-layout Program transaction report", + ), + block_map=block_map, + params=params, + diagnostics=diagnostics, + histories=histories, + cache=cache, + profiler=_common_exact( + (report.profiler for _row, _blocks, report in children), + where="multi-layout Program profiler report", + ), + clocks=clocks, + level_relations=level_relations, + flux_ledger=flux_ledger, + synchronization=synchronization, + temporal=_common_exact( + (report.temporal for _row, _blocks, report in children), + where="multi-layout Program temporal report", + ), + ) + + def installed_program_hash(self) -> str: + """Return the domain-separated identity of the exact installed Program set.""" + return self.program_report().program_hash + def state_global(self, block: str) -> Any: return self.executor_for_block(block).state_global(block) diff --git a/python/pops/runtime/inspection.py b/python/pops/runtime/inspection.py index dcc71a420..6b2160c42 100644 --- a/python/pops/runtime/inspection.py +++ b/python/pops/runtime/inspection.py @@ -204,8 +204,14 @@ def _program(sim: Any) -> Any: ("installed"/"hash") are preserved, with the richer transaction/block-map/parameter/history/cache summary folded in from the same report.""" - from pops.runtime.program_report import build_program_report - report = build_program_report(sim) + from pops.runtime.program_report import ProgramRuntimeReport, build_program_report + + provider = getattr(sim, "program_report", None) + report = provider() if callable(provider) else build_program_report(sim) + if type(report) is not ProgramRuntimeReport: + raise TypeError( + "runtime inspection requires the canonical ProgramRuntimeReport" + ) return { "installed": report.installed, "hash": report.program_hash, From 59856acc7ce1c045e42a972a3b9d2f6b286b86f4 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:25:46 +0200 Subject: [PATCH 25/62] runtime(m4): expose complete program inspection --- python/pops/runtime/inspection.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/pops/runtime/inspection.py b/python/pops/runtime/inspection.py index 6b2160c42..ae66e5bba 100644 --- a/python/pops/runtime/inspection.py +++ b/python/pops/runtime/inspection.py @@ -218,9 +218,15 @@ def _program(sim: Any) -> Any: "step_transaction": dict(report.step_transaction), "block_map": list(report.block_map), "params": [dict(row) for row in report.params], + "diagnostics": dict(report.diagnostics), "histories": [dict(row) for row in report.histories], "cache": [dict(row) for row in report.cache], "profiler": dict(report.profiler), + "clocks": [dict(row) for row in report.clocks], + "level_relations": [dict(row) for row in report.level_relations], + "flux_ledger": [dict(row) for row in report.flux_ledger], + "synchronization": [dict(row) for row in report.synchronization], + "temporal": dict(report.temporal), } From 7ab57de20b28fab2f145def5d8dfd8ae90630486 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:25:46 +0200 Subject: [PATCH 26/62] test(m4): prove runtime instance report parity --- .../runtime/test_multi_layout_runtime.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/tests/python/integration/runtime/test_multi_layout_runtime.py b/tests/python/integration/runtime/test_multi_layout_runtime.py index 48a72f56c..80511e16e 100644 --- a/tests/python/integration/runtime/test_multi_layout_runtime.py +++ b/tests/python/integration/runtime/test_multi_layout_runtime.py @@ -372,6 +372,160 @@ def __getattr__(self, name): assert len(inspection.instance["installed_components"]) == 1 +def test_uniform_amr_and_multi_layout_share_complete_runtime_instance_contract( + compiled_multi_layout, +): + from pops.runtime._runtime_instance import RuntimeInstance + from tests.python.integration.runtime.test_dsl_runtime_params import ( + DT as SINGLE_LAYOUT_DT, + _resolved_analytic_initial_parameter_case, + ) + + executions = [] + for label, target in (("uniform", "system"), ("amr", "amr_system")): + resolved, amplitude = _resolved_analytic_initial_parameter_case( + target=target + ) + artifact = pops.compile(resolved) + runtime = pops.bind( + artifact, + params={amplitude: 1.0}, + resources={"execution_context": artifact_execution_context(artifact)}, + ) + report = pops.run( + runtime, + t_end=SINGLE_LAYOUT_DT, + max_steps=1, + console=False, + ) + executions.append((label, runtime, artifact, report)) + + ( + multi, + multi_artifact, + coarse_layout_id, + fine_layout_id, + mapping_id, + _u_fine, + _u_coarse, + ) = _bind( + compiled_multi_layout + ) + multi_report = pops.run( + multi, + t_end=DT, + max_steps=1, + console=False, + ) + executions.append(("multi-layout", multi, multi_artifact, multi_report)) + + expected_layout_counts = {"uniform": 1, "amr": 1, "multi-layout": 2} + expected_level_counts = {"uniform": 1, "amr": 2, "multi-layout": 1} + expected_runtime_kinds = { + "uniform": "uniform", + "amr": "adaptive", + "multi-layout": "uniform", + } + expected_final_times = { + "uniform": SINGLE_LAYOUT_DT, + "amr": SINGLE_LAYOUT_DT, + "multi-layout": DT, + } + schemas = set() + + for label, runtime, artifact, report in executions: + assert type(runtime) is RuntimeInstance + assert type(report) is pops.RunReport + assert report.accepted_steps == 1 + assert report.rejected_steps == 0 + assert report.final_time == expected_final_times[label] + assert report.final_macro_step == 1 + assert report.stop_reason is pops.RunStopReason.TARGET_TIME_REACHED + assert report.run_identity == runtime.last_run_identity + assert report.bind_identity == runtime.bind_identity + assert report.execution_identity == runtime._execution_context.identity + assert report.artifact_identity == artifact.artifact_identity + assert report.artifact_identity == runtime.bound_snapshot.artifact_identity + assert report.field_providers == () + assert runtime.time() == report.final_time + assert runtime.macro_step() == report.final_macro_step + assert runtime.n_levels() == expected_level_counts[label] + + inspection = runtime.inspect().to_dict() + instance = inspection["instance"] + program = runtime.program_report().to_dict() + report_data = report.to_data() + assert inspection["runtime"] == expected_runtime_kinds[label] + assert inspection["clock"] == { + "time": runtime.time(), + "macro_step": runtime.macro_step(), + } + assert inspection["blocks"] == list(runtime.block_names()) + assert inspection["bound_snapshot"] == runtime.bound_snapshot.to_dict() + assert instance["bind_identity"] == runtime.bind_identity.to_data() + assert instance["artifact_identity"] == artifact.artifact_identity.to_data() + assert instance["consumer_graph"] == runtime.consumer_graph.to_data() + assert instance["consumer_cursors"] == runtime.consumer_cursors.to_data() + assert instance["last_run_identity"] == report.run_identity.to_data() + assert len(instance["layout_plan"]["layouts"]) == expected_layout_counts[label] + assert program["installed"] is True + assert program["program_hash"] == runtime.installed_program_hash() + assert inspection["program"]["installed"] == program["installed"] + assert inspection["program"]["hash"] == program["program_hash"] + for name in ( + "step_transaction", + "block_map", + "params", + "diagnostics", + "histories", + "cache", + "profiler", + "clocks", + "level_relations", + "flux_ledger", + "synchronization", + "temporal", + ): + assert inspection["program"][name] == program[name] + assert all( + np.isfinite(runtime.integral(block)) + for block in runtime.block_names() + ) + + native_transaction = runtime._executor._last_step_transaction_report + assert ( + native_transaction.status, + native_transaction.phase, + native_transaction.action, + ) == ("accepted", "commit", "commit") + assert native_transaction.staged_effects + assert ( + native_transaction.committed_effects + == native_transaction.staged_effects + ) + assert native_transaction.rolled_back_effects == () + schemas.add( + ( + tuple(sorted(report_data)), + tuple(sorted(inspection)), + tuple(sorted(instance)), + tuple(sorted(program)), + ) + ) + + assert len(schemas) == 1 + multi_program = multi.program_report() + assert len(multi_program.program_hash) == 64 + assert tuple(sorted(multi_program.block_map)) == (0, 1) + assert {row["program_block"] for row in multi_program.params} == {0, 1} + assert {row["block"] for row in multi_program.params} == {"coarse", "tracer"} + assert {row["layout_id"] for row in multi_program.params} == { + coarse_layout_id, + fine_layout_id, + } + assert multi._executor.mapping_report() == {mapping_id: 1} + + def test_multi_layout_checkpoint_restart_restores_every_layout_and_mapping_count( compiled_multi_layout, tmp_path ): From af658ebfd7fa3de80077013869560977f267c5fe Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:27:23 +0200 Subject: [PATCH 27/62] gate(m4): select complete runtime instance proof --- tests/gates/m4_runtime_io.toml | 19 ++--- .../architecture/test_m4_runtime_io_gate.py | 80 ++++++++++++++++++- 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 83aa60801..a58da0936 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -23,17 +23,6 @@ evidence_paths = [ "scripts/run_m4_gate.py", ] -[[deferred]] -issue = "ADC-684" -requirement = "runtime_instance" -polarity = "positive" -reason = "Uniform, AMR, and multi-layout executions are covered separately and only partially inspect reports; one complete public-contract and report-parity proof is still absent." -evidence_paths = [ - "tests/python/integration/native_loader/test_external_component_package.py", - "tests/python/integration/runtime/test_multi_layout_runtime.py", - "tests/python/integration/runtime/test_shared_interface_runtime.py", -] - # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, # optional imports, skip/xfail, non-exact CTest selectors, duplicate proofs, @@ -191,6 +180,14 @@ kind = "pytest" target = "runtime_instance" nodeid = "tests/python/integration/runtime/test_shared_interface_runtime.py::test_runtime_instance_executes_one_two_sided_shared_flux" +[[check]] +issue = "ADC-684" +requirement = "runtime_instance" +polarity = "positive" +kind = "pytest" +target = "runtime_instance" +nodeid = "tests/python/integration/runtime/test_multi_layout_runtime.py::test_uniform_amr_and_multi_layout_share_complete_runtime_instance_contract" + [[check]] issue = "ADC-684" requirement = "runtime_instance" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 7930a0dac..b1806cb85 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -38,8 +38,8 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 2 - assert len(data["check"]) >= 49 + assert len(data["deferred"]) == 1 + assert len(data["check"]) == 50 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -56,7 +56,6 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): (row["issue"], row["requirement"], row["polarity"]) for row in data["deferred"] } == { - ("ADC-684", "runtime_instance", "positive"), ("ADC-687", "gate_execution", "positive"), } @@ -161,6 +160,17 @@ def test_m4_gate_pins_real_runtime_instance_and_positive_checkpoint_proofs(): "test_runtime_instance_executes_one_two_sided_shared_flux" ), } in checks + assert { + "issue": "ADC-684", + "requirement": "runtime_instance", + "polarity": "positive", + "kind": "pytest", + "target": "runtime_instance", + "nodeid": ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_uniform_amr_and_multi_layout_share_complete_runtime_instance_contract" + ), + } in checks assert { "issue": "ADC-684", "requirement": "runtime_instance", @@ -295,6 +305,70 @@ def test_m4_runtime_refusal_uses_a_real_prepared_component_without_step_wrapper( assert not any(isinstance(node, ast.ClassDef) for node in ast.walk(function)) +def test_m4_runtime_positive_compiles_all_layout_kinds_without_test_doubles(): + data, errors = _load_runner().audit_manifest(MANIFEST) + assert not errors + nodeid = ( + "tests/python/integration/runtime/test_multi_layout_runtime.py::" + "test_uniform_amr_and_multi_layout_share_complete_runtime_instance_contract" + ) + assert [ + row for row in data["check"] if row.get("nodeid") == nodeid + ] == [{ + "issue": "ADC-684", + "requirement": "runtime_instance", + "polarity": "positive", + "kind": "pytest", + "target": "runtime_instance", + "nodeid": nodeid, + }] + + path = ROOT / "tests/python/integration/runtime/test_multi_layout_runtime.py" + tree = ast.parse(path.read_text(encoding="utf-8")) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name + == "test_uniform_amr_and_multi_layout_share_complete_runtime_instance_contract" + ) + calls = { + ( + node.func.id + if isinstance(node.func, ast.Name) + else node.func.attr + if isinstance(node.func, ast.Attribute) + else "" + ) + for node in ast.walk(function) + if isinstance(node, ast.Call) + } + assert {"compile", "bind", "run", "program_report", "inspect", "integral"} <= calls + labels = { + node.value + for node in ast.walk(function) + if isinstance(node, ast.Constant) and isinstance(node.value, str) + } + assert {"uniform", "amr", "multi-layout"} <= labels + names = {node.id for node in ast.walk(function) if isinstance(node, ast.Name)} + assert names.isdisjoint( + { + "FailFirstStep", + "_RankLocalFailureTarget", + "Mock", + "MagicMock", + "SimpleNamespace", + "monkeypatch", + } + ) + attributes = { + node.attr for node in ast.walk(function) if isinstance(node, ast.Attribute) + } + assert "_native_step_target" not in attributes + assert "_engines" not in attributes + assert not any(isinstance(node, ast.ClassDef) for node in ast.walk(function)) + + def test_m4_gate_pins_real_writer_refusal_without_publication_fakes(): data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors From 7425d1920c3a19d43975ee44ff2b962c3eca5345 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:27:23 +0200 Subject: [PATCH 28/62] docs(m4): record runtime instance report parity --- docs/design/m4-conformance-gate.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 48669bf10..e3c2c473c 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -3,7 +3,8 @@ The current status is **AUDITED OPEN**. The ledger in `tests/gates/m4_runtime_io.toml` records exact executable evidence for ADC-679 through ADC-687 and exact deferred gaps. It deliberately does not -claim M4 closure while any `[[deferred]]` row remains. +claim M4 closure while any `[[deferred]]` row remains. The current ledger has +exactly 50 executable checks and one deferred requirement. The source audit already authenticates real proofs for: @@ -21,6 +22,9 @@ The source audit already authenticates real proofs for: - real Uniform and AMR writer transactions, a real multi-layout transfer, a real two-writer collision with complete ConsumerGraph compensation, and a positive multi-layout checkpoint/restart; +- one complete RuntimeInstance contract proof across compiled Uniform, AMR, + and multi-layout execution, including RunReport and Program/inspection + parity; - a prepared native FieldSolver whose invalid first result is refused through RuntimeInstance with exact accepted-state rollback and a successful retry; - accepted scientific publication, diagnostics, two-rank collective HDF5, @@ -28,11 +32,10 @@ The source audit already authenticates real proofs for: This evidence is intentionally narrower than the final ADC-687 acceptance contract. The deferred rows name the missing polarity and the nearby source -that must not be mistaken for closure. They currently cover: +that must not be mistaken for closure. The only remaining gap is: - a CI lane that installs every mandatory dependency, including VTK, and executes every selected pytest and CTest proof with zero skips; -- complete Uniform, AMR, and multi-layout public-contract/report parity. ## Exact output evidence @@ -91,6 +94,18 @@ fault is removed, the same prepared component returns a finite result and the unchanged RuntimeInstance accepts the retry. The selected test defines no step wrapper and never replaces a native engine or step target. +The positive RuntimeInstance proof is also a compiled route. It builds and +executes one Uniform artifact, one AMR artifact, and one two-layout artifact +with a native conservative Transfer. Every execution returns the exact public +`RuntimeInstance` and `RunReport` types with aligned artifact, bind, execution, +run, clock, step, and transaction evidence. The multi-layout executor +authenticates each installed child Program, creates one domain-separated hash +for the ordered Program set, and projects local block/parameter/cache metadata +into deterministic layout-qualified report rows. Runtime inspection consumes +that same complete `ProgramRuntimeReport`; the selected test proves direct and +inspection parity without a wrapper, fake engine, replaced step target, or +monkeypatch. + ## Gate modes The architecture CI runs: From 9ead54b8a534cd04816c400f297f55f4e10bb86b Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:36:14 +0200 Subject: [PATCH 29/62] gate(m4): expose exact native target plan --- scripts/run_m4_gate.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/run_m4_gate.py b/scripts/run_m4_gate.py index e437aef38..5fb6ff486 100644 --- a/scripts/run_m4_gate.py +++ b/scripts/run_m4_gate.py @@ -737,6 +737,19 @@ def _run_ctest(build_dir: Path, target: str, selector: str) -> None: raise subprocess.CalledProcessError(completed.returncode, command) +def _required_ctest_targets(checks: Iterable[dict]) -> tuple[str, ...]: + """Return the exact native build targets needed by the selected CTest proofs.""" + return tuple( + sorted( + { + row["target"].split("@", 1)[1] + for row in checks + if row["kind"] == "ctest" + } + ) + ) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) @@ -747,6 +760,11 @@ def main(argv: list[str] | None = None) -> int: help="verify exact evidence and explicit gaps without claiming M4 closure", ) mode.add_argument("--check-only", action="store_true") + mode.add_argument( + "--list-ctest-targets", + action="store_true", + help="print the exact native targets required by a closed manifest", + ) parser.add_argument("--python-only", action="store_true") parser.add_argument("--build-dir", type=Path, default=ROOT / "build-mpi") parser.add_argument("--mpi-exec", default="mpiexec") @@ -763,10 +781,23 @@ def main(argv: list[str] | None = None) -> int: return 2 checks = data["check"] + if args.list_ctest_targets: + targets = _required_ctest_targets(checks) + if not targets: + print("M4 gate selects no CTest build target", file=sys.stderr) + return 2 + print("\n".join(targets)) + return 0 print( "M4 gate source matrix: %s (%d executable, %d deferred)" % ( - "AUDITED OPEN" if args.audit_only else "CLOSED", + ( + "AUDITED OPEN" + if data["deferred"] + else "AUDITED CLOSED" + ) + if args.audit_only + else "CLOSED", len(checks), len(data["deferred"]), ) From d1c0d037e8507451ebd656f3c4486fba0416451a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:37:15 +0200 Subject: [PATCH 30/62] ci(m4): execute complete installed runtime gate --- .github/workflows/ci.yml | 54 +++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cbeed497..12ecd77f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,12 +225,18 @@ jobs: - 'tests/CMakeLists.txt' - 'tests/test_manifest.toml' - 'scripts/ci_select_tests.py' + # The M4 closure is executed inside the installed MPI/VTK lane below. Any edit to + # its ledger, runner, source fence, or workflow must therefore require this job on + # the PR that changes the proof, never wait for a later nightly run. + - 'tests/gates/m4_runtime_io.toml' + - 'tests/python/architecture/test_m4_runtime_io_gate.py' + - 'scripts/run_m4_gate.py' + - '.github/workflows/ci.yml' - 'cmake/**' - 'CMakeLists.txt' - 'CMakePresets.json' - # Les changements de workflows/actions CI sont valides par revue + lint YAML local, - # et ne compilent pas Kokkos par defaut. Ajouter `ci-kokkos` pour forcer les gates - # Serial, ou `ci-full` pour MPI + OpenMP. + # Les autres changements de workflows/actions CI sont valides par revue + lint YAML + # local. Le workflow CI lui-meme fait exception car il porte la lane M4 executable. # full : un push master ne lance la suite COMPLETE (MPI + Kokkos OpenMP) que si un # chemin build/backend a bouge. Conservateur a dessein -- couvre tout ce qui peut affecter # l'artefact compile OU le chemin DSL/production exerce par le job kokkos-openmp (dsl.py et @@ -1598,7 +1604,7 @@ jobs: # The native build, processor-grouped CTest plan, and Python MPI contract # fence run sequentially. Each C++ launch retains its configured bounded # TIMEOUT; grouping removes PROCESSORS head-of-line blocking without skips. - timeout-minutes: 70 + timeout-minutes: 180 needs: [set-mode, changes, gate-mpi-prewarm] # Suite complete, ou PR qui modifie directement le chemin distribue/MPI. if: needs.set-mode.outputs.mpi_required == 'true' @@ -1635,7 +1641,7 @@ jobs: sudo apt-get install -y --no-install-recommends \ ccache libeigen3-dev libhdf5-openmpi-dev libopenmpi-dev ninja-build openmpi-bin \ pybind11-dev python3-dev python3-h5py \ - python3-numpy python3-pytest + python3-numpy python3-pytest python3-vtk9 - name: Resolve runner and compiler cache identity id: kokkos-platform @@ -1720,7 +1726,7 @@ jobs: test -s build-mpi/mpi-ctest-groups.tsv - name: Configure + build (MPI + Kokkos Serial) - timeout-minutes: 22 + timeout-minutes: 35 # Flags : preset ci-mpi (source unique, cf. CMakePresets.json) ; Kokkos_ROOT vient de # $KOKKOS_PREFIX (env du job, install en cache). ccache auto-detecte. run: | @@ -1775,6 +1781,10 @@ jobs: --build-dir build-mpi \ --verify-contracts "${compile_contracts[@]}" read -r -a mpi_targets <<< "${{ steps.mpi-test-plan.outputs.cpp_label_targets }}" + mapfile -t m4_targets < <( + /usr/bin/python3 scripts/run_m4_gate.py --list-ctest-targets + ) + test "${#m4_targets[@]}" -gt 0 export NINJA_STATUS='[%f/%t elapsed=%es active=%r] ' # The monolithic Python module link is memory-heavy. Keep it isolated # from test compilation/linking so a small hosted runner cannot evict @@ -1783,6 +1793,8 @@ jobs: cmake --build --preset ci-mpi --parallel 1 --target _pops run_with_heartbeat "MPI native test build" 8m \ cmake --build --preset ci-mpi --parallel 4 --target "${mpi_targets[@]}" + run_with_heartbeat "M4 native test build" 10m \ + cmake --build --preset ci-mpi --parallel 4 --target "${m4_targets[@]}" - name: Installed package smoke (MPI-only + collective HDF5) run: | @@ -1938,6 +1950,36 @@ jobs: timeout --signal=TERM --kill-after=30s 25m \ /usr/bin/python3 -m pytest -q -ra --maxfail=1 "$mpi_orchestrator" done < build-mpi/python-mpi-orchestrators.txt + + - name: M4 complete native runtime and scientific I/O gate + timeout-minutes: 45 + env: + PYTHONPATH: ${{ github.workspace }}/build-mpi/python-package:${{ github.workspace }} + POPS_INCLUDE: ${{ github.workspace }}/include + POPS_KOKKOS_ROOT: ${{ github.workspace }}/.kokkos-install + Kokkos_ROOT: ${{ github.workspace }}/.kokkos-install + POPS_CACHE_DIR: ${{ github.workspace }}/.pops-ci/m4-dsl-cache + POPS_KEEP_GENERATED: "1" + POPS_REQUIRE_MPI_TESTS: "1" + POPS_REQUIRE_NATIVE_TESTS: "1" + run: | + # These readers are mandatory capabilities of this lane. Imports happen before the gate + # so a missing apt module cannot masquerade as a scientific skip. + /usr/bin/python3 - <<'PY' + import h5py + import numpy + from vtkmodules.vtkIOXML import ( + vtkXMLPUnstructuredGridReader, + vtkXMLUnstructuredGridReader, + ) + + print("M4 readers:", numpy.__version__, h5py.__version__) + print(vtkXMLPUnstructuredGridReader, vtkXMLUnstructuredGridReader) + PY + /usr/bin/python3 scripts/run_m4_gate.py \ + --build-dir build-mpi \ + --mpi-exec mpiexec + - name: ccache stats (MPI) if: always() run: ccache -s From 947a3678875d10a34d38406f834fecc63375cc8c Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:39:33 +0200 Subject: [PATCH 31/62] gate(m4): close required execution ledger --- .github/workflows/ci.yml | 2 +- tests/gates/m4_runtime_io.toml | 19 ++- .../architecture/test_m4_runtime_io_gate.py | 135 ++++++++++++++---- 3 files changed, 114 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12ecd77f4..f89a9094b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -722,7 +722,7 @@ jobs: run: python3 scripts/run_m3_gate.py --check-only - name: M4 native runtime and scientific I/O gate manifest - run: python3 scripts/run_m4_gate.py --audit-only + run: python3 scripts/run_m4_gate.py --check-only - name: Generated component catalog env: diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index a58da0936..52b643dff 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -12,16 +12,7 @@ issues = [ "ADC-687", ] -[[deferred]] -issue = "ADC-687" -requirement = "gate_execution" -polarity = "positive" -reason = "CI only audits the source ledger; no required lane installs VTK and executes every selected pytest and CTest proof with zero skips." -evidence_paths = [ - ".github/workflows/ci.yml", - "environment.yml", - "scripts/run_m4_gate.py", -] +deferred = [] # This is an exact evidence ledger, not a list of nearby suites. Every row names # one source-registered proof. The runner rejects mock fixtures/imports, @@ -341,6 +332,14 @@ kind = "pytest" target = "diagnostics" nodeid = "tests/python/unit/output/test_exact_writers.py::test_composite_integrals_refuses_non_cartesian_cell_measure" +[[check]] +issue = "ADC-687" +requirement = "gate_execution" +polarity = "positive" +kind = "pytest" +target = "gate_execution" +nodeid = "tests/python/architecture/test_m4_runtime_io_gate.py::test_m4_required_ci_lane_executes_the_complete_installed_gate" + [[check]] issue = "ADC-687" requirement = "external_solver" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index b1806cb85..e327e1aab 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -33,13 +33,13 @@ def _mutated_manifest(tmp_path: Path, old: str, new: str) -> Path: return path -def test_m4_manifest_is_an_audited_open_exact_matrix(): +def test_m4_manifest_is_a_closed_exact_matrix(): runner = _load_runner() data, errors = runner.audit_manifest(MANIFEST) assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) - assert len(data["deferred"]) == 1 - assert len(data["check"]) == 50 + assert data["deferred"] == [] + assert len(data["check"]) == 51 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -55,16 +55,13 @@ def test_m4_manifest_is_an_audited_open_exact_matrix(): assert { (row["issue"], row["requirement"], row["polarity"]) for row in data["deferred"] - } == { - ("ADC-687", "gate_execution", "positive"), - } + } == set() _, closure_errors = runner.validate_manifest(MANIFEST) - assert len(closure_errors) == len(data["deferred"]) - assert all("remains deferred" in error for error in closure_errors) + assert closure_errors == [] -def test_m4_cli_reports_open_and_check_only_refuses_closure(): +def test_m4_cli_reports_closed_and_check_only_accepts_source_contract(): audit = subprocess.run( [sys.executable, str(RUNNER), "--audit-only"], cwd=ROOT, @@ -74,7 +71,7 @@ def test_m4_cli_reports_open_and_check_only_refuses_closure(): check=False, ) assert audit.returncode == 0 - assert "M4 gate source matrix: AUDITED OPEN" in audit.stdout + assert "M4 gate source matrix: AUDITED CLOSED" in audit.stdout closure = subprocess.run( [sys.executable, str(RUNNER), "--check-only"], @@ -84,9 +81,94 @@ def test_m4_cli_reports_open_and_check_only_refuses_closure(): stderr=subprocess.STDOUT, check=False, ) - assert closure.returncode == 2 - assert "M4 gate is incomplete or invalid" in closure.stdout - assert "remains deferred" in closure.stdout + assert closure.returncode == 0 + assert "M4 gate source matrix: CLOSED" in closure.stdout + + +def test_m4_required_ci_lane_executes_the_complete_installed_gate(): + data, errors = _load_runner().audit_manifest(MANIFEST) + assert not errors + nodeid = ( + "tests/python/architecture/test_m4_runtime_io_gate.py::" + "test_m4_required_ci_lane_executes_the_complete_installed_gate" + ) + assert [ + row for row in data["check"] if row.get("nodeid") == nodeid + ] == [{ + "issue": "ADC-687", + "requirement": "gate_execution", + "polarity": "positive", + "kind": "pytest", + "target": "gate_execution", + "nodeid": nodeid, + }] + + workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + mpi_job = workflow.split("\n mpi:\n", 1)[1] + mpi_job = mpi_job.split("\n gate-openmp-prewarm:\n", 1)[0] + assert "if: needs.set-mode.outputs.mpi_required == 'true'" in mpi_job + assert "python3-vtk9" in mpi_job + assert "/usr/bin/python3 scripts/run_m4_gate.py --list-ctest-targets" in mpi_job + assert 'cmake --build --preset ci-mpi --parallel 4 --target "${m4_targets[@]}"' in mpi_job + + complete = mpi_job.split( + "- name: M4 complete native runtime and scientific I/O gate", 1 + )[1] + complete = complete.split("- name: ccache stats (MPI)", 1)[0] + assert "POPS_REQUIRE_MPI_TESTS: \"1\"" in complete + assert "POPS_REQUIRE_NATIVE_TESTS: \"1\"" in complete + assert "vtkXMLPUnstructuredGridReader" in complete + assert "vtkXMLUnstructuredGridReader" in complete + assert "/usr/bin/python3 scripts/run_m4_gate.py \\" in complete + assert "--build-dir build-mpi" in complete + assert "--mpi-exec mpiexec" in complete + assert "--audit-only" not in complete + assert "--python-only" not in complete + assert "continue-on-error" not in complete + + aggregator = workflow.split("\n gate:\n", 1)[1] + aggregator = aggregator.split("\n mpi:\n", 1)[0] + assert "mpi" in aggregator.split("needs:", 1)[1].splitlines()[0] + assert '--gate mpi "${{ needs.mpi.result }}"' in aggregator + assert '"${{ needs.set-mode.outputs.mpi_required }}"' in aggregator + + mpi_filter = workflow.split("\n mpi:\n", 1)[1] + mpi_filter = mpi_filter.split("\n # full", 1)[0] + for protected_path in ( + "tests/gates/m4_runtime_io.toml", + "tests/python/architecture/test_m4_runtime_io_gate.py", + "scripts/run_m4_gate.py", + ".github/workflows/ci.yml", + ): + assert "'%s'" % protected_path in mpi_filter + + +def test_m4_closed_gate_lists_every_exact_native_build_target(): + runner = _load_runner() + data, errors = runner.validate_manifest(MANIFEST) + assert not errors + expected = ( + "test_amr_native_loader", + "test_brick_catalog", + "test_component_interfaces", + "test_flux_interfaces", + "test_mpi_hdf5_collective", + "test_native_loader_param_overflow", + "test_platform_manifest", + "test_program_context_contract", + ) + assert runner._required_ctest_targets(data["check"]) == expected + + listed = subprocess.run( + [sys.executable, str(RUNNER), "--list-ctest-targets"], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + assert listed.returncode == 0 + assert tuple(listed.stdout.splitlines()) == expected def test_m4_gate_pins_every_external_component_family(): @@ -422,7 +504,7 @@ def test_m4_gate_pins_real_writer_refusal_without_publication_fakes(): ) -def test_m4_gate_keeps_real_tamper_capacity_proofs_and_defers_runtime_gaps(): +def test_m4_gate_keeps_real_tamper_and_capacity_refusals(): data, errors = _load_runner().audit_manifest(MANIFEST) assert not errors @@ -553,14 +635,14 @@ def test_m4_gate_pins_complete_program_only_dispatch_and_fallback_fences(): workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") job = workflow.split("\n gate-python-architecture:\n", 1)[1] job = job.split("\n gate-python-build:\n", 1)[0] - command = "run: python3 scripts/run_m4_gate.py --audit-only" + command = "run: python3 scripts/run_m4_gate.py --check-only" assert [line.strip() for line in job.splitlines()].count(command) == 1 - assert "run: python3 scripts/run_m4_gate.py --check-only" not in job + assert "run: python3 scripts/run_m4_gate.py --audit-only" not in job documentation = ( ROOT / "docs/design/m4-conformance-gate.md" ).read_text(encoding="utf-8") - assert "current status is **AUDITED OPEN**" in documentation + assert "current status is **CLOSED AND CI-EXECUTED**" in documentation assert "four serial proofs" in documentation @@ -680,23 +762,14 @@ def test_m4_mpi_entrypoint_accepts_only_the_required_prerequisite_guard(): assert not runner._has_authenticated_mpi_guard(untrusted) -def test_m4_gate_rejects_every_explicit_deferred_gap(): +def test_m4_gate_has_no_explicit_deferred_gap(): runner = _load_runner() data, audit_errors = runner.audit_manifest(MANIFEST) assert not audit_errors - assert data["deferred"] + assert data["deferred"] == [] _, errors = runner.validate_manifest(MANIFEST) - expected = { - "%s/%s/%s" % (row["issue"], row["requirement"], row["polarity"]) - for row in data["deferred"] - } - observed = { - error.split(" remains deferred:", 1)[0] - for error in errors - if " remains deferred:" in error - } - assert observed == expected + assert errors == [] def test_m4_required_pytest_execution_rejects_junit_skips(monkeypatch): @@ -766,7 +839,7 @@ def ctest_with_a_skip(command, **kwargs): assert calls == 2 -def test_m4_check_only_refuses_open_ledger_before_launcher_or_build(monkeypatch): +def test_m4_check_only_accepts_closed_ledger_without_launcher_or_build(monkeypatch): runner = _load_runner() def forbidden_call(*_args, **_kwargs): @@ -775,4 +848,4 @@ def forbidden_call(*_args, **_kwargs): monkeypatch.setattr(runner.shutil, "which", forbidden_call) monkeypatch.setattr(runner.subprocess, "run", forbidden_call) - assert runner.main(["--check-only"]) == 2 + assert runner.main(["--check-only"]) == 0 From 1c97d74fd944997b9e5e41b52b13b5d2d701dacb Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:39:33 +0200 Subject: [PATCH 32/62] docs(m4): record executable gate closure --- docs/design/m4-conformance-gate.md | 70 ++++++++++++++++-------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index e3c2c473c..7fa6c573c 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -1,10 +1,11 @@ # M4 native runtime and scientific I/O conformance gate -The current status is **AUDITED OPEN**. The ledger in +The current status is **CLOSED AND CI-EXECUTED**. The ledger in `tests/gates/m4_runtime_io.toml` records exact executable evidence for -ADC-679 through ADC-687 and exact deferred gaps. It deliberately does not -claim M4 closure while any `[[deferred]]` row remains. The current ledger has -exactly 50 executable checks and one deferred requirement. +ADC-679 through ADC-687. It contains exactly 51 executable checks and +`deferred = []`. Closure is accepted only for a commit whose required MPI job +successfully executes the complete installed gate; source audit alone is not +the acceptance evidence. The source audit already authenticates real proofs for: @@ -30,12 +31,12 @@ The source audit already authenticates real proofs for: - accepted scientific publication, diagnostics, two-rank collective HDF5, and a two-rank PVD/PVTU/rank-VTU hierarchy reopened by native VTK readers. -This evidence is intentionally narrower than the final ADC-687 acceptance -contract. The deferred rows name the missing polarity and the nearby source -that must not be mistaken for closure. The only remaining gap is: - -- a CI lane that installs every mandatory dependency, including VTK, and - executes every selected pytest and CTest proof with zero skips; +The required Ubuntu 24.04 MPI lane installs Open MPI, parallel HDF5, NumPy, +h5py, pytest, and the native VTK Python readers. It builds the MPI-enabled +extension plus every exact CTest target selected by the ledger, then runs the +complete gate. The global required-check aggregator rejects a skipped, failed, +cancelled, or timed-out MPI lane whenever the M4 runner, ledger, source fence, +or CI workflow changes. ## Exact output evidence @@ -53,7 +54,8 @@ There are four serial proofs that are real and remain selected: An additional HDF5 refusal mutates a dataset with h5py and proves that the authenticated PoPS reader rejects it. These tests contain no optional import or skip. That makes their dependencies mandatory wherever the executable gate -runs; it does not prove that CI currently provisions those dependencies. +runs; the required MPI lane provisions and imports those readers before +launching the matrix. The selected two-rank ParaView entrypoint starts from the standard `.pvd` catalogue, preserves its exact temporal ordering, and requires the native VTK @@ -61,8 +63,8 @@ parallel reader to assemble every referenced `.pvtu`. It also reopens every rank-local `.vtu` directly with VTK and checks its geometry, public arrays, component name, and `TimeValue`. VTK imports are unconditional in the required MPI lane: `POPS_REQUIRE_MPI_TESTS=1` turns an absent reader into a test failure. -The separate `gate_execution` gap remains open until CI provisions VTK and -executes this selected entrypoint rather than auditing only its source. +The selected `gate_execution` proof authenticates that exact CI route, and the +same required job executes the entrypoint rather than auditing only its source. The strict-checkpoint refusal is also provider-backed. A correctly sealed AMR checkpoint with an inconsistent dynamic accepted-ledger claim passes the real @@ -108,33 +110,37 @@ monkeypatch. ## Gate modes -The architecture CI runs: +The source-only architecture CI checks that the ledger is closed: ```bash -python scripts/run_m4_gate.py --audit-only +python scripts/run_m4_gate.py --check-only ``` -`--audit-only` verifies the exact nodeids, CTest selectors, manifest ownership, -deferred-gap schema, and source-level anti-skip rules. It prints -`AUDITED OPEN` and launches no compiler, test, MPI process, or native reader. +`--check-only` verifies the exact nodeids, CTest selectors, manifest ownership, +empty deferred-gap ledger, and source-level anti-skip rules without launching +a compiler, test, MPI process, or native reader. `--audit-only` performs the +same structural audit and reports `AUDITED CLOSED`. -The closure check is intentionally red while the ledger is open: +The installed MPI lane asks the same closed manifest for its exact native build +targets: ```bash -python scripts/run_m4_gate.py --check-only +python scripts/run_m4_gate.py --list-ctest-targets ``` -`--check-only` rejects every remaining deferred row and exits nonzero before -launching anything. Running the script without either audit flag, or with -`--python-only`, is also fail-closed until all deferred gaps are replaced by -real selected proofs. +It then invokes the complete executable gate, with no audit-only or +Python-only reduction: -Once `deferred = []` is honestly restored, the full command requires an -MPI-enabled build containing every selected CTest and environments with NumPy, -h5py, and VTK. Every pytest and CTest execution must emit a JUnit report with -zero skipped or xfailed proofs. +```bash +/usr/bin/python3 scripts/run_m4_gate.py \ + --build-dir build-mpi \ + --mpi-exec mpiexec +``` -Each deferred row contains `issue`, `requirement`, `polarity`, a precise -`reason`, and existing `evidence_paths`. The validator rejects malformed or -duplicate gaps, wildcard selectors, missing manifest ownership, optional -pytest imports, skip/xfail markers, mock fixtures/imports, and disabled CTests. +The full command requires the MPI-enabled extension, every selected CTest +target, NumPy, h5py, and VTK. Every selected pytest and CTest execution emits a +JUnit report and fails on any skipped or xfailed proof. A future limitation +must be restored as an explicit `[[deferred]]` row; the validator rejects +malformed or duplicate gaps, wildcard selectors, missing manifest ownership, +optional pytest imports, skip/xfail markers, mock fixtures/imports, and +disabled CTests. From 0d8c0158fd7ed9bf1fc6aeff63025dfa873cc189 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:16 +0200 Subject: [PATCH 33/62] feat(runtime): retain automatic balance evidence per attempt --- .../runtime/program/program_runtime_state.hpp | 71 +++++++++++++++++++ src/runtime/amr/amr_system.cpp | 3 + src/runtime/system/system_impl.hpp | 3 + 3 files changed, 77 insertions(+) diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index a2ac587c9..e27984c07 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -143,6 +143,29 @@ struct HistoryManager { } }; +/// Attempt-local native balance evidence emitted by one exact runtime operator. +/// +/// The coordinate deliberately remains independent of a user-facing BalanceLedger route: native +/// operators know their qualified runtime block, hierarchy level and conservative component, while +/// the route-to-quantity selector is a separate planning authority. Keeping both identities +/// separate prevents a reflux correction from being silently relabelled as a complete balance. +struct AutomaticBalanceKey { + int runtime_block = -1; + int level = -1; + int component = -1; + std::string term; + + friend bool operator<(const AutomaticBalanceKey& left, const AutomaticBalanceKey& right) { + if (left.runtime_block != right.runtime_block) + return left.runtime_block < right.runtime_block; + if (left.level != right.level) + return left.level < right.level; + if (left.component != right.component) + return left.component < right.component; + return left.term < right.term; + } +}; + /// The compiled time-Program runtime state, extracted from the System / AmrSystem god-object (ADC-594). /// /// A plain aggregate: the owning Impl embeds ONE instance and routes every Program seam through it. The @@ -272,6 +295,11 @@ struct ProgramRuntimeState { /// consumers read it while the facade's outer transaction still retains U^n, so a missing term /// cannot silently reuse the preceding step. std::map step_balance_terms_; + /// Native operator contributions captured only for a due Balance attempt. These values are keyed + /// by their physical runtime coordinate instead of a user ledger route and are therefore not read + /// by accepted_balance_terms(). The owning facade snapshots this map with the rest of the attempt, + /// so rejection cannot leak automatic evidence into a retry. + std::map automatic_balance_terms_; /// Attempt-local outer accepted-step target used by ConsumerGraph-fused balance guards. Program /// substeps temporarily publish their window-start macro step through the facade, so generated /// balance code must not infer the public target from `macro_step()+1`. @@ -791,6 +819,14 @@ struct ProgramRuntimeState { throw std::invalid_argument(runtime + " requires one canonical five-term balance name"); } + static void require_automatic_balance_term(const std::string& term, const std::string& runtime) { + static constexpr std::array kTerms{"outward_boundary_flux", "sources", + "reflux", "projection"}; + if (std::find(kTerms.begin(), kTerms.end(), std::string_view(term)) == kTerms.end()) + throw std::invalid_argument(runtime + + " requires one native operator balance contribution name"); + } + /// Record a compiled-Program scalar. Ordinary P.record_scalar names remain inspectable after the /// step with last-write-wins semantics. The balance namespace has a separate typed sink. void record_diagnostic(const std::string& name, Real value) { @@ -816,6 +852,40 @@ struct ProgramRuntimeState { entry->second += value; } + /// Whether a compiled Program has actually emitted a due Balance route in this attempt. + /// + /// Generated balance records are cadence-guarded before their reductions. Reflux executes after + /// the Program body, so observing a non-empty authored mailbox here avoids every extra native + /// reduction on an off-cadence or replay step without introducing a second scheduler. + [[nodiscard]] bool automatic_balance_capture_due() const noexcept { + return !balance_replay_active_ && !step_balance_terms_.empty(); + } + + /// Accumulate one signed, metric-integrated native operator contribution. + /// + /// This is intentionally not accepted_balance_terms(): automatic evidence remains qualified by + /// block/level/component until a resolved quantity selector proves which BalanceLedger route owns + /// it. The separation is fail-closed and lets boundary/source/projection producers join the same + /// mailbox later without fabricating missing terms. + void record_automatic_balance_term(int runtime_block, int level, int component, + const std::string& term, Real value, + const std::string& runtime) { + if (!automatic_balance_capture_due()) + throw std::logic_error(runtime + + "::record_automatic_balance_term requires a due authored balance"); + if (runtime_block < 0 || level < 0 || component < 0) + throw std::invalid_argument( + runtime + "::record_automatic_balance_term requires non-negative coordinates"); + require_automatic_balance_term(term, runtime + "::record_automatic_balance_term"); + if (!std::isfinite(static_cast(value))) + throw std::invalid_argument(runtime + + "::record_automatic_balance_term requires a finite value"); + auto [entry, inserted] = automatic_balance_terms_.try_emplace( + AutomaticBalanceKey{runtime_block, level, component, term}, value); + if (!inserted) + entry->second += value; + } + /// Read the named diagnostic, FAIL-LOUD if the Program never recorded it. @p runtime names the /// Program subsystem setter in the message (not a generic getter). @throws std::out_of_range. Real diagnostic(const std::string& name, const std::string& runtime) const { @@ -833,6 +903,7 @@ struct ProgramRuntimeState { void begin_step_projection_report() { step_projections_.clear(); step_balance_terms_.clear(); + automatic_balance_terms_.clear(); balance_due_window_active_ = false; balance_due_target_step_ = 0; balance_step_completed_ = false; diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index d84124499..cf5d1d318 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -462,6 +462,7 @@ struct AmrSystem::Impl { int cadence_clock_restore_macro_step = 0; std::map program_diagnostics; std::map step_balance_terms; + std::map automatic_balance_terms; bool balance_step_completed = false; bool balance_program_was_due = false; pops::runtime::program::CacheManager cache; @@ -509,6 +510,7 @@ struct AmrSystem::Impl { cadence_clock_restore_macro_step = impl.program_.cadence_clock_restore_macro_step_; copy_value_map_into(program_diagnostics, impl.program_.diagnostics_); copy_value_map_into(step_balance_terms, impl.program_.step_balance_terms_); + copy_value_map_into(automatic_balance_terms, impl.program_.automatic_balance_terms_); balance_step_completed = impl.program_.balance_step_completed_; balance_program_was_due = impl.program_.balance_program_was_due_; // AMR currently owns its native cache/history rings inside AmrRuntime. These two shared @@ -542,6 +544,7 @@ struct AmrSystem::Impl { impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; copy_value_map_into(impl.program_.diagnostics_, program_diagnostics); copy_value_map_into(impl.program_.step_balance_terms_, step_balance_terms); + copy_value_map_into(impl.program_.automatic_balance_terms_, automatic_balance_terms); impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; diff --git a/src/runtime/system/system_impl.hpp b/src/runtime/system/system_impl.hpp index 13abe58bd..40fddf113 100644 --- a/src/runtime/system/system_impl.hpp +++ b/src/runtime/system/system_impl.hpp @@ -617,6 +617,7 @@ struct System::Impl { int cadence_clock_restore_macro_step; std::map program_diagnostics; std::map step_balance_terms; + std::map automatic_balance_terms; bool balance_step_completed; bool balance_program_was_due; pops::runtime::program::CacheManager cache; @@ -643,6 +644,7 @@ struct System::Impl { cadence_clock_restore_macro_step(impl.program_.cadence_clock_restore_macro_step_), program_diagnostics(impl.program_.diagnostics_), step_balance_terms(impl.program_.step_balance_terms_), + automatic_balance_terms(impl.program_.automatic_balance_terms_), balance_step_completed(impl.program_.balance_step_completed_), balance_program_was_due(impl.program_.balance_program_was_due_), cache(impl.program_.cache_), @@ -677,6 +679,7 @@ struct System::Impl { impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; impl.program_.diagnostics_ = program_diagnostics; impl.program_.step_balance_terms_ = step_balance_terms; + impl.program_.automatic_balance_terms_ = automatic_balance_terms; impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; From f2135ace5ee757b2ab61290761f65597a621d1dc Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:22 +0200 Subject: [PATCH 34/62] feat(amr): extract signed reflux balance corrections --- .../time/amr/levels/amr_patch_range.hpp | 17 +++++++++++++ .../time/amr/levels/amr_subcycling.hpp | 5 +++- .../pops/runtime/amr/amr_program_reflux.hpp | 11 +++++--- .../runtime/program/amr_program_context.hpp | 25 ++++++++++++++++--- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/include/pops/numerics/time/amr/levels/amr_patch_range.hpp b/include/pops/numerics/time/amr/levels/amr_patch_range.hpp index 02a52a46b..82d8d1bf0 100644 --- a/include/pops/numerics/time/amr/levels/amr_patch_range.hpp +++ b/include/pops/numerics/time/amr/levels/amr_patch_range.hpp @@ -468,6 +468,23 @@ struct FluxRegister { device_fence(); all_reduce_sum_inplace(buf.data(), buf.size(), communicator); } + /// Sum the already-gathered sparse correction by conservative component. + /// + /// RefluxStorage is pinned host storage shared with device kernels. The fence makes the gathered + /// register host-readable; every communicator rank then traverses the same compact global order, + /// so this adds no second collective and produces the exact state increment applied below. + [[nodiscard]] std::vector component_sums(Real cell_measure) const { + if (!std::isfinite(static_cast(cell_measure)) || cell_measure <= Real(0)) + throw std::invalid_argument( + "FluxRegister component sum requires a finite positive cell measure"); + device_fence(); + std::vector result(static_cast(nc), Real(0)); + const std::size_t components = static_cast(nc); + for (std::size_t offset = 0; offset < buf.size(); offset += components) + for (std::size_t component = 0; component < components; ++component) + result[component] += cell_measure * buf[offset + component]; + return result; + } [[nodiscard]] std::size_t lookup_capacity() const noexcept { return cell_lookup.capacity(); } [[nodiscard]] std::size_t covered_cell_count() const noexcept { return cell_lookup.size(); } diff --git a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp index 12292338a..0b4bde30e 100644 --- a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp +++ b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp @@ -824,7 +824,8 @@ class PreparedAmrProgramRefluxTransition { template void synchronize_integrated(MultiFab& parent_state, Real dx, Real dy, const CoarseStripRange& coarse_role, const FineStripRange& fine_role, - const CommunicatorView& communicator) { + const CommunicatorView& communicator, + std::vector* integrated_state_correction = nullptr) { validate_communicator_(communicator); using CoarseStrip = typename CoarseStripRange::value_type; using FineStrip = typename FineStripRange::value_type; @@ -882,6 +883,8 @@ class PreparedAmrProgramRefluxTransition { ncomp_); } correction_.gather(communicator); + if (integrated_state_correction != nullptr) + *integrated_state_correction = correction_.component_sums(dx * dy); for (int local_parent = 0; local_parent < parent_state.local_size(); ++local_parent) for_each_cell(parent_state.box(local_parent), detail::ApplyRefluxRegisterKernel{parent_state.fab(local_parent).array(), diff --git a/include/pops/runtime/amr/amr_program_reflux.hpp b/include/pops/runtime/amr/amr_program_reflux.hpp index 96ab1bd65..3e79b5af4 100644 --- a/include/pops/runtime/amr/amr_program_reflux.hpp +++ b/include/pops/runtime/amr/amr_program_reflux.hpp @@ -525,14 +525,19 @@ inline void sample_fine_role_strip(const MultiFab& state, const MultiFab& Fx, co /// per (cell,direction) (ADC-636 ownership: each C/F face is owned by the rank holding the covering fine /// patch), so the gather is associativity-free -> distributed == replicated bit-for-bit. inline void route_reflux_program(AmrRuntime& eng, std::size_t b, int k, const EdgeFlux& coarse_role, - const EdgeFlux& fine_role) { + const EdgeFlux& fine_role, + std::vector* integrated_state_correction = nullptr) { MultiFab& Uc = eng.level_state(b, k - 1); // the PARENT (coarse) live state we correct const BoxArray child_ba = eng.level_state(b, k).box_array(); // GLOBAL level-k patches - if (child_ba.size() == 0) + if (child_ba.size() == 0) { + if (integrated_state_correction != nullptr) + integrated_state_correction->assign(static_cast(Uc.ncomp()), Real(0)); return; + } const Geometry gc = eng.level_geom(k - 1); eng.prepared_reflux_transition(b, k).synchronize_integrated( - Uc, gc.dx(), gc.dy(), coarse_role.coarse, fine_role.fine, world_communicator_view()); + Uc, gc.dx(), gc.dy(), coarse_role.coarse, fine_role.fine, world_communicator_view(), + integrated_state_correction); } } // namespace detail diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index c22140dc3..05127b51c 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -1115,7 +1115,8 @@ class AmrProgramContext : public ProgramExecutionServices { amr::ClockStamp sync_clock = accepted; sync_clock.level = parent; for (int b = 0; b < n_blocks(); ++b) { - const std::size_t sb = static_cast(sys_block(b)); + const int runtime_block = sys_block(b); + const std::size_t sb = static_cast(runtime_block); if (capturing()) { sync_report_.push_back({parent, child, b, SyncPhase::Reflux, sync_clock}); const EdgeFlux coarse_role = reflux_flux_from_ledger_(b, parent, ledger_begin, ledger_end); @@ -1123,8 +1124,26 @@ class AmrProgramContext : public ProgramExecutionServices { if (coarse_role.empty() != fine_role.empty()) throw std::runtime_error( "AMR conservative ledger contains only one side of a parent/child flux pair"); - if (!coarse_role.empty()) - pops::detail::route_reflux_program(*eng_, sb, child, coarse_role, fine_role); + const bool capture_balance = + facade_->program_runtime_state_().automatic_balance_capture_due(); + std::vector integrated_reflux; + if (!coarse_role.empty()) { + pops::detail::route_reflux_program(*eng_, sb, child, coarse_role, fine_role, + capture_balance ? &integrated_reflux : nullptr); + } else if (capture_balance) { + integrated_reflux.assign(static_cast(eng_->level_state(sb, parent).ncomp()), + Real(0)); + } + if (capture_balance) { + const int components = eng_->level_state(sb, parent).ncomp(); + if (integrated_reflux.size() != static_cast(components)) + throw std::runtime_error( + "AMR automatic reflux balance contribution changed component width"); + for (int component = 0; component < components; ++component) + facade_->program_runtime_state_().record_automatic_balance_term( + runtime_block, parent, component, "reflux", + integrated_reflux[static_cast(component)], "AmrProgramContext"); + } } sync_report_.push_back({parent, child, b, SyncPhase::AverageDown, sync_clock}); eng_->average_down_level(sb, child); From 31d21432d373be50ff0416bf66332f7469bbc84b Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:26 +0200 Subject: [PATCH 35/62] test(architecture): fence automatic reflux balance evidence --- .../test_automatic_reflux_balance_fence.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/python/architecture/test_automatic_reflux_balance_fence.py diff --git a/tests/python/architecture/test_automatic_reflux_balance_fence.py b/tests/python/architecture/test_automatic_reflux_balance_fence.py new file mode 100644 index 000000000..7b2c866a6 --- /dev/null +++ b/tests/python/architecture/test_automatic_reflux_balance_fence.py @@ -0,0 +1,99 @@ +"""ADC-686: automatic reflux evidence stays exact, sparse, and fail-closed.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +PROGRAM_STATE = ( + ROOT / "include" / "pops" / "runtime" / "program" / "program_runtime_state.hpp" +) +AMR_CONTEXT = ( + ROOT / "include" / "pops" / "runtime" / "program" / "amr_program_context.hpp" +) +AMR_REFLUX = ROOT / "include" / "pops" / "runtime" / "amr" / "amr_program_reflux.hpp" +AMR_SUBCYCLING = ( + ROOT / "include" / "pops" / "numerics" / "time" / "amr" / "levels" + / "amr_subcycling.hpp" +) +AMR_PATCH_RANGE = ( + ROOT / "include" / "pops" / "numerics" / "time" / "amr" / "levels" + / "amr_patch_range.hpp" +) +UNIFORM_IMPL = ROOT / "src" / "runtime" / "system" / "system_impl.hpp" +AMR_IMPL = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_automatic_balance_mailbox_is_attempt_local_and_not_a_route_fallback() -> None: + state = PROGRAM_STATE.read_text() + assert "struct AutomaticBalanceKey" in state + assert "std::map automatic_balance_terms_;" in state + assert "automatic_balance_terms_.clear();" in state + assert "record_automatic_balance_term(" in state + assert "automatic_balance_capture_due()" in state + + accepted = _between( + state, + "std::map accepted_balance_terms(", + "void begin_balance_due_window(", + ) + assert "step_balance_terms_" in accepted + assert "automatic_balance_terms_" not in accepted + + uniform = UNIFORM_IMPL.read_text() + adaptive = AMR_IMPL.read_text() + for source in (uniform, adaptive): + assert "automatic_balance_terms" in source + assert "impl.program_.automatic_balance_terms_" in source + + +def test_reflux_integral_comes_from_the_gathered_sparse_correction() -> None: + register = AMR_PATCH_RANGE.read_text() + component_sums = _between( + register, + "[[nodiscard]] std::vector component_sums(", + "[[nodiscard]] std::size_t lookup_capacity()", + ) + assert "device_fence();" in component_sums + assert "cell_measure * buf[offset + component]" in component_sums + assert "all_reduce" not in component_sums + + transition = AMR_SUBCYCLING.read_text() + synchronize = _between( + transition, + "void synchronize_integrated(", + "\n private:", + ) + assert synchronize.index("correction_.gather(communicator);") < synchronize.index( + "correction_.component_sums(dx * dy)" + ) + assert synchronize.index("correction_.component_sums(dx * dy)") < synchronize.index( + "ApplyRefluxRegisterKernel" + ) + + route = AMR_REFLUX.read_text() + routing = _between(route, "inline void route_reflux_program(", "\n}\n\n} // namespace detail") + assert "std::vector* integrated_state_correction = nullptr" in routing + assert "integrated_state_correction);" in routing + + +def test_amr_records_reflux_before_average_down_only_when_balance_is_due() -> None: + context = AMR_CONTEXT.read_text() + synchronize = _between( + context, + "void synchronize_level_pair_(", + "void finalize_history_rotation_()", + ) + assert "automatic_balance_capture_due()" in synchronize + assert "record_automatic_balance_term(" in synchronize + assert '"reflux"' in synchronize + assert "reduce_sum(" not in synchronize + assert synchronize.index("route_reflux_program(") < synchronize.index( + "record_automatic_balance_term(" + ) + assert synchronize.index("record_automatic_balance_term(") < synchronize.index( + "SyncPhase::AverageDown" + ) From 9ad7aba00a64e37aed2da135d9bb1f667e085f39 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:02:43 +0200 Subject: [PATCH 36/62] feat(runtime): capture due projection balance evidence (ADC-686) --- .../runtime/program/amr_program_context.hpp | 44 ++++++++++++++++++- .../pops/runtime/program/program_context.hpp | 24 ++++++++++ .../program/program_execution_services.hpp | 32 +++++++++++++- .../runtime/program/program_runtime_state.hpp | 33 +++++++++++--- python/pops/codegen/program_balance_due.py | 7 +++ src/runtime/amr/amr_system.cpp | 3 ++ src/runtime/system/system_impl.hpp | 3 ++ 7 files changed, 137 insertions(+), 9 deletions(-) diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index 05127b51c..ac1fa24fc 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -36,6 +36,7 @@ #include #include #include // AmrRuntime (the engine the driver wraps) +#include #include #include // GridContext (per-level Schur assembly seam, ADC-633) #include // AmrSystem (the facade: params / block map / engine) @@ -3164,6 +3165,45 @@ class AmrProgramContext : public ProgramExecutionServices { void program_execution_apply_projection_(int runtime_block, MultiFab& state) const { eng_->project_level_state(static_cast(runtime_block), level_, state); } + std::optional> program_execution_projection_balance_integrals_( + int program_block, const MultiFab& state) const { + const std::size_t runtime_block = static_cast(sys_block(program_block)); + if (level_ < 0 || level_ >= nlev()) + throw std::out_of_range("AMR Program projection balance active level is out of range"); + const MultiFab& live = eng_->level_state(runtime_block, level_); + if (state.box_array().boxes() != live.box_array().boxes() || + state.dmap().ranks() != live.dmap().ranks() || state.ncomp() != live.ncomp() || + state.n_grow() != live.n_grow() || state.local_size() != live.local_size()) + throw std::invalid_argument( + "AMR Program projection balance candidate changed its exact level layout"); + + std::vector views; + views.reserve(static_cast(nlev())); + for (int level = 0; level < nlev(); ++level) { + const Geometry geometry = eng_->level_geom(level); + const MultiFab* values = level == level_ ? &state : &eng_->level_state(runtime_block, level); + views.push_back({values, geometry.dx(), geometry.dy()}); + } + const int next = level_ + 1 < nlev() ? level_ + 1 : -1; + MultiFab mask = pops::runtime::amr::composite_detail::active_mask(views, level_, next); + std::vector result(static_cast(state.ncomp()), 0.0); + for (int component = 0; component < state.ncomp(); ++component) + result[static_cast(component)] = + static_cast(pops::runtime::amr::composite_detail::local_sum( + state, mask, component, pops::runtime::amr::composite_detail::CompositeSumKind::Sum)); + if (!eng_->level_is_replicated(level_)) + all_reduce_sum_inplace(result.data(), result.size()); + const Geometry geometry = eng_->level_geom(level_); + const double cell_measure = + static_cast(geometry.dx()) * static_cast(geometry.dy()); + if (!std::isfinite(cell_measure) || cell_measure <= 0.0) + throw std::runtime_error( + "AMR Program projection balance requires a positive finite cell measure"); + std::vector integrated(result.size(), Real(0)); + for (std::size_t component = 0; component < result.size(); ++component) + integrated[component] = static_cast(cell_measure * result[component]); + return integrated; + } Real program_execution_hmin_() const { return eng_->level_hmin(level_); } Real program_execution_max_wave_speed_(int runtime_block, const MultiFab& state) const { return eng_->level_max_speed(static_cast(runtime_block), level_, state); @@ -3273,8 +3313,8 @@ class AmrProgramContext : public ProgramExecutionServices { const HistoryRegistration& registration) const { return pops::detail::AmrHistoryOps::initialized(*eng_, registration.name); } - double program_execution_history_slot_dt_storage_( - const HistoryRegistration& registration, int lag) const { + double program_execution_history_slot_dt_storage_(const HistoryRegistration& registration, + int lag) const { return pops::detail::AmrHistoryOps::slot_dt(*eng_, registration.name, lag); } void program_execution_set_history_initialized_storage_(const HistoryRegistration& registration, diff --git a/include/pops/runtime/program/program_context.hpp b/include/pops/runtime/program/program_context.hpp index d4780ec9d..fab5a6518 100644 --- a/include/pops/runtime/program/program_context.hpp +++ b/include/pops/runtime/program/program_context.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -660,6 +661,29 @@ class ProgramContext : public ProgramExecutionServices { void program_execution_apply_projection_(int runtime_block, MultiFab& state) const { sys_->block_project(runtime_block, state); } + std::optional> program_execution_projection_balance_integrals_( + int program_block, const MultiFab& state) const { + // The public polar diagnostic path has no exact per-cell volume provider yet. Keep automatic + // evidence absent instead of relabelling Cartesian dx*dy as a polar measure; authored balance + // terms remain available and the future selector must fail closed on this missing producer. + if (sys_->program_is_polar()) + return std::nullopt; + const GridContext context = program_execution_block_grid_context_(program_block); + const Real cell_measure = context.geom.dx() * context.geom.dy(); + if (!std::isfinite(static_cast(cell_measure)) || cell_measure <= Real(0)) + throw std::runtime_error( + "Uniform Program projection balance requires a positive finite cell measure"); + RelativeCellMeasure measure; + if (context.domain_mask != nullptr) { + measure.active_cells = context.domain_mask; + measure.inverse_volume_fraction = context.eb_inverse_volume_fraction; + } + std::vector result(static_cast(state.ncomp()), Real(0)); + for (int component = 0; component < state.ncomp(); ++component) + result[static_cast(component)] = + cell_measure * pops::reduce_sum(state, component, measure); + return result; + } Real program_execution_hmin_() const { return sys_->cfl_min_dx(); } Real program_execution_max_wave_speed_(int runtime_block, const MultiFab& state) const { return sys_->block_max_speed(runtime_block, state); diff --git a/include/pops/runtime/program/program_execution_services.hpp b/include/pops/runtime/program/program_execution_services.hpp index 93bd66963..e7d1943a1 100644 --- a/include/pops/runtime/program/program_execution_services.hpp +++ b/include/pops/runtime/program/program_execution_services.hpp @@ -502,9 +502,33 @@ class ProgramExecutionServices { /// Project one candidate state through the exact authored block closure. /// /// Program-to-runtime block qualification is topology-independent. The provider owns only the - /// Uniform or level-qualified native projection call. + /// Uniform or level-qualified native projection call. When a generated Balance route is due, the + /// provider also supplies exact metric-integrated component values before and after projection; + /// their signed delta stays qualified by runtime block/level/component in the attempt mailbox. void apply_projection(int block, MultiFab& state) const { - provider_().program_execution_apply_projection_(sys_block(block), state); + const int runtime_block = sys_block(block); + ProgramRuntimeState& runtime = program_runtime_state_(); + if (!runtime.automatic_balance_capture_due()) { + provider_().program_execution_apply_projection_(runtime_block, state); + return; + } + const std::optional> before = + provider_().program_execution_projection_balance_integrals_(block, state); + provider_().program_execution_apply_projection_(runtime_block, state); + if (!before) + return; + const std::optional> after = + provider_().program_execution_projection_balance_integrals_(block, state); + if (!after || before->size() != after->size() || + before->size() != static_cast(state.ncomp())) + throw std::runtime_error( + "Program projection balance provider changed its conservative component width"); + const int level = program_resource_field_level(); + for (int component = 0; component < state.ncomp(); ++component) + runtime.record_automatic_balance_term(runtime_block, level, component, "projection", + (*after)[static_cast(component)] - + (*before)[static_cast(component)], + "ProgramExecutionServices"); } /// Minimum physical cell size used by the native CFL authority. @@ -1397,6 +1421,10 @@ class ProgramExecutionServices { provider_().program_execution_record_balance_term_(route, term, value); } + void note_automatic_balance_capture_due(bool due) const { + program_runtime_state_().note_automatic_balance_capture_due(due, "ProgramExecutionServices"); + } + void note_step_projection(const std::string& name) const { program_runtime_state_().note_step_projection(name); } diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index e27984c07..406faaeed 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -300,6 +300,11 @@ struct ProgramRuntimeState { /// by accepted_balance_terms(). The owning facade snapshots this map with the rest of the attempt, /// so rejection cannot leak automatic evidence into a retry. std::map automatic_balance_terms_; + /// Monotone attempt-local decision emitted by generated code before any Program operator runs. + /// It is the OR of the exact ConsumerGraph-derived route decisions for this public step. Keeping + /// this separate from step_balance_terms_ lets projection operators execute before their later + /// Program.record_balance sinks without losing due automatic evidence. + bool automatic_balance_due_ = false; /// Attempt-local outer accepted-step target used by ConsumerGraph-fused balance guards. Program /// substeps temporarily publish their window-start macro step through the facade, so generated /// balance code must not infer the public target from `macro_step()+1`. @@ -852,13 +857,30 @@ struct ProgramRuntimeState { entry->second += value; } - /// Whether a compiled Program has actually emitted a due Balance route in this attempt. + /// Whether generated code proved that at least one Balance route is due in this attempt. /// - /// Generated balance records are cadence-guarded before their reductions. Reflux executes after - /// the Program body, so observing a non-empty authored mailbox here avoids every extra native - /// reduction on an off-cadence or replay step without introducing a second scheduler. + /// The exact ConsumerGraph-derived decision is emitted before any Program operator, so both an + /// in-body projection and post-body reflux observe the same cadence without a second scheduler. [[nodiscard]] bool automatic_balance_capture_due() const noexcept { - return !balance_replay_active_ && !step_balance_terms_.empty(); + return !balance_replay_active_ && automatic_balance_due_; + } + + /// Publish one generated ConsumerGraph due decision before Program operators execute. + /// + /// Several compiled Program invocations may share one outer accepted-step window. The marker is + /// therefore monotone inside an attempt and is reset only at attempt entry. Static-false routes + /// emit no call, so a run without Balance consumers retains no generated hot-path branch. + void note_automatic_balance_capture_due(bool due, const std::string& runtime) { + if (balance_replay_active_) { + if (due) + throw std::logic_error(runtime + + "::note_automatic_balance_capture_due cannot enable replay capture"); + return; + } + if (!balance_due_window_active_) + throw std::logic_error( + runtime + "::note_automatic_balance_capture_due requires an active public-step window"); + automatic_balance_due_ = automatic_balance_due_ || due; } /// Accumulate one signed, metric-integrated native operator contribution. @@ -904,6 +926,7 @@ struct ProgramRuntimeState { step_projections_.clear(); step_balance_terms_.clear(); automatic_balance_terms_.clear(); + automatic_balance_due_ = false; balance_due_window_active_ = false; balance_due_target_step_ = 0; balance_step_completed_ = false; diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py index 59c323b8a..1a57dafe5 100644 --- a/python/pops/codegen/program_balance_due.py +++ b/python/pops/codegen/program_balance_due.py @@ -212,6 +212,7 @@ def emit_balance_due_guards( if type(lowering) is not BalanceDueLowering: raise TypeError("balance due guard emission requires BalanceDueLowering") contract = json.dumps(lowering.contract.token) + automatic_tokens = [] for index, (route, periods) in enumerate(sorted(lowering.route_periods.items())): if not periods: token = "false" @@ -223,7 +224,13 @@ def emit_balance_due_guards( ] token = "balance_due_%d" % index lines.append("const bool %s = (%s);" % (token, " || ".join(calls))) + automatic_tokens.append(token) var[("balance_due_route", route)] = token + if automatic_tokens: + lines.append( + "ctx.note_automatic_balance_capture_due(%s);" + % (" || ".join(automatic_tokens)) + ) var[("balance_guarded_values",)] = lowering.guarded_values var[("balance_record_routes",)] = lowering.record_routes diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index cf5d1d318..3e0403973 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -463,6 +463,7 @@ struct AmrSystem::Impl { std::map program_diagnostics; std::map step_balance_terms; std::map automatic_balance_terms; + bool automatic_balance_due = false; bool balance_step_completed = false; bool balance_program_was_due = false; pops::runtime::program::CacheManager cache; @@ -511,6 +512,7 @@ struct AmrSystem::Impl { copy_value_map_into(program_diagnostics, impl.program_.diagnostics_); copy_value_map_into(step_balance_terms, impl.program_.step_balance_terms_); copy_value_map_into(automatic_balance_terms, impl.program_.automatic_balance_terms_); + automatic_balance_due = impl.program_.automatic_balance_due_; balance_step_completed = impl.program_.balance_step_completed_; balance_program_was_due = impl.program_.balance_program_was_due_; // AMR currently owns its native cache/history rings inside AmrRuntime. These two shared @@ -545,6 +547,7 @@ struct AmrSystem::Impl { copy_value_map_into(impl.program_.diagnostics_, program_diagnostics); copy_value_map_into(impl.program_.step_balance_terms_, step_balance_terms); copy_value_map_into(impl.program_.automatic_balance_terms_, automatic_balance_terms); + impl.program_.automatic_balance_due_ = automatic_balance_due; impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; diff --git a/src/runtime/system/system_impl.hpp b/src/runtime/system/system_impl.hpp index 40fddf113..dfcde1e1c 100644 --- a/src/runtime/system/system_impl.hpp +++ b/src/runtime/system/system_impl.hpp @@ -618,6 +618,7 @@ struct System::Impl { std::map program_diagnostics; std::map step_balance_terms; std::map automatic_balance_terms; + bool automatic_balance_due; bool balance_step_completed; bool balance_program_was_due; pops::runtime::program::CacheManager cache; @@ -645,6 +646,7 @@ struct System::Impl { program_diagnostics(impl.program_.diagnostics_), step_balance_terms(impl.program_.step_balance_terms_), automatic_balance_terms(impl.program_.automatic_balance_terms_), + automatic_balance_due(impl.program_.automatic_balance_due_), balance_step_completed(impl.program_.balance_step_completed_), balance_program_was_due(impl.program_.balance_program_was_due_), cache(impl.program_.cache_), @@ -680,6 +682,7 @@ struct System::Impl { impl.program_.diagnostics_ = program_diagnostics; impl.program_.step_balance_terms_ = step_balance_terms; impl.program_.automatic_balance_terms_ = automatic_balance_terms; + impl.program_.automatic_balance_due_ = automatic_balance_due; impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; From ab41987f9d52ac88f4d83ef8e9e303fb21647a9a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:02:57 +0200 Subject: [PATCH 37/62] test(balance): fence projection evidence cadence (ADC-686) --- .../runtime/test_program_runtime.cpp | 25 ++++ ...test_automatic_projection_balance_fence.py | 115 ++++++++++++++++++ .../python/unit/time/test_time_ops_polish.py | 11 +- 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 tests/python/architecture/test_automatic_projection_balance_fence.py diff --git a/tests/cpp/integration/runtime/test_program_runtime.cpp b/tests/cpp/integration/runtime/test_program_runtime.cpp index f36e55e23..7dbdd6202 100644 --- a/tests/cpp/integration/runtime/test_program_runtime.cpp +++ b/tests/cpp/integration/runtime/test_program_runtime.cpp @@ -139,6 +139,31 @@ TEST(ProgramRuntime, BalanceDueWindowUsesTheOuterAcceptedStepAndCleansUpOnFailur EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 4, "test"), std::logic_error); } +TEST(ProgramRuntime, AutomaticBalanceDueMarkerIsAttemptLocalMonotoneAndReplaySafe) { + runtime::program::ProgramRuntimeState state; + + EXPECT_FALSE(state.automatic_balance_capture_due()); + EXPECT_THROW(state.note_automatic_balance_capture_due(true, "test"), std::logic_error); + state.run_balance_due_window(0, "test", [&] { + state.note_automatic_balance_capture_due(false, "test"); + EXPECT_FALSE(state.automatic_balance_capture_due()); + state.note_automatic_balance_capture_due(true, "test"); + EXPECT_TRUE(state.automatic_balance_capture_due()); + state.note_automatic_balance_capture_due(false, "test"); + EXPECT_TRUE(state.automatic_balance_capture_due()); + }); + EXPECT_TRUE(state.automatic_balance_capture_due()); + + state.begin_step_projection_report(); + EXPECT_FALSE(state.automatic_balance_capture_due()); + state.run_balance_replay("test", [&] { + state.note_automatic_balance_capture_due(false, "test"); + EXPECT_FALSE(state.automatic_balance_capture_due()); + EXPECT_THROW(state.note_automatic_balance_capture_due(true, "test"), std::logic_error); + }); + EXPECT_FALSE(state.automatic_balance_capture_due()); +} + TEST(ProgramRuntime, SelectiveReplayCompilesBalanceOffAndRestoresTheGuard) { runtime::program::ProgramRuntimeState state; const std::string contract = "pops.balance-due-contract.v1:sha256:" + std::string(64, '3'); diff --git a/tests/python/architecture/test_automatic_projection_balance_fence.py b/tests/python/architecture/test_automatic_projection_balance_fence.py new file mode 100644 index 000000000..9c0e6e5fd --- /dev/null +++ b/tests/python/architecture/test_automatic_projection_balance_fence.py @@ -0,0 +1,115 @@ +"""ADC-686: projection balance evidence is due-only, metric, and still private.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +PROGRAM_STATE = ROOT / "include" / "pops" / "runtime" / "program" / "program_runtime_state.hpp" +EXECUTION_SERVICES = ( + ROOT / "include" / "pops" / "runtime" / "program" / "program_execution_services.hpp" +) +UNIFORM_CONTEXT = ROOT / "include" / "pops" / "runtime" / "program" / "program_context.hpp" +AMR_CONTEXT = ROOT / "include" / "pops" / "runtime" / "program" / "amr_program_context.hpp" +BALANCE_CODEGEN = ROOT / "python" / "pops" / "codegen" / "program_balance_due.py" +UNIFORM_IMPL = ROOT / "src" / "runtime" / "system" / "system_impl.hpp" +AMR_IMPL = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_generated_due_marker_precedes_operators_and_is_attempt_local() -> None: + codegen = BALANCE_CODEGEN.read_text() + emit = _between( + codegen, + "def emit_balance_due_guards(", + "\ndef balance_value_due_expression(", + ) + assert "automatic_tokens = []" in emit + assert "if automatic_tokens:" in emit + assert "ctx.note_automatic_balance_capture_due(%s);" in emit + + state = PROGRAM_STATE.read_text() + assert "bool automatic_balance_due_ = false;" in state + capture_due = _between( + state, + "[[nodiscard]] bool automatic_balance_capture_due() const noexcept", + "/// Accumulate one signed, metric-integrated native operator contribution.", + ) + assert "!balance_replay_active_ && automatic_balance_due_" in capture_due + assert "automatic_balance_due_ = automatic_balance_due_ || due;" in capture_due + + attempt_entry = _between( + state, + "void begin_step_projection_report()", + "void note_step_projection(", + ) + assert "automatic_balance_due_ = false;" in attempt_entry + + uniform = UNIFORM_IMPL.read_text() + adaptive = AMR_IMPL.read_text() + for source in (uniform, adaptive): + assert "automatic_balance_due" in source + assert "impl.program_.automatic_balance_due_" in source + + +def test_projection_delta_is_captured_only_when_due_and_stays_qualified() -> None: + services = EXECUTION_SERVICES.read_text() + projection = _between( + services, + "void apply_projection(int block, MultiFab& state) const", + "/// Minimum physical cell size used by the native CFL authority.", + ) + assert "if (!runtime.automatic_balance_capture_due())" in projection + assert projection.count("program_execution_projection_balance_integrals_") == 2 + assert projection.index("const std::optional> before") < projection.index( + "program_execution_apply_projection_" + ) + assert projection.index("program_execution_apply_projection_") < projection.index( + "const std::optional> after" + ) + assert "record_automatic_balance_term(" in projection + assert '"projection"' in projection + assert "runtime_block, level, component" in projection + + state = PROGRAM_STATE.read_text() + accepted = _between( + state, + "std::map accepted_balance_terms(", + "void begin_balance_due_window(", + ) + assert "automatic_balance_terms_" not in accepted + + +def test_uniform_projection_evidence_uses_exact_available_measure() -> None: + context = UNIFORM_CONTEXT.read_text() + provider = _between( + context, + "std::optional> program_execution_projection_balance_integrals_(", + "Real program_execution_hmin_() const", + ) + assert "if (sys_->program_is_polar())" in provider + assert "return std::nullopt;" in provider + assert "context.geom.dx() * context.geom.dy()" in provider + assert "RelativeCellMeasure measure;" in provider + assert "measure.active_cells = context.domain_mask;" in provider + assert "measure.inverse_volume_fraction = context.eb_inverse_volume_fraction;" in provider + assert "pops::reduce_sum(state, component, measure)" in provider + + +def test_amr_projection_evidence_excludes_covered_cells_and_reduces_once() -> None: + context = AMR_CONTEXT.read_text() + provider = _between( + context, + "std::optional> program_execution_projection_balance_integrals_(", + "Real program_execution_hmin_() const", + ) + assert "active_mask(views, level_, next)" in provider + assert "CompositeSumKind::Sum" in provider + assert "local_sum(" in provider + assert "if (!eng_->level_is_replicated(level_))" in provider + assert provider.count("all_reduce_sum_inplace(") == 1 + assert "geometry.dx()) * static_cast(geometry.dy())" in provider + assert "state.n_grow() != live.n_grow()" in provider + assert "state.local_size() != live.local_size()" in provider diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index d3f8a996c..b996b6950 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -360,6 +360,10 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): source = emit_cpp_program(P, balance_due_contract=contract) assert source.count("ctx.record_balance_term(") == 5 assert source.count("ctx.balance_consumer_is_due(") == 1 + assert source.count("ctx.note_automatic_balance_capture_due(") == 1 + assert source.index("ctx.note_automatic_balance_capture_due(") < source.index( + "ctx.record_balance_term(" + ) assert '"%s", 3)' % route.token in source assert "? (ctx.sum_component(" in source assert "ctx.record_scalar(" not in source @@ -367,12 +371,11 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): unreachable_source = emit_cpp_program( P, - balance_due_contract=_balance_due_contract( - route, every(1 << 31, clock=P.clock) - ), + balance_due_contract=_balance_due_contract(route, every(1 << 31, clock=P.clock)), ) assert "2147483648" not in unreachable_source assert "ctx.balance_consumer_is_due(" not in unreachable_source + assert "ctx.note_automatic_balance_capture_due(" not in unreachable_source def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): @@ -420,6 +423,7 @@ def test_record_balance_elides_native_collectives_without_a_consumer(t): source = emit_cpp_program(P) assert "ctx.balance_consumer_is_due(" not in source + assert "ctx.note_automatic_balance_capture_due(" not in source assert "ctx.record_balance_term(" not in source assert "(false) ? (ctx.sum_component(" in source @@ -480,6 +484,7 @@ def test_record_balance_physical_time_cadence_stays_conservatively_due(t): ) assert source.count("ctx.balance_consumer_is_due(") == 1 + assert source.count("ctx.note_automatic_balance_capture_due(") == 1 assert '"%s", 1)' % route.token in source assert source.count("ctx.record_balance_term(") == 5 From b2c7a90be7b5a9283fd5fa890bbbea10fb9ea35a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:03:03 +0200 Subject: [PATCH 38/62] docs(balance): describe qualified projection evidence (ADC-686) --- docs/design/exact-output-consumers.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 33b265125..45e587f90 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -498,12 +498,22 @@ balance reductions are not yet skipped. This fallback can add work but cannot su evidence. A zero-step run has no accepted native occurrence: its coincident start/end moment cannot publish an accepted-step consumer, including `Balance`. -This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot -produce its actual reflux or projection increment cannot declare `Balance`. In particular, the -generic automatic extraction of AMR reflux/projection contributions from the internal native -operator ledgers remains separate work. On an adaptive layout the recorded values must already be -composite and coverage-corrected; an ordinary sum of every per-level state would double-count -covered coarse cells. Neither `Balance` nor `BalanceTerms` silently claims otherwise. +This public route still consumes explicit evidence: a Program that cannot produce every actual term +cannot declare `Balance`. Native operator instrumentation is deliberately kept in a separate, +qualified attempt-local mailbox until a resolved quantity selector can prove which +`BalanceLedger` route owns each block/level/component contribution. Generated code publishes the OR +of the exact due route decisions before the first Program operator; the marker is monotone for the +attempt, disabled during replay, and reset at attempt entry. Consequently off-cadence steps do not +pay for automatic operator reductions. + +That private mailbox currently captures the signed AMR reflux correction and the before/after +projection delta. Uniform Cartesian projection uses the authenticated cell measure and embedded +boundary mask; AMR projection excludes covered coarse cells and performs one component-vector +collective per participating level. Polar projection stays absent because no exact per-cell polar +volume provider exists on this path. Automatic physical-boundary flux and source evidence are also +not yet producers. None of these private values is read by `accepted_balance_terms()`, so this +instrumentation does not silently complete an authored five-term balance or widen the public +contract. Checkpoint remains a separate restart effect. These consumers do not define a checkpoint schema or reader and do not call the scientific-output manifest a restart identity. The checkpoint provider From 3247df3c5284399ff3cbbf9701451a5ce333a9f4 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:03:33 +0200 Subject: [PATCH 39/62] test(balance): distinguish projection fast path ordering (ADC-686) --- .../test_automatic_projection_balance_fence.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/python/architecture/test_automatic_projection_balance_fence.py b/tests/python/architecture/test_automatic_projection_balance_fence.py index 9c0e6e5fd..8ce73ec53 100644 --- a/tests/python/architecture/test_automatic_projection_balance_fence.py +++ b/tests/python/architecture/test_automatic_projection_balance_fence.py @@ -63,10 +63,10 @@ def test_projection_delta_is_captured_only_when_due_and_stays_qualified() -> Non ) assert "if (!runtime.automatic_balance_capture_due())" in projection assert projection.count("program_execution_projection_balance_integrals_") == 2 - assert projection.index("const std::optional> before") < projection.index( - "program_execution_apply_projection_" - ) - assert projection.index("program_execution_apply_projection_") < projection.index( + due_projection = projection.split( + "const std::optional> before", 1 + )[1] + assert due_projection.index("program_execution_apply_projection_") < due_projection.index( "const std::optional> after" ) assert "record_automatic_balance_term(" in projection From c76e7c6f0db0e385cc2288d957acb6d964a267b0 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:49:46 +0200 Subject: [PATCH 40/62] feat(balance): select native ledger terms explicitly (ADC-686) --- python/pops/_balance_contract.py | 56 ++++++++++++++++++++-- python/pops/_balance_due_contract.py | 29 +++++++++-- python/pops/codegen/program_balance_due.py | 30 ++++++------ python/pops/diagnostics/measures.py | 12 ++++- python/pops/output/_consumer_contracts.py | 26 +++++++++- python/pops/time/_program/contract.py | 4 +- python/pops/time/_program/diagnostics.py | 56 ++++++++++++++++++---- 7 files changed, 176 insertions(+), 37 deletions(-) diff --git a/python/pops/_balance_contract.py b/python/pops/_balance_contract.py index 2fcf86671..14499db5a 100644 --- a/python/pops/_balance_contract.py +++ b/python/pops/_balance_contract.py @@ -31,31 +31,79 @@ def _canonical_name(value: Any, *, where: str) -> str: class BalanceLedger: """Identity joining one Program-authored discrete balance to one consumer. - The ledger does not contain values. :meth:`Program.record_balance` writes the five reduced - scalars into the current native step-attempt mailbox, while + The ledger does not contain values. :meth:`Program.record_balance` writes the explicitly + authored reduced scalars into the current native step-attempt mailbox. A ledger may delegate + ``reflux`` and/or ``projection`` to exact native operators for one typed component role, while :class:`pops.diagnostics.Balance` selects the same identity after that attempt has advanced successfully. """ name: str + role: Any = None + component: int | None = None + automatic_terms: tuple[str, ...] = () identity: Identity = field(init=False) __pops_ir_immutable__ = True def __post_init__(self) -> None: name = _canonical_name(self.name, where="BalanceLedger.name") + role = None + if self.role is not None: + from pops.physics.roles import native_role_token + + try: + role = native_role_token(self.role) + except TypeError as exc: + raise TypeError( + "BalanceLedger.role must be a typed pops.physics.roles.ComponentRole" + ) from exc + if not isinstance(self.automatic_terms, tuple): + raise TypeError("BalanceLedger.automatic_terms must be a tuple") + automatic_terms = tuple(sorted(set(self.automatic_terms))) + if len(automatic_terms) != len(self.automatic_terms): + raise ValueError("BalanceLedger.automatic_terms must be unique") + unsupported = set(automatic_terms).difference({"reflux", "projection"}) + if unsupported: + raise ValueError( + "BalanceLedger automatic native producers currently support only " + "reflux and projection; got %s" % sorted(unsupported) + ) + component = self.component + if automatic_terms and component is None: + component = 0 + if component is not None and (type(component) is not int or component < 0): + raise TypeError("BalanceLedger.component must be a non-negative int or None") object.__setattr__(self, "name", name) + object.__setattr__(self, "component", component) + object.__setattr__(self, "automatic_terms", automatic_terms) + payload: dict[str, Any] = {"schema_version": 1, "name": name} + if role is not None: + payload["role"] = role + if component is not None: + payload["component"] = component + if automatic_terms: + payload["automatic_terms"] = list(automatic_terms) object.__setattr__( self, "identity", - make_identity("balance-ledger", {"schema_version": 1, "name": name}), + make_identity("balance-ledger", payload), ) def to_data(self) -> dict[str, Any]: - return { + data = { "schema_version": 1, "name": self.name, "identity": self.identity.to_data(), } + if self.role is not None: + from pops.physics.roles import native_role_token + + data["role"] = native_role_token(self.role) + if self.component is not None: + data["component"] = self.component + if self.automatic_terms: + data["automatic_terms"] = list(self.automatic_terms) + return data def route_identity(self, block: Any) -> Identity: from pops.problem.handles import BlockHandle diff --git a/python/pops/_balance_due_contract.py b/python/pops/_balance_due_contract.py index 3052caaf7..f2bf15186 100644 --- a/python/pops/_balance_due_contract.py +++ b/python/pops/_balance_due_contract.py @@ -51,6 +51,7 @@ class BalanceDueRoute: route: Identity consumers: tuple[BalanceDueConsumer, ...] + automatic_terms: tuple[str, ...] = () def __post_init__(self) -> None: object.__setattr__( @@ -75,12 +76,21 @@ def __post_init__(self) -> None: if len(identities) != len(set(identities)): raise ValueError("BalanceDueRoute contains a duplicate consumer") object.__setattr__(self, "consumers", consumers) + if not isinstance(self.automatic_terms, tuple): + raise TypeError("BalanceDueRoute.automatic_terms must be a tuple") + if self.automatic_terms != tuple(sorted(set(self.automatic_terms))): + raise ValueError("BalanceDueRoute.automatic_terms must be sorted and unique") + if set(self.automatic_terms).difference({"reflux", "projection"}): + raise ValueError("BalanceDueRoute names an unavailable automatic balance producer") def to_data(self) -> dict[str, Any]: - return { + data = { "route": self.route.to_data(), "consumers": [value.to_data() for value in self.consumers], } + if self.automatic_terms: + data["automatic_terms"] = list(self.automatic_terms) + return data def accepted_step_periods(self) -> tuple[int, ...]: """Return exact native periods, conservatively using period one when unprovable. @@ -161,7 +171,9 @@ def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: raise TypeError( "BalanceDueContract requires an exact resolved ConsumerGraph or None" ) - by_route: dict[str, tuple[Identity, list[BalanceDueConsumer]]] = {} + by_route: dict[ + str, tuple[Identity, list[BalanceDueConsumer], tuple[str, ...]] + ] = {} for manifest in graph.nodes: for quantity in manifest.diagnostic_quantities: for operation in quantity.execution["operations"]: @@ -173,15 +185,22 @@ def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: "balance-ledger-route", where="accepted balance operation route", ) - existing = by_route.setdefault(route.token, (route, [])) + automatic_terms = tuple(operation.get("automatic_terms", ())) + existing = by_route.setdefault( + route.token, (route, [], automatic_terms) + ) + if existing[2] != automatic_terms: + raise ValueError( + "one balance route cannot select different automatic producers" + ) existing[1].append( BalanceDueConsumer(manifest.identity, manifest.schedule) ) return cls( graph.identity, tuple( - BalanceDueRoute(route, tuple(consumers)) - for route, consumers in by_route.values() + BalanceDueRoute(route, tuple(consumers), automatic_terms) + for route, consumers, automatic_terms in by_route.values() ), ) diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py index 1a57dafe5..2d2c2336b 100644 --- a/python/pops/codegen/program_balance_due.py +++ b/python/pops/codegen/program_balance_due.py @@ -99,15 +99,6 @@ def _program_balance_records( ) by_term[term] = value record_routes[value.id] = route.token - expected = set(BALANCE_TERM_NAMES) - for route, by_term in terms.items(): - if set(by_term) != expected: - missing = sorted(expected.difference(by_term)) - extra = sorted(set(by_term).difference(expected)) - raise ValueError( - "Program balance route %s must record exactly five terms; missing=%s extra=%s" - % (route, missing, extra) - ) return operations, record_routes, terms @@ -118,13 +109,23 @@ def validate_balance_due_contract(program: Any, contract: Any) -> None: "balance due validation requires an exact BalanceDueContract" ) _operations, _records, terms = _program_balance_records(program) - missing = sorted( - row.route.token for row in contract.routes if row.route.token not in terms - ) - if missing: + failures = [] + for row in contract.routes: + expected = set(BALANCE_TERM_NAMES).difference(row.automatic_terms) + actual = set(terms.get(row.route.token, {})) + if actual != expected: + failures.append( + "%s missing=%s extra=%s" + % ( + row.route.token, + sorted(expected.difference(actual)), + sorted(actual.difference(expected)), + ) + ) + if failures: raise ValueError( "ConsumerGraph Balance routes have no Program.record_balance producer: %s" - % ", ".join(missing) + % "; ".join(failures) ) @@ -137,6 +138,7 @@ def prepare_balance_due_lowering( raise TypeError( "balance due lowering requires an exact BalanceDueContract" ) + validate_balance_due_contract(program, contract) operations, record_routes, terms = _program_balance_records(program) route_periods = { route: ( diff --git a/python/pops/diagnostics/measures.py b/python/pops/diagnostics/measures.py index bf4c36cf8..d595d5398 100644 --- a/python/pops/diagnostics/measures.py +++ b/python/pops/diagnostics/measures.py @@ -334,7 +334,7 @@ def __init__( ) if block is None: raise TypeError("Balance(block=...) requires an exact physics BlockHandle") - super().__init__(block=block, role=None, cadence=cadence) + super().__init__(block=block, role=ledger.role, cadence=cadence) self.ledger = ledger def options(self) -> dict: @@ -346,11 +346,19 @@ def diagnostic_execution(self) -> dict[str, Any]: route = self.ledger.route_identity(self.block) return { "schema_version": 1, - "role": None, + "role": _role_name(self.ledger.role), "operations": [ { **_operation("balance", "accepted_balance"), "balance_route": route.token, + **( + { + "automatic_terms": list(self.ledger.automatic_terms), + "balance_component": self.ledger.component, + } + if self.ledger.automatic_terms + else {} + ), }, ], "conservation": None, diff --git a/python/pops/output/_consumer_contracts.py b/python/pops/output/_consumer_contracts.py index 6d4f7517c..a0323c9b2 100644 --- a/python/pops/output/_consumer_contracts.py +++ b/python/pops/output/_consumer_contracts.py @@ -343,6 +343,9 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: expected = {"name", "reduction", "transform", "metric_weighted"} if reduction == "accepted_balance": expected.add("balance_route") + if "automatic_terms" in operation: + expected.add("automatic_terms") + expected.add("balance_component") if set(operation) != expected: raise TypeError("%s has an unknown schema" % where) name = _text(operation["name"], "%s.name" % where) @@ -373,6 +376,27 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: "accepted balance route must use the version-1 balance-ledger-route identity" ) row["balance_route"] = route.token + automatic_terms = operation.get("automatic_terms", ()) + if not isinstance(automatic_terms, (tuple, list)): + raise TypeError("%s.automatic_terms must be a sequence" % where) + automatic_terms = tuple(automatic_terms) + if automatic_terms != tuple(sorted(set(automatic_terms))): + raise ValueError( + "%s.automatic_terms must be sorted and unique" % where + ) + unsupported = set(automatic_terms).difference({"reflux", "projection"}) + if unsupported: + raise ValueError( + "%s.automatic_terms names an unavailable native producer" % where + ) + if automatic_terms: + row["automatic_terms"] = list(automatic_terms) + component = operation["balance_component"] + if type(component) is not int or component < 0: + raise TypeError( + "%s.balance_component must be a non-negative int" % where + ) + row["balance_component"] = component normalized.append(row) if len({row["name"] for row in normalized}) != len(normalized): raise ValueError("DiagnosticQuantity execution operation names must be unique") @@ -383,8 +407,6 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: raise ValueError( "accepted balance evidence must be the sole diagnostic execution operation" ) - if has_accepted_balance and role is not None: - raise ValueError("accepted balance evidence cannot select one component role") conservation = value["conservation"] normalized_conservation = None if conservation is not None: diff --git a/python/pops/time/_program/contract.py b/python/pops/time/_program/contract.py index 4e39deeaf..cc262ddf4 100644 --- a/python/pops/time/_program/contract.py +++ b/python/pops/time/_program/contract.py @@ -208,8 +208,8 @@ def record_balance( storage_change: Any, outward_boundary_flux: Any, sources: Any, - reflux: Any, - projection: Any, + reflux: Any = None, + projection: Any = None, ) -> tuple[Any, ...]: ... # --- solve / commit / board sugar (_ProgramSolve) --- diff --git a/python/pops/time/_program/diagnostics.py b/python/pops/time/_program/diagnostics.py index 75d3ab00f..16bf5a501 100644 --- a/python/pops/time/_program/diagnostics.py +++ b/python/pops/time/_program/diagnostics.py @@ -34,17 +34,19 @@ def record_balance( storage_change: Any, outward_boundary_flux: Any, sources: Any, - reflux: Any, - projection: Any, + reflux: Any = None, + projection: Any = None, ) -> tuple[ProgramValue, ...]: """Publish one exact five-term balance into the current native attempt. - Every term is a signed, time-integrated increment for this Program invocation and - must be an additive global Program reduction (sum/dot), or scalar arithmetic composed - exclusively from such reductions and exact literals. The native mailbox accumulates - these increments across cadence substeps in the same public macro-step. Raw Python values, - extrema/norm reductions, and rank-local runtime scalars are rejected. The five records are - attempt-local: a rejected step or consumer rollback cannot leave evidence for a later sample. + Every explicitly authored term is a signed, time-integrated increment for this Program + invocation and must be an additive global Program reduction (sum/dot), or scalar arithmetic + composed exclusively from such reductions and exact literals. A ledger that explicitly + delegates ``reflux`` or ``projection`` to its native producer requires the corresponding + argument to remain ``None``. The native mailbox accumulates all increments across cadence + substeps in the same public macro-step. Raw Python values, extrema/norm reductions, and + rank-local runtime scalars are rejected. The records are attempt-local: a rejected step or + consumer rollback cannot leave evidence for a later sample. """ from pops._balance_contract import ( BALANCE_TERM_NAMES, @@ -90,10 +92,47 @@ def require_reduced(value: Any, term: str, seen: set[int]) -> ProgramValue: "only from global reductions; got scalar op %r" % (term, value.op) ) + automatic = set(ledger.automatic_terms) + for name in automatic: + if supplied[name] is not None: + raise ValueError( + "record_balance %s is owned by the ledger's native automatic producer; " + "leave it as None" % name + ) terms = { name: require_reduced(supplied[name], name, set()) for name in BALANCE_TERM_NAMES + if name not in automatic } + if automatic: + expected_component = ledger.component + + def reduced_components( + value: ProgramValue, term: str, seen: set[int] + ) -> set[int]: + if value.id in seen: + return set() + seen.add(value.id) + if value.op == "reduce": + component = value.attrs.get("comp") + if value.attrs.get("kind") != "sum" or type(component) is not int: + raise ValueError( + "record_balance %s must use component-qualified sum reductions " + "when native terms are selected" % term + ) + return {component} + components: set[int] = set() + for item in value.inputs: + components.update(reduced_components(item, term, seen)) + return components + + for name, value in terms.items(): + components = reduced_components(value, name, set()) + if components != {expected_component}: + raise ValueError( + "record_balance %s selects components %s but the native ledger owns " + "component %d" % (name, sorted(components), expected_component) + ) blocks = {value.block for value in terms.values()} if None in blocks or len(blocks) != 1: raise ValueError( @@ -114,6 +153,7 @@ def require_reduced(value: Any, term: str, seen: set[int]) -> ProgramValue: terms[name].block, ) for name in BALANCE_TERM_NAMES + if name in terms ) @atomic_authoring From 5412a954ce7a3a7c16d5b67bc44b729ac67f3616 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:50:03 +0200 Subject: [PATCH 41/62] feat(runtime): resolve qualified automatic balance evidence (ADC-686) --- include/pops/runtime/amr_system.hpp | 4 + .../runtime/program/program_runtime_state.hpp | 88 +++++++++++++++++++ include/pops/runtime/system.hpp | 4 + python/bindings/core/init/init_amr.cpp | 3 + python/bindings/core/init/init_system.cpp | 3 + python/pops/_pops.pyi | 16 ++++ python/pops/runtime/_runtime_consumers.py | 74 ++++++++++++++-- src/runtime/amr/amr_system.cpp | 22 +++++ src/runtime/system/system_program.cpp | 17 ++++ 9 files changed, 224 insertions(+), 7 deletions(-) diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index 29bdfe8fe..06d7c10ab 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -903,6 +903,10 @@ class AmrSystem { /// Five current-attempt scalars for one typed balance route. RuntimeInstance calls this only /// inside its active outer accepted-step transaction; missing/stale/non-finite evidence fails. POPS_EXPORT std::map accepted_balance_terms(const std::string& route) const; + /// The same accepted route with selected attempt-local native reflux/projection producers. + POPS_EXPORT std::map selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const; POPS_EXPORT void begin_step_projection_report(); POPS_EXPORT void note_step_projection(const std::string& name); POPS_EXPORT std::vector consume_step_projections(); diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index 406faaeed..436a5fe3c 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -969,6 +969,94 @@ struct ProgramRuntimeState { return result; } + /// Resolve one public Balance route against exact native operator coordinates. + /// + /// Explicit Program records remain authoritative for every term not listed in @p automatic_terms. + /// Reflux and projection may instead be selected from the attempt-local native mailbox. The + /// selector is complete and owner-qualified: one runtime block, one conservative component and + /// the full active contiguous hierarchy. A selected producer must have published every expected + /// coordinate; missing evidence and duplicate Program/native authority fail instead of becoming + /// zero or reusing a stale value. + std::map selected_accepted_balance_terms( + const std::string& route, int runtime_block, int component, const std::vector& levels, + const std::vector& automatic_terms, const std::string& runtime) const { + static constexpr std::array kTerms{"storage_change", "outward_boundary_flux", + "sources", "reflux", "projection"}; + require_balance_route(route, runtime + "::_selected_accepted_balance_terms"); + if (runtime_block < 0 || component < 0) + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms requires non-negative coordinates"); + if (levels.empty() || levels.front() < 0 || + std::adjacent_find(levels.begin(), levels.end(), + [](int left, int right) { return right != left + 1; }) != levels.end()) + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms requires a non-empty contiguous hierarchy"); + if (!std::is_sorted(automatic_terms.begin(), automatic_terms.end()) || + std::adjacent_find(automatic_terms.begin(), automatic_terms.end()) != automatic_terms.end()) + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms requires sorted unique automatic terms"); + for (const std::string& term : automatic_terms) + if (term != "reflux" && term != "projection") + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms has no native producer for '" + term + + "'"); + + std::map result; + if (step_balance_terms_.empty() && balance_step_completed_ && !balance_program_was_due_) { + for (const char* term : kTerms) + result.emplace(term, Real(0)); + return result; + } + for (const char* term_value : kTerms) { + const std::string term = term_value; + const bool automatic = + std::binary_search(automatic_terms.begin(), automatic_terms.end(), term); + const std::string record = "pops.balance-term.v1:" + route + ":" + term; + const auto authored = step_balance_terms_.find(record); + if (!automatic) { + if (authored == step_balance_terms_.end()) + throw std::runtime_error( + runtime + + "::_selected_accepted_balance_terms: current native attempt omitted term '" + term + + "'; Program.record_balance must publish every non-automatic term"); + if (!std::isfinite(static_cast(authored->second))) + throw std::runtime_error( + runtime + + "::_selected_accepted_balance_terms: current native attempt produced " + "non-finite term '" + + term + "'"); + result.emplace(term, authored->second); + continue; + } + if (authored != step_balance_terms_.end()) + throw std::runtime_error(runtime + "::_selected_accepted_balance_terms: term '" + term + + "' has both Program and native producer authority"); + + Real value = Real(0); + const std::size_t expected = term == "reflux" ? levels.size() - 1 : levels.size(); + for (std::size_t index = 0; index < expected; ++index) { + const AutomaticBalanceKey key{runtime_block, levels[index], component, term}; + const auto found = automatic_balance_terms_.find(key); + if (found == automatic_balance_terms_.end()) + throw std::runtime_error( + runtime + "::_selected_accepted_balance_terms: native producer omitted term '" + + term + "' at level " + std::to_string(levels[index])); + if (!std::isfinite(static_cast(found->second))) + throw std::runtime_error( + runtime + + "::_selected_accepted_balance_terms: native producer returned non-finite " + "term '" + + term + "'"); + value += found->second; + } + if (!std::isfinite(static_cast(value))) + throw std::runtime_error( + runtime + "::_selected_accepted_balance_terms: native term accumulation overflowed"); + result.emplace(term, value); + } + return result; + } + void begin_balance_due_window(int accepted_macro_step, const std::string& runtime) { if (balance_due_window_active_) throw std::logic_error(runtime + " balance due window is already active"); diff --git a/include/pops/runtime/system.hpp b/include/pops/runtime/system.hpp index 579275ff9..63837952b 100644 --- a/include/pops/runtime/system.hpp +++ b/include/pops/runtime/system.hpp @@ -1240,6 +1240,10 @@ class System { /// Five current-attempt scalars for one typed balance route. RuntimeInstance calls this only /// inside its active outer accepted-step transaction; missing/stale/non-finite evidence fails. POPS_EXPORT std::map accepted_balance_terms(const std::string& route) const; + /// The same accepted route with selected attempt-local native reflux/projection producers. + POPS_EXPORT std::map selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const; POPS_EXPORT void begin_step_projection_report(); POPS_EXPORT void note_step_projection(const std::string& name); POPS_EXPORT std::vector consume_step_projections(); diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index cbde10c19..3336d071b 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -824,6 +824,9 @@ void bind_amr_program(py::class_& cls) { .def("program_diagnostic", &AmrSystem::program_diagnostic, py::arg("name")) .def("program_diagnostics", &AmrSystem::program_diagnostics) .def("_accepted_balance_terms", &AmrSystem::accepted_balance_terms, py::arg("route")) + .def("_selected_accepted_balance_terms", &AmrSystem::selected_accepted_balance_terms, + py::arg("route"), py::arg("block"), py::arg("component"), py::arg("levels"), + py::arg("automatic_terms")) .def("_consume_step_projections", &AmrSystem::consume_step_projections) .def("record_program_diagnostic", &AmrSystem::record_program_diagnostic, py::arg("name"), py::arg("value")) diff --git a/python/bindings/core/init/init_system.cpp b/python/bindings/core/init/init_system.cpp index cca9cc0ed..51c260046 100644 --- a/python/bindings/core/init/init_system.cpp +++ b/python/bindings/core/init/init_system.cpp @@ -355,6 +355,9 @@ void bind_system_program(py::class_& cls) { .def("program_diagnostic", &System::program_diagnostic, py::arg("name")) .def("program_diagnostics", &System::program_diagnostics) .def("_accepted_balance_terms", &System::accepted_balance_terms, py::arg("route")) + .def("_selected_accepted_balance_terms", &System::selected_accepted_balance_terms, + py::arg("route"), py::arg("block"), py::arg("component"), py::arg("levels"), + py::arg("automatic_terms")) .def("_consume_step_projections", &System::consume_step_projections) // ADC-542: the native collective reduction over a named block the diagnostics driver drives to // fire a declared typed measure (Norm / Integral / MinMax) each cadence tick, and the sink the diff --git a/python/pops/_pops.pyi b/python/pops/_pops.pyi index 4b8c5d49d..0c6ffdc46 100644 --- a/python/pops/_pops.pyi +++ b/python/pops/_pops.pyi @@ -277,6 +277,14 @@ class System: def solve_fields(self) -> _SolveReport: ... def _consume_step_projections(self) -> list[str]: ... def _accepted_balance_terms(self, route: str) -> dict[str, float]: ... + def _selected_accepted_balance_terms( + self, + route: str, + block: str, + component: int, + levels: list[int], + automatic_terms: list[str], + ) -> dict[str, float]: ... def output_state_local_pieces( self, block: str, level: int ) -> tuple[dict[str, object], ...]: ... @@ -297,6 +305,14 @@ class AmrSystem: def configured_n_levels(self) -> int: ... def _consume_step_projections(self) -> list[str]: ... def _accepted_balance_terms(self, route: str) -> dict[str, float]: ... + def _selected_accepted_balance_terms( + self, + route: str, + block: str, + component: int, + levels: list[int], + automatic_terms: list[str], + ) -> dict[str, float]: ... def materialize_program_restart_histories( self, payload: bytes, diff --git a/python/pops/runtime/_runtime_consumers.py b/python/pops/runtime/_runtime_consumers.py index a0d040ee1..872cb4466 100644 --- a/python/pops/runtime/_runtime_consumers.py +++ b/python/pops/runtime/_runtime_consumers.py @@ -2433,9 +2433,44 @@ def _validate_diagnostic_providers(self) -> None: if reductions == {"accepted_balance"}: if len(quantity.execution["operations"]) != 1: raise ValueError("accepted balance requires exactly one native evidence route") - if quantity.execution["role"] is not None: - raise ValueError("accepted balance route cannot carry a component role") - if not callable(getattr(engine, "_accepted_balance_terms", None)): + operation, = quantity.execution["operations"] + automatic_terms = tuple(operation.get("automatic_terms", ())) + if automatic_terms: + if not callable( + getattr(engine, "_selected_accepted_balance_terms", None) + ): + raise NotImplementedError( + "automatic balance terms require native " + "_selected_accepted_balance_terms(...)" + ) + component = operation["balance_component"] + if component >= len(names): + raise ValueError( + "automatic balance component %d is outside block %r width %d" + % (component, block, len(names)) + ) + if quantity.execution["role"] is not None: + role_component, _ = self._diagnostic_component( + names, roles, quantity.execution["role"] + ) + if role_component != component: + raise ValueError( + "automatic balance role selects component %d but ledger " + "declares component %d" % (role_component, component) + ) + if "reflux" in automatic_terms and not layout.adaptive: + raise NotImplementedError( + "automatic reflux balance requires an adaptive hierarchy" + ) + if ( + "projection" in automatic_terms + and layout.geometry.cell_measure != CARTESIAN_CELL_AREA + ): + raise NotImplementedError( + "automatic projection balance requires exact Cartesian cell " + "measure support" + ) + elif not callable(getattr(engine, "_accepted_balance_terms", None)): raise NotImplementedError( "balance diagnostic requires native _accepted_balance_terms(route)" ) @@ -2556,14 +2591,31 @@ def _native_diagnostic_reduction( return float(cast(Any, native)(block, kind, component)), False @staticmethod - def _native_balance_terms(engine: Any, route: str) -> Any: + def _native_balance_terms( + engine: Any, + route: str, + *, + block: str, + component: int, + levels: tuple[int, ...], + automatic_terms: tuple[str, ...], + ) -> Any: """Read one current-attempt balance tuple from the native transaction mailbox.""" from pops.output.diagnostics import BalanceTerms - native = getattr(engine, "_accepted_balance_terms", None) + native_name = ( + "_selected_accepted_balance_terms" + if automatic_terms + else "_accepted_balance_terms" + ) + native = getattr(engine, native_name, None) if not callable(native): raise RuntimeError("installed runtime has no accepted balance evidence provider") - raw = native(route) + raw = ( + native(route, block, component, list(levels), list(automatic_terms)) + if automatic_terms + else native(route) + ) required = { "storage_change", "outward_boundary_flux", @@ -2612,8 +2664,16 @@ def _diagnostic_values( if "accepted_balance" in skip_reductions: continue operation, = execution["operations"] + automatic_terms = tuple(operation.get("automatic_terms", ())) + component = operation.get("balance_component", 0) balance = self._native_balance_terms( - engine, operation["balance_route"]) + engine, + operation["balance_route"], + block=block, + component=component, + levels=levels, + automatic_terms=automatic_terms, + ) terms = { "storage_change": balance.storage_change, "outward_boundary_flux": balance.outward_boundary_flux, diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index 3e0403973..22050acba 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -3579,6 +3579,28 @@ std::map AmrSystem::accepted_balance_terms(const std::strin "transaction"); return p_->program_.accepted_balance_terms(route, "AmrSystem"); } +std::map AmrSystem::selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const { + if (!p_->external_step_transaction_active_ || p_->external_step_transaction_committed_) + throw std::runtime_error( + "AmrSystem::_selected_accepted_balance_terms requires an active uncommitted external step " + "transaction"); + if (!p_->runtime) + throw std::runtime_error( + "AmrSystem::_selected_accepted_balance_terms requires an installed AMR runtime"); + const std::size_t runtime_block = p_->block_index_or_throw(block); + if (component < 0 || component >= p_->runtime->block_n_vars(runtime_block)) + throw std::out_of_range( + "AmrSystem::_selected_accepted_balance_terms component is out of range"); + if (levels.empty() || std::any_of(levels.begin(), levels.end(), [&](int level) { + return level < 0 || level >= p_->runtime->nlev(); + })) + throw std::out_of_range( + "AmrSystem::_selected_accepted_balance_terms level is out of active hierarchy range"); + return p_->program_.selected_accepted_balance_terms( + route, static_cast(runtime_block), component, levels, automatic_terms, "AmrSystem"); +} void AmrSystem::begin_step_projection_report() { p_->program_.begin_step_projection_report(); } diff --git a/src/runtime/system/system_program.cpp b/src/runtime/system/system_program.cpp index ee3c308f2..6f1ceb023 100644 --- a/src/runtime/system/system_program.cpp +++ b/src/runtime/system/system_program.cpp @@ -435,6 +435,23 @@ std::map System::accepted_balance_terms(const std::string& ro "System::_accepted_balance_terms requires an active uncommitted external step transaction"); return p_->program_.accepted_balance_terms(route, "System"); } +std::map System::selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const { + if (!p_->external_step_transaction_ || p_->external_step_transaction_committed_) + throw std::runtime_error( + "System::_selected_accepted_balance_terms requires an active uncommitted external step " + "transaction"); + const int runtime_block = p_->index(block); + const auto& state = p_->find(block); + if (component < 0 || component >= state.ncomp) + throw std::out_of_range("System::_selected_accepted_balance_terms component is out of range"); + if (levels != std::vector{0}) + throw std::invalid_argument( + "System::_selected_accepted_balance_terms requires exactly uniform level 0"); + return p_->program_.selected_accepted_balance_terms(route, runtime_block, component, levels, + automatic_terms, "System"); +} void System::begin_step_projection_report() { p_->program_.begin_step_projection_report(); } From 7a57c4d6c068f90e7c9d928a8361a305dc2dd5a3 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:50:24 +0200 Subject: [PATCH 42/62] test(balance): prove qualified native term selection (ADC-686) --- .../runtime/test_program_runtime.cpp | 31 ++++++++ ...test_automatic_projection_balance_fence.py | 6 +- .../test_automatic_reflux_balance_fence.py | 8 ++- ...qualified_automatic_balance_route_fence.py | 72 +++++++++++++++++++ ...est_async_scientific_output_diagnostics.py | 38 ++++++++++ .../unit/runtime/test_consumer_authoring.py | 39 ++++++++++ .../unit/runtime/test_diagnostics_typed.py | 33 +++++++++ .../python/unit/time/test_time_ops_polish.py | 62 +++++++++++++++- 8 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 tests/python/architecture/test_qualified_automatic_balance_route_fence.py diff --git a/tests/cpp/integration/runtime/test_program_runtime.cpp b/tests/cpp/integration/runtime/test_program_runtime.cpp index 7dbdd6202..a70d1481f 100644 --- a/tests/cpp/integration/runtime/test_program_runtime.cpp +++ b/tests/cpp/integration/runtime/test_program_runtime.cpp @@ -164,6 +164,37 @@ TEST(ProgramRuntime, AutomaticBalanceDueMarkerIsAttemptLocalMonotoneAndReplaySaf EXPECT_FALSE(state.automatic_balance_capture_due()); } +TEST(ProgramRuntime, SelectedAutomaticBalanceTermsRequireCompleteQualifiedEvidence) { + runtime::program::ProgramRuntimeState state; + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '5'); + state.begin_step_projection_report(); + state.run_balance_due_window(0, "test", [&] { + state.note_automatic_balance_capture_due(true, "test"); + state.record_balance_term(route, "storage_change", 1.0, "test"); + state.record_balance_term(route, "outward_boundary_flux", 2.0, "test"); + state.record_balance_term(route, "sources", 3.0, "test"); + state.record_automatic_balance_term(2, 0, 1, "projection", 0.25, "test"); + state.record_automatic_balance_term(2, 1, 1, "projection", 0.75, "test"); + state.record_automatic_balance_term(2, 0, 1, "reflux", 0.5, "test"); + }); + state.complete_balance_step(true); + + const auto selected = + state.selected_accepted_balance_terms(route, 2, 1, {0, 1}, {"projection", "reflux"}, "test"); + EXPECT_EQ(selected.at("storage_change"), 1.0); + EXPECT_EQ(selected.at("outward_boundary_flux"), 2.0); + EXPECT_EQ(selected.at("sources"), 3.0); + EXPECT_EQ(selected.at("projection"), 1.0); + EXPECT_EQ(selected.at("reflux"), 0.5); + + EXPECT_THROW((void)state.selected_accepted_balance_terms(route, 2, 1, {0, 1, 2}, + {"projection", "reflux"}, "test"), + std::runtime_error); + EXPECT_THROW((void)state.selected_accepted_balance_terms(route, 2, 1, {0, 2}, + {"projection", "reflux"}, "test"), + std::invalid_argument); +} + TEST(ProgramRuntime, SelectiveReplayCompilesBalanceOffAndRestoresTheGuard) { runtime::program::ProgramRuntimeState state; const std::string contract = "pops.balance-due-contract.v1:sha256:" + std::string(64, '3'); diff --git a/tests/python/architecture/test_automatic_projection_balance_fence.py b/tests/python/architecture/test_automatic_projection_balance_fence.py index 8ce73ec53..14064e070 100644 --- a/tests/python/architecture/test_automatic_projection_balance_fence.py +++ b/tests/python/architecture/test_automatic_projection_balance_fence.py @@ -79,7 +79,11 @@ def test_projection_delta_is_captured_only_when_due_and_stays_qualified() -> Non "std::map accepted_balance_terms(", "void begin_balance_due_window(", ) - assert "automatic_balance_terms_" not in accepted + explicit_only = accepted.split( + "std::map selected_accepted_balance_terms(", 1 + )[0] + assert "automatic_balance_terms_" not in explicit_only + assert "automatic_balance_terms_" in accepted def test_uniform_projection_evidence_uses_exact_available_measure() -> None: diff --git a/tests/python/architecture/test_automatic_reflux_balance_fence.py b/tests/python/architecture/test_automatic_reflux_balance_fence.py index 7b2c866a6..2b300d152 100644 --- a/tests/python/architecture/test_automatic_reflux_balance_fence.py +++ b/tests/python/architecture/test_automatic_reflux_balance_fence.py @@ -40,8 +40,12 @@ def test_automatic_balance_mailbox_is_attempt_local_and_not_a_route_fallback() - "std::map accepted_balance_terms(", "void begin_balance_due_window(", ) - assert "step_balance_terms_" in accepted - assert "automatic_balance_terms_" not in accepted + explicit_only = accepted.split( + "std::map selected_accepted_balance_terms(", 1 + )[0] + assert "step_balance_terms_" in explicit_only + assert "automatic_balance_terms_" not in explicit_only + assert "automatic_balance_terms_" in accepted uniform = UNIFORM_IMPL.read_text() adaptive = AMR_IMPL.read_text() diff --git a/tests/python/architecture/test_qualified_automatic_balance_route_fence.py b/tests/python/architecture/test_qualified_automatic_balance_route_fence.py new file mode 100644 index 000000000..71768e4c8 --- /dev/null +++ b/tests/python/architecture/test_qualified_automatic_balance_route_fence.py @@ -0,0 +1,72 @@ +"""ADC-686: public Balance routes select qualified native evidence fail-closed.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +LEDGER = ROOT / "python" / "pops" / "_balance_contract.py" +MEASURES = ROOT / "python" / "pops" / "diagnostics" / "measures.py" +CONSUMERS = ROOT / "python" / "pops" / "runtime" / "_runtime_consumers.py" +PROGRAM_STATE = ( + ROOT / "include" / "pops" / "runtime" / "program" / "program_runtime_state.hpp" +) +SYSTEM = ROOT / "src" / "runtime" / "system" / "system_program.cpp" +AMR = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" +SYSTEM_BINDING = ROOT / "python" / "bindings" / "core" / "init" / "init_system.cpp" +AMR_BINDING = ROOT / "python" / "bindings" / "core" / "init" / "init_amr.cpp" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_public_ledger_owns_role_and_exact_automatic_term_selection() -> None: + ledger = LEDGER.read_text() + assert "role: Any = None" in ledger + assert "component: int | None = None" in ledger + assert "automatic_terms: tuple[str, ...] = ()" in ledger + assert '{"reflux", "projection"}' in ledger + + measures = MEASURES.read_text() + balance = _between(measures, "class Balance(_Measure):", "class ConservationCheck") + assert "role=ledger.role" in balance + assert '"automatic_terms": list(self.ledger.automatic_terms)' in balance + assert '"balance_component": self.ledger.component' in balance + + +def test_runtime_uses_selected_native_entrypoint_only_for_delegated_terms() -> None: + consumers = CONSUMERS.read_text() + native = _between( + consumers, + "def _native_balance_terms(", + "def _diagnostic_values(", + ) + assert '"_selected_accepted_balance_terms"' in native + assert 'if automatic_terms' in native + assert "native(route, block, component, list(levels), list(automatic_terms))" in native + assert "else native(route)" in native + + for binding in (SYSTEM_BINDING, AMR_BINDING): + assert '"_selected_accepted_balance_terms"' in binding.read_text() + + +def test_native_selector_requires_complete_owner_level_component_evidence() -> None: + state = PROGRAM_STATE.read_text() + selector = _between( + state, + "std::map selected_accepted_balance_terms(", + "void begin_balance_due_window(", + ) + assert "AutomaticBalanceKey key{runtime_block, levels[index], component, term}" in selector + assert "native producer omitted term" in selector + assert "both Program and native producer authority" in selector + assert 'term == "reflux" ? levels.size() - 1 : levels.size()' in selector + + uniform = SYSTEM.read_text() + assert "const int runtime_block = p_->index(block);" in uniform + assert "levels != std::vector{0}" in uniform + + adaptive = AMR.read_text() + assert "const std::size_t runtime_block = p_->block_index_or_throw(block);" in adaptive + assert "p_->runtime->block_n_vars(runtime_block)" in adaptive + assert "p_->runtime->nlev()" in adaptive diff --git a/tests/python/unit/output/test_async_scientific_output_diagnostics.py b/tests/python/unit/output/test_async_scientific_output_diagnostics.py index 6a1298553..7fb6b856a 100644 --- a/tests/python/unit/output/test_async_scientific_output_diagnostics.py +++ b/tests/python/unit/output/test_async_scientific_output_diagnostics.py @@ -30,6 +30,7 @@ from pops.output._restart_provider import RestartAuthority from pops.output._writers.common import writer_session_authority from pops.problem.handles import BlockHandle +from pops.runtime._runtime_consumers import RuntimeConsumerPublisher from pops.runtime._runtime_instance import RuntimeInstance from pops.time import Clock, every from tests.python.support.layout_plan import cartesian_grid @@ -282,6 +283,43 @@ def _accepted_balance_terms(self, route): } +def test_selected_native_balance_forwards_exact_owner_coordinates(): + class _SelectedExecutor: + def __init__(self): + self.call = None + + def _selected_accepted_balance_terms( + self, route, block, component, levels, automatic_terms + ): + self.call = (route, block, component, levels, automatic_terms) + return { + "storage_change": 7.0, + "outward_boundary_flux": 2.0, + "sources": 3.0, + "reflux": 1.0, + "projection": 0.5, + } + + executor = _SelectedExecutor() + terms = RuntimeConsumerPublisher._native_balance_terms( + executor, + "route", + block="fluid", + component=2, + levels=(0, 1), + automatic_terms=("projection", "reflux"), + ) + + assert executor.call == ( + "route", + "fluid", + 2, + [0, 1], + ["projection", "reflux"], + ) + assert terms.residual == pytest.approx(4.5) + + def _async_balance_runtime(tmp_path: Path): base = _install() mode = _scientific_output_mode(base.artifact) diff --git a/tests/python/unit/runtime/test_consumer_authoring.py b/tests/python/unit/runtime/test_consumer_authoring.py index c5d2ed660..73569e27f 100644 --- a/tests/python/unit/runtime/test_consumer_authoring.py +++ b/tests/python/unit/runtime/test_consumer_authoring.py @@ -293,6 +293,45 @@ def test_balance_consumer_resolves_one_exact_native_ledger_route(): assert contract.identity.domain == "balance-due-contract" +def test_balance_consumer_retains_native_term_selector_in_due_contract(): + case, block, state = _case() + clock = Clock("macro", owner=case.owner_path) + schedule = every(4, clock=clock) + ledger = BalanceLedger( + "mass-native", automatic_terms=("projection", "reflux") + ) + graph = ConsumerGraph.from_consumers(( + ScientificOutput( + format=ParaView(), + schedule=schedule, + fields=(state,), + diagnostics=(Balance(ledger, block=block),), + target="state/native-balance", + ), + )) + case.consumers(graph) + pops.validate(case) + subjects = case.layout_subjects() + layout = normalize_layout_plan( + Uniform(cartesian_grid(n=8)), + owner=case.owner_path.canonical(), + states=subjects.states, + fields=subjects.fields, + blocks=subjects.blocks, + handle_resolver=case.resolve, + ) + + resolved = graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()) + quantity, = resolved.nodes[0].diagnostic_quantities + operation, = quantity.execution["operations"] + route = ledger.route_identity(case.resolve(block)) + contract = BalanceDueContract.from_consumer_graph(resolved) + + assert operation["automatic_terms"] == ("projection", "reflux") + assert operation["balance_component"] == 0 + assert contract.route(route.token).automatic_terms == ("projection", "reflux") + + def test_balance_consumer_refuses_a_schedule_that_can_fire_at_start(): case, block, state = _case() clock = Clock("macro", owner=case.owner_path) diff --git a/tests/python/unit/runtime/test_diagnostics_typed.py b/tests/python/unit/runtime/test_diagnostics_typed.py index a2670174d..b93c3ea36 100644 --- a/tests/python/unit/runtime/test_diagnostics_typed.py +++ b/tests/python/unit/runtime/test_diagnostics_typed.py @@ -126,6 +126,39 @@ def test_balance_uses_one_typed_native_attempt_route(): ConservationCheck(balance).diagnostic_execution() +def test_balance_ledger_selects_exact_native_component_terms(): + ledger = BalanceLedger( + "mass-native", + role=Density(), + automatic_terms=("projection", "reflux"), + ) + balance = Balance(ledger, block=_NE_BLOCK) + execution = balance.diagnostic_execution() + operation, = execution["operations"] + + assert execution["role"] == "Density" + assert operation["automatic_terms"] == ["projection", "reflux"] + assert operation["balance_component"] == 0 + assert balance.options()["role"] == "Density" + assert ledger.to_data()["role"] == "Density" + assert ledger.to_data()["component"] == 0 + assert ledger.to_data()["automatic_terms"] == ["projection", "reflux"] + assert ledger.identity != BalanceLedger("mass-native").identity + + with pytest.raises(TypeError, match="ComponentRole"): + BalanceLedger("bad-role", role="Density") + reordered = BalanceLedger( + "canonical-order", automatic_terms=("reflux", "projection") + ) + assert reordered.automatic_terms == ("projection", "reflux") + with pytest.raises(ValueError, match="must be unique"): + BalanceLedger("duplicate", automatic_terms=("reflux", "reflux")) + with pytest.raises(ValueError, match="only reflux and projection"): + BalanceLedger("bad-producer", automatic_terms=("sources",)) + with pytest.raises(TypeError, match="non-negative int"): + BalanceLedger("bad-component", component=-1, automatic_terms=("projection",)) + + # --- Integral / MinMax ------------------------------------------------------------------ def test_integral_is_a_sum_reduction(): mass = Integral(role=Density()) diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index b996b6950..788566df8 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -59,7 +59,7 @@ def t(): return time -def _balance_due_contract(route, *schedules): +def _balance_due_contract(route, *schedules, automatic_terms=()): return BalanceDueContract( make_identity("consumer-graph", {"test": "balance-due"}), ( @@ -72,6 +72,7 @@ def _balance_due_contract(route, *schedules): ) for index, schedule in enumerate(schedules) ), + automatic_terms, ), ), ) @@ -378,6 +379,65 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): assert "ctx.note_automatic_balance_capture_due(" not in unreachable_source +def test_record_balance_delegates_selected_native_terms_without_placeholders(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("native-balance-terms") + U = typed_state(P, "blk") + total = P.sum(U) + ledger = BalanceLedger( + "mass-native", automatic_terms=("projection", "reflux") + ) + records = P.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=total * 2.0, + sources=total * 3.0, + ) + route = ledger.route_identity(U.block) + assert tuple(record.attrs["term"] for record in records) == ( + "storage_change", + "outward_boundary_flux", + "sources", + ) + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + contract = _balance_due_contract( + route, + every(2, clock=P.clock), + automatic_terms=("projection", "reflux"), + ) + source = emit_cpp_program(P, balance_due_contract=contract) + assert source.count("ctx.record_balance_term(") == 3 + assert source.count("ctx.note_automatic_balance_capture_due(") == 1 + + P_bad = t.Program("duplicate-native-balance-term") + U_bad = typed_state(P_bad, "blk") + total_bad = P_bad.sum(U_bad) + with pytest.raises(ValueError, match="owned by.*native automatic producer"): + P_bad.record_balance( + ledger, + storage_change=total_bad, + outward_boundary_flux=total_bad, + sources=total_bad, + projection=total_bad, + ) + + component_ledger = BalanceLedger( + "component-one", + component=1, + automatic_terms=("projection",), + ) + with pytest.raises(ValueError, match="selects components.*component 1"): + P_bad.record_balance( + component_ledger, + storage_change=total_bad, + outward_boundary_flux=total_bad, + sources=total_bad, + reflux=total_bad, + ) + + def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): from pops.diagnostics import BalanceLedger From e7e70f66b97466fc3af661a33df86623e6635e95 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:50:33 +0200 Subject: [PATCH 43/62] docs(balance): define automatic ledger authority (ADC-686) --- docs/design/exact-output-consumers.md | 72 +++++++++++++++++++-------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 45e587f90..6f74b73d8 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -53,6 +53,28 @@ ScientificOutput( ) ``` +When the native runtime owns an AMR reflux correction and/or an authored projection, the ledger can +delegate those exact terms instead of requiring zero placeholders. `component` is the exact +conservative index shared by the explicit Program sums and native evidence (it defaults to zero for +a scalar state); the optional typed role is checked against that index at bind: + +```python +from pops.physics.roles import Density + +mass = BalanceLedger( + "mass", + role=Density(), + component=0, + automatic_terms=("projection", "reflux"), +) +program.record_balance( + mass, + storage_change=storage_increment, + outward_boundary_flux=boundary_flux_increment, + sources=source_increment, +) +``` + Le fournisseur possède l'extension. Une cible comme `solution/tracer.vtu` est refusée dès l'authoring, avant le bind ; elle empêcherait le changement de format et entrerait en collision au deuxième échantillon. Chaque pas accepté dû publie immédiatement un fichier distinct sous le chemin @@ -451,15 +473,20 @@ schedule and transaction. Its reductions are completed on the simulation thread the post-commit worker receives only immutable arrays and scalar payloads, never the native mailbox or communicator facade. -Each argument to `record_balance` is a signed, time-integrated native Program sum/dot reduction, -or scalar arithmetic composed only from such reductions and exact literals. +Each non-automatic argument to `record_balance` is a signed, time-integrated native Program sum/dot +reduction, or scalar arithmetic composed only from such reductions and exact literals. When any +term is delegated to a native producer, every explicit term must instead be composed from +component-qualified `sum` reductions for the ledger's exact `component`; an all-state dot product +cannot be reconciled with one component's reflux/projection evidence. The reported residual is `storage_change + outward_boundary_flux - sources - reflux - projection`. The native attempt mailbox accumulates repeated cadence/substep invocations, rejects missing or non-finite terms, and is cleared before the next attempt. The consumer reads it only while the outer accepted-step transaction still retains the pre-step image. Python therefore packages the five returned scalars and residual but never traverses arrays, invents a zero term, or reuses a -previous step. A rejected attempt or failed consumer publication restores the mailbox with the -rest of the native transaction. +previous step. Selected automatic terms are resolved by exact runtime block, active hierarchy level +and conservative component. A missing coordinate, a non-finite value, or simultaneous Program and +native authority for one term fails the accepted transaction. A rejected attempt or failed consumer +publication restores both mailboxes with the rest of the native transaction. The `pops.balance-term` namespace is reserved. Ordinary `Program.record_scalar(...)` authoring and the Python runtime diagnostic binding both reject it; generated `record_balance` code reaches a @@ -476,8 +503,9 @@ by an OR of their exact accepted-step periods. `Always` and `when(True)` are per The compiler traces the complete reduction/scalar chain rather than scheduling only the terminal records. If a value is also consumed by an ordinary Program diagnostic or another non-balance operation, that shared producer remains unconditional so cadence fusion cannot change unrelated -semantics. A `Balance` consumer with no matching five-term `Program.record_balance` producer fails -before native code generation. Program stride/substeps use one attempt-local outer accepted-step +semantics. A `Balance` consumer with no complete matching `Program.record_balance` producer for all +non-automatic terms fails before native code generation. Program stride/substeps use one +attempt-local outer accepted-step target, so every substep of one due public step sees the same decision and accumulates into the same attempt mailbox. The cadence is authored once as part of the Program identity, for example `program.cadence(substeps=2, stride=3)`, then authenticated and installed before runtime freeze on @@ -498,22 +526,22 @@ balance reductions are not yet skipped. This fallback can add work but cannot su evidence. A zero-step run has no accepted native occurrence: its coincident start/end moment cannot publish an accepted-step consumer, including `Balance`. -This public route still consumes explicit evidence: a Program that cannot produce every actual term -cannot declare `Balance`. Native operator instrumentation is deliberately kept in a separate, -qualified attempt-local mailbox until a resolved quantity selector can prove which -`BalanceLedger` route owns each block/level/component contribution. Generated code publishes the OR -of the exact due route decisions before the first Program operator; the marker is monotone for the -attempt, disabled during replay, and reset at attempt entry. Consequently off-cadence steps do not -pay for automatic operator reductions. - -That private mailbox currently captures the signed AMR reflux correction and the before/after -projection delta. Uniform Cartesian projection uses the authenticated cell measure and embedded -boundary mask; AMR projection excludes covered coarse cells and performs one component-vector -collective per participating level. Polar projection stays absent because no exact per-cell polar -volume provider exists on this path. Automatic physical-boundary flux and source evidence are also -not yet producers. None of these private values is read by `accepted_balance_terms()`, so this -instrumentation does not silently complete an authored five-term balance or widen the public -contract. +The selected public route now consumes signed AMR reflux corrections and before/after projection +deltas from the separate qualified attempt mailbox. Uniform Cartesian projection uses the +authenticated cell measure and embedded-boundary mask; AMR projection excludes covered coarse cells +and performs one component-vector collective per participating level. A reflux selection requires +an adaptive hierarchy and expects one contribution for every active parent/fine interface; +projection expects one for every selected active level. Generated code publishes the OR of the exact +due route decisions before the first Program operator; the marker is monotone for the attempt, +disabled during replay, and reset at attempt entry. Consequently off-cadence steps do not pay for +automatic operator reductions. + +The capability remains deliberately bounded. Polar projection is rejected because no exact +per-cell polar volume provider exists on this path. Automatic physical-boundary flux and source +evidence are not yet producers and therefore remain explicit `Program.record_balance` arguments. +The native selector never substitutes a missing automatic value with zero (except the exact reflux +identity for a hierarchy with no coarse/fine interface), and the legacy all-explicit ledger route +retains its original identity and behavior. Checkpoint remains a separate restart effect. These consumers do not define a checkpoint schema or reader and do not call the scientific-output manifest a restart identity. The checkpoint provider From ed81c0989fd0bb9abf98eedaab1dd96cb3d5eb38 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 00:19:45 +0200 Subject: [PATCH 44/62] gate(m4): select qualified automatic balance proof --- docs/design/m4-conformance-gate.md | 5 +++-- tests/gates/m4_runtime_io.toml | 8 ++++++++ tests/python/architecture/test_m4_runtime_io_gate.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 7fa6c573c..79d6a4e5d 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -2,7 +2,7 @@ The current status is **CLOSED AND CI-EXECUTED**. The ledger in `tests/gates/m4_runtime_io.toml` records exact executable evidence for -ADC-679 through ADC-687. It contains exactly 51 executable checks and +ADC-679 through ADC-687. It contains exactly 52 executable checks and `deferred = []`. Closure is accepted only for a commit whose required MPI job successfully executes the complete installed gate; source audit alone is not the acceptance evidence. @@ -28,7 +28,8 @@ The source audit already authenticates real proofs for: parity; - a prepared native FieldSolver whose invalid first result is refused through RuntimeInstance with exact accepted-state rollback and a successful retry; -- accepted scientific publication, diagnostics, two-rank collective HDF5, +- accepted scientific publication, diagnostics including qualified native + projection/reflux term selection, two-rank collective HDF5, and a two-rank PVD/PVTU/rank-VTU hierarchy reopened by native VTK readers. The required Ubuntu 24.04 MPI lane installs Open MPI, parallel HDF5, NumPy, diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 52b643dff..095979fc6 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -324,6 +324,14 @@ kind = "ctest" target = "diagnostics@test_program_context_contract" test_regex = "^ProgramContextContract\\.AcceptedBalanceEvidenceIsCurrentAttemptExactAndFailClosed$" +[[check]] +issue = "ADC-686" +requirement = "diagnostics" +polarity = "positive" +kind = "ctest" +target = "diagnostics@test_program_runtime" +test_regex = "^ProgramRuntime\\.SelectedAutomaticBalanceTermsRequireCompleteQualifiedEvidence$" + [[check]] issue = "ADC-686" requirement = "diagnostics" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index e327e1aab..47d6093a8 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -39,7 +39,7 @@ def test_m4_manifest_is_a_closed_exact_matrix(): assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) assert data["deferred"] == [] - assert len(data["check"]) == 51 + assert len(data["check"]) == 52 assert data["issues"] == [ "ADC-679", "ADC-680", From bb89bf343b26209ac8f3e498175698670eaaa4d3 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 00:20:46 +0200 Subject: [PATCH 45/62] test(m4): build selected program runtime proof --- tests/python/architecture/test_m4_runtime_io_gate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 47d6093a8..6a9f37960 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -156,6 +156,7 @@ def test_m4_closed_gate_lists_every_exact_native_build_target(): "test_native_loader_param_overflow", "test_platform_manifest", "test_program_context_contract", + "test_program_runtime", ) assert runner._required_ctest_targets(data["check"]) == expected From cb81112a10d35896ca0be9025805a261202438d8 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 00:48:59 +0200 Subject: [PATCH 46/62] test(ci): align architecture proofs with complete M4 gate --- .../architecture/test_ci_impacted_selection.py | 5 ++++- .../test_program_execution_services.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/python/architecture/test_ci_impacted_selection.py b/tests/python/architecture/test_ci_impacted_selection.py index 72108fb5f..0d20d810b 100644 --- a/tests/python/architecture/test_ci_impacted_selection.py +++ b/tests/python/architecture/test_ci_impacted_selection.py @@ -955,7 +955,10 @@ def test_ci_required_gate_aggregates_full_matrix_and_mpi_path_changes(): assert "selected_count=$(python3 -c" in mpi_block assert "selected ${selected_count}/${expected} launches" in mpi_block assert "ctest --preset ci-mpi --output-on-failure --parallel 4 --no-tests=error" in mpi_block - assert "timeout-minutes: 70" in mpi_block + # The complete M4 installed-package gate now runs after the native MPI, + # Python MPI and collective-HDF5 matrices in this same required job. Keep + # the outer watchdog aligned with that complete sequential contract. + assert "timeout-minutes: 180" in mpi_block assert "timeout-minutes: 35" in mpi_block assert '/usr/bin/python3 -u "$mpi_test"' in mpi_block assert "mpiexec -n \"$mpi_ranks\"" not in mpi_block diff --git a/tests/python/architecture/test_program_execution_services.py b/tests/python/architecture/test_program_execution_services.py index 217fffe8b..d701fb0a2 100644 --- a/tests/python/architecture/test_program_execution_services.py +++ b/tests/python/architecture/test_program_execution_services.py @@ -364,8 +364,11 @@ def test_contexts_expose_explicit_provider_hooks_for_the_shared_surface(): "program_execution_set_field_parameters_", "program_execution_set_field_kernel_", ): - assert source.count(hook) == 1, ( - "%s must provide exactly one explicit provider hook %s" % (context, hook) + definitions = re.findall( + rf"(?m)^ [^\n;=]*\b{re.escape(hook)}\s*\(", source + ) + assert len(definitions) == 1, ( + "%s must define exactly one explicit provider hook %s" % (context, hook) ) @@ -668,7 +671,12 @@ def test_shared_projection_maps_the_program_block_once_and_leaves_native_dispatc shared = _read(SHARED) uniform = _read(UNIFORM) amr = _read(AMR) - assert "program_execution_apply_projection_(sys_block(block), state)" in shared + projection = shared.split("void apply_projection(int block, MultiFab& state) const {", 1)[ + 1 + ].split("\n }", 1)[0] + assert projection.count("const int runtime_block = sys_block(block);") == 1 + assert "program_execution_apply_projection_(runtime_block, state)" in projection + assert "program_execution_apply_projection_(sys_block(block), state)" not in projection assert "sys_->block_project(runtime_block, state);" in uniform assert ( "eng_->project_level_state(static_cast(runtime_block), level_, state);" in amr From 3a6ee80eb1178799c5c46b06df155cf02dfc697d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 00:53:23 +0200 Subject: [PATCH 47/62] test(m4): prove fixed binary tamper refusal --- docs/design/m4-conformance-gate.md | 2 +- tests/gates/m4_runtime_io.toml | 8 ++++++ .../architecture/test_m4_runtime_io_gate.py | 6 ++++- .../unit/codegen/test_component_packages.py | 26 +++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 79d6a4e5d..96402396e 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -2,7 +2,7 @@ The current status is **CLOSED AND CI-EXECUTED**. The ledger in `tests/gates/m4_runtime_io.toml` records exact executable evidence for -ADC-679 through ADC-687. It contains exactly 52 executable checks and +ADC-679 through ADC-687. It contains exactly 53 executable checks and `deferred = []`. Closure is accepted only for a commit whose required MPI job successfully executes the complete installed gate; source audit alone is not the acceptance evidence. diff --git a/tests/gates/m4_runtime_io.toml b/tests/gates/m4_runtime_io.toml index 095979fc6..2e9d3e664 100644 --- a/tests/gates/m4_runtime_io.toml +++ b/tests/gates/m4_runtime_io.toml @@ -372,6 +372,14 @@ kind = "ctest" target = "tamper_capability_abi@test_amr_native_loader" test_regex = "^test_amr_native_loader\\.RefusesComponentBuiltForAnotherNativeAbi$" +[[check]] +issue = "ADC-687" +requirement = "tamper_capability_abi" +polarity = "refusal" +kind = "pytest" +target = "tamper_capability_abi" +nodeid = "tests/python/unit/codegen/test_component_packages.py::test_fixed_binary_bytes_are_authenticated_before_package_use" + [[check]] issue = "ADC-687" requirement = "legacy_stepper_retirement" diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 6a9f37960..dc70c7a43 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -39,7 +39,7 @@ def test_m4_manifest_is_a_closed_exact_matrix(): assert not errors, "M4 gate audit is structurally invalid:\n " + "\n ".join(errors) assert data["deferred"] == [] - assert len(data["check"]) == 52 + assert len(data["check"]) == 53 assert data["issues"] == [ "ADC-679", "ADC-680", @@ -523,6 +523,10 @@ def test_m4_gate_keeps_real_tamper_and_capacity_refusals(): r"^test_native_loader_param_overflow\.Runs$", r"^test_amr_native_loader\.RefusesComponentBuiltForAnotherNativeAbi$", r"^PlatformManifest\.UnknownCapabilityRefusesBeforeKernel$", + ( + "tests/python/unit/codegen/test_component_packages.py::" + "test_fixed_binary_bytes_are_authenticated_before_package_use" + ), } <= refusals assert ( "tests/python/unit/codegen/test_component_manifest_v2.py::" diff --git a/tests/python/unit/codegen/test_component_packages.py b/tests/python/unit/codegen/test_component_packages.py index 3e987b6a2..d5733d824 100644 --- a/tests/python/unit/codegen/test_component_packages.py +++ b/tests/python/unit/codegen/test_component_packages.py @@ -121,6 +121,32 @@ def test_fixed_binary_cannot_claim_template_genericity(): assert error.value.code == "fixed_generic_claim" +def test_fixed_binary_bytes_are_authenticated_before_package_use(tmp_path): + platform = proven_serial_manifest( + backend="aot-component", target="component", abi="headers|clang|c++20") + component = _manifest(generic=False) + binary = b"authenticated-fixed-component" + data = build_fixed_binary_manifest( + components={"average": component}, + platform=platform, + binary_path="average.so", + binary=binary, + symbols=("pops_component_interface_v1",), + ) + binary_path = tmp_path / "average.so" + manifest_path = tmp_path / "average.pops.json" + binary_path.write_bytes(binary) + manifest_path.write_text(json.dumps(data), encoding="utf-8") + + package = load(manifest_path) + assert package.binary == binary + + binary_path.write_bytes(binary + b"-tampered") + with pytest.raises(ComponentPackageError) as error: + load(manifest_path) + assert error.value.code == "binary_digest" + + def test_compiled_registry_refuses_source_values_and_freezes(): registry = CompiledArtifactRegistry() with pytest.raises(TypeError, match="CompiledComponentArtifact"): From b81150cd4403d01db54a8f0585be08f66fcbefff Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 00:54:14 +0200 Subject: [PATCH 48/62] docs(m4): separate source closure from CI evidence --- docs/design/m4-conformance-gate.md | 4 ++-- tests/python/architecture/test_m4_runtime_io_gate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 96402396e..2b88ef7e0 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -1,9 +1,9 @@ # M4 native runtime and scientific I/O conformance gate -The current status is **CLOSED AND CI-EXECUTED**. The ledger in +The evidence ledger is **SOURCE-CLOSED AND REQUIRED BY CI**. The ledger in `tests/gates/m4_runtime_io.toml` records exact executable evidence for ADC-679 through ADC-687. It contains exactly 53 executable checks and -`deferred = []`. Closure is accepted only for a commit whose required MPI job +`deferred = []`. Milestone closure is accepted only for a commit whose required MPI job successfully executes the complete installed gate; source audit alone is not the acceptance evidence. diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index dc70c7a43..2d2170505 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -647,7 +647,7 @@ def test_m4_gate_pins_complete_program_only_dispatch_and_fallback_fences(): documentation = ( ROOT / "docs/design/m4-conformance-gate.md" ).read_text(encoding="utf-8") - assert "current status is **CLOSED AND CI-EXECUTED**" in documentation + assert "evidence ledger is **SOURCE-CLOSED AND REQUIRED BY CI**" in documentation assert "four serial proofs" in documentation From 3cd73709dd2f57fed2b3b51c4485e334a81de10d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:41:07 +0200 Subject: [PATCH 49/62] fix(runtime): expose exact installed Program metadata --- python/bindings/core/init/init_amr.cpp | 6 ++++++ python/bindings/core/init/init_system.cpp | 9 +++++++++ python/pops/runtime/program_report.py | 8 ++++++-- .../integration/runtime/test_multi_layout_runtime.py | 8 ++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index 3336d071b..deb8be1ab 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -789,6 +789,12 @@ void bind_amr_program(py::class_& cls) { // IR hash of the installed compiled Program (the .so's pops_program_hash), or "" if none. Parity // System::installed_program_hash (the checkpoint guard). .def("installed_program_hash", &AmrSystem::installed_program_hash) + // Exact Program-index -> AMR-block-index map established by name at install. Expose only + // immutable report metadata, never a structural mutation route. + .def("program_block_map", &AmrSystem::program_block_map) + .def("program_param_count", [](const AmrSystem& system, int program_block) { + return system.program_params(program_block).count; + }, py::arg("program_block")) .def("program_accepted_state", [](const AmrSystem& s) { const auto bytes = s.program_accepted_state(); diff --git a/python/bindings/core/init/init_system.cpp b/python/bindings/core/init/init_system.cpp index 51c260046..7a3ad83a3 100644 --- a/python/bindings/core/init/init_system.cpp +++ b/python/bindings/core/init/init_system.cpp @@ -339,6 +339,15 @@ void bind_system_program(py::class_& cls) { // ADC-406b: IR hash of the installed compiled Program (the .so's pops_program_hash), or "" if // none. sim.checkpoint records it; sim.restart rejects a restart against a DIFFERENT Program. .def("installed_program_hash", &System::installed_program_hash) + // Exact Program-index -> System-index map established by name during install_program. The + // structured runtime report consumes this owned native fact; an empty Python-side fallback + // must never be mistaken for an identity map in a sliced multi-layout Program. + .def("program_block_map", &System::program_block_map) + // Metadata-only parameter occupancy for ProgramRuntimeReport. Keep the fixed-size values + // private while exposing the native count that proves every compiled carrier was installed. + .def("program_param_count", [](const System& system, int program_block) { + return system.program_params(program_block).count; + }, py::arg("program_block")) // ADC-592: runtime freeze lifecycle. mark_bound() (called LAST by the Python bind flow) freezes // the composition -> every structural setter then rejects; lifecycle_state() reports // assembling / bound / running (running derived from macro_step()). diff --git a/python/pops/runtime/program_report.py b/python/pops/runtime/program_report.py index 6194a5727..08e17c48e 100644 --- a/python/pops/runtime/program_report.py +++ b/python/pops/runtime/program_report.py @@ -126,8 +126,12 @@ def _params(sim: Any) -> Any: block_map = list(_call(sim, "program_block_map", []) or []) prog_blocks = list(range(len(block_map))) if block_map else [0] for prog_block in prog_blocks: - rp = _call(sim, "program_params", None, prog_block) - count = getattr(rp, "count", None) if rp is not None else None + count = _call(sim, "program_param_count", None, prog_block) + if count is None: + # Compatibility for report-only authorities used by downstream integrations. Native + # System and AmrSystem expose program_param_count directly, without publishing values. + rp = _call(sim, "program_params", None, prog_block) + count = getattr(rp, "count", None) if rp is not None else None rows.append({"program_block": prog_block, "count": count, "limit": limit}) return rows diff --git a/tests/python/integration/runtime/test_multi_layout_runtime.py b/tests/python/integration/runtime/test_multi_layout_runtime.py index 80511e16e..f1a165473 100644 --- a/tests/python/integration/runtime/test_multi_layout_runtime.py +++ b/tests/python/integration/runtime/test_multi_layout_runtime.py @@ -470,6 +470,14 @@ def test_uniform_amr_and_multi_layout_share_complete_runtime_instance_contract( assert len(instance["layout_plan"]["layouts"]) == expected_layout_counts[label] assert program["installed"] is True assert program["program_hash"] == runtime.installed_program_hash() + assert len(program["block_map"]) == len(runtime.block_names()) + assert tuple(sorted(program["block_map"])) == tuple( + range(len(runtime.block_names())) + ) + assert all( + type(row["count"]) is int and 0 <= row["count"] <= row["limit"] + for row in program["params"] + ) assert inspection["program"]["installed"] == program["installed"] assert inspection["program"]["hash"] == program["program_hash"] for name in ( From 701b80795adaa3e52d686a8564caaaddb5ab1036 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:41:12 +0200 Subject: [PATCH 50/62] fix(fields): admit authenticated singleton MPI solvers --- .../prepared_field_solver_component.hpp | 19 +++++++++++++++---- .../test_external_field_solver_runtime.py | 6 +++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/pops/runtime/system/prepared_field_solver_component.hpp b/include/pops/runtime/system/prepared_field_solver_component.hpp index 7eecc7ad0..2ee2c1ac3 100644 --- a/include/pops/runtime/system/prepared_field_solver_component.hpp +++ b/include/pops/runtime/system/prepared_field_solver_component.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -63,8 +64,8 @@ struct FieldTopologyReportRow { /// materialized once from replicated patch metadata and reused for every solve. A solve sends every /// local patch view in one request and calls the component exactly once on every participating rank, /// including ranks with zero local patches. The currently proven System route is host-resident, -/// serial, Cartesian, cell-centered and full-material; unsupported execution/layout facts are -/// rejected before either component can mutate the solution. +/// serial or singleton-MPI, Cartesian, cell-centered and full-material; unsupported +/// execution/layout facts are rejected before either component can mutate the solution. class PreparedFieldSolverComponent final { public: PreparedFieldSolverComponent(PreparedFieldSolverSpec spec, @@ -566,10 +567,20 @@ class PreparedFieldSolverComponent final { throw std::invalid_argument("prepared external field solver specification is incomplete"); const auto execution = spec_.execution->view(); component::validate_execution_context(execution); + const std::string communicator_identity(execution.communicator_identity); + bool singleton_mpi = false; +#ifdef POPS_HAS_MPI + if (communicator_identity == "MPI_COMM_WORLD") { + const CommunicatorView communicator{ + MPI_Comm_f2c(static_cast(execution.communicator_f_handle))}; + singleton_mpi = communicator.active() && communicator.size() == 1; + } +#endif if (execution.memory_space != POPS_MEMORY_SPACE_HOST_V1 || - std::string(execution.communicator_identity) != "serial") + (communicator_identity != "serial" && !singleton_mpi)) throw std::invalid_argument( - "external FieldSolver v2 System adapter currently proves host/serial execution only"); + "external FieldSolver v2 System adapter currently proves host serial or singleton-MPI " + "execution only"); const auto& topology_api = topology_component_->api(); const auto& solver_api = solver_component_->api(); if (topology_api.component_id == nullptr || topology_api.manifest_identity == nullptr || diff --git a/tests/python/integration/native_loader/test_external_field_solver_runtime.py b/tests/python/integration/native_loader/test_external_field_solver_runtime.py index f1f40d1f0..4d902fc9a 100644 --- a/tests/python/integration/native_loader/test_external_field_solver_runtime.py +++ b/tests/python/integration/native_loader/test_external_field_solver_runtime.py @@ -15,6 +15,7 @@ passive_field_model, resolve_periodic_field_program, ) +from tests.python.support.native_execution_context import artifact_execution_context def _manifest(name, interface, parameters=()): @@ -314,6 +315,7 @@ def test_external_field_pair_executes_and_reports_materialized_topology(tmp_path simulation = pops.bind( artifact, initial_state={"material": np.ones((1, 8, 8), dtype=np.float64)}, + resources={"execution_context": artifact_execution_context(artifact)}, ) slot, = simulation.field_provider_slots() before = simulation.inspect().to_dict()["instance"]["field_providers"] @@ -379,9 +381,11 @@ def test_real_prepared_field_solver_failure_rolls_back_runtime_instance_and_retr target="system", n=8, field_solver=provider, components=(topology, solver)) + artifact = pops.compile(resolved) simulation = pops.bind( - pops.compile(resolved), + artifact, initial_state={"material": np.ones((1, 8, 8), dtype=np.float64)}, + resources={"execution_context": artifact_execution_context(artifact)}, ) slot, = simulation.field_provider_slots() accepted_before = { From d5391b4b56e7829b11b2256f7cd4f4aa4f870bc9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:41:16 +0200 Subject: [PATCH 51/62] test(m4): honor the installed consumer context --- .../runtime/test_consumer_transactions.py | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/tests/python/unit/runtime/test_consumer_transactions.py b/tests/python/unit/runtime/test_consumer_transactions.py index 9bdc4d645..072da4c1c 100644 --- a/tests/python/unit/runtime/test_consumer_transactions.py +++ b/tests/python/unit/runtime/test_consumer_transactions.py @@ -47,6 +47,14 @@ from tests.python.unit.runtime.test_runtime_planning import _install, _manifest +def _output_mode(runtime) -> ParallelMode: + return ( + ParallelMode.SERIAL + if runtime.communication.communicator_id == "serial" + else ParallelMode.PER_RANK + ) + + def _runtime(*, collective: bool = False): install = _install() requirements = () @@ -77,7 +85,7 @@ def _manifest_for( resource: str = "state:u", dependency: Handle | None = None, action=None, - parallel_mode=ParallelMode.SERIAL, + parallel_mode=None, ) -> ConsumerManifest: owner = OwnerPath.consumer("adc-685") handle = Handle(name, kind="consumer", owner=owner) @@ -89,6 +97,8 @@ def _manifest_for( dependencies = (dependency,) if dependency is not None else () if action is None: action = FailRun() + if parallel_mode is None: + parallel_mode = _output_mode(runtime) return ConsumerManifest( handle=handle, kind=ConsumerKind.SCIENTIFIC_OUTPUT, @@ -97,7 +107,9 @@ def _manifest_for( target_uri="file:///adc-685/%s" % name, output_format=( HDF5(mode=ParallelMode.COLLECTIVE) - if parallel_mode is ParallelMode.COLLECTIVE else NPZ()), + if parallel_mode is ParallelMode.COLLECTIVE + else NPZ(mode=parallel_mode) + ), parallel_mode=parallel_mode, dependencies=dependencies, failure_action=action, @@ -221,6 +233,24 @@ def test_graph_and_plan_are_semantic_and_insertion_order_independent(): def test_distributed_modes_require_a_nonserial_context_before_planning(parallel_mode): _, serial_runtime = _runtime() clock = Clock("solution", owner=OwnerPath.consumer("adc-685-collective")) + if serial_runtime.communication.communicator_id != "serial": + # The installed MPI package cannot manufacture a serial ExecutionContext. Prove the + # inverse mismatch against its real context instead; the serial CI route exercises every + # distributed mode below. + manifest = replace( + _manifest_for(serial_runtime, "serial", clock), + output_format=NPZ(mode=ParallelMode.SERIAL), + parallel_mode=ParallelMode.SERIAL, + ) + with pytest.raises(RuntimePlanningError) as error: + plan_accepted_side_effects( + serial_runtime, ConsumerGraph((manifest,)), _moment(clock) + ) + assert error.value.code == "serial_consumer_requires_serial_context" + assert error.value.evidence == { + "communicator": serial_runtime.communication.communicator_id + } + return output_format = ( HDF5(mode=parallel_mode) if parallel_mode is not ParallelMode.PER_RANK @@ -314,14 +344,15 @@ def test_stale_field_requires_explicit_policy_and_records_recompute_without_solv runtime.calls[0].layout_id, field_context=context, ) + parallel_mode = _output_mode(runtime) manifest = ConsumerManifest( Handle("field-output", kind="consumer", owner=OwnerPath.consumer("adc-685-field")), ConsumerKind.SCIENTIFIC_OUTPUT, (quantity,), Schedule(Every(AcceptedStep(clock), 1)), "file:///adc-685/field", - NPZ(), - ParallelMode.SERIAL, + NPZ(mode=parallel_mode), + parallel_mode, ) moment = _moment(clock, step=2, layouts=(layout,)) with pytest.raises(RuntimePlanningError) as error: From 5abcaa247a278b45e474e8a1351554f6006101ee Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:41:20 +0200 Subject: [PATCH 52/62] docs(m4): state installed report and solver bounds --- docs/design/m4-conformance-gate.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 2b88ef7e0..80c152fa4 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -97,6 +97,11 @@ fault is removed, the same prepared component returns a finite result and the unchanged RuntimeInstance accepts the retry. The selected test defines no step wrapper and never replaces a native engine or step target. +That installed proof uses the MPI-enabled module with a one-rank +`MPI_COMM_WORLD`. The System adapter authenticates and accepts this singleton +communicator explicitly; it still refuses multi-rank external FieldSolver +execution until a collective distributed solve contract is proved. + The positive RuntimeInstance proof is also a compiled route. It builds and executes one Uniform artifact, one AMR artifact, and one two-layout artifact with a native conservative Transfer. Every execution returns the exact public @@ -104,10 +109,12 @@ with a native conservative Transfer. Every execution returns the exact public run, clock, step, and transaction evidence. The multi-layout executor authenticates each installed child Program, creates one domain-separated hash for the ordered Program set, and projects local block/parameter/cache metadata -into deterministic layout-qualified report rows. Runtime inspection consumes -that same complete `ProgramRuntimeReport`; the selected test proves direct and -inspection parity without a wrapper, fake engine, replaced step target, or -monkeypatch. +into deterministic layout-qualified report rows. The block bijection and +parameter occupancy come from the installed native `program_block_map()` and +`program_param_count()` accessors; the report does not infer an identity map +from missing bindings. Runtime inspection consumes that same complete +`ProgramRuntimeReport`; the selected test proves direct and inspection parity +without a wrapper, fake engine, replaced step target, or monkeypatch. ## Gate modes From 1a5f81aa6e614178b2ef6a7e3631f9650342985d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:49:14 +0200 Subject: [PATCH 53/62] fix(fields): hide provisional topology after rollback --- docs/design/m4-conformance-gate.md | 12 +++++++----- include/pops/runtime/system/system_field_solver.hpp | 10 ++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/design/m4-conformance-gate.md b/docs/design/m4-conformance-gate.md index 80c152fa4..eb17085d9 100644 --- a/docs/design/m4-conformance-gate.md +++ b/docs/design/m4-conformance-gate.md @@ -91,11 +91,13 @@ fault marker makes its solve report convergence while returning non-finite values, so the production field validation fails inside the native Program step. RuntimeInstance must restore the conservative state, field potential, accepted clock, macro-step, temporal -authority, consumer cursors, reports, and provider evidence exactly. The -component's prepared state is not mutated by this failure. After the external -fault is removed, the same prepared component returns a finite result and the -unchanged RuntimeInstance accepts the retry. The selected test defines no step -wrapper and never replaces a native engine or step target. +authority, consumer cursors, reports, and accepted provider evidence exactly. +The immutable prepared component stays installed and may reuse its private +topology cache, but that provisional cache is not published as accepted +materialization. After the external fault is removed, the same prepared +component returns a finite result and the unchanged RuntimeInstance accepts the +retry. The selected test defines no step wrapper and never replaces a native +engine or step target. That installed proof uses the MPI-enabled module with a one-rank `MPI_COMM_WORLD`. The System adapter authenticates and accepts this singleton diff --git a/include/pops/runtime/system/system_field_solver.hpp b/include/pops/runtime/system/system_field_solver.hpp index 646f6c3ac..f26eddfa5 100644 --- a/include/pops/runtime/system/system_field_solver.hpp +++ b/include/pops/runtime/system/system_field_solver.hpp @@ -1680,8 +1680,6 @@ class SystemFieldSolver { program_boundary_baselines_; std::set candidate_program_boundary_slots_; bool program_boundary_install_active_ = false; - std::map> - external_field_components_; EllipticBackendRegistry elliptic_registry_; std::shared_ptr nullspace_provider_registry_; @@ -1942,7 +1940,6 @@ class SystemFieldSolver { auto component = std::make_shared( std::move(spec), std::move(topology), std::move(solver)); register_elliptic_provider(slot, std::make_unique(component)); - external_field_components_[slot] = component; if (found == named_field_plans_.end()) return std::string(component->provider_identity()); found->second.backend_provider_identity = slot; @@ -1963,11 +1960,12 @@ class SystemFieldSolver { auto field = named_fields_.find(slot); if (field != named_fields_.end() && field->second.backend) return field->second.backend->topology_report(); - auto external = external_field_components_.find(slot); - if (external != external_field_components_.end()) - return external->second->topology_report(); if (named_field_plans_.find(slot) == named_field_plans_.end()) throw std::runtime_error("unknown qualified field provider slot"); + // A failed attempt may leave an immutable prepared component's private topology cache warm + // for retry, while restore_step_snapshot() correctly removes the provisional backend from the + // accepted runtime. Inspection reports accepted materialization only: never leak that private + // cache as published provider evidence before a backend belongs to the accepted state. return {}; } From c6a82836c0f330a5727cf9751d6ea8be4d56ef58 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:51:58 +0200 Subject: [PATCH 54/62] style(bindings): format installed report accessors --- python/bindings/core/init/init_amr.cpp | 9 ++++++--- python/bindings/core/init/init_system.cpp | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index deb8be1ab..15a3a1ef2 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -792,9 +792,12 @@ void bind_amr_program(py::class_& cls) { // Exact Program-index -> AMR-block-index map established by name at install. Expose only // immutable report metadata, never a structural mutation route. .def("program_block_map", &AmrSystem::program_block_map) - .def("program_param_count", [](const AmrSystem& system, int program_block) { - return system.program_params(program_block).count; - }, py::arg("program_block")) + .def( + "program_param_count", + [](const AmrSystem& system, int program_block) { + return system.program_params(program_block).count; + }, + py::arg("program_block")) .def("program_accepted_state", [](const AmrSystem& s) { const auto bytes = s.program_accepted_state(); diff --git a/python/bindings/core/init/init_system.cpp b/python/bindings/core/init/init_system.cpp index 7a3ad83a3..5c8f1c9d6 100644 --- a/python/bindings/core/init/init_system.cpp +++ b/python/bindings/core/init/init_system.cpp @@ -345,9 +345,12 @@ void bind_system_program(py::class_& cls) { .def("program_block_map", &System::program_block_map) // Metadata-only parameter occupancy for ProgramRuntimeReport. Keep the fixed-size values // private while exposing the native count that proves every compiled carrier was installed. - .def("program_param_count", [](const System& system, int program_block) { - return system.program_params(program_block).count; - }, py::arg("program_block")) + .def( + "program_param_count", + [](const System& system, int program_block) { + return system.program_params(program_block).count; + }, + py::arg("program_block")) // ADC-592: runtime freeze lifecycle. mark_bound() (called LAST by the Python bind flow) freezes // the composition -> every structural setter then rejects; lifecycle_state() reports // assembling / bound / running (running derived from macro_step()). From 61e6c715f5e2fd39a44e0b7704c2a18ea8db1ed6 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 02:04:23 +0200 Subject: [PATCH 55/62] fix(fields): transact accepted topology evidence --- .../runtime/system/system_field_solver.hpp | 43 ++++++++++++++----- src/runtime/system/system_fields.cpp | 13 +++--- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/include/pops/runtime/system/system_field_solver.hpp b/include/pops/runtime/system/system_field_solver.hpp index f26eddfa5..d75817925 100644 --- a/include/pops/runtime/system/system_field_solver.hpp +++ b/include/pops/runtime/system/system_field_solver.hpp @@ -313,6 +313,8 @@ class SystemFieldSolver { RuntimeDiagnosticsReport diagnostics; std::map named_potentials; std::vector named_unbuilt; + std::map> + named_topology_reports; }; [[nodiscard]] static bool same_publication_layout(const MultiFab& lhs, @@ -333,6 +335,7 @@ class SystemFieldSolver { if (phi_src_polar_) out.polar_source = *phi_src_polar_; for (auto& item : named_fields_) { + out.named_topology_reports.emplace(item.first, item.second.published_topology_report); if (!item.second.backend) { out.named_unbuilt.push_back(item.first); continue; @@ -362,9 +365,13 @@ class SystemFieldSolver { } phi_src_polar_ = snapshot.polar_source; for (auto& item : named_fields_) { + const auto topology = snapshot.named_topology_reports.find(item.first); + if (topology == snapshot.named_topology_reports.end()) + throw std::logic_error("System field snapshot lost accepted topology evidence"); if (std::find(snapshot.named_unbuilt.begin(), snapshot.named_unbuilt.end(), item.first) != snapshot.named_unbuilt.end()) { invalidate_named_backend_(item.second); + item.second.published_topology_report = topology->second; continue; } const auto saved = snapshot.named_potentials.find(item.first); @@ -373,6 +380,7 @@ class SystemFieldSolver { ensure_named_backend(item.second, item.first); item.second.backend->restore(saved->second); } + item.second.published_topology_report = topology->second; } diagnostics_ = snapshot.diagnostics; } @@ -418,6 +426,8 @@ class SystemFieldSolver { if (snapshot.named_potentials.size() + snapshot.named_unbuilt.size() != named_fields_.size()) return false; for (const auto& [name, field] : named_fields_) { + if (snapshot.named_topology_reports.find(name) == snapshot.named_topology_reports.end()) + return false; const auto saved = snapshot.named_potentials.find(name); const bool was_unbuilt = std::find(snapshot.named_unbuilt.begin(), snapshot.named_unbuilt.end(), name) != @@ -448,6 +458,10 @@ class SystemFieldSolver { const auto saved = snapshot.named_potentials.find(name); if (saved != snapshot.named_potentials.end()) PureFieldAlgebra::copy_allocated(field.backend->phi(), saved->second); + const auto topology = snapshot.named_topology_reports.find(name); + if (topology == snapshot.named_topology_reports.end()) + std::terminate(); + field.published_topology_report.swap(topology->second); } std::swap(diagnostics_.schema_version, snapshot.diagnostics.schema_version); diagnostics_.source.swap(snapshot.diagnostics.source); @@ -1525,6 +1539,7 @@ class SystemFieldSolver { FieldSolveConfig plan{}; std::vector prepared_providers; std::unique_ptr backend; + std::vector published_topology_report; std::optional contribution_scratch; std::optional published_phi_scratch; std::optional published_aux_scratch; @@ -1646,9 +1661,7 @@ class SystemFieldSolver { throw std::logic_error("System Program-install rollback lost a named field"); field.has_plan = saved->second.has_plan; field.plan = std::move(saved->second.plan); - field.backend.reset(); - field.nullspace_ready = false; - field.nullspace_workspace.reset(); + invalidate_named_backend_(field); } program_boundary_baselines_ = std::move(snapshot.boundary_baselines); candidate_program_boundary_slots_ = std::move(snapshot.candidate_boundary_slots); @@ -1658,6 +1671,7 @@ class SystemFieldSolver { static void invalidate_named_backend_(NamedField& field) { field.backend.reset(); + field.published_topology_report.clear(); field.nullspace_ready = false; field.nullspace_workspace.reset(); } @@ -1958,15 +1972,22 @@ class SystemFieldSolver { std::vector topology_report( const std::string& slot) const { auto field = named_fields_.find(slot); - if (field != named_fields_.end() && field->second.backend) - return field->second.backend->topology_report(); - if (named_field_plans_.find(slot) == named_field_plans_.end()) + if (field == named_fields_.end() && named_field_plans_.find(slot) == named_field_plans_.end()) throw std::runtime_error("unknown qualified field provider slot"); - // A failed attempt may leave an immutable prepared component's private topology cache warm - // for retry, while restore_step_snapshot() correctly removes the provisional backend from the - // accepted runtime. Inspection reports accepted materialization only: never leak that private - // cache as published provider evidence before a backend belongs to the accepted state. - return {}; + return field == named_fields_.end() ? std::vector{} + : field->second.published_topology_report; + } + + /// Stage live backend topology as candidate publication evidence. The surrounding + /// FieldPublicationSnapshot restores the previously accepted rows until SolveOutcome::accept(), + /// so a failed solve may retain a private warm cache without making it observable. + void stage_named_topology_reports() { + for (auto& [name, field] : named_fields_) { + (void)name; + field.published_topology_report = field.backend + ? field.backend->topology_report() + : std::vector{}; + } } template diff --git a/src/runtime/system/system_fields.cpp b/src/runtime/system/system_fields.cpp index 460c04b42..fda0379ea 100644 --- a/src/runtime/system/system_fields.cpp +++ b/src/runtime/system/system_fields.cpp @@ -233,12 +233,11 @@ SolveOutcome System::solve_fields_from_state(const std::string& field, int block }); } -SolveOutcome System::solve_fields_from_blocks( - const std::string& field, const std::vector& U_stages) { +SolveOutcome System::solve_fields_from_blocks(const std::string& field, + const std::vector& U_stages) { prepare_named_field_publication_storage_(field); - return run_field_publication_outcome_([this, &field, &U_stages]() { - return solve_fields_from_blocks_in_place_(field, U_stages); - }); + return run_field_publication_outcome_( + [this, &field, &U_stages]() { return solve_fields_from_blocks_in_place_(field, U_stages); }); } void System::prepare_default_field_publication_storage_() { @@ -352,6 +351,7 @@ void System::stage_field_publication_candidate() { if (!p_->field_publication_active_ || !p_->accepted_field_publication_ || p_->field_publication_candidate_ready_) throw std::logic_error("System field publication has no unique active candidate slot"); + p_->fields_.stage_named_topology_reports(); if (p_->candidate_field_publication_) p_->candidate_field_publication_->capture(*p_); else @@ -365,8 +365,7 @@ void System::validate_field_publication_candidate() { !p_->candidate_field_publication_ || !p_->field_publication_candidate_ready_) throw std::logic_error("System field publication has no staged candidate"); if (!p_->candidate_field_publication_->publication_layout_matches(*p_)) - throw std::logic_error( - "System field publication snapshot layout changed before Accept"); + throw std::logic_error("System field publication snapshot layout changed before Accept"); } void System::accept_field_publication_candidate() noexcept { From 95290009358aa7f94172a9e4abe17d22132791d8 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 02:13:15 +0200 Subject: [PATCH 56/62] fix(fields): preserve prepared solver views on rollback --- include/pops/runtime/system/system_field_solver.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/pops/runtime/system/system_field_solver.hpp b/include/pops/runtime/system/system_field_solver.hpp index d75817925..37b9586ee 100644 --- a/include/pops/runtime/system/system_field_solver.hpp +++ b/include/pops/runtime/system/system_field_solver.hpp @@ -830,7 +830,11 @@ class SystemFieldSolver { return FieldDistribution::Distributed; } [[nodiscard]] MultiFab snapshot() override { return MultiFab(phi_); } - void restore(const MultiFab& value) override { phi_ = value; } + void restore(const MultiFab& value) override { + // The prepared FieldSolver request borrows phi_'s stable storage. Rollback restores values + // without replacing that allocation, so the cached ABI views remain valid for an exact retry. + PureFieldAlgebra::copy_allocated(phi_, value); + } void configure_boundary(FieldSolveConfig& plan) override { if (plan.has_boundary_kernel) throw std::runtime_error( From 0a90bfec582a294b3766f0b0e894d4e2ea60daf0 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:06:00 +0200 Subject: [PATCH 57/62] fix(fields): preserve singleton provider diagnostics --- .../pops/runtime/system/system_field_solver.hpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/include/pops/runtime/system/system_field_solver.hpp b/include/pops/runtime/system/system_field_solver.hpp index 37b9586ee..f13da0545 100644 --- a/include/pops/runtime/system/system_field_solver.hpp +++ b/include/pops/runtime/system/system_field_solver.hpp @@ -2980,15 +2980,21 @@ class SystemFieldSolver { template void require_collective_named_phase_(std::string_view phase, Phase&& action) const { - bool failed = false; + std::exception_ptr local_failure; try { std::forward(action)(); } catch (...) { - failed = true; - } - if (all_reduce_max(failed ? 1L : 0L) != 0) + local_failure = std::current_exception(); + } + if (all_reduce_max(local_failure ? 1L : 0L) != 0) { + // A singleton execution has no remote failure to hide. Preserve the provider's exact + // diagnostic instead of replacing it with a collective summary; multi-rank execution still + // reports one rank-independent error after every participant reaches the reduction. + if (n_ranks() == 1 && local_failure) + std::rethrow_exception(local_failure); throw std::runtime_error("System: named field " + std::string(phase) + " failed on at least one communicator rank"); + } } void require_collective_field_providers_(NamedField& field) { From c7f713366838ca5ef3365384a466f97899e41c43 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:16:33 +0200 Subject: [PATCH 58/62] fix(fields): validate prepared patch source identity --- include/pops/runtime/system/prepared_field_solver_component.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pops/runtime/system/prepared_field_solver_component.hpp b/include/pops/runtime/system/prepared_field_solver_component.hpp index 2ee2c1ac3..a7b8d26b5 100644 --- a/include/pops/runtime/system/prepared_field_solver_component.hpp +++ b/include/pops/runtime/system/prepared_field_solver_component.hpp @@ -439,7 +439,7 @@ class PreparedFieldSolverComponent final { geometry.ylo + static_cast(box.lo[1]) * geometry.dy() || patch.cell_spacing[0] != geometry.dx() || patch.cell_spacing[1] != geometry.dy() || patch.layout_identity == nullptr || patch.patch_identity == nullptr || - materialized_layout_identity_ != patch.layout_identity || + spec_.source_layout_identity != patch.layout_identity || patch_identities_[index] != patch.patch_identity) throw std::runtime_error( "prepared external field topology cannot be reused after a layout change"); From d1890b1082af6c9b745eb45c9272974dc286e2ec Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:30:26 +0200 Subject: [PATCH 59/62] fix(runtime): narrow parameter report indices --- python/pops/runtime/_multi_layout_executor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pops/runtime/_multi_layout_executor.py b/python/pops/runtime/_multi_layout_executor.py index 890e418bf..fdeb07be3 100644 --- a/python/pops/runtime/_multi_layout_executor.py +++ b/python/pops/runtime/_multi_layout_executor.py @@ -443,11 +443,15 @@ def _ordered_program_reports(self) -> tuple[tuple[Any, tuple[str, ...], Any], .. or not isinstance(index, int) for index in parameter_blocks ) - or tuple(sorted(parameter_blocks)) != tuple(range(len(local_map))) ): raise RuntimeError( "multi-layout child Program parameter report is not exact" ) + exact_parameter_blocks = cast(tuple[int, ...], parameter_blocks) + if tuple(sorted(exact_parameter_blocks)) != tuple(range(len(local_map))): + raise RuntimeError( + "multi-layout child Program parameter report is not exact" + ) rows.append((layout_program, engine_blocks, report)) return tuple(rows) From 3d0ef5c9768e91b1aa171db5f52cab1625e1e0d2 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:34:48 +0200 Subject: [PATCH 60/62] test(consumers): authenticate distributed fake receipts --- tests/python/unit/runtime/test_consumer_transactions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/python/unit/runtime/test_consumer_transactions.py b/tests/python/unit/runtime/test_consumer_transactions.py index 072da4c1c..ceef81318 100644 --- a/tests/python/unit/runtime/test_consumer_transactions.py +++ b/tests/python/unit/runtime/test_consumer_transactions.py @@ -150,11 +150,19 @@ def publish(self): self.publisher.temporaries.remove(self.temp_id) artifact = "artifact-%s" % self.effect.payload.identity.hexdigest[:12] self.publisher.artifacts.add(artifact) + mode = self.effect.target.parallel_mode + rank_artifacts = () + if mode is ParallelMode.PER_RANK: + rank_artifacts = tuple( + (rank, "%s-r%d" % (artifact, rank)) for rank in range(2) + ) return PublicationReceipt( self.effect.identity, self.effect.payload.identity, "test-publisher", artifact, + parallel_mode=mode, + rank_artifacts=rank_artifacts, ) def discard(self): From 9e4ed7f67529fc074baec1c7e01adc26f71382cc Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:47:40 +0200 Subject: [PATCH 61/62] fix(m4): carry repository imports into MPI proofs --- scripts/run_m4_gate.py | 11 ++++++++++- tests/python/architecture/test_m4_runtime_io_gate.py | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/run_m4_gate.py b/scripts/run_m4_gate.py index 5fb6ff486..615bbbd92 100644 --- a/scripts/run_m4_gate.py +++ b/scripts/run_m4_gate.py @@ -136,7 +136,7 @@ def _forbidden_python_markers(node: ast.AST) -> list[str]: ): markers.append(name) elif isinstance(child, (ast.Import, ast.ImportFrom)): - module = child.module if isinstance(child, ast.ImportFrom) else "" + module = (child.module or "") if isinstance(child, ast.ImportFrom) else "" names = [alias.name for alias in child.names] if module.startswith(("unittest.mock", "pytest_mock")) or any( name.startswith(("unittest.mock", "pytest_mock")) for name in names @@ -640,6 +640,15 @@ def _required_environment() -> dict[str, str]: environment = os.environ.copy() environment["POPS_REQUIRE_MPI_TESTS"] = "1" environment["POPS_REQUIRE_NATIVE_TESTS"] = "1" + root = str(ROOT) + inherited = environment.get("PYTHONPATH", "") + python_path = [root] + python_path.extend( + entry + for entry in inherited.split(os.pathsep) + if entry and entry != root + ) + environment["PYTHONPATH"] = os.pathsep.join(python_path) return environment diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index 2d2170505..bc0e0b913 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -758,7 +758,9 @@ def test_m4_mpi_entrypoint_accepts_only_the_required_prerequisite_guard(): "tests/python/integration/mpi/test_scientific_output_mpi.py::" "_validate_paraview" ) - assert runner._required_environment()["POPS_REQUIRE_MPI_TESTS"] == "1" + environment = runner._required_environment() + assert environment["POPS_REQUIRE_MPI_TESTS"] == "1" + assert str(ROOT) in environment["PYTHONPATH"].split(runner.os.pathsep) trusted = ast.parse( "from tests.python.support.requirements import require_mpi_or_skip\n" ) From c53b3d0da0993d83829aaa20526e3f56dcc98d10 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 10:58:52 +0200 Subject: [PATCH 62/62] ci(m4): bypass broken OpenMPI OMPIO path --- .github/workflows/ci.yml | 5 +++++ tests/python/architecture/test_m4_runtime_io_gate.py | 1 + 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f89a9094b..c33edc616 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1962,6 +1962,11 @@ jobs: POPS_KEEP_GENERATED: "1" POPS_REQUIRE_MPI_TESTS: "1" POPS_REQUIRE_NATIVE_TESTS: "1" + # Ubuntu 24.04 OpenMPI 4 OMPIO selects sharedfp/lockedfile during + # HDF5 MPI_File_open and aborts inside its fortified sprintf path. + # ROMIO is the packaged OpenMPI MPI-IO component and exercises the + # same collective HDF5 contract without that implementation defect. + OMPI_MCA_io: "^ompio" run: | # These readers are mandatory capabilities of this lane. Imports happen before the gate # so a missing apt module cannot masquerade as a scientific skip. diff --git a/tests/python/architecture/test_m4_runtime_io_gate.py b/tests/python/architecture/test_m4_runtime_io_gate.py index bc0e0b913..59b57ea29 100644 --- a/tests/python/architecture/test_m4_runtime_io_gate.py +++ b/tests/python/architecture/test_m4_runtime_io_gate.py @@ -117,6 +117,7 @@ def test_m4_required_ci_lane_executes_the_complete_installed_gate(): complete = complete.split("- name: ccache stats (MPI)", 1)[0] assert "POPS_REQUIRE_MPI_TESTS: \"1\"" in complete assert "POPS_REQUIRE_NATIVE_TESTS: \"1\"" in complete + assert 'OMPI_MCA_io: "^ompio"' in complete assert "vtkXMLPUnstructuredGridReader" in complete assert "vtkXMLUnstructuredGridReader" in complete assert "/usr/bin/python3 scripts/run_m4_gate.py \\" in complete