[ADC-689] Prove parity of the installed public API

.github/workflows/release.yml

@@ -61,18 +61,21 @@ jobs:
           wheels=("$RUNNER_TEMP"/wheelhouse/pops-*.whl)
           test "${#wheels[@]}" -eq 1
           evidence="$RUNNER_TEMP/pops-final-evidence.json"
+          public_api_evidence="$RUNNER_TEMP/pops-final-evidence-public-api.json"
+          python scripts/run_final_gate.py --wheel "${wheels[0]}" --evidence "$evidence"
           python scripts/prove_public_api_parity.py \
             --wheel "${wheels[0]}" \
-            --evidence "$RUNNER_TEMP/pops-final-evidence-public-api.json"
-          python scripts/run_final_gate.py --wheel "${wheels[0]}" --evidence "$evidence"
+            --installed \
+            --evidence "$public_api_evidence"
           python - <<'PY'
           from pops.runtime_environment import runtime_environment_report
           report = runtime_environment_report()
           assert report["kokkos_backend"] == "Serial", report
           assert report["mpi_compiled"] is False, report
           PY
           python scripts/release_preflight.py \
-            --release --tag "$GITHUB_REF_NAME" --installed --evidence "$evidence"
+            --release --tag "$GITHUB_REF_NAME" --installed --evidence "$evidence" \
+            --public-api-evidence "$public_api_evidence"
 
       - name: Retain authenticated release evidence
         uses: actions/upload-artifact@v7

docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md

@@ -1536,14 +1536,23 @@ dans `examples/final/`. Chaque script doit :
 
 ## 14. Gate de conformance finale
 
-Le job de release commence par
-`scripts/prove_public_api_parity.py --wheel <wheel> --evidence <chemin-hors-checkout>`.
-Cette preuve compare octet par octet tous les fichiers Python et de typage (`*.py`, `*.pyi`,
-`py.typed`) du checkout et du wheel retenu, puis importe séparément les deux arbres dans des
-interpréteurs isolés. Les deux snapshots doivent exposer la même racine publique, les mêmes
-signatures et annotations, un `Case` explicite, des handles qualifiés distincts et
+Le job de release exécute d'abord
+`scripts/run_final_gate.py --wheel <wheel> --evidence <chemin-hors-checkout>`. Ce gate installe
+l'artefact exact avant que
+`scripts/prove_public_api_parity.py --wheel <wheel> --installed --evidence <autre-chemin-hors-checkout>`
+ne résolve la distribution installée avec `importlib.metadata`, sans importer `pops` dans le
+processus du gate. Le chemin résolu doit être extérieur au checkout. La preuve compare octet par
+octet tous les fichiers Python et de typage (`*.py`, `*.pyi`, `py.typed`) du checkout, du wheel
+retenu et du package installé, puis importe séparément les trois arbres dans des interpréteurs
+isolés. Les trois snapshots doivent exposer la même racine publique, les mêmes signatures et
+annotations, un `Case` explicite, des handles qualifiés distincts et
 authoring/validation/inspection sans chargement de `_pops`. Un ancien nom public, un fichier de
-typage absent ou une divergence source/wheel bloque la publication.
+typage absent, un chemin provenant du checkout ou une divergence source/wheel/installé bloque la
+publication. La preuve authentifie aussi le `Name`, la `Version` et le digest du `METADATA` de la
+distribution installée contre ceux du wheel. Enfin `release_preflight.py` reçoit cette evidence via
+`--public-api-evidence` et vérifie son producteur, le SHA-256 du wheel et le chemin du package contre
+le même runtime installé que l'evidence finale ; une evidence de parité issue d'un autre wheel ou
+d'une autre installation ne peut donc pas être réutilisée.
 
 Une release ne peut être déclarée conforme que par
 `scripts/run_final_gate.py --evidence <chemin-hors-checkout>`. La commande exige un checkout propre,

docs/docmap.toml

@@ -85,6 +85,7 @@ depends_on = [
   "python/pops/problem/problem.py",
   "python/pops/time/_program/api.py",
   "scripts/prove_public_api_parity.py",
+  "scripts/release_preflight.py",
   ".github/workflows/release.yml",
   "examples/final/EXEMPLE_SPEC_FINALE_ADVECTION_SCALAIRE_COMPLET.py",
   "examples/final/EXEMPLE_SPEC_FINALE_MULTIPHYSIQUE_CORE.py",

scripts/prove_public_api_parity.py

@@ -5,7 +5,10 @@
 
 import argparse
 from collections.abc import Mapping, Sequence
+from email import policy
+from email.parser import BytesParser
 import hashlib
+import importlib.metadata
 import json
 from pathlib import Path, PurePosixPath
 import subprocess
@@ -17,7 +20,7 @@
 
 ROOT = Path(__file__).resolve().parents[1]
 SOURCE_PACKAGE = ROOT / "python" / "pops"
-PROOF_SCHEMA_VERSION = 1
+PROOF_SCHEMA_VERSION = 3
 TYPED_PAYLOAD_SUFFIXES = (".py", ".pyi")
 PUBLIC_ROOT = (
     "Model",
@@ -148,6 +151,7 @@ def _symbol(name):
 snapshot = {
     "public": public,
     "symbols": {name: _symbol(name) for name in public},
+    "package_version": pops.__version__,
     "case_is_explicit_type": True,
     "qualified_handles": True,
     "pure_authoring": True,
@@ -174,9 +178,9 @@ def _is_typed_payload(relative: str) -> bool:
     return path.name == "py.typed" or path.suffix in TYPED_PAYLOAD_SUFFIXES
 
 
-def _source_manifest(package: Path = SOURCE_PACKAGE) -> dict[str, str]:
+def _typed_manifest(package: Path, *, label: str) -> dict[str, str]:
     if not package.is_dir():
-        raise PublicApiParityError("source package is absent: %s" % package)
+        raise PublicApiParityError("%s package is absent: %s" % (label, package))
     manifest = {
         path.relative_to(package).as_posix(): _sha256(path)
         for path in sorted(package.rglob("*"))
@@ -186,7 +190,7 @@ def _source_manifest(package: Path = SOURCE_PACKAGE) -> dict[str, str]:
     }
     required = {"__init__.py", "_pops.pyi", "py.typed"}
     if not required.issubset(manifest):
-        raise PublicApiParityError("source package lacks its root API or typing payload")
+        raise PublicApiParityError("%s package lacks its root API or typing payload" % label)
     return manifest
 
 
@@ -210,6 +214,38 @@ def _wheel_manifest(archive: zipfile.ZipFile) -> dict[str, str]:
     return manifest
 
 
+def _distribution_identity(payload: bytes, *, label: str) -> dict[str, str]:
+    try:
+        metadata = BytesParser(policy=policy.default).parsebytes(payload)
+    except (TypeError, ValueError) as exc:
+        raise PublicApiParityError("%s distribution METADATA is unreadable" % label) from exc
+    name = metadata.get("Name")
+    version = metadata.get("Version")
+    if not isinstance(name, str) or not name.strip() \
+            or not isinstance(version, str) or not version.strip():
+        raise PublicApiParityError(
+            "%s distribution METADATA has no exact Name/Version" % label)
+    normalized = name.strip().lower().replace("_", "-").replace(".", "-")
+    if normalized != "pops":
+        raise PublicApiParityError("%s distribution name is not PoPS" % label)
+    return {
+        "name": name.strip(),
+        "version": version.strip(),
+        "metadata_sha256": _sha256_bytes(payload),
+    }
+
+
+def _wheel_distribution_identity(archive: zipfile.ZipFile) -> dict[str, str]:
+    names = [info.filename for info in archive.infolist() if not info.is_dir()]
+    if len(names) != len(set(names)):
+        raise PublicApiParityError("release wheel contains duplicate members")
+    metadata_names = [name for name in names if name.endswith(".dist-info/METADATA")]
+    if len(metadata_names) != 1:
+        raise PublicApiParityError("release wheel has no unique distribution METADATA")
+    return _distribution_identity(
+        archive.read(metadata_names[0]), label="wheel")
+
+
 def _safe_extract(archive: zipfile.ZipFile, destination: Path) -> None:
     for info in archive.infolist():
         relative = PurePosixPath(info.filename)
@@ -251,51 +287,150 @@ def _canonical_sha256(payload: Mapping[str, Any]) -> str:
     return _sha256_bytes(encoded)
 
 
-def build_proof(wheel: Path) -> dict[str, Any]:
+def _require_manifest_parity(
+    reference: Mapping[str, str],
+    candidate: Mapping[str, str],
+    *,
+    label: str,
+) -> None:
+    if candidate == reference:
+        return
+    missing = sorted(set(reference) - set(candidate))
+    extra = sorted(set(candidate) - set(reference))
+    changed = sorted(
+        name
+        for name in set(reference) & set(candidate)
+        if reference[name] != candidate[name]
+    )
+    raise PublicApiParityError(
+        "%s Python/typing payload differs from source "
+        "(missing=%s, extra=%s, changed=%s)"
+        % (label, missing[:8], extra[:8], changed[:8])
+    )
+
+
+def _installed_package_from_distribution() -> tuple[Path, dict[str, str]]:
+    try:
+        distribution = importlib.metadata.distribution("PoPS")
+    except importlib.metadata.PackageNotFoundError as exc:
+        raise PublicApiParityError("the PoPS distribution is not installed") from exc
+    files = distribution.files
+    if files is None:
+        raise PublicApiParityError("the installed PoPS distribution has no file inventory")
+    metadata_files = [
+        row
+        for row in files
+        if PurePosixPath(str(row)).as_posix().endswith(".dist-info/METADATA")
+    ]
+    if len(metadata_files) != 1:
+        raise PublicApiParityError(
+            "the installed PoPS distribution has no unique METADATA")
+    metadata_path = Path(distribution.locate_file(metadata_files[0])).resolve()
+    if not metadata_path.is_file():
+        raise PublicApiParityError("the installed PoPS distribution METADATA is absent")
+    identity = _distribution_identity(metadata_path.read_bytes(), label="installed")
+    package_initializers = [
+        row for row in files if PurePosixPath(str(row)).as_posix() == "pops/__init__.py"
+    ]
+    if len(package_initializers) != 1:
+        raise PublicApiParityError(
+            "the installed PoPS distribution has no unique pops/__init__.py")
+    package = Path(distribution.locate_file(package_initializers[0])).resolve().parent
+    if not package.is_dir():
+        raise PublicApiParityError("the installed PoPS package directory is absent")
+    try:
+        package.relative_to(ROOT)
+    except ValueError:
+        return package, identity
+    raise PublicApiParityError(
+        "the installed-package proof resolved inside the source checkout: %s" % package)
+
+
+def build_proof(
+    wheel: Path,
+    *,
+    installed_package: Path | None = None,
+    installed_distribution: Mapping[str, str] | None = None,
+) -> dict[str, Any]:
     """Compare one exact wheel archive with the current source checkout."""
     retained = wheel.expanduser().resolve()
     if retained.suffix != ".whl" or not retained.is_file():
         raise PublicApiParityError("release artifact is not one readable wheel")
-    source_manifest = _source_manifest()
+    source_manifest = _typed_manifest(SOURCE_PACKAGE, label="source")
+    installed = None if installed_package is None else installed_package.expanduser().resolve()
+    if installed_distribution is not None and installed is None:
+        raise PublicApiParityError(
+            "installed distribution identity requires an installed package")
+    if installed is not None:
+        try:
+            installed.relative_to(ROOT)
+        except ValueError:
+            pass
+        else:
+            raise PublicApiParityError(
+                "the installed-package proof resolved inside the source checkout: %s" % installed)
     try:
         with tempfile.TemporaryDirectory(prefix="pops-public-api-") as temporary:
             extracted = Path(temporary)
             with zipfile.ZipFile(retained) as archive:
+                wheel_distribution = _wheel_distribution_identity(archive)
                 wheel_manifest = _wheel_manifest(archive)
-                if wheel_manifest != source_manifest:
-                    missing = sorted(set(source_manifest) - set(wheel_manifest))
-                    extra = sorted(set(wheel_manifest) - set(source_manifest))
-                    changed = sorted(
-                        name
-                        for name in set(source_manifest) & set(wheel_manifest)
-                        if source_manifest[name] != wheel_manifest[name]
-                    )
-                    raise PublicApiParityError(
-                        "wheel Python/typing payload differs from source "
-                        "(missing=%s, extra=%s, changed=%s)"
-                        % (missing[:8], extra[:8], changed[:8])
-                    )
+                _require_manifest_parity(
+                    source_manifest, wheel_manifest, label="wheel")
                 _safe_extract(archive, extracted)
             source_snapshot = _snapshot(SOURCE_PACKAGE.parent)
             wheel_snapshot = _snapshot(extracted)
+            if installed is not None:
+                installed_manifest = _typed_manifest(installed, label="installed")
+                _require_manifest_parity(
+                    source_manifest, installed_manifest, label="installed")
+                installed_snapshot = _snapshot(installed.parent)
     except (OSError, zipfile.BadZipFile) as exc:
         raise PublicApiParityError("release wheel is unreadable: %s" % exc) from exc
     if wheel_snapshot != source_snapshot:
         raise PublicApiParityError("wheel and source public API snapshots differ")
+    if source_snapshot.get("package_version") != wheel_distribution["version"]:
+        raise PublicApiParityError(
+            "source public API version differs from wheel distribution METADATA")
+    if installed is not None and installed_snapshot != source_snapshot:
+        raise PublicApiParityError("installed and source public API snapshots differ")
+    if installed_distribution is not None:
+        exact_installed_distribution = dict(installed_distribution)
+        if set(exact_installed_distribution) != {"name", "version", "metadata_sha256"}:
+            raise PublicApiParityError("installed distribution identity is malformed")
+        if exact_installed_distribution != wheel_distribution:
+            raise PublicApiParityError(
+                "installed distribution identity differs from wheel METADATA")
     if tuple(source_snapshot["public"]) != PUBLIC_ROOT:
         raise PublicApiParityError("public API snapshot differs from the final root contract")
-    return {
+    proof = {
         "schema_version": PROOF_SCHEMA_VERSION,
+        "producer": {
+            "script": "scripts/prove_public_api_parity.py",
+            "sha256": _sha256(Path(__file__).resolve()),
+        },
         "wheel_path": str(retained),
         "wheel_sha256": _sha256(retained),
+        "distribution": wheel_distribution,
         "typed_payload_files": len(source_manifest),
         "typed_payload_sha256": _canonical_sha256(source_manifest),
         "public_api_sha256": _canonical_sha256(source_snapshot),
         "public_names": source_snapshot["public"],
         "pure_authoring": source_snapshot["pure_authoring"],
         "qualified_handles": source_snapshot["qualified_handles"],
         "py_typed": source_snapshot["py_typed"],
+        "installed": installed is not None,
+        "installed_distribution": (
+            None if installed_distribution is None else dict(installed_distribution)
+        ),
     }
+    if installed is not None:
+        proof.update({
+            "installed_package": str(installed),
+            "installed_typed_payload_sha256": _canonical_sha256(installed_manifest),
+            "installed_public_api_sha256": _canonical_sha256(installed_snapshot),
+        })
+    return proof
 
 
 def _write_evidence(path: Path, proof: Mapping[str, Any]) -> None:
@@ -321,10 +456,23 @@ def _write_evidence(path: Path, proof: Mapping[str, Any]) -> None:
 def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--wheel", required=True, type=Path)
+    parser.add_argument(
+        "--installed",
+        action="store_true",
+        help="also prove the importlib.metadata-resolved installed distribution outside checkout",
+    )
     parser.add_argument("--evidence", type=Path)
     args = parser.parse_args(argv)
     try:
-        proof = build_proof(args.wheel)
+        if args.installed:
+            installed, installed_distribution = _installed_package_from_distribution()
+        else:
+            installed, installed_distribution = None, None
+        proof = build_proof(
+            args.wheel,
+            installed_package=installed,
+            installed_distribution=installed_distribution,
+        )
         if args.evidence is not None:
             _write_evidence(args.evidence, proof)
     except (PublicApiParityError, OSError, ValueError) as exc: